mkpreempt.go

     1  // Copyright 2019 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build ignore
     6  
     7  // mkpreempt generates the asyncPreempt functions for each
     8  // architecture.
     9  package main
    10  
    11  import (
    12  	"bytes"
    13  	"flag"
    14  	"fmt"
    15  	"go/format"
    16  	"io"
    17  	"log"
    18  	"os"
    19  	"strings"
    20  )
    21  
    22  // Copied from cmd/compile/internal/ssa/gen/*Ops.go
    23  
    24  var regNames386 = []string{
    25  	"AX",
    26  	"CX",
    27  	"DX",
    28  	"BX",
    29  	"SP",
    30  	"BP",
    31  	"SI",
    32  	"DI",
    33  	"X0",
    34  	"X1",
    35  	"X2",
    36  	"X3",
    37  	"X4",
    38  	"X5",
    39  	"X6",
    40  	"X7",
    41  }
    42  
    43  var regNamesAMD64 = []string{
    44  	"AX",
    45  	"CX",
    46  	"DX",
    47  	"BX",
    48  	"SP",
    49  	"BP",
    50  	"SI",
    51  	"DI",
    52  	"R8",
    53  	"R9",
    54  	"R10",
    55  	"R11",
    56  	"R12",
    57  	"R13",
    58  	"R14",
    59  	"R15",
    60  	"X0",
    61  	"X1",
    62  	"X2",
    63  	"X3",
    64  	"X4",
    65  	"X5",
    66  	"X6",
    67  	"X7",
    68  	"X8",
    69  	"X9",
    70  	"X10",
    71  	"X11",
    72  	"X12",
    73  	"X13",
    74  	"X14",
    75  	"X15",
    76  }
    77  
    78  var arches = map[string]func(g *gen){
    79  	"386":     gen386,
    80  	"amd64":   genAMD64,
    81  	"arm":     genARM,
    82  	"arm64":   genARM64,
    83  	"loong64": genLoong64,
    84  	"mips64x": func(g *gen) { genMIPS(g, true) },
    85  	"mipsx":   func(g *gen) { genMIPS(g, false) },
    86  	"ppc64x":  genPPC64,
    87  	"riscv64": genRISCV64,
    88  	"s390x":   genS390X,
    89  	"wasm":    genWasm,
    90  }
    91  var beLe = map[string]bool{"mips64x": true, "mipsx": true, "ppc64x": true}
    92  
    93  func main() {
    94  	flag.Parse()
    95  	if flag.NArg() > 0 {
    96  		for _, arch := range flag.Args() {
    97  			genFn, ok := arches[arch]
    98  			if !ok {
    99  				log.Fatalf("unknown arch %s", arch)
   100  			}
   101  			g := gen{os.Stdout, arch}
   102  			g.asmHeader()
   103  			genFn(&g)
   104  		}
   105  		return
   106  	}
   107  
   108  	for arch, genFn := range arches {
   109  		f, err := os.Create(fmt.Sprintf("preempt_%s.s", arch))
   110  		if err != nil {
   111  			log.Fatal(err)
   112  		}
   113  		g := gen{f, arch}
   114  		g.asmHeader()
   115  		genFn(&g)
   116  		if err := f.Close(); err != nil {
   117  			log.Fatal(err)
   118  		}
   119  	}
   120  }
   121  
   122  type gen struct {
   123  	w      io.Writer
   124  	goarch string
   125  }
   126  
   127  func (g *gen) commonHeader() {
   128  	fmt.Fprintf(g.w, "// Code generated by mkpreempt.go; DO NOT EDIT.\n\n")
   129  	if beLe[g.goarch] {
   130  		base := g.goarch[:len(g.goarch)-1]
   131  		fmt.Fprintf(g.w, "//go:build %s || %sle\n\n", base, base)
   132  	}
   133  }
   134  
   135  func (g *gen) asmHeader() {
   136  	g.commonHeader()
   137  	fmt.Fprintf(g.w, "#include \"go_asm.h\"\n")
   138  	if g.goarch == "amd64" {
   139  		fmt.Fprintf(g.w, "#include \"go_tls.h\"\n")
   140  		fmt.Fprintf(g.w, "#include \"asm_amd64.h\"\n")
   141  	}
   142  	fmt.Fprintf(g.w, "#include \"textflag.h\"\n\n")
   143  	fmt.Fprintf(g.w, "TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0\n")
   144  }
   145  
   146  func (g *gen) p(f string, args ...any) {
   147  	fmted := fmt.Sprintf(f, args...)
   148  	fmt.Fprintf(g.w, "\t%s\n", strings.ReplaceAll(fmted, "\n", "\n\t"))
   149  }
   150  
   151  func (g *gen) label(l string) {
   152  	fmt.Fprintf(g.w, "%s\n", l)
   153  }
   154  
   155  // writeXRegs writes an architecture xregs file.
   156  func writeXRegs(arch string, l *layout) {
   157  	var code bytes.Buffer
   158  	g := gen{&code, arch}
   159  	g.commonHeader()
   160  	fmt.Fprintf(g.w, `
   161  package runtime
   162  
   163  type xRegs struct {
   164  `)
   165  	pos := 0
   166  	for _, reg := range l.regs {
   167  		if reg.pos != pos {
   168  			log.Fatalf("padding not implemented")
   169  		}
   170  		typ := fmt.Sprintf("[%d]byte", reg.size)
   171  		switch {
   172  		case reg.size == 4 && reg.pos%4 == 0:
   173  			typ = "uint32"
   174  		case reg.size == 8 && reg.pos%8 == 0:
   175  			typ = "uint64"
   176  		}
   177  		fmt.Fprintf(g.w, "\t%s %s\n", reg.reg, typ)
   178  		pos += reg.size
   179  	}
   180  	fmt.Fprintf(g.w, "}\n")
   181  
   182  	path := fmt.Sprintf("preempt_%s.go", arch)
   183  	b, err := format.Source(code.Bytes())
   184  	if err != nil {
   185  		log.Fatalf("formatting %s: %s", path, err)
   186  	}
   187  	if err := os.WriteFile(path, b, 0666); err != nil {
   188  		log.Fatal(err)
   189  	}
   190  }
   191  
   192  type layout struct {
   193  	stack int
   194  	regs  []regPos
   195  	sp    string // stack pointer register
   196  }
   197  
   198  type regPos struct {
   199  	pos, size int
   200  
   201  	saveOp    string
   202  	restoreOp string
   203  	reg       string
   204  
   205  	// If this register requires special save and restore, these
   206  	// give those operations with a %d placeholder for the stack
   207  	// offset.
   208  	save, restore string
   209  }
   210  
   211  func (l *layout) add(op, reg string, size int) {
   212  	l.regs = append(l.regs, regPos{saveOp: op, restoreOp: op, reg: reg, pos: l.stack, size: size})
   213  	l.stack += size
   214  }
   215  
   216  func (l *layout) add2(sop, rop, reg string, size int) {
   217  	l.regs = append(l.regs, regPos{saveOp: sop, restoreOp: rop, reg: reg, pos: l.stack, size: size})
   218  	l.stack += size
   219  }
   220  
   221  func (l *layout) addSpecial(save, restore string, size int) {
   222  	l.regs = append(l.regs, regPos{save: save, restore: restore, pos: l.stack, size: size})
   223  	l.stack += size
   224  }
   225  
   226  func (l *layout) save(g *gen) {
   227  	for _, reg := range l.regs {
   228  		if reg.save != "" {
   229  			g.p(reg.save, reg.pos)
   230  		} else {
   231  			g.p("%s %s, %d(%s)", reg.saveOp, reg.reg, reg.pos, l.sp)
   232  		}
   233  	}
   234  }
   235  
   236  func (l *layout) restore(g *gen) {
   237  	for i := len(l.regs) - 1; i >= 0; i-- {
   238  		reg := l.regs[i]
   239  		if reg.restore != "" {
   240  			g.p(reg.restore, reg.pos)
   241  		} else {
   242  			g.p("%s %d(%s), %s", reg.restoreOp, reg.pos, l.sp, reg.reg)
   243  		}
   244  	}
   245  }
   246  
   247  func gen386(g *gen) {
   248  	p := g.p
   249  
   250  	p("PUSHFL")
   251  	// Save general purpose registers.
   252  	var l = layout{sp: "SP"}
   253  	for _, reg := range regNames386 {
   254  		if reg == "SP" || strings.HasPrefix(reg, "X") {
   255  			continue
   256  		}
   257  		l.add("MOVL", reg, 4)
   258  	}
   259  
   260  	softfloat := "GO386_softfloat"
   261  
   262  	// Save SSE state only if supported.
   263  	lSSE := layout{stack: l.stack, sp: "SP"}
   264  	for i := 0; i < 8; i++ {
   265  		lSSE.add("MOVUPS", fmt.Sprintf("X%d", i), 16)
   266  	}
   267  
   268  	p("ADJSP $%d", lSSE.stack)
   269  	p("NOP SP")
   270  	l.save(g)
   271  	p("#ifndef %s", softfloat)
   272  	lSSE.save(g)
   273  	p("#endif")
   274  	p("CALL ·asyncPreempt2(SB)")
   275  	p("#ifndef %s", softfloat)
   276  	lSSE.restore(g)
   277  	p("#endif")
   278  	l.restore(g)
   279  	p("ADJSP $%d", -lSSE.stack)
   280  
   281  	p("POPFL")
   282  	p("RET")
   283  }
   284  
   285  func genAMD64(g *gen) {
   286  	const xReg = "AX" // *xRegState
   287  
   288  	p, label := g.p, g.label
   289  
   290  	// Assign stack offsets.
   291  	var l = layout{sp: "SP"}
   292  	for _, reg := range regNamesAMD64 {
   293  		if reg == "SP" || reg == "BP" {
   294  			continue
   295  		}
   296  		if !strings.HasPrefix(reg, "X") {
   297  			l.add("MOVQ", reg, 8)
   298  		}
   299  	}
   300  	// Create layouts for X, Y, and Z registers.
   301  	const (
   302  		numXRegs = 16
   303  		numZRegs = 16 // TODO: If we start using upper registers, change to 32
   304  		numKRegs = 8
   305  	)
   306  	lZRegs := layout{sp: xReg} // Non-GP registers
   307  	lXRegs, lYRegs := lZRegs, lZRegs
   308  	for i := range numZRegs {
   309  		lZRegs.add("VMOVDQU64", fmt.Sprintf("Z%d", i), 512/8)
   310  		if i < numXRegs {
   311  			// Use SSE-only instructions for X registers.
   312  			lXRegs.add("MOVUPS", fmt.Sprintf("X%d", i), 128/8)
   313  			lYRegs.add("VMOVDQU", fmt.Sprintf("Y%d", i), 256/8)
   314  		}
   315  	}
   316  	for i := range numKRegs {
   317  		lZRegs.add("KMOVQ", fmt.Sprintf("K%d", i), 8)
   318  	}
   319  	// The Z layout is the most general, so we line up the others with that one.
   320  	// We don't have to do this, but it results in a nice Go type. If we split
   321  	// this into multiple types, we probably should stop doing this.
   322  	for i := range lXRegs.regs {
   323  		lXRegs.regs[i].pos = lZRegs.regs[i].pos
   324  		lYRegs.regs[i].pos = lZRegs.regs[i].pos
   325  	}
   326  	writeXRegs(g.goarch, &lZRegs)
   327  
   328  	p("PUSHQ BP")
   329  	p("MOVQ SP, BP")
   330  	p("// Save flags before clobbering them")
   331  	p("PUSHFQ")
   332  	p("// obj doesn't understand ADD/SUB on SP, but does understand ADJSP")
   333  	p("ADJSP $%d", l.stack)
   334  	p("// But vet doesn't know ADJSP, so suppress vet stack checking")
   335  	p("NOP SP")
   336  
   337  	p("// Save GPs")
   338  	l.save(g)
   339  
   340  	// In general, the limitations on asynchronous preemption mean we only
   341  	// preempt in ABIInternal code. However, there's at least one exception to
   342  	// this: when we're in an open-coded transition between an ABIInternal
   343  	// function and an ABI0 call. We could more carefully arrange unsafe points
   344  	// to avoid ever landing in ABI0, but it's easy to just make this code not
   345  	// sensitive to the ABI we're preempting. The CALL to asyncPreempt2 will
   346  	// ensure we're in ABIInternal register state.
   347  	p("// Save extended register state to p.xRegs.scratch")
   348  	p("// Don't make assumptions about ABI register state. See mkpreempt.go")
   349  	p("get_tls(CX)")
   350  	p("MOVQ g(CX), R14")
   351  	p("MOVQ g_m(R14), %s", xReg)
   352  	p("MOVQ m_p(%s), %s", xReg, xReg)
   353  	p("LEAQ (p_xRegs+xRegPerP_scratch)(%s), %s", xReg, xReg)
   354  
   355  	// Which registers do we need to save?
   356  	p("#ifdef GOEXPERIMENT_simd")
   357  	p("CMPB internal∕cpu·X86+const_offsetX86HasAVX512(SB), $1")
   358  	p("JE saveAVX512")
   359  	p("CMPB internal∕cpu·X86+const_offsetX86HasAVX2(SB), $1")
   360  	p("JE saveAVX2")
   361  	p("#endif")
   362  
   363  	// No features. Assume only SSE.
   364  	label("saveSSE:")
   365  	lXRegs.save(g)
   366  	p("JMP preempt")
   367  
   368  	label("saveAVX2:")
   369  	lYRegs.save(g)
   370  	p("JMP preempt")
   371  
   372  	label("saveAVX512:")
   373  	lZRegs.save(g)
   374  	p("JMP preempt")
   375  
   376  	label("preempt:")
   377  	p("CALL ·asyncPreempt2(SB)")
   378  
   379  	p("// Restore non-GPs from *p.xRegs.cache")
   380  	p("MOVQ g_m(R14), %s", xReg)
   381  	p("MOVQ m_p(%s), %s", xReg, xReg)
   382  	p("MOVQ (p_xRegs+xRegPerP_cache)(%s), %s", xReg, xReg)
   383  
   384  	p("#ifdef GOEXPERIMENT_simd")
   385  	p("CMPB internal∕cpu·X86+const_offsetX86HasAVX512(SB), $1")
   386  	p("JE restoreAVX512")
   387  	p("CMPB internal∕cpu·X86+const_offsetX86HasAVX2(SB), $1")
   388  	p("JE restoreAVX2")
   389  	p("#endif")
   390  
   391  	label("restoreSSE:")
   392  	lXRegs.restore(g)
   393  	p("JMP restoreGPs")
   394  
   395  	label("restoreAVX2:")
   396  	lYRegs.restore(g)
   397  	p("JMP restoreGPs")
   398  
   399  	label("restoreAVX512:")
   400  	lZRegs.restore(g)
   401  	p("JMP restoreGPs")
   402  
   403  	label("restoreGPs:")
   404  	p("// Restore GPs")
   405  	l.restore(g)
   406  	p("ADJSP $%d", -l.stack)
   407  	p("POPFQ")
   408  	p("POPQ BP")
   409  	p("RET")
   410  }
   411  
   412  func genARM(g *gen) {
   413  	p := g.p
   414  
   415  	// Add integer registers R0-R12.
   416  	// R13 (SP), R14 (LR), R15 (PC) are special and not saved here.
   417  	var l = layout{sp: "R13", stack: 4} // add LR slot
   418  	for i := 0; i <= 12; i++ {
   419  		reg := fmt.Sprintf("R%d", i)
   420  		if i == 10 {
   421  			continue // R10 is g register, no need to save/restore
   422  		}
   423  		l.add("MOVW", reg, 4)
   424  	}
   425  	// Add flag register.
   426  	l.addSpecial(
   427  		"MOVW CPSR, R0\nMOVW R0, %d(R13)",
   428  		"MOVW %d(R13), R0\nMOVW R0, CPSR",
   429  		4)
   430  
   431  	// Add floating point registers F0-F15 and flag register.
   432  	var lfp = layout{stack: l.stack, sp: "R13"}
   433  	lfp.addSpecial(
   434  		"MOVW FPCR, R0\nMOVW R0, %d(R13)",
   435  		"MOVW %d(R13), R0\nMOVW R0, FPCR",
   436  		4)
   437  	for i := 0; i <= 15; i++ {
   438  		reg := fmt.Sprintf("F%d", i)
   439  		lfp.add("MOVD", reg, 8)
   440  	}
   441  
   442  	p("MOVW.W R14, -%d(R13)", lfp.stack) // allocate frame, save LR
   443  	l.save(g)
   444  	p("MOVB ·goarmsoftfp(SB), R0\nCMP $0, R0\nBNE nofp") // test goarmsoftfp, and skip FP registers if goarmsoftfp!=0.
   445  	lfp.save(g)
   446  	g.label("nofp:")
   447  	p("CALL ·asyncPreempt2(SB)")
   448  	p("MOVB ·goarmsoftfp(SB), R0\nCMP $0, R0\nBNE nofp2") // test goarmsoftfp, and skip FP registers if goarmsoftfp!=0.
   449  	lfp.restore(g)
   450  	g.label("nofp2:")
   451  	l.restore(g)
   452  
   453  	p("MOVW %d(R13), R14", lfp.stack)     // sigctxt.pushCall pushes LR on stack, restore it
   454  	p("MOVW.P %d(R13), R15", lfp.stack+4) // load PC, pop frame (including the space pushed by sigctxt.pushCall)
   455  	p("UNDEF")                            // shouldn't get here
   456  }
   457  
   458  func genARM64(g *gen) {
   459  	p := g.p
   460  	// Add integer registers R0-R26
   461  	// R27 (REGTMP), R28 (g), R29 (FP), R30 (LR), R31 (SP) are special
   462  	// and not saved here.
   463  	var l = layout{sp: "RSP", stack: 8} // add slot to save PC of interrupted instruction
   464  	for i := 0; i < 26; i += 2 {
   465  		if i == 18 {
   466  			i--
   467  			continue // R18 is not used, skip
   468  		}
   469  		reg := fmt.Sprintf("(R%d, R%d)", i, i+1)
   470  		l.add2("STP", "LDP", reg, 16)
   471  	}
   472  	// Add flag registers.
   473  	l.addSpecial(
   474  		"MOVD NZCV, R0\nMOVD R0, %d(RSP)",
   475  		"MOVD %d(RSP), R0\nMOVD R0, NZCV",
   476  		8)
   477  	l.addSpecial(
   478  		"MOVD FPSR, R0\nMOVD R0, %d(RSP)",
   479  		"MOVD %d(RSP), R0\nMOVD R0, FPSR",
   480  		8)
   481  	// TODO: FPCR? I don't think we'll change it, so no need to save.
   482  	// Add floating point registers F0-F31.
   483  	for i := 0; i < 31; i += 2 {
   484  		reg := fmt.Sprintf("(F%d, F%d)", i, i+1)
   485  		l.add2("FSTPD", "FLDPD", reg, 16)
   486  	}
   487  	if l.stack%16 != 0 {
   488  		l.stack += 8 // SP needs 16-byte alignment
   489  	}
   490  
   491  	// allocate frame, save PC of interrupted instruction (in LR)
   492  	p("MOVD R30, %d(RSP)", -l.stack)
   493  	p("SUB $%d, RSP", l.stack)
   494  	p("MOVD R29, -8(RSP)") // save frame pointer (only used on Linux)
   495  	p("SUB $8, RSP, R29")  // set up new frame pointer
   496  	// On iOS, save the LR again after decrementing SP. We run the
   497  	// signal handler on the G stack (as it doesn't support sigaltstack),
   498  	// so any writes below SP may be clobbered.
   499  	p("#ifdef GOOS_ios")
   500  	p("MOVD R30, (RSP)")
   501  	p("#endif")
   502  
   503  	l.save(g)
   504  	p("CALL ·asyncPreempt2(SB)")
   505  	l.restore(g)
   506  
   507  	p("MOVD %d(RSP), R30", l.stack) // sigctxt.pushCall has pushed LR (at interrupt) on stack, restore it
   508  	p("MOVD -8(RSP), R29")          // restore frame pointer
   509  	p("MOVD (RSP), R27")            // load PC to REGTMP
   510  	p("ADD $%d, RSP", l.stack+16)   // pop frame (including the space pushed by sigctxt.pushCall)
   511  	p("RET (R27)")
   512  }
   513  
   514  func genMIPS(g *gen, _64bit bool) {
   515  	p := g.p
   516  
   517  	mov := "MOVW"
   518  	movf := "MOVF"
   519  	add := "ADD"
   520  	sub := "SUB"
   521  	r28 := "R28"
   522  	regsize := 4
   523  	softfloat := "GOMIPS_softfloat"
   524  	if _64bit {
   525  		mov = "MOVV"
   526  		movf = "MOVD"
   527  		add = "ADDV"
   528  		sub = "SUBV"
   529  		r28 = "RSB"
   530  		regsize = 8
   531  		softfloat = "GOMIPS64_softfloat"
   532  	}
   533  
   534  	// Add integer registers R1-R22, R24-R25, R28
   535  	// R0 (zero), R23 (REGTMP), R29 (SP), R30 (g), R31 (LR) are special,
   536  	// and not saved here. R26 and R27 are reserved by kernel and not used.
   537  	var l = layout{sp: "R29", stack: regsize} // add slot to save PC of interrupted instruction (in LR)
   538  	for i := 1; i <= 25; i++ {
   539  		if i == 23 {
   540  			continue // R23 is REGTMP
   541  		}
   542  		reg := fmt.Sprintf("R%d", i)
   543  		l.add(mov, reg, regsize)
   544  	}
   545  	l.add(mov, r28, regsize)
   546  	l.addSpecial(
   547  		mov+" HI, R1\n"+mov+" R1, %d(R29)",
   548  		mov+" %d(R29), R1\n"+mov+" R1, HI",
   549  		regsize)
   550  	l.addSpecial(
   551  		mov+" LO, R1\n"+mov+" R1, %d(R29)",
   552  		mov+" %d(R29), R1\n"+mov+" R1, LO",
   553  		regsize)
   554  
   555  	// Add floating point control/status register FCR31 (FCR0-FCR30 are irrelevant)
   556  	var lfp = layout{sp: "R29", stack: l.stack}
   557  	lfp.addSpecial(
   558  		mov+" FCR31, R1\n"+mov+" R1, %d(R29)",
   559  		mov+" %d(R29), R1\n"+mov+" R1, FCR31",
   560  		regsize)
   561  	// Add floating point registers F0-F31.
   562  	for i := 0; i <= 31; i++ {
   563  		reg := fmt.Sprintf("F%d", i)
   564  		lfp.add(movf, reg, regsize)
   565  	}
   566  
   567  	// allocate frame, save PC of interrupted instruction (in LR)
   568  	p(mov+" R31, -%d(R29)", lfp.stack)
   569  	p(sub+" $%d, R29", lfp.stack)
   570  
   571  	l.save(g)
   572  	p("#ifndef %s", softfloat)
   573  	lfp.save(g)
   574  	p("#endif")
   575  	p("CALL ·asyncPreempt2(SB)")
   576  	p("#ifndef %s", softfloat)
   577  	lfp.restore(g)
   578  	p("#endif")
   579  	l.restore(g)
   580  
   581  	p(mov+" %d(R29), R31", lfp.stack)     // sigctxt.pushCall has pushed LR (at interrupt) on stack, restore it
   582  	p(mov + " (R29), R23")                // load PC to REGTMP
   583  	p(add+" $%d, R29", lfp.stack+regsize) // pop frame (including the space pushed by sigctxt.pushCall)
   584  	p("JMP (R23)")
   585  }
   586  
   587  func genLoong64(g *gen) {
   588  	p := g.p
   589  
   590  	mov := "MOVV"
   591  	movf := "MOVD"
   592  	add := "ADDV"
   593  	sub := "SUBV"
   594  	regsize := 8
   595  
   596  	// Add integer registers r4-r21 r23-r29 r31
   597  	// R0 (zero), R30 (REGTMP), R2 (tp), R3 (SP), R22 (g), R1 (LR) are special,
   598  	var l = layout{sp: "R3", stack: regsize} // add slot to save PC of interrupted instruction (in LR)
   599  	for i := 4; i <= 31; i++ {
   600  		if i == 22 || i == 30 {
   601  			continue
   602  		}
   603  		reg := fmt.Sprintf("R%d", i)
   604  		l.add(mov, reg, regsize)
   605  	}
   606  
   607  	// Add floating point registers F0-F31.
   608  	for i := 0; i <= 31; i++ {
   609  		reg := fmt.Sprintf("F%d", i)
   610  		l.add(movf, reg, regsize)
   611  	}
   612  
   613  	// save/restore FCC0
   614  	l.addSpecial(
   615  		mov+" FCC0, R4\n"+mov+" R4, %d(R3)",
   616  		mov+" %d(R3), R4\n"+mov+" R4, FCC0",
   617  		regsize)
   618  
   619  	// allocate frame, save PC of interrupted instruction (in LR)
   620  	p(mov+" R1, -%d(R3)", l.stack)
   621  	p(sub+" $%d, R3", l.stack)
   622  
   623  	l.save(g)
   624  	p("CALL ·asyncPreempt2(SB)")
   625  	l.restore(g)
   626  
   627  	p(mov+" %d(R3), R1", l.stack)      // sigctxt.pushCall has pushed LR (at interrupt) on stack, restore it
   628  	p(mov + " (R3), R30")              // load PC to REGTMP
   629  	p(add+" $%d, R3", l.stack+regsize) // pop frame (including the space pushed by sigctxt.pushCall)
   630  	p("JMP (R30)")
   631  }
   632  
   633  func genPPC64(g *gen) {
   634  	p := g.p
   635  
   636  	// Add integer registers R3-R29
   637  	// R0 (zero), R1 (SP), R30 (g) are special and not saved here.
   638  	// R2 (TOC pointer in PIC mode), R12 (function entry address in PIC mode) have been saved in sigctxt.pushCall.
   639  	// R31 (REGTMP) will be saved manually.
   640  	var l = layout{sp: "R1", stack: 32 + 8} // MinFrameSize on PPC64, plus one word for saving R31
   641  	for i := 3; i <= 29; i++ {
   642  		if i == 12 || i == 13 {
   643  			// R12 has been saved in sigctxt.pushCall.
   644  			// R13 is TLS pointer, not used by Go code. we must NOT
   645  			// restore it, otherwise if we parked and resumed on a
   646  			// different thread we'll mess up TLS addresses.
   647  			continue
   648  		}
   649  		reg := fmt.Sprintf("R%d", i)
   650  		l.add("MOVD", reg, 8)
   651  	}
   652  	l.addSpecial(
   653  		"MOVW CR, R31\nMOVW R31, %d(R1)",
   654  		"MOVW %d(R1), R31\nMOVFL R31, $0xff", // this is MOVW R31, CR
   655  		8)                                    // CR is 4-byte wide, but just keep the alignment
   656  	l.addSpecial(
   657  		"MOVD XER, R31\nMOVD R31, %d(R1)",
   658  		"MOVD %d(R1), R31\nMOVD R31, XER",
   659  		8)
   660  	// Add floating point registers F0-F31.
   661  	for i := 0; i <= 31; i++ {
   662  		reg := fmt.Sprintf("F%d", i)
   663  		l.add("FMOVD", reg, 8)
   664  	}
   665  	// Add floating point control/status register FPSCR.
   666  	l.addSpecial(
   667  		"MOVFL FPSCR, F0\nFMOVD F0, %d(R1)",
   668  		"FMOVD %d(R1), F0\nMOVFL F0, FPSCR",
   669  		8)
   670  
   671  	p("MOVD R31, -%d(R1)", l.stack-32) // save R31 first, we'll use R31 for saving LR
   672  	p("MOVD LR, R31")
   673  	p("MOVDU R31, -%d(R1)", l.stack) // allocate frame, save PC of interrupted instruction (in LR)
   674  
   675  	l.save(g)
   676  	p("CALL ·asyncPreempt2(SB)")
   677  	l.restore(g)
   678  
   679  	p("MOVD %d(R1), R31", l.stack) // sigctxt.pushCall has pushed LR, R2, R12 (at interrupt) on stack, restore them
   680  	p("MOVD R31, LR")
   681  	p("MOVD %d(R1), R2", l.stack+8)
   682  	p("MOVD %d(R1), R12", l.stack+16)
   683  	p("MOVD (R1), R31") // load PC to CTR
   684  	p("MOVD R31, CTR")
   685  	p("MOVD 32(R1), R31")        // restore R31
   686  	p("ADD $%d, R1", l.stack+32) // pop frame (including the space pushed by sigctxt.pushCall)
   687  	p("JMP (CTR)")
   688  }
   689  
   690  func genRISCV64(g *gen) {
   691  	p := g.p
   692  
   693  	// X0 (zero), X1 (LR), X2 (SP), X3 (GP), X4 (TP), X27 (g), X31 (TMP) are special.
   694  	var l = layout{sp: "X2", stack: 8}
   695  
   696  	// Add integer registers (X5-X26, X28-30).
   697  	for i := 5; i < 31; i++ {
   698  		if i == 27 {
   699  			continue
   700  		}
   701  		reg := fmt.Sprintf("X%d", i)
   702  		l.add("MOV", reg, 8)
   703  	}
   704  
   705  	// Add floating point registers (F0-F31).
   706  	for i := 0; i <= 31; i++ {
   707  		reg := fmt.Sprintf("F%d", i)
   708  		l.add("MOVD", reg, 8)
   709  	}
   710  
   711  	p("MOV X1, -%d(X2)", l.stack)
   712  	p("SUB $%d, X2", l.stack)
   713  	l.save(g)
   714  	p("CALL ·asyncPreempt2(SB)")
   715  	l.restore(g)
   716  	p("MOV %d(X2), X1", l.stack)
   717  	p("MOV (X2), X31")
   718  	p("ADD $%d, X2", l.stack+8)
   719  	p("JMP (X31)")
   720  }
   721  
   722  func genS390X(g *gen) {
   723  	p := g.p
   724  
   725  	// Add integer registers R0-R12
   726  	// R13 (g), R14 (LR), R15 (SP) are special, and not saved here.
   727  	// Saving R10 (REGTMP) is not necessary, but it is saved anyway.
   728  	var l = layout{sp: "R15", stack: 16} // add slot to save PC of interrupted instruction and flags
   729  	l.addSpecial(
   730  		"STMG R0, R12, %d(R15)",
   731  		"LMG %d(R15), R0, R12",
   732  		13*8)
   733  	// Add floating point registers F0-F31.
   734  	for i := 0; i <= 15; i++ {
   735  		reg := fmt.Sprintf("F%d", i)
   736  		l.add("FMOVD", reg, 8)
   737  	}
   738  
   739  	// allocate frame, save PC of interrupted instruction (in LR) and flags (condition code)
   740  	p("IPM R10") // save flags upfront, as ADD will clobber flags
   741  	p("MOVD R14, -%d(R15)", l.stack)
   742  	p("ADD $-%d, R15", l.stack)
   743  	p("MOVW R10, 8(R15)") // save flags
   744  
   745  	l.save(g)
   746  	p("CALL ·asyncPreempt2(SB)")
   747  	l.restore(g)
   748  
   749  	p("MOVD %d(R15), R14", l.stack)    // sigctxt.pushCall has pushed LR (at interrupt) on stack, restore it
   750  	p("ADD $%d, R15", l.stack+8)       // pop frame (including the space pushed by sigctxt.pushCall)
   751  	p("MOVWZ -%d(R15), R10", l.stack)  // load flags to REGTMP
   752  	p("TMLH R10, $(3<<12)")            // restore flags
   753  	p("MOVD -%d(R15), R10", l.stack+8) // load PC to REGTMP
   754  	p("JMP (R10)")
   755  }
   756  
   757  func genWasm(g *gen) {
   758  	p := g.p
   759  	p("// No async preemption on wasm")
   760  	p("UNDEF")
   761  }
   762  
   763  func notImplemented(g *gen) {
   764  	p := g.p
   765  	p("// Not implemented yet")
   766  	p("JMP ·abort(SB)")
   767  }
   768
View as plain text