Source file src/cmd/compile/internal/amd64/ssa.go

     1  // Copyright 2016 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package amd64
     6  
     7  import (
     8  	"fmt"
     9  	"math"
    10  
    11  	"cmd/compile/internal/base"
    12  	"cmd/compile/internal/ir"
    13  	"cmd/compile/internal/logopt"
    14  	"cmd/compile/internal/objw"
    15  	"cmd/compile/internal/ssa"
    16  	"cmd/compile/internal/ssagen"
    17  	"cmd/compile/internal/types"
    18  	"cmd/internal/obj"
    19  	"cmd/internal/obj/x86"
    20  	"internal/abi"
    21  )
    22  
    23  // ssaMarkMoves marks any MOVXconst ops that need to avoid clobbering flags.
    24  func ssaMarkMoves(s *ssagen.State, b *ssa.Block) {
    25  	flive := b.FlagsLiveAtEnd
    26  	for _, c := range b.ControlValues() {
    27  		flive = c.Type.IsFlags() || flive
    28  	}
    29  	for i := len(b.Values) - 1; i >= 0; i-- {
    30  		v := b.Values[i]
    31  		if flive && (v.Op == ssa.OpAMD64MOVLconst || v.Op == ssa.OpAMD64MOVQconst) {
    32  			// The "mark" is any non-nil Aux value.
    33  			v.Aux = ssa.AuxMark
    34  		}
    35  		if v.Type.IsFlags() {
    36  			flive = false
    37  		}
    38  		for _, a := range v.Args {
    39  			if a.Type.IsFlags() {
    40  				flive = true
    41  			}
    42  		}
    43  	}
    44  }
    45  
    46  // loadByType returns the load instruction of the given type.
    47  func loadByType(t *types.Type) obj.As {
    48  	// Avoid partial register write
    49  	if !t.IsFloat() {
    50  		switch t.Size() {
    51  		case 1:
    52  			return x86.AMOVBLZX
    53  		case 2:
    54  			return x86.AMOVWLZX
    55  		}
    56  	}
    57  	// Otherwise, there's no difference between load and store opcodes.
    58  	return storeByType(t)
    59  }
    60  
    61  // storeByType returns the store instruction of the given type.
    62  func storeByType(t *types.Type) obj.As {
    63  	width := t.Size()
    64  	if t.IsFloat() {
    65  		switch width {
    66  		case 4:
    67  			return x86.AMOVSS
    68  		case 8:
    69  			return x86.AMOVSD
    70  		}
    71  	} else {
    72  		switch width {
    73  		case 1:
    74  			return x86.AMOVB
    75  		case 2:
    76  			return x86.AMOVW
    77  		case 4:
    78  			return x86.AMOVL
    79  		case 8:
    80  			return x86.AMOVQ
    81  		case 16:
    82  			return x86.AMOVUPS
    83  		}
    84  	}
    85  	panic(fmt.Sprintf("bad store type %v", t))
    86  }
    87  
    88  // moveByType returns the reg->reg move instruction of the given type.
    89  func moveByType(t *types.Type) obj.As {
    90  	if t.IsFloat() {
    91  		// Moving the whole sse2 register is faster
    92  		// than moving just the correct low portion of it.
    93  		// There is no xmm->xmm move with 1 byte opcode,
    94  		// so use movups, which has 2 byte opcode.
    95  		return x86.AMOVUPS
    96  	} else {
    97  		switch t.Size() {
    98  		case 1:
    99  			// Avoids partial register write
   100  			return x86.AMOVL
   101  		case 2:
   102  			return x86.AMOVL
   103  		case 4:
   104  			return x86.AMOVL
   105  		case 8:
   106  			return x86.AMOVQ
   107  		case 16:
   108  			return x86.AMOVUPS // int128s are in SSE registers
   109  		default:
   110  			panic(fmt.Sprintf("bad int register width %d:%v", t.Size(), t))
   111  		}
   112  	}
   113  }
   114  
   115  // opregreg emits instructions for
   116  //
   117  //	dest := dest(To) op src(From)
   118  //
   119  // and also returns the created obj.Prog so it
   120  // may be further adjusted (offset, scale, etc).
   121  func opregreg(s *ssagen.State, op obj.As, dest, src int16) *obj.Prog {
   122  	p := s.Prog(op)
   123  	p.From.Type = obj.TYPE_REG
   124  	p.To.Type = obj.TYPE_REG
   125  	p.To.Reg = dest
   126  	p.From.Reg = src
   127  	return p
   128  }
   129  
   130  // memIdx fills out a as an indexed memory reference for v.
   131  // It assumes that the base register and the index register
   132  // are v.Args[0].Reg() and v.Args[1].Reg(), respectively.
   133  // The caller must still use gc.AddAux/gc.AddAux2 to handle v.Aux as necessary.
   134  func memIdx(a *obj.Addr, v *ssa.Value) {
   135  	r, i := v.Args[0].Reg(), v.Args[1].Reg()
   136  	a.Type = obj.TYPE_MEM
   137  	a.Scale = v.Op.Scale()
   138  	if a.Scale == 1 && i == x86.REG_SP {
   139  		r, i = i, r
   140  	}
   141  	a.Reg = r
   142  	a.Index = i
   143  }
   144  
   145  // DUFFZERO consists of repeated blocks of 4 MOVUPSs + LEAQ,
   146  // See runtime/mkduff.go.
   147  const (
   148  	dzBlocks    = 16 // number of MOV/ADD blocks
   149  	dzBlockLen  = 4  // number of clears per block
   150  	dzBlockSize = 23 // size of instructions in a single block
   151  	dzMovSize   = 5  // size of single MOV instruction w/ offset
   152  	dzLeaqSize  = 4  // size of single LEAQ instruction
   153  	dzClearStep = 16 // number of bytes cleared by each MOV instruction
   154  )
   155  
   156  func duffStart(size int64) int64 {
   157  	x, _ := duff(size)
   158  	return x
   159  }
   160  func duffAdj(size int64) int64 {
   161  	_, x := duff(size)
   162  	return x
   163  }
   164  
   165  // duff returns the offset (from duffzero, in bytes) and pointer adjust (in bytes)
   166  // required to use the duffzero mechanism for a block of the given size.
   167  func duff(size int64) (int64, int64) {
   168  	if size < 32 || size > 1024 || size%dzClearStep != 0 {
   169  		panic("bad duffzero size")
   170  	}
   171  	steps := size / dzClearStep
   172  	blocks := steps / dzBlockLen
   173  	steps %= dzBlockLen
   174  	off := dzBlockSize * (dzBlocks - blocks)
   175  	var adj int64
   176  	if steps != 0 {
   177  		off -= dzLeaqSize
   178  		off -= dzMovSize * steps
   179  		adj -= dzClearStep * (dzBlockLen - steps)
   180  	}
   181  	return off, adj
   182  }
   183  
   184  func getgFromTLS(s *ssagen.State, r int16) {
   185  	// See the comments in cmd/internal/obj/x86/obj6.go
   186  	// near CanUse1InsnTLS for a detailed explanation of these instructions.
   187  	if x86.CanUse1InsnTLS(base.Ctxt) {
   188  		// MOVQ (TLS), r
   189  		p := s.Prog(x86.AMOVQ)
   190  		p.From.Type = obj.TYPE_MEM
   191  		p.From.Reg = x86.REG_TLS
   192  		p.To.Type = obj.TYPE_REG
   193  		p.To.Reg = r
   194  	} else {
   195  		// MOVQ TLS, r
   196  		// MOVQ (r)(TLS*1), r
   197  		p := s.Prog(x86.AMOVQ)
   198  		p.From.Type = obj.TYPE_REG
   199  		p.From.Reg = x86.REG_TLS
   200  		p.To.Type = obj.TYPE_REG
   201  		p.To.Reg = r
   202  		q := s.Prog(x86.AMOVQ)
   203  		q.From.Type = obj.TYPE_MEM
   204  		q.From.Reg = r
   205  		q.From.Index = x86.REG_TLS
   206  		q.From.Scale = 1
   207  		q.To.Type = obj.TYPE_REG
   208  		q.To.Reg = r
   209  	}
   210  }
   211  
   212  func ssaGenValue(s *ssagen.State, v *ssa.Value) {
   213  	switch v.Op {
   214  	case ssa.OpAMD64VFMADD231SD, ssa.OpAMD64VFMADD231SS:
   215  		p := s.Prog(v.Op.Asm())
   216  		p.From = obj.Addr{Type: obj.TYPE_REG, Reg: v.Args[2].Reg()}
   217  		p.To = obj.Addr{Type: obj.TYPE_REG, Reg: v.Reg()}
   218  		p.AddRestSourceReg(v.Args[1].Reg())
   219  	case ssa.OpAMD64ADDQ, ssa.OpAMD64ADDL:
   220  		r := v.Reg()
   221  		r1 := v.Args[0].Reg()
   222  		r2 := v.Args[1].Reg()
   223  		switch {
   224  		case r == r1:
   225  			p := s.Prog(v.Op.Asm())
   226  			p.From.Type = obj.TYPE_REG
   227  			p.From.Reg = r2
   228  			p.To.Type = obj.TYPE_REG
   229  			p.To.Reg = r
   230  		case r == r2:
   231  			p := s.Prog(v.Op.Asm())
   232  			p.From.Type = obj.TYPE_REG
   233  			p.From.Reg = r1
   234  			p.To.Type = obj.TYPE_REG
   235  			p.To.Reg = r
   236  		default:
   237  			var asm obj.As
   238  			if v.Op == ssa.OpAMD64ADDQ {
   239  				asm = x86.ALEAQ
   240  			} else {
   241  				asm = x86.ALEAL
   242  			}
   243  			p := s.Prog(asm)
   244  			p.From.Type = obj.TYPE_MEM
   245  			p.From.Reg = r1
   246  			p.From.Scale = 1
   247  			p.From.Index = r2
   248  			p.To.Type = obj.TYPE_REG
   249  			p.To.Reg = r
   250  		}
   251  	// 2-address opcode arithmetic
   252  	case ssa.OpAMD64SUBQ, ssa.OpAMD64SUBL,
   253  		ssa.OpAMD64MULQ, ssa.OpAMD64MULL,
   254  		ssa.OpAMD64ANDQ, ssa.OpAMD64ANDL,
   255  		ssa.OpAMD64ORQ, ssa.OpAMD64ORL,
   256  		ssa.OpAMD64XORQ, ssa.OpAMD64XORL,
   257  		ssa.OpAMD64SHLQ, ssa.OpAMD64SHLL,
   258  		ssa.OpAMD64SHRQ, ssa.OpAMD64SHRL, ssa.OpAMD64SHRW, ssa.OpAMD64SHRB,
   259  		ssa.OpAMD64SARQ, ssa.OpAMD64SARL, ssa.OpAMD64SARW, ssa.OpAMD64SARB,
   260  		ssa.OpAMD64ROLQ, ssa.OpAMD64ROLL, ssa.OpAMD64ROLW, ssa.OpAMD64ROLB,
   261  		ssa.OpAMD64RORQ, ssa.OpAMD64RORL, ssa.OpAMD64RORW, ssa.OpAMD64RORB,
   262  		ssa.OpAMD64ADDSS, ssa.OpAMD64ADDSD, ssa.OpAMD64SUBSS, ssa.OpAMD64SUBSD,
   263  		ssa.OpAMD64MULSS, ssa.OpAMD64MULSD, ssa.OpAMD64DIVSS, ssa.OpAMD64DIVSD,
   264  		ssa.OpAMD64MINSS, ssa.OpAMD64MINSD,
   265  		ssa.OpAMD64POR, ssa.OpAMD64PXOR,
   266  		ssa.OpAMD64BTSL, ssa.OpAMD64BTSQ,
   267  		ssa.OpAMD64BTCL, ssa.OpAMD64BTCQ,
   268  		ssa.OpAMD64BTRL, ssa.OpAMD64BTRQ,
   269  		ssa.OpAMD64PCMPEQB, ssa.OpAMD64PSIGNB,
   270  		ssa.OpAMD64PUNPCKLBW:
   271  		opregreg(s, v.Op.Asm(), v.Reg(), v.Args[1].Reg())
   272  
   273  	case ssa.OpAMD64PSHUFLW:
   274  		p := s.Prog(v.Op.Asm())
   275  		imm := v.AuxInt
   276  		if imm < 0 || imm > 255 {
   277  			v.Fatalf("Invalid source selection immediate")
   278  		}
   279  		p.From.Offset = imm
   280  		p.From.Type = obj.TYPE_CONST
   281  		p.AddRestSourceReg(v.Args[0].Reg())
   282  		p.To.Type = obj.TYPE_REG
   283  		p.To.Reg = v.Reg()
   284  
   285  	case ssa.OpAMD64PSHUFBbroadcast:
   286  		// PSHUFB with a control mask of zero copies byte 0 to all
   287  		// bytes in the register.
   288  		//
   289  		// X15 is always zero with ABIInternal.
   290  		if s.ABI != obj.ABIInternal {
   291  			// zero X15 manually
   292  			opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
   293  		}
   294  
   295  		p := s.Prog(v.Op.Asm())
   296  		p.From.Type = obj.TYPE_REG
   297  		p.To.Type = obj.TYPE_REG
   298  		p.To.Reg = v.Reg()
   299  		p.From.Reg = x86.REG_X15
   300  
   301  	case ssa.OpAMD64SHRDQ, ssa.OpAMD64SHLDQ:
   302  		p := s.Prog(v.Op.Asm())
   303  		lo, hi, bits := v.Args[0].Reg(), v.Args[1].Reg(), v.Args[2].Reg()
   304  		p.From.Type = obj.TYPE_REG
   305  		p.From.Reg = bits
   306  		p.To.Type = obj.TYPE_REG
   307  		p.To.Reg = lo
   308  		p.AddRestSourceReg(hi)
   309  
   310  	case ssa.OpAMD64BLSIQ, ssa.OpAMD64BLSIL,
   311  		ssa.OpAMD64BLSMSKQ, ssa.OpAMD64BLSMSKL,
   312  		ssa.OpAMD64BLSRQ, ssa.OpAMD64BLSRL:
   313  		p := s.Prog(v.Op.Asm())
   314  		p.From.Type = obj.TYPE_REG
   315  		p.From.Reg = v.Args[0].Reg()
   316  		p.To.Type = obj.TYPE_REG
   317  		switch v.Op {
   318  		case ssa.OpAMD64BLSRQ, ssa.OpAMD64BLSRL:
   319  			p.To.Reg = v.Reg0()
   320  		default:
   321  			p.To.Reg = v.Reg()
   322  		}
   323  
   324  	case ssa.OpAMD64ANDNQ, ssa.OpAMD64ANDNL:
   325  		p := s.Prog(v.Op.Asm())
   326  		p.From.Type = obj.TYPE_REG
   327  		p.From.Reg = v.Args[0].Reg()
   328  		p.To.Type = obj.TYPE_REG
   329  		p.To.Reg = v.Reg()
   330  		p.AddRestSourceReg(v.Args[1].Reg())
   331  
   332  	case ssa.OpAMD64SARXL, ssa.OpAMD64SARXQ,
   333  		ssa.OpAMD64SHLXL, ssa.OpAMD64SHLXQ,
   334  		ssa.OpAMD64SHRXL, ssa.OpAMD64SHRXQ:
   335  		p := opregreg(s, v.Op.Asm(), v.Reg(), v.Args[1].Reg())
   336  		p.AddRestSourceReg(v.Args[0].Reg())
   337  
   338  	case ssa.OpAMD64SHLXLload, ssa.OpAMD64SHLXQload,
   339  		ssa.OpAMD64SHRXLload, ssa.OpAMD64SHRXQload,
   340  		ssa.OpAMD64SARXLload, ssa.OpAMD64SARXQload:
   341  		p := opregreg(s, v.Op.Asm(), v.Reg(), v.Args[1].Reg())
   342  		m := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[0].Reg()}
   343  		ssagen.AddAux(&m, v)
   344  		p.AddRestSource(m)
   345  
   346  	case ssa.OpAMD64SHLXLloadidx1, ssa.OpAMD64SHLXLloadidx4, ssa.OpAMD64SHLXLloadidx8,
   347  		ssa.OpAMD64SHRXLloadidx1, ssa.OpAMD64SHRXLloadidx4, ssa.OpAMD64SHRXLloadidx8,
   348  		ssa.OpAMD64SARXLloadidx1, ssa.OpAMD64SARXLloadidx4, ssa.OpAMD64SARXLloadidx8,
   349  		ssa.OpAMD64SHLXQloadidx1, ssa.OpAMD64SHLXQloadidx8,
   350  		ssa.OpAMD64SHRXQloadidx1, ssa.OpAMD64SHRXQloadidx8,
   351  		ssa.OpAMD64SARXQloadidx1, ssa.OpAMD64SARXQloadidx8:
   352  		p := opregreg(s, v.Op.Asm(), v.Reg(), v.Args[2].Reg())
   353  		m := obj.Addr{Type: obj.TYPE_MEM}
   354  		memIdx(&m, v)
   355  		ssagen.AddAux(&m, v)
   356  		p.AddRestSource(m)
   357  
   358  	case ssa.OpAMD64DIVQU, ssa.OpAMD64DIVLU, ssa.OpAMD64DIVWU:
   359  		// Arg[0] (the dividend) is in AX.
   360  		// Arg[1] (the divisor) can be in any other register.
   361  		// Result[0] (the quotient) is in AX.
   362  		// Result[1] (the remainder) is in DX.
   363  		r := v.Args[1].Reg()
   364  
   365  		// Zero extend dividend.
   366  		opregreg(s, x86.AXORL, x86.REG_DX, x86.REG_DX)
   367  
   368  		// Issue divide.
   369  		p := s.Prog(v.Op.Asm())
   370  		p.From.Type = obj.TYPE_REG
   371  		p.From.Reg = r
   372  
   373  	case ssa.OpAMD64DIVQ, ssa.OpAMD64DIVL, ssa.OpAMD64DIVW:
   374  		// Arg[0] (the dividend) is in AX.
   375  		// Arg[1] (the divisor) can be in any other register.
   376  		// Result[0] (the quotient) is in AX.
   377  		// Result[1] (the remainder) is in DX.
   378  		r := v.Args[1].Reg()
   379  
   380  		var opCMP, opNEG, opSXD obj.As
   381  		switch v.Op {
   382  		case ssa.OpAMD64DIVQ:
   383  			opCMP, opNEG, opSXD = x86.ACMPQ, x86.ANEGQ, x86.ACQO
   384  		case ssa.OpAMD64DIVL:
   385  			opCMP, opNEG, opSXD = x86.ACMPL, x86.ANEGL, x86.ACDQ
   386  		case ssa.OpAMD64DIVW:
   387  			opCMP, opNEG, opSXD = x86.ACMPW, x86.ANEGW, x86.ACWD
   388  		}
   389  
   390  		// CPU faults upon signed overflow, which occurs when the most
   391  		// negative int is divided by -1. Handle divide by -1 as a special case.
   392  		var j1, j2 *obj.Prog
   393  		if ssa.DivisionNeedsFixUp(v) {
   394  			c := s.Prog(opCMP)
   395  			c.From.Type = obj.TYPE_REG
   396  			c.From.Reg = r
   397  			c.To.Type = obj.TYPE_CONST
   398  			c.To.Offset = -1
   399  
   400  			// Divisor is not -1, proceed with normal division.
   401  			j1 = s.Prog(x86.AJNE)
   402  			j1.To.Type = obj.TYPE_BRANCH
   403  
   404  			// Divisor is -1, manually compute quotient and remainder via fixup code.
   405  			// n / -1 = -n
   406  			n1 := s.Prog(opNEG)
   407  			n1.To.Type = obj.TYPE_REG
   408  			n1.To.Reg = x86.REG_AX
   409  
   410  			// n % -1 == 0
   411  			opregreg(s, x86.AXORL, x86.REG_DX, x86.REG_DX)
   412  
   413  			// TODO(khr): issue only the -1 fixup code we need.
   414  			// For instance, if only the quotient is used, no point in zeroing the remainder.
   415  
   416  			// Skip over normal division.
   417  			j2 = s.Prog(obj.AJMP)
   418  			j2.To.Type = obj.TYPE_BRANCH
   419  		}
   420  
   421  		// Sign extend dividend and perform division.
   422  		p := s.Prog(opSXD)
   423  		if j1 != nil {
   424  			j1.To.SetTarget(p)
   425  		}
   426  		p = s.Prog(v.Op.Asm())
   427  		p.From.Type = obj.TYPE_REG
   428  		p.From.Reg = r
   429  
   430  		if j2 != nil {
   431  			j2.To.SetTarget(s.Pc())
   432  		}
   433  
   434  	case ssa.OpAMD64HMULQ, ssa.OpAMD64HMULL, ssa.OpAMD64HMULQU, ssa.OpAMD64HMULLU:
   435  		// the frontend rewrites constant division by 8/16/32 bit integers into
   436  		// HMUL by a constant
   437  		// SSA rewrites generate the 64 bit versions
   438  
   439  		// Arg[0] is already in AX as it's the only register we allow
   440  		// and DX is the only output we care about (the high bits)
   441  		p := s.Prog(v.Op.Asm())
   442  		p.From.Type = obj.TYPE_REG
   443  		p.From.Reg = v.Args[1].Reg()
   444  
   445  		// IMULB puts the high portion in AH instead of DL,
   446  		// so move it to DL for consistency
   447  		if v.Type.Size() == 1 {
   448  			m := s.Prog(x86.AMOVB)
   449  			m.From.Type = obj.TYPE_REG
   450  			m.From.Reg = x86.REG_AH
   451  			m.To.Type = obj.TYPE_REG
   452  			m.To.Reg = x86.REG_DX
   453  		}
   454  
   455  	case ssa.OpAMD64MULQU, ssa.OpAMD64MULLU:
   456  		// Arg[0] is already in AX as it's the only register we allow
   457  		// results lo in AX
   458  		p := s.Prog(v.Op.Asm())
   459  		p.From.Type = obj.TYPE_REG
   460  		p.From.Reg = v.Args[1].Reg()
   461  
   462  	case ssa.OpAMD64MULQU2:
   463  		// Arg[0] is already in AX as it's the only register we allow
   464  		// results hi in DX, lo in AX
   465  		p := s.Prog(v.Op.Asm())
   466  		p.From.Type = obj.TYPE_REG
   467  		p.From.Reg = v.Args[1].Reg()
   468  
   469  	case ssa.OpAMD64DIVQU2:
   470  		// Arg[0], Arg[1] are already in Dx, AX, as they're the only registers we allow
   471  		// results q in AX, r in DX
   472  		p := s.Prog(v.Op.Asm())
   473  		p.From.Type = obj.TYPE_REG
   474  		p.From.Reg = v.Args[2].Reg()
   475  
   476  	case ssa.OpAMD64AVGQU:
   477  		// compute (x+y)/2 unsigned.
   478  		// Do a 64-bit add, the overflow goes into the carry.
   479  		// Shift right once and pull the carry back into the 63rd bit.
   480  		p := s.Prog(x86.AADDQ)
   481  		p.From.Type = obj.TYPE_REG
   482  		p.To.Type = obj.TYPE_REG
   483  		p.To.Reg = v.Reg()
   484  		p.From.Reg = v.Args[1].Reg()
   485  		p = s.Prog(x86.ARCRQ)
   486  		p.From.Type = obj.TYPE_CONST
   487  		p.From.Offset = 1
   488  		p.To.Type = obj.TYPE_REG
   489  		p.To.Reg = v.Reg()
   490  
   491  	case ssa.OpAMD64ADDQcarry, ssa.OpAMD64ADCQ:
   492  		r := v.Reg0()
   493  		r0 := v.Args[0].Reg()
   494  		r1 := v.Args[1].Reg()
   495  		switch r {
   496  		case r0:
   497  			p := s.Prog(v.Op.Asm())
   498  			p.From.Type = obj.TYPE_REG
   499  			p.From.Reg = r1
   500  			p.To.Type = obj.TYPE_REG
   501  			p.To.Reg = r
   502  		case r1:
   503  			p := s.Prog(v.Op.Asm())
   504  			p.From.Type = obj.TYPE_REG
   505  			p.From.Reg = r0
   506  			p.To.Type = obj.TYPE_REG
   507  			p.To.Reg = r
   508  		default:
   509  			v.Fatalf("output not in same register as an input %s", v.LongString())
   510  		}
   511  
   512  	case ssa.OpAMD64SUBQborrow, ssa.OpAMD64SBBQ:
   513  		p := s.Prog(v.Op.Asm())
   514  		p.From.Type = obj.TYPE_REG
   515  		p.From.Reg = v.Args[1].Reg()
   516  		p.To.Type = obj.TYPE_REG
   517  		p.To.Reg = v.Reg0()
   518  
   519  	case ssa.OpAMD64ADDQconstcarry, ssa.OpAMD64ADCQconst, ssa.OpAMD64SUBQconstborrow, ssa.OpAMD64SBBQconst:
   520  		p := s.Prog(v.Op.Asm())
   521  		p.From.Type = obj.TYPE_CONST
   522  		p.From.Offset = v.AuxInt
   523  		p.To.Type = obj.TYPE_REG
   524  		p.To.Reg = v.Reg0()
   525  
   526  	case ssa.OpAMD64ADDQconst, ssa.OpAMD64ADDLconst:
   527  		r := v.Reg()
   528  		a := v.Args[0].Reg()
   529  		if r == a {
   530  			switch v.AuxInt {
   531  			case 1:
   532  				var asm obj.As
   533  				// Software optimization manual recommends add $1,reg.
   534  				// But inc/dec is 1 byte smaller. ICC always uses inc
   535  				// Clang/GCC choose depending on flags, but prefer add.
   536  				// Experiments show that inc/dec is both a little faster
   537  				// and make a binary a little smaller.
   538  				if v.Op == ssa.OpAMD64ADDQconst {
   539  					asm = x86.AINCQ
   540  				} else {
   541  					asm = x86.AINCL
   542  				}
   543  				p := s.Prog(asm)
   544  				p.To.Type = obj.TYPE_REG
   545  				p.To.Reg = r
   546  				return
   547  			case -1:
   548  				var asm obj.As
   549  				if v.Op == ssa.OpAMD64ADDQconst {
   550  					asm = x86.ADECQ
   551  				} else {
   552  					asm = x86.ADECL
   553  				}
   554  				p := s.Prog(asm)
   555  				p.To.Type = obj.TYPE_REG
   556  				p.To.Reg = r
   557  				return
   558  			case 0x80:
   559  				// 'SUBQ $-0x80, r' is shorter to encode than
   560  				// and functionally equivalent to 'ADDQ $0x80, r'.
   561  				asm := x86.ASUBL
   562  				if v.Op == ssa.OpAMD64ADDQconst {
   563  					asm = x86.ASUBQ
   564  				}
   565  				p := s.Prog(asm)
   566  				p.From.Type = obj.TYPE_CONST
   567  				p.From.Offset = -0x80
   568  				p.To.Type = obj.TYPE_REG
   569  				p.To.Reg = r
   570  				return
   571  
   572  			}
   573  			p := s.Prog(v.Op.Asm())
   574  			p.From.Type = obj.TYPE_CONST
   575  			p.From.Offset = v.AuxInt
   576  			p.To.Type = obj.TYPE_REG
   577  			p.To.Reg = r
   578  			return
   579  		}
   580  		var asm obj.As
   581  		if v.Op == ssa.OpAMD64ADDQconst {
   582  			asm = x86.ALEAQ
   583  		} else {
   584  			asm = x86.ALEAL
   585  		}
   586  		p := s.Prog(asm)
   587  		p.From.Type = obj.TYPE_MEM
   588  		p.From.Reg = a
   589  		p.From.Offset = v.AuxInt
   590  		p.To.Type = obj.TYPE_REG
   591  		p.To.Reg = r
   592  
   593  	case ssa.OpAMD64CMOVQEQ, ssa.OpAMD64CMOVLEQ, ssa.OpAMD64CMOVWEQ,
   594  		ssa.OpAMD64CMOVQLT, ssa.OpAMD64CMOVLLT, ssa.OpAMD64CMOVWLT,
   595  		ssa.OpAMD64CMOVQNE, ssa.OpAMD64CMOVLNE, ssa.OpAMD64CMOVWNE,
   596  		ssa.OpAMD64CMOVQGT, ssa.OpAMD64CMOVLGT, ssa.OpAMD64CMOVWGT,
   597  		ssa.OpAMD64CMOVQLE, ssa.OpAMD64CMOVLLE, ssa.OpAMD64CMOVWLE,
   598  		ssa.OpAMD64CMOVQGE, ssa.OpAMD64CMOVLGE, ssa.OpAMD64CMOVWGE,
   599  		ssa.OpAMD64CMOVQHI, ssa.OpAMD64CMOVLHI, ssa.OpAMD64CMOVWHI,
   600  		ssa.OpAMD64CMOVQLS, ssa.OpAMD64CMOVLLS, ssa.OpAMD64CMOVWLS,
   601  		ssa.OpAMD64CMOVQCC, ssa.OpAMD64CMOVLCC, ssa.OpAMD64CMOVWCC,
   602  		ssa.OpAMD64CMOVQCS, ssa.OpAMD64CMOVLCS, ssa.OpAMD64CMOVWCS,
   603  		ssa.OpAMD64CMOVQGTF, ssa.OpAMD64CMOVLGTF, ssa.OpAMD64CMOVWGTF,
   604  		ssa.OpAMD64CMOVQGEF, ssa.OpAMD64CMOVLGEF, ssa.OpAMD64CMOVWGEF:
   605  		p := s.Prog(v.Op.Asm())
   606  		p.From.Type = obj.TYPE_REG
   607  		p.From.Reg = v.Args[1].Reg()
   608  		p.To.Type = obj.TYPE_REG
   609  		p.To.Reg = v.Reg()
   610  
   611  	case ssa.OpAMD64CMOVQNEF, ssa.OpAMD64CMOVLNEF, ssa.OpAMD64CMOVWNEF:
   612  		// Flag condition: ^ZERO || PARITY
   613  		// Generate:
   614  		//   CMOV*NE  SRC,DST
   615  		//   CMOV*PS  SRC,DST
   616  		p := s.Prog(v.Op.Asm())
   617  		p.From.Type = obj.TYPE_REG
   618  		p.From.Reg = v.Args[1].Reg()
   619  		p.To.Type = obj.TYPE_REG
   620  		p.To.Reg = v.Reg()
   621  		var q *obj.Prog
   622  		if v.Op == ssa.OpAMD64CMOVQNEF {
   623  			q = s.Prog(x86.ACMOVQPS)
   624  		} else if v.Op == ssa.OpAMD64CMOVLNEF {
   625  			q = s.Prog(x86.ACMOVLPS)
   626  		} else {
   627  			q = s.Prog(x86.ACMOVWPS)
   628  		}
   629  		q.From.Type = obj.TYPE_REG
   630  		q.From.Reg = v.Args[1].Reg()
   631  		q.To.Type = obj.TYPE_REG
   632  		q.To.Reg = v.Reg()
   633  
   634  	case ssa.OpAMD64CMOVQEQF, ssa.OpAMD64CMOVLEQF, ssa.OpAMD64CMOVWEQF:
   635  		// Flag condition: ZERO && !PARITY
   636  		// Generate:
   637  		//   MOV      SRC,TMP
   638  		//   CMOV*NE  DST,TMP
   639  		//   CMOV*PC  TMP,DST
   640  		//
   641  		// TODO(rasky): we could generate:
   642  		//   CMOV*NE  DST,SRC
   643  		//   CMOV*PC  SRC,DST
   644  		// But this requires a way for regalloc to know that SRC might be
   645  		// clobbered by this instruction.
   646  		t := v.RegTmp()
   647  		opregreg(s, moveByType(v.Type), t, v.Args[1].Reg())
   648  
   649  		p := s.Prog(v.Op.Asm())
   650  		p.From.Type = obj.TYPE_REG
   651  		p.From.Reg = v.Reg()
   652  		p.To.Type = obj.TYPE_REG
   653  		p.To.Reg = t
   654  		var q *obj.Prog
   655  		if v.Op == ssa.OpAMD64CMOVQEQF {
   656  			q = s.Prog(x86.ACMOVQPC)
   657  		} else if v.Op == ssa.OpAMD64CMOVLEQF {
   658  			q = s.Prog(x86.ACMOVLPC)
   659  		} else {
   660  			q = s.Prog(x86.ACMOVWPC)
   661  		}
   662  		q.From.Type = obj.TYPE_REG
   663  		q.From.Reg = t
   664  		q.To.Type = obj.TYPE_REG
   665  		q.To.Reg = v.Reg()
   666  
   667  	case ssa.OpAMD64MULQconst, ssa.OpAMD64MULLconst:
   668  		r := v.Reg()
   669  		p := s.Prog(v.Op.Asm())
   670  		p.From.Type = obj.TYPE_CONST
   671  		p.From.Offset = v.AuxInt
   672  		p.To.Type = obj.TYPE_REG
   673  		p.To.Reg = r
   674  		p.AddRestSourceReg(v.Args[0].Reg())
   675  
   676  	case ssa.OpAMD64ANDQconst:
   677  		asm := v.Op.Asm()
   678  		// If the constant is positive and fits into 32 bits, use ANDL.
   679  		// This saves a few bytes of encoding.
   680  		if 0 <= v.AuxInt && v.AuxInt <= (1<<32-1) {
   681  			asm = x86.AANDL
   682  		}
   683  		p := s.Prog(asm)
   684  		p.From.Type = obj.TYPE_CONST
   685  		p.From.Offset = v.AuxInt
   686  		p.To.Type = obj.TYPE_REG
   687  		p.To.Reg = v.Reg()
   688  
   689  	case ssa.OpAMD64SUBQconst, ssa.OpAMD64SUBLconst,
   690  		ssa.OpAMD64ANDLconst,
   691  		ssa.OpAMD64ORQconst, ssa.OpAMD64ORLconst,
   692  		ssa.OpAMD64XORQconst, ssa.OpAMD64XORLconst,
   693  		ssa.OpAMD64SHLQconst, ssa.OpAMD64SHLLconst,
   694  		ssa.OpAMD64SHRQconst, ssa.OpAMD64SHRLconst, ssa.OpAMD64SHRWconst, ssa.OpAMD64SHRBconst,
   695  		ssa.OpAMD64SARQconst, ssa.OpAMD64SARLconst, ssa.OpAMD64SARWconst, ssa.OpAMD64SARBconst,
   696  		ssa.OpAMD64ROLQconst, ssa.OpAMD64ROLLconst, ssa.OpAMD64ROLWconst, ssa.OpAMD64ROLBconst:
   697  		p := s.Prog(v.Op.Asm())
   698  		p.From.Type = obj.TYPE_CONST
   699  		p.From.Offset = v.AuxInt
   700  		p.To.Type = obj.TYPE_REG
   701  		p.To.Reg = v.Reg()
   702  	case ssa.OpAMD64SBBQcarrymask, ssa.OpAMD64SBBLcarrymask:
   703  		r := v.Reg()
   704  		p := s.Prog(v.Op.Asm())
   705  		p.From.Type = obj.TYPE_REG
   706  		p.From.Reg = r
   707  		p.To.Type = obj.TYPE_REG
   708  		p.To.Reg = r
   709  	case ssa.OpAMD64LEAQ1, ssa.OpAMD64LEAQ2, ssa.OpAMD64LEAQ4, ssa.OpAMD64LEAQ8,
   710  		ssa.OpAMD64LEAL1, ssa.OpAMD64LEAL2, ssa.OpAMD64LEAL4, ssa.OpAMD64LEAL8,
   711  		ssa.OpAMD64LEAW1, ssa.OpAMD64LEAW2, ssa.OpAMD64LEAW4, ssa.OpAMD64LEAW8:
   712  		p := s.Prog(v.Op.Asm())
   713  		memIdx(&p.From, v)
   714  		o := v.Reg()
   715  		p.To.Type = obj.TYPE_REG
   716  		p.To.Reg = o
   717  		if v.AuxInt != 0 && v.Aux == nil {
   718  			// Emit an additional LEA to add the displacement instead of creating a slow 3 operand LEA.
   719  			switch v.Op {
   720  			case ssa.OpAMD64LEAQ1, ssa.OpAMD64LEAQ2, ssa.OpAMD64LEAQ4, ssa.OpAMD64LEAQ8:
   721  				p = s.Prog(x86.ALEAQ)
   722  			case ssa.OpAMD64LEAL1, ssa.OpAMD64LEAL2, ssa.OpAMD64LEAL4, ssa.OpAMD64LEAL8:
   723  				p = s.Prog(x86.ALEAL)
   724  			case ssa.OpAMD64LEAW1, ssa.OpAMD64LEAW2, ssa.OpAMD64LEAW4, ssa.OpAMD64LEAW8:
   725  				p = s.Prog(x86.ALEAW)
   726  			}
   727  			p.From.Type = obj.TYPE_MEM
   728  			p.From.Reg = o
   729  			p.To.Type = obj.TYPE_REG
   730  			p.To.Reg = o
   731  		}
   732  		ssagen.AddAux(&p.From, v)
   733  	case ssa.OpAMD64LEAQ, ssa.OpAMD64LEAL, ssa.OpAMD64LEAW:
   734  		p := s.Prog(v.Op.Asm())
   735  		p.From.Type = obj.TYPE_MEM
   736  		p.From.Reg = v.Args[0].Reg()
   737  		ssagen.AddAux(&p.From, v)
   738  		p.To.Type = obj.TYPE_REG
   739  		p.To.Reg = v.Reg()
   740  	case ssa.OpAMD64CMPQ, ssa.OpAMD64CMPL, ssa.OpAMD64CMPW, ssa.OpAMD64CMPB,
   741  		ssa.OpAMD64TESTQ, ssa.OpAMD64TESTL, ssa.OpAMD64TESTW, ssa.OpAMD64TESTB,
   742  		ssa.OpAMD64BTL, ssa.OpAMD64BTQ:
   743  		opregreg(s, v.Op.Asm(), v.Args[1].Reg(), v.Args[0].Reg())
   744  	case ssa.OpAMD64UCOMISS, ssa.OpAMD64UCOMISD:
   745  		// Go assembler has swapped operands for UCOMISx relative to CMP,
   746  		// must account for that right here.
   747  		opregreg(s, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg())
   748  	case ssa.OpAMD64CMPQconst, ssa.OpAMD64CMPLconst, ssa.OpAMD64CMPWconst, ssa.OpAMD64CMPBconst:
   749  		p := s.Prog(v.Op.Asm())
   750  		p.From.Type = obj.TYPE_REG
   751  		p.From.Reg = v.Args[0].Reg()
   752  		p.To.Type = obj.TYPE_CONST
   753  		p.To.Offset = v.AuxInt
   754  	case ssa.OpAMD64BTLconst, ssa.OpAMD64BTQconst,
   755  		ssa.OpAMD64TESTQconst, ssa.OpAMD64TESTLconst, ssa.OpAMD64TESTWconst, ssa.OpAMD64TESTBconst,
   756  		ssa.OpAMD64BTSQconst,
   757  		ssa.OpAMD64BTCQconst,
   758  		ssa.OpAMD64BTRQconst:
   759  		op := v.Op
   760  		if op == ssa.OpAMD64BTQconst && v.AuxInt < 32 {
   761  			// Emit 32-bit version because it's shorter
   762  			op = ssa.OpAMD64BTLconst
   763  		}
   764  		p := s.Prog(op.Asm())
   765  		p.From.Type = obj.TYPE_CONST
   766  		p.From.Offset = v.AuxInt
   767  		p.To.Type = obj.TYPE_REG
   768  		p.To.Reg = v.Args[0].Reg()
   769  	case ssa.OpAMD64CMPQload, ssa.OpAMD64CMPLload, ssa.OpAMD64CMPWload, ssa.OpAMD64CMPBload:
   770  		p := s.Prog(v.Op.Asm())
   771  		p.From.Type = obj.TYPE_MEM
   772  		p.From.Reg = v.Args[0].Reg()
   773  		ssagen.AddAux(&p.From, v)
   774  		p.To.Type = obj.TYPE_REG
   775  		p.To.Reg = v.Args[1].Reg()
   776  	case ssa.OpAMD64CMPQconstload, ssa.OpAMD64CMPLconstload, ssa.OpAMD64CMPWconstload, ssa.OpAMD64CMPBconstload:
   777  		sc := v.AuxValAndOff()
   778  		p := s.Prog(v.Op.Asm())
   779  		p.From.Type = obj.TYPE_MEM
   780  		p.From.Reg = v.Args[0].Reg()
   781  		ssagen.AddAux2(&p.From, v, sc.Off64())
   782  		p.To.Type = obj.TYPE_CONST
   783  		p.To.Offset = sc.Val64()
   784  	case ssa.OpAMD64CMPQloadidx8, ssa.OpAMD64CMPQloadidx1, ssa.OpAMD64CMPLloadidx4, ssa.OpAMD64CMPLloadidx1, ssa.OpAMD64CMPWloadidx2, ssa.OpAMD64CMPWloadidx1, ssa.OpAMD64CMPBloadidx1:
   785  		p := s.Prog(v.Op.Asm())
   786  		memIdx(&p.From, v)
   787  		ssagen.AddAux(&p.From, v)
   788  		p.To.Type = obj.TYPE_REG
   789  		p.To.Reg = v.Args[2].Reg()
   790  	case ssa.OpAMD64CMPQconstloadidx8, ssa.OpAMD64CMPQconstloadidx1, ssa.OpAMD64CMPLconstloadidx4, ssa.OpAMD64CMPLconstloadidx1, ssa.OpAMD64CMPWconstloadidx2, ssa.OpAMD64CMPWconstloadidx1, ssa.OpAMD64CMPBconstloadidx1:
   791  		sc := v.AuxValAndOff()
   792  		p := s.Prog(v.Op.Asm())
   793  		memIdx(&p.From, v)
   794  		ssagen.AddAux2(&p.From, v, sc.Off64())
   795  		p.To.Type = obj.TYPE_CONST
   796  		p.To.Offset = sc.Val64()
   797  	case ssa.OpAMD64MOVLconst, ssa.OpAMD64MOVQconst:
   798  		x := v.Reg()
   799  
   800  		// If flags aren't live (indicated by v.Aux == nil),
   801  		// then we can rewrite MOV $0, AX into XOR AX, AX.
   802  		if v.AuxInt == 0 && v.Aux == nil {
   803  			opregreg(s, x86.AXORL, x, x)
   804  			break
   805  		}
   806  
   807  		asm := v.Op.Asm()
   808  		// Use MOVL to move a small constant into a register
   809  		// when the constant is positive and fits into 32 bits.
   810  		if 0 <= v.AuxInt && v.AuxInt <= (1<<32-1) {
   811  			// The upper 32bit are zeroed automatically when using MOVL.
   812  			asm = x86.AMOVL
   813  		}
   814  		p := s.Prog(asm)
   815  		p.From.Type = obj.TYPE_CONST
   816  		p.From.Offset = v.AuxInt
   817  		p.To.Type = obj.TYPE_REG
   818  		p.To.Reg = x
   819  	case ssa.OpAMD64MOVSSconst, ssa.OpAMD64MOVSDconst:
   820  		x := v.Reg()
   821  		p := s.Prog(v.Op.Asm())
   822  		p.From.Type = obj.TYPE_FCONST
   823  		p.From.Val = math.Float64frombits(uint64(v.AuxInt))
   824  		p.To.Type = obj.TYPE_REG
   825  		p.To.Reg = x
   826  	case ssa.OpAMD64MOVQload, ssa.OpAMD64MOVLload, ssa.OpAMD64MOVWload, ssa.OpAMD64MOVBload, ssa.OpAMD64MOVOload,
   827  		ssa.OpAMD64MOVSSload, ssa.OpAMD64MOVSDload, ssa.OpAMD64MOVBQSXload, ssa.OpAMD64MOVWQSXload, ssa.OpAMD64MOVLQSXload,
   828  		ssa.OpAMD64MOVBEQload, ssa.OpAMD64MOVBELload:
   829  		p := s.Prog(v.Op.Asm())
   830  		p.From.Type = obj.TYPE_MEM
   831  		p.From.Reg = v.Args[0].Reg()
   832  		ssagen.AddAux(&p.From, v)
   833  		p.To.Type = obj.TYPE_REG
   834  		p.To.Reg = v.Reg()
   835  	case ssa.OpAMD64MOVBloadidx1, ssa.OpAMD64MOVWloadidx1, ssa.OpAMD64MOVLloadidx1, ssa.OpAMD64MOVQloadidx1, ssa.OpAMD64MOVSSloadidx1, ssa.OpAMD64MOVSDloadidx1,
   836  		ssa.OpAMD64MOVQloadidx8, ssa.OpAMD64MOVSDloadidx8, ssa.OpAMD64MOVLloadidx8, ssa.OpAMD64MOVLloadidx4, ssa.OpAMD64MOVSSloadidx4, ssa.OpAMD64MOVWloadidx2,
   837  		ssa.OpAMD64MOVBELloadidx1, ssa.OpAMD64MOVBELloadidx4, ssa.OpAMD64MOVBELloadidx8, ssa.OpAMD64MOVBEQloadidx1, ssa.OpAMD64MOVBEQloadidx8:
   838  		p := s.Prog(v.Op.Asm())
   839  		memIdx(&p.From, v)
   840  		ssagen.AddAux(&p.From, v)
   841  		p.To.Type = obj.TYPE_REG
   842  		p.To.Reg = v.Reg()
   843  	case ssa.OpAMD64MOVQstore, ssa.OpAMD64MOVSSstore, ssa.OpAMD64MOVSDstore, ssa.OpAMD64MOVLstore, ssa.OpAMD64MOVWstore, ssa.OpAMD64MOVBstore, ssa.OpAMD64MOVOstore,
   844  		ssa.OpAMD64ADDQmodify, ssa.OpAMD64SUBQmodify, ssa.OpAMD64ANDQmodify, ssa.OpAMD64ORQmodify, ssa.OpAMD64XORQmodify,
   845  		ssa.OpAMD64ADDLmodify, ssa.OpAMD64SUBLmodify, ssa.OpAMD64ANDLmodify, ssa.OpAMD64ORLmodify, ssa.OpAMD64XORLmodify,
   846  		ssa.OpAMD64MOVBEQstore, ssa.OpAMD64MOVBELstore, ssa.OpAMD64MOVBEWstore:
   847  		p := s.Prog(v.Op.Asm())
   848  		p.From.Type = obj.TYPE_REG
   849  		p.From.Reg = v.Args[1].Reg()
   850  		p.To.Type = obj.TYPE_MEM
   851  		p.To.Reg = v.Args[0].Reg()
   852  		ssagen.AddAux(&p.To, v)
   853  	case ssa.OpAMD64MOVBstoreidx1, ssa.OpAMD64MOVWstoreidx1, ssa.OpAMD64MOVLstoreidx1, ssa.OpAMD64MOVQstoreidx1, ssa.OpAMD64MOVSSstoreidx1, ssa.OpAMD64MOVSDstoreidx1,
   854  		ssa.OpAMD64MOVQstoreidx8, ssa.OpAMD64MOVSDstoreidx8, ssa.OpAMD64MOVLstoreidx8, ssa.OpAMD64MOVSSstoreidx4, ssa.OpAMD64MOVLstoreidx4, ssa.OpAMD64MOVWstoreidx2,
   855  		ssa.OpAMD64ADDLmodifyidx1, ssa.OpAMD64ADDLmodifyidx4, ssa.OpAMD64ADDLmodifyidx8, ssa.OpAMD64ADDQmodifyidx1, ssa.OpAMD64ADDQmodifyidx8,
   856  		ssa.OpAMD64SUBLmodifyidx1, ssa.OpAMD64SUBLmodifyidx4, ssa.OpAMD64SUBLmodifyidx8, ssa.OpAMD64SUBQmodifyidx1, ssa.OpAMD64SUBQmodifyidx8,
   857  		ssa.OpAMD64ANDLmodifyidx1, ssa.OpAMD64ANDLmodifyidx4, ssa.OpAMD64ANDLmodifyidx8, ssa.OpAMD64ANDQmodifyidx1, ssa.OpAMD64ANDQmodifyidx8,
   858  		ssa.OpAMD64ORLmodifyidx1, ssa.OpAMD64ORLmodifyidx4, ssa.OpAMD64ORLmodifyidx8, ssa.OpAMD64ORQmodifyidx1, ssa.OpAMD64ORQmodifyidx8,
   859  		ssa.OpAMD64XORLmodifyidx1, ssa.OpAMD64XORLmodifyidx4, ssa.OpAMD64XORLmodifyidx8, ssa.OpAMD64XORQmodifyidx1, ssa.OpAMD64XORQmodifyidx8,
   860  		ssa.OpAMD64MOVBEWstoreidx1, ssa.OpAMD64MOVBEWstoreidx2, ssa.OpAMD64MOVBELstoreidx1, ssa.OpAMD64MOVBELstoreidx4, ssa.OpAMD64MOVBELstoreidx8, ssa.OpAMD64MOVBEQstoreidx1, ssa.OpAMD64MOVBEQstoreidx8:
   861  		p := s.Prog(v.Op.Asm())
   862  		p.From.Type = obj.TYPE_REG
   863  		p.From.Reg = v.Args[2].Reg()
   864  		memIdx(&p.To, v)
   865  		ssagen.AddAux(&p.To, v)
   866  	case ssa.OpAMD64ADDQconstmodify, ssa.OpAMD64ADDLconstmodify:
   867  		sc := v.AuxValAndOff()
   868  		off := sc.Off64()
   869  		val := sc.Val()
   870  		if val == 1 || val == -1 {
   871  			var asm obj.As
   872  			if v.Op == ssa.OpAMD64ADDQconstmodify {
   873  				if val == 1 {
   874  					asm = x86.AINCQ
   875  				} else {
   876  					asm = x86.ADECQ
   877  				}
   878  			} else {
   879  				if val == 1 {
   880  					asm = x86.AINCL
   881  				} else {
   882  					asm = x86.ADECL
   883  				}
   884  			}
   885  			p := s.Prog(asm)
   886  			p.To.Type = obj.TYPE_MEM
   887  			p.To.Reg = v.Args[0].Reg()
   888  			ssagen.AddAux2(&p.To, v, off)
   889  			break
   890  		}
   891  		fallthrough
   892  	case ssa.OpAMD64ANDQconstmodify, ssa.OpAMD64ANDLconstmodify, ssa.OpAMD64ORQconstmodify, ssa.OpAMD64ORLconstmodify,
   893  		ssa.OpAMD64XORQconstmodify, ssa.OpAMD64XORLconstmodify,
   894  		ssa.OpAMD64BTSQconstmodify, ssa.OpAMD64BTRQconstmodify, ssa.OpAMD64BTCQconstmodify:
   895  		sc := v.AuxValAndOff()
   896  		off := sc.Off64()
   897  		val := sc.Val64()
   898  		p := s.Prog(v.Op.Asm())
   899  		p.From.Type = obj.TYPE_CONST
   900  		p.From.Offset = val
   901  		p.To.Type = obj.TYPE_MEM
   902  		p.To.Reg = v.Args[0].Reg()
   903  		ssagen.AddAux2(&p.To, v, off)
   904  
   905  	case ssa.OpAMD64MOVQstoreconst, ssa.OpAMD64MOVLstoreconst, ssa.OpAMD64MOVWstoreconst, ssa.OpAMD64MOVBstoreconst:
   906  		p := s.Prog(v.Op.Asm())
   907  		p.From.Type = obj.TYPE_CONST
   908  		sc := v.AuxValAndOff()
   909  		p.From.Offset = sc.Val64()
   910  		p.To.Type = obj.TYPE_MEM
   911  		p.To.Reg = v.Args[0].Reg()
   912  		ssagen.AddAux2(&p.To, v, sc.Off64())
   913  	case ssa.OpAMD64MOVOstoreconst:
   914  		sc := v.AuxValAndOff()
   915  		if sc.Val() != 0 {
   916  			v.Fatalf("MOVO for non zero constants not implemented: %s", v.LongString())
   917  		}
   918  
   919  		if s.ABI != obj.ABIInternal {
   920  			// zero X15 manually
   921  			opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
   922  		}
   923  		p := s.Prog(v.Op.Asm())
   924  		p.From.Type = obj.TYPE_REG
   925  		p.From.Reg = x86.REG_X15
   926  		p.To.Type = obj.TYPE_MEM
   927  		p.To.Reg = v.Args[0].Reg()
   928  		ssagen.AddAux2(&p.To, v, sc.Off64())
   929  
   930  	case ssa.OpAMD64MOVQstoreconstidx1, ssa.OpAMD64MOVQstoreconstidx8, ssa.OpAMD64MOVLstoreconstidx1, ssa.OpAMD64MOVLstoreconstidx4, ssa.OpAMD64MOVWstoreconstidx1, ssa.OpAMD64MOVWstoreconstidx2, ssa.OpAMD64MOVBstoreconstidx1,
   931  		ssa.OpAMD64ADDLconstmodifyidx1, ssa.OpAMD64ADDLconstmodifyidx4, ssa.OpAMD64ADDLconstmodifyidx8, ssa.OpAMD64ADDQconstmodifyidx1, ssa.OpAMD64ADDQconstmodifyidx8,
   932  		ssa.OpAMD64ANDLconstmodifyidx1, ssa.OpAMD64ANDLconstmodifyidx4, ssa.OpAMD64ANDLconstmodifyidx8, ssa.OpAMD64ANDQconstmodifyidx1, ssa.OpAMD64ANDQconstmodifyidx8,
   933  		ssa.OpAMD64ORLconstmodifyidx1, ssa.OpAMD64ORLconstmodifyidx4, ssa.OpAMD64ORLconstmodifyidx8, ssa.OpAMD64ORQconstmodifyidx1, ssa.OpAMD64ORQconstmodifyidx8,
   934  		ssa.OpAMD64XORLconstmodifyidx1, ssa.OpAMD64XORLconstmodifyidx4, ssa.OpAMD64XORLconstmodifyidx8, ssa.OpAMD64XORQconstmodifyidx1, ssa.OpAMD64XORQconstmodifyidx8:
   935  		p := s.Prog(v.Op.Asm())
   936  		p.From.Type = obj.TYPE_CONST
   937  		sc := v.AuxValAndOff()
   938  		p.From.Offset = sc.Val64()
   939  		switch {
   940  		case p.As == x86.AADDQ && p.From.Offset == 1:
   941  			p.As = x86.AINCQ
   942  			p.From.Type = obj.TYPE_NONE
   943  		case p.As == x86.AADDQ && p.From.Offset == -1:
   944  			p.As = x86.ADECQ
   945  			p.From.Type = obj.TYPE_NONE
   946  		case p.As == x86.AADDL && p.From.Offset == 1:
   947  			p.As = x86.AINCL
   948  			p.From.Type = obj.TYPE_NONE
   949  		case p.As == x86.AADDL && p.From.Offset == -1:
   950  			p.As = x86.ADECL
   951  			p.From.Type = obj.TYPE_NONE
   952  		}
   953  		memIdx(&p.To, v)
   954  		ssagen.AddAux2(&p.To, v, sc.Off64())
   955  	case ssa.OpAMD64MOVLQSX, ssa.OpAMD64MOVWQSX, ssa.OpAMD64MOVBQSX, ssa.OpAMD64MOVLQZX, ssa.OpAMD64MOVWQZX, ssa.OpAMD64MOVBQZX,
   956  		ssa.OpAMD64CVTTSS2SL, ssa.OpAMD64CVTTSD2SL, ssa.OpAMD64CVTTSS2SQ, ssa.OpAMD64CVTTSD2SQ,
   957  		ssa.OpAMD64CVTSS2SD, ssa.OpAMD64CVTSD2SS, ssa.OpAMD64VPBROADCASTB, ssa.OpAMD64PMOVMSKB:
   958  		opregreg(s, v.Op.Asm(), v.Reg(), v.Args[0].Reg())
   959  	case ssa.OpAMD64CVTSL2SD, ssa.OpAMD64CVTSQ2SD, ssa.OpAMD64CVTSQ2SS, ssa.OpAMD64CVTSL2SS:
   960  		r := v.Reg()
   961  		// Break false dependency on destination register.
   962  		opregreg(s, x86.AXORPS, r, r)
   963  		opregreg(s, v.Op.Asm(), r, v.Args[0].Reg())
   964  	case ssa.OpAMD64MOVQi2f, ssa.OpAMD64MOVQf2i, ssa.OpAMD64MOVLi2f, ssa.OpAMD64MOVLf2i:
   965  		var p *obj.Prog
   966  		switch v.Op {
   967  		case ssa.OpAMD64MOVQi2f, ssa.OpAMD64MOVQf2i:
   968  			p = s.Prog(x86.AMOVQ)
   969  		case ssa.OpAMD64MOVLi2f, ssa.OpAMD64MOVLf2i:
   970  			p = s.Prog(x86.AMOVL)
   971  		}
   972  		p.From.Type = obj.TYPE_REG
   973  		p.From.Reg = v.Args[0].Reg()
   974  		p.To.Type = obj.TYPE_REG
   975  		p.To.Reg = v.Reg()
   976  	case ssa.OpAMD64ADDQload, ssa.OpAMD64ADDLload, ssa.OpAMD64SUBQload, ssa.OpAMD64SUBLload,
   977  		ssa.OpAMD64ANDQload, ssa.OpAMD64ANDLload, ssa.OpAMD64ORQload, ssa.OpAMD64ORLload,
   978  		ssa.OpAMD64XORQload, ssa.OpAMD64XORLload, ssa.OpAMD64ADDSDload, ssa.OpAMD64ADDSSload,
   979  		ssa.OpAMD64SUBSDload, ssa.OpAMD64SUBSSload, ssa.OpAMD64MULSDload, ssa.OpAMD64MULSSload,
   980  		ssa.OpAMD64DIVSDload, ssa.OpAMD64DIVSSload:
   981  		p := s.Prog(v.Op.Asm())
   982  		p.From.Type = obj.TYPE_MEM
   983  		p.From.Reg = v.Args[1].Reg()
   984  		ssagen.AddAux(&p.From, v)
   985  		p.To.Type = obj.TYPE_REG
   986  		p.To.Reg = v.Reg()
   987  	case ssa.OpAMD64ADDLloadidx1, ssa.OpAMD64ADDLloadidx4, ssa.OpAMD64ADDLloadidx8, ssa.OpAMD64ADDQloadidx1, ssa.OpAMD64ADDQloadidx8,
   988  		ssa.OpAMD64SUBLloadidx1, ssa.OpAMD64SUBLloadidx4, ssa.OpAMD64SUBLloadidx8, ssa.OpAMD64SUBQloadidx1, ssa.OpAMD64SUBQloadidx8,
   989  		ssa.OpAMD64ANDLloadidx1, ssa.OpAMD64ANDLloadidx4, ssa.OpAMD64ANDLloadidx8, ssa.OpAMD64ANDQloadidx1, ssa.OpAMD64ANDQloadidx8,
   990  		ssa.OpAMD64ORLloadidx1, ssa.OpAMD64ORLloadidx4, ssa.OpAMD64ORLloadidx8, ssa.OpAMD64ORQloadidx1, ssa.OpAMD64ORQloadidx8,
   991  		ssa.OpAMD64XORLloadidx1, ssa.OpAMD64XORLloadidx4, ssa.OpAMD64XORLloadidx8, ssa.OpAMD64XORQloadidx1, ssa.OpAMD64XORQloadidx8,
   992  		ssa.OpAMD64ADDSSloadidx1, ssa.OpAMD64ADDSSloadidx4, ssa.OpAMD64ADDSDloadidx1, ssa.OpAMD64ADDSDloadidx8,
   993  		ssa.OpAMD64SUBSSloadidx1, ssa.OpAMD64SUBSSloadidx4, ssa.OpAMD64SUBSDloadidx1, ssa.OpAMD64SUBSDloadidx8,
   994  		ssa.OpAMD64MULSSloadidx1, ssa.OpAMD64MULSSloadidx4, ssa.OpAMD64MULSDloadidx1, ssa.OpAMD64MULSDloadidx8,
   995  		ssa.OpAMD64DIVSSloadidx1, ssa.OpAMD64DIVSSloadidx4, ssa.OpAMD64DIVSDloadidx1, ssa.OpAMD64DIVSDloadidx8:
   996  		p := s.Prog(v.Op.Asm())
   997  
   998  		r, i := v.Args[1].Reg(), v.Args[2].Reg()
   999  		p.From.Type = obj.TYPE_MEM
  1000  		p.From.Scale = v.Op.Scale()
  1001  		if p.From.Scale == 1 && i == x86.REG_SP {
  1002  			r, i = i, r
  1003  		}
  1004  		p.From.Reg = r
  1005  		p.From.Index = i
  1006  
  1007  		ssagen.AddAux(&p.From, v)
  1008  		p.To.Type = obj.TYPE_REG
  1009  		p.To.Reg = v.Reg()
  1010  
  1011  	case ssa.OpAMD64LoweredZero:
  1012  		if s.ABI != obj.ABIInternal {
  1013  			// zero X15 manually
  1014  			opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
  1015  		}
  1016  		ptrReg := v.Args[0].Reg()
  1017  		n := v.AuxInt
  1018  		if n < 16 {
  1019  			v.Fatalf("Zero too small %d", n)
  1020  		}
  1021  		zero16 := func(off int64) {
  1022  			zero16(s, ptrReg, off)
  1023  		}
  1024  
  1025  		// Generate zeroing instructions.
  1026  		var off int64
  1027  		for n >= 16 {
  1028  			zero16(off)
  1029  			off += 16
  1030  			n -= 16
  1031  		}
  1032  		if n != 0 {
  1033  			// use partially overlapped write.
  1034  			// TODO: n <= 8, use smaller write?
  1035  			zero16(off + n - 16)
  1036  		}
  1037  
  1038  	case ssa.OpAMD64LoweredZeroLoop:
  1039  		if s.ABI != obj.ABIInternal {
  1040  			// zero X15 manually
  1041  			opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
  1042  		}
  1043  		ptrReg := v.Args[0].Reg()
  1044  		countReg := v.RegTmp()
  1045  		n := v.AuxInt
  1046  		loopSize := int64(64)
  1047  		if n < 3*loopSize {
  1048  			// - a loop count of 0 won't work.
  1049  			// - a loop count of 1 is useless.
  1050  			// - a loop count of 2 is a code size ~tie
  1051  			//     4 instructions to implement the loop
  1052  			//     4 instructions in the loop body
  1053  			//   vs
  1054  			//     8 instructions in the straightline code
  1055  			//   Might as well use straightline code.
  1056  			v.Fatalf("ZeroLoop size too small %d", n)
  1057  		}
  1058  		zero16 := func(off int64) {
  1059  			zero16(s, ptrReg, off)
  1060  		}
  1061  
  1062  		// Put iteration count in a register.
  1063  		//   MOVL    $n, countReg
  1064  		p := s.Prog(x86.AMOVL)
  1065  		p.From.Type = obj.TYPE_CONST
  1066  		p.From.Offset = n / loopSize
  1067  		p.To.Type = obj.TYPE_REG
  1068  		p.To.Reg = countReg
  1069  		cntInit := p
  1070  
  1071  		// Zero loopSize bytes starting at ptrReg.
  1072  		for i := range loopSize / 16 {
  1073  			zero16(i * 16)
  1074  		}
  1075  		//   ADDQ    $loopSize, ptrReg
  1076  		p = s.Prog(x86.AADDQ)
  1077  		p.From.Type = obj.TYPE_CONST
  1078  		p.From.Offset = loopSize
  1079  		p.To.Type = obj.TYPE_REG
  1080  		p.To.Reg = ptrReg
  1081  		//   DECL    countReg
  1082  		p = s.Prog(x86.ADECL)
  1083  		p.To.Type = obj.TYPE_REG
  1084  		p.To.Reg = countReg
  1085  		// Jump to first instruction in loop if we're not done yet.
  1086  		//   JNE     head
  1087  		p = s.Prog(x86.AJNE)
  1088  		p.To.Type = obj.TYPE_BRANCH
  1089  		p.To.SetTarget(cntInit.Link)
  1090  
  1091  		// Multiples of the loop size are now done.
  1092  		n %= loopSize
  1093  
  1094  		// Write any fractional portion.
  1095  		var off int64
  1096  		for n >= 16 {
  1097  			zero16(off)
  1098  			off += 16
  1099  			n -= 16
  1100  		}
  1101  		if n != 0 {
  1102  			// Use partially-overlapping write.
  1103  			// TODO: n <= 8, use smaller write?
  1104  			zero16(off + n - 16)
  1105  		}
  1106  
  1107  	case ssa.OpAMD64DUFFCOPY:
  1108  		p := s.Prog(obj.ADUFFCOPY)
  1109  		p.To.Type = obj.TYPE_ADDR
  1110  		p.To.Sym = ir.Syms.Duffcopy
  1111  		if v.AuxInt%16 != 0 {
  1112  			v.Fatalf("bad DUFFCOPY AuxInt %v", v.AuxInt)
  1113  		}
  1114  		p.To.Offset = 14 * (64 - v.AuxInt/16)
  1115  		// 14 and 64 are magic constants.  14 is the number of bytes to encode:
  1116  		//	MOVUPS	(SI), X0
  1117  		//	ADDQ	$16, SI
  1118  		//	MOVUPS	X0, (DI)
  1119  		//	ADDQ	$16, DI
  1120  		// and 64 is the number of such blocks. See src/runtime/duff_amd64.s:duffcopy.
  1121  
  1122  	case ssa.OpCopy: // TODO: use MOVQreg for reg->reg copies instead of OpCopy?
  1123  		if v.Type.IsMemory() {
  1124  			return
  1125  		}
  1126  		x := v.Args[0].Reg()
  1127  		y := v.Reg()
  1128  		if x != y {
  1129  			opregreg(s, moveByType(v.Type), y, x)
  1130  		}
  1131  	case ssa.OpLoadReg:
  1132  		if v.Type.IsFlags() {
  1133  			v.Fatalf("load flags not implemented: %v", v.LongString())
  1134  			return
  1135  		}
  1136  		p := s.Prog(loadByType(v.Type))
  1137  		ssagen.AddrAuto(&p.From, v.Args[0])
  1138  		p.To.Type = obj.TYPE_REG
  1139  		p.To.Reg = v.Reg()
  1140  
  1141  	case ssa.OpStoreReg:
  1142  		if v.Type.IsFlags() {
  1143  			v.Fatalf("store flags not implemented: %v", v.LongString())
  1144  			return
  1145  		}
  1146  		p := s.Prog(storeByType(v.Type))
  1147  		p.From.Type = obj.TYPE_REG
  1148  		p.From.Reg = v.Args[0].Reg()
  1149  		ssagen.AddrAuto(&p.To, v)
  1150  	case ssa.OpAMD64LoweredHasCPUFeature:
  1151  		p := s.Prog(x86.AMOVBLZX)
  1152  		p.From.Type = obj.TYPE_MEM
  1153  		ssagen.AddAux(&p.From, v)
  1154  		p.To.Type = obj.TYPE_REG
  1155  		p.To.Reg = v.Reg()
  1156  	case ssa.OpArgIntReg, ssa.OpArgFloatReg:
  1157  		// The assembler needs to wrap the entry safepoint/stack growth code with spill/unspill
  1158  		// The loop only runs once.
  1159  		for _, ap := range v.Block.Func.RegArgs {
  1160  			// Pass the spill/unspill information along to the assembler, offset by size of return PC pushed on stack.
  1161  			addr := ssagen.SpillSlotAddr(ap, x86.REG_SP, v.Block.Func.Config.PtrSize)
  1162  			s.FuncInfo().AddSpill(
  1163  				obj.RegSpill{Reg: ap.Reg, Addr: addr, Unspill: loadByType(ap.Type), Spill: storeByType(ap.Type)})
  1164  		}
  1165  		v.Block.Func.RegArgs = nil
  1166  		ssagen.CheckArgReg(v)
  1167  	case ssa.OpAMD64LoweredGetClosurePtr:
  1168  		// Closure pointer is DX.
  1169  		ssagen.CheckLoweredGetClosurePtr(v)
  1170  	case ssa.OpAMD64LoweredGetG:
  1171  		if s.ABI == obj.ABIInternal {
  1172  			v.Fatalf("LoweredGetG should not appear in ABIInternal")
  1173  		}
  1174  		r := v.Reg()
  1175  		getgFromTLS(s, r)
  1176  	case ssa.OpAMD64CALLstatic, ssa.OpAMD64CALLtail:
  1177  		if s.ABI == obj.ABI0 && v.Aux.(*ssa.AuxCall).Fn.ABI() == obj.ABIInternal {
  1178  			// zeroing X15 when entering ABIInternal from ABI0
  1179  			opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
  1180  			// set G register from TLS
  1181  			getgFromTLS(s, x86.REG_R14)
  1182  		}
  1183  		if v.Op == ssa.OpAMD64CALLtail {
  1184  			s.TailCall(v)
  1185  			break
  1186  		}
  1187  		s.Call(v)
  1188  		if s.ABI == obj.ABIInternal && v.Aux.(*ssa.AuxCall).Fn.ABI() == obj.ABI0 {
  1189  			// zeroing X15 when entering ABIInternal from ABI0
  1190  			opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
  1191  			// set G register from TLS
  1192  			getgFromTLS(s, x86.REG_R14)
  1193  		}
  1194  	case ssa.OpAMD64CALLclosure, ssa.OpAMD64CALLinter:
  1195  		s.Call(v)
  1196  
  1197  	case ssa.OpAMD64LoweredGetCallerPC:
  1198  		p := s.Prog(x86.AMOVQ)
  1199  		p.From.Type = obj.TYPE_MEM
  1200  		p.From.Offset = -8 // PC is stored 8 bytes below first parameter.
  1201  		p.From.Name = obj.NAME_PARAM
  1202  		p.To.Type = obj.TYPE_REG
  1203  		p.To.Reg = v.Reg()
  1204  
  1205  	case ssa.OpAMD64LoweredGetCallerSP:
  1206  		// caller's SP is the address of the first arg
  1207  		mov := x86.AMOVQ
  1208  		if types.PtrSize == 4 {
  1209  			mov = x86.AMOVL
  1210  		}
  1211  		p := s.Prog(mov)
  1212  		p.From.Type = obj.TYPE_ADDR
  1213  		p.From.Offset = -base.Ctxt.Arch.FixedFrameSize // 0 on amd64, just to be consistent with other architectures
  1214  		p.From.Name = obj.NAME_PARAM
  1215  		p.To.Type = obj.TYPE_REG
  1216  		p.To.Reg = v.Reg()
  1217  
  1218  	case ssa.OpAMD64LoweredWB:
  1219  		p := s.Prog(obj.ACALL)
  1220  		p.To.Type = obj.TYPE_MEM
  1221  		p.To.Name = obj.NAME_EXTERN
  1222  		// AuxInt encodes how many buffer entries we need.
  1223  		p.To.Sym = ir.Syms.GCWriteBarrier[v.AuxInt-1]
  1224  
  1225  	case ssa.OpAMD64LoweredPanicBoundsRR, ssa.OpAMD64LoweredPanicBoundsRC, ssa.OpAMD64LoweredPanicBoundsCR, ssa.OpAMD64LoweredPanicBoundsCC:
  1226  		// Compute the constant we put in the PCData entry for this call.
  1227  		code, signed := ssa.BoundsKind(v.AuxInt).Code()
  1228  		xIsReg := false
  1229  		yIsReg := false
  1230  		xVal := 0
  1231  		yVal := 0
  1232  		switch v.Op {
  1233  		case ssa.OpAMD64LoweredPanicBoundsRR:
  1234  			xIsReg = true
  1235  			xVal = int(v.Args[0].Reg() - x86.REG_AX)
  1236  			yIsReg = true
  1237  			yVal = int(v.Args[1].Reg() - x86.REG_AX)
  1238  		case ssa.OpAMD64LoweredPanicBoundsRC:
  1239  			xIsReg = true
  1240  			xVal = int(v.Args[0].Reg() - x86.REG_AX)
  1241  			c := v.Aux.(ssa.PanicBoundsC).C
  1242  			if c >= 0 && c <= abi.BoundsMaxConst {
  1243  				yVal = int(c)
  1244  			} else {
  1245  				// Move constant to a register
  1246  				yIsReg = true
  1247  				if yVal == xVal {
  1248  					yVal = 1
  1249  				}
  1250  				p := s.Prog(x86.AMOVQ)
  1251  				p.From.Type = obj.TYPE_CONST
  1252  				p.From.Offset = c
  1253  				p.To.Type = obj.TYPE_REG
  1254  				p.To.Reg = x86.REG_AX + int16(yVal)
  1255  			}
  1256  		case ssa.OpAMD64LoweredPanicBoundsCR:
  1257  			yIsReg = true
  1258  			yVal := int(v.Args[0].Reg() - x86.REG_AX)
  1259  			c := v.Aux.(ssa.PanicBoundsC).C
  1260  			if c >= 0 && c <= abi.BoundsMaxConst {
  1261  				xVal = int(c)
  1262  			} else {
  1263  				// Move constant to a register
  1264  				xIsReg = true
  1265  				if xVal == yVal {
  1266  					xVal = 1
  1267  				}
  1268  				p := s.Prog(x86.AMOVQ)
  1269  				p.From.Type = obj.TYPE_CONST
  1270  				p.From.Offset = c
  1271  				p.To.Type = obj.TYPE_REG
  1272  				p.To.Reg = x86.REG_AX + int16(xVal)
  1273  			}
  1274  		case ssa.OpAMD64LoweredPanicBoundsCC:
  1275  			c := v.Aux.(ssa.PanicBoundsCC).Cx
  1276  			if c >= 0 && c <= abi.BoundsMaxConst {
  1277  				xVal = int(c)
  1278  			} else {
  1279  				// Move constant to a register
  1280  				xIsReg = true
  1281  				p := s.Prog(x86.AMOVQ)
  1282  				p.From.Type = obj.TYPE_CONST
  1283  				p.From.Offset = c
  1284  				p.To.Type = obj.TYPE_REG
  1285  				p.To.Reg = x86.REG_AX + int16(xVal)
  1286  			}
  1287  			c = v.Aux.(ssa.PanicBoundsCC).Cy
  1288  			if c >= 0 && c <= abi.BoundsMaxConst {
  1289  				yVal = int(c)
  1290  			} else {
  1291  				// Move constant to a register
  1292  				yIsReg = true
  1293  				yVal = 1
  1294  				p := s.Prog(x86.AMOVQ)
  1295  				p.From.Type = obj.TYPE_CONST
  1296  				p.From.Offset = c
  1297  				p.To.Type = obj.TYPE_REG
  1298  				p.To.Reg = x86.REG_AX + int16(yVal)
  1299  			}
  1300  		}
  1301  		c := abi.BoundsEncode(code, signed, xIsReg, yIsReg, xVal, yVal)
  1302  
  1303  		p := s.Prog(obj.APCDATA)
  1304  		p.From.SetConst(abi.PCDATA_PanicBounds)
  1305  		p.To.SetConst(int64(c))
  1306  		p = s.Prog(obj.ACALL)
  1307  		p.To.Type = obj.TYPE_MEM
  1308  		p.To.Name = obj.NAME_EXTERN
  1309  		p.To.Sym = ir.Syms.PanicBounds
  1310  
  1311  	case ssa.OpAMD64NEGQ, ssa.OpAMD64NEGL,
  1312  		ssa.OpAMD64BSWAPQ, ssa.OpAMD64BSWAPL,
  1313  		ssa.OpAMD64NOTQ, ssa.OpAMD64NOTL:
  1314  		p := s.Prog(v.Op.Asm())
  1315  		p.To.Type = obj.TYPE_REG
  1316  		p.To.Reg = v.Reg()
  1317  
  1318  	case ssa.OpAMD64NEGLflags:
  1319  		p := s.Prog(v.Op.Asm())
  1320  		p.To.Type = obj.TYPE_REG
  1321  		p.To.Reg = v.Reg0()
  1322  
  1323  	case ssa.OpAMD64ADDQconstflags, ssa.OpAMD64ADDLconstflags:
  1324  		p := s.Prog(v.Op.Asm())
  1325  		p.From.Type = obj.TYPE_CONST
  1326  		p.From.Offset = v.AuxInt
  1327  		// Note: the inc/dec instructions do not modify
  1328  		// the carry flag like add$1 / sub$1 do.
  1329  		// We currently never use the CF/OF flags from
  1330  		// these instructions, so that is ok.
  1331  		switch {
  1332  		case p.As == x86.AADDQ && p.From.Offset == 1:
  1333  			p.As = x86.AINCQ
  1334  			p.From.Type = obj.TYPE_NONE
  1335  		case p.As == x86.AADDQ && p.From.Offset == -1:
  1336  			p.As = x86.ADECQ
  1337  			p.From.Type = obj.TYPE_NONE
  1338  		case p.As == x86.AADDL && p.From.Offset == 1:
  1339  			p.As = x86.AINCL
  1340  			p.From.Type = obj.TYPE_NONE
  1341  		case p.As == x86.AADDL && p.From.Offset == -1:
  1342  			p.As = x86.ADECL
  1343  			p.From.Type = obj.TYPE_NONE
  1344  		}
  1345  		p.To.Type = obj.TYPE_REG
  1346  		p.To.Reg = v.Reg0()
  1347  
  1348  	case ssa.OpAMD64BSFQ, ssa.OpAMD64BSRQ, ssa.OpAMD64BSFL, ssa.OpAMD64BSRL, ssa.OpAMD64SQRTSD, ssa.OpAMD64SQRTSS:
  1349  		p := s.Prog(v.Op.Asm())
  1350  		p.From.Type = obj.TYPE_REG
  1351  		p.From.Reg = v.Args[0].Reg()
  1352  		p.To.Type = obj.TYPE_REG
  1353  		switch v.Op {
  1354  		case ssa.OpAMD64BSFQ, ssa.OpAMD64BSRQ:
  1355  			p.To.Reg = v.Reg0()
  1356  		case ssa.OpAMD64BSFL, ssa.OpAMD64BSRL, ssa.OpAMD64SQRTSD, ssa.OpAMD64SQRTSS:
  1357  			p.To.Reg = v.Reg()
  1358  		}
  1359  	case ssa.OpAMD64LoweredRound32F, ssa.OpAMD64LoweredRound64F:
  1360  		// input is already rounded
  1361  	case ssa.OpAMD64ROUNDSD:
  1362  		p := s.Prog(v.Op.Asm())
  1363  		val := v.AuxInt
  1364  		// 0 means math.RoundToEven, 1 Floor, 2 Ceil, 3 Trunc
  1365  		if val < 0 || val > 3 {
  1366  			v.Fatalf("Invalid rounding mode")
  1367  		}
  1368  		p.From.Offset = val
  1369  		p.From.Type = obj.TYPE_CONST
  1370  		p.AddRestSourceReg(v.Args[0].Reg())
  1371  		p.To.Type = obj.TYPE_REG
  1372  		p.To.Reg = v.Reg()
  1373  	case ssa.OpAMD64POPCNTQ, ssa.OpAMD64POPCNTL,
  1374  		ssa.OpAMD64TZCNTQ, ssa.OpAMD64TZCNTL,
  1375  		ssa.OpAMD64LZCNTQ, ssa.OpAMD64LZCNTL:
  1376  		if v.Args[0].Reg() != v.Reg() {
  1377  			// POPCNT/TZCNT/LZCNT have a false dependency on the destination register on Intel cpus.
  1378  			// TZCNT/LZCNT problem affects pre-Skylake models. See discussion at https://gcc.gnu.org/bugzilla/show_bug.cgi?id=62011#c7.
  1379  			// Xor register with itself to break the dependency.
  1380  			opregreg(s, x86.AXORL, v.Reg(), v.Reg())
  1381  		}
  1382  		p := s.Prog(v.Op.Asm())
  1383  		p.From.Type = obj.TYPE_REG
  1384  		p.From.Reg = v.Args[0].Reg()
  1385  		p.To.Type = obj.TYPE_REG
  1386  		p.To.Reg = v.Reg()
  1387  
  1388  	case ssa.OpAMD64SETEQ, ssa.OpAMD64SETNE,
  1389  		ssa.OpAMD64SETL, ssa.OpAMD64SETLE,
  1390  		ssa.OpAMD64SETG, ssa.OpAMD64SETGE,
  1391  		ssa.OpAMD64SETGF, ssa.OpAMD64SETGEF,
  1392  		ssa.OpAMD64SETB, ssa.OpAMD64SETBE,
  1393  		ssa.OpAMD64SETORD, ssa.OpAMD64SETNAN,
  1394  		ssa.OpAMD64SETA, ssa.OpAMD64SETAE,
  1395  		ssa.OpAMD64SETO:
  1396  		p := s.Prog(v.Op.Asm())
  1397  		p.To.Type = obj.TYPE_REG
  1398  		p.To.Reg = v.Reg()
  1399  
  1400  	case ssa.OpAMD64SETEQstore, ssa.OpAMD64SETNEstore,
  1401  		ssa.OpAMD64SETLstore, ssa.OpAMD64SETLEstore,
  1402  		ssa.OpAMD64SETGstore, ssa.OpAMD64SETGEstore,
  1403  		ssa.OpAMD64SETBstore, ssa.OpAMD64SETBEstore,
  1404  		ssa.OpAMD64SETAstore, ssa.OpAMD64SETAEstore:
  1405  		p := s.Prog(v.Op.Asm())
  1406  		p.To.Type = obj.TYPE_MEM
  1407  		p.To.Reg = v.Args[0].Reg()
  1408  		ssagen.AddAux(&p.To, v)
  1409  
  1410  	case ssa.OpAMD64SETEQstoreidx1, ssa.OpAMD64SETNEstoreidx1,
  1411  		ssa.OpAMD64SETLstoreidx1, ssa.OpAMD64SETLEstoreidx1,
  1412  		ssa.OpAMD64SETGstoreidx1, ssa.OpAMD64SETGEstoreidx1,
  1413  		ssa.OpAMD64SETBstoreidx1, ssa.OpAMD64SETBEstoreidx1,
  1414  		ssa.OpAMD64SETAstoreidx1, ssa.OpAMD64SETAEstoreidx1:
  1415  		p := s.Prog(v.Op.Asm())
  1416  		memIdx(&p.To, v)
  1417  		ssagen.AddAux(&p.To, v)
  1418  
  1419  	case ssa.OpAMD64SETNEF:
  1420  		t := v.RegTmp()
  1421  		p := s.Prog(v.Op.Asm())
  1422  		p.To.Type = obj.TYPE_REG
  1423  		p.To.Reg = v.Reg()
  1424  		q := s.Prog(x86.ASETPS)
  1425  		q.To.Type = obj.TYPE_REG
  1426  		q.To.Reg = t
  1427  		// ORL avoids partial register write and is smaller than ORQ, used by old compiler
  1428  		opregreg(s, x86.AORL, v.Reg(), t)
  1429  
  1430  	case ssa.OpAMD64SETEQF:
  1431  		t := v.RegTmp()
  1432  		p := s.Prog(v.Op.Asm())
  1433  		p.To.Type = obj.TYPE_REG
  1434  		p.To.Reg = v.Reg()
  1435  		q := s.Prog(x86.ASETPC)
  1436  		q.To.Type = obj.TYPE_REG
  1437  		q.To.Reg = t
  1438  		// ANDL avoids partial register write and is smaller than ANDQ, used by old compiler
  1439  		opregreg(s, x86.AANDL, v.Reg(), t)
  1440  
  1441  	case ssa.OpAMD64InvertFlags:
  1442  		v.Fatalf("InvertFlags should never make it to codegen %v", v.LongString())
  1443  	case ssa.OpAMD64FlagEQ, ssa.OpAMD64FlagLT_ULT, ssa.OpAMD64FlagLT_UGT, ssa.OpAMD64FlagGT_ULT, ssa.OpAMD64FlagGT_UGT:
  1444  		v.Fatalf("Flag* ops should never make it to codegen %v", v.LongString())
  1445  	case ssa.OpAMD64AddTupleFirst32, ssa.OpAMD64AddTupleFirst64:
  1446  		v.Fatalf("AddTupleFirst* should never make it to codegen %v", v.LongString())
  1447  	case ssa.OpAMD64REPSTOSQ:
  1448  		s.Prog(x86.AREP)
  1449  		s.Prog(x86.ASTOSQ)
  1450  	case ssa.OpAMD64REPMOVSQ:
  1451  		s.Prog(x86.AREP)
  1452  		s.Prog(x86.AMOVSQ)
  1453  	case ssa.OpAMD64LoweredNilCheck:
  1454  		// Issue a load which will fault if the input is nil.
  1455  		// TODO: We currently use the 2-byte instruction TESTB AX, (reg).
  1456  		// Should we use the 3-byte TESTB $0, (reg) instead? It is larger
  1457  		// but it doesn't have false dependency on AX.
  1458  		// Or maybe allocate an output register and use MOVL (reg),reg2 ?
  1459  		// That trades clobbering flags for clobbering a register.
  1460  		p := s.Prog(x86.ATESTB)
  1461  		p.From.Type = obj.TYPE_REG
  1462  		p.From.Reg = x86.REG_AX
  1463  		p.To.Type = obj.TYPE_MEM
  1464  		p.To.Reg = v.Args[0].Reg()
  1465  		if logopt.Enabled() {
  1466  			logopt.LogOpt(v.Pos, "nilcheck", "genssa", v.Block.Func.Name)
  1467  		}
  1468  		if base.Debug.Nil != 0 && v.Pos.Line() > 1 { // v.Pos.Line()==1 in generated wrappers
  1469  			base.WarnfAt(v.Pos, "generated nil check")
  1470  		}
  1471  	case ssa.OpAMD64MOVBatomicload, ssa.OpAMD64MOVLatomicload, ssa.OpAMD64MOVQatomicload:
  1472  		p := s.Prog(v.Op.Asm())
  1473  		p.From.Type = obj.TYPE_MEM
  1474  		p.From.Reg = v.Args[0].Reg()
  1475  		ssagen.AddAux(&p.From, v)
  1476  		p.To.Type = obj.TYPE_REG
  1477  		p.To.Reg = v.Reg0()
  1478  	case ssa.OpAMD64XCHGB, ssa.OpAMD64XCHGL, ssa.OpAMD64XCHGQ:
  1479  		p := s.Prog(v.Op.Asm())
  1480  		p.From.Type = obj.TYPE_REG
  1481  		p.From.Reg = v.Reg0()
  1482  		p.To.Type = obj.TYPE_MEM
  1483  		p.To.Reg = v.Args[1].Reg()
  1484  		ssagen.AddAux(&p.To, v)
  1485  	case ssa.OpAMD64XADDLlock, ssa.OpAMD64XADDQlock:
  1486  		s.Prog(x86.ALOCK)
  1487  		p := s.Prog(v.Op.Asm())
  1488  		p.From.Type = obj.TYPE_REG
  1489  		p.From.Reg = v.Reg0()
  1490  		p.To.Type = obj.TYPE_MEM
  1491  		p.To.Reg = v.Args[1].Reg()
  1492  		ssagen.AddAux(&p.To, v)
  1493  	case ssa.OpAMD64CMPXCHGLlock, ssa.OpAMD64CMPXCHGQlock:
  1494  		if v.Args[1].Reg() != x86.REG_AX {
  1495  			v.Fatalf("input[1] not in AX %s", v.LongString())
  1496  		}
  1497  		s.Prog(x86.ALOCK)
  1498  		p := s.Prog(v.Op.Asm())
  1499  		p.From.Type = obj.TYPE_REG
  1500  		p.From.Reg = v.Args[2].Reg()
  1501  		p.To.Type = obj.TYPE_MEM
  1502  		p.To.Reg = v.Args[0].Reg()
  1503  		ssagen.AddAux(&p.To, v)
  1504  		p = s.Prog(x86.ASETEQ)
  1505  		p.To.Type = obj.TYPE_REG
  1506  		p.To.Reg = v.Reg0()
  1507  	case ssa.OpAMD64ANDBlock, ssa.OpAMD64ANDLlock, ssa.OpAMD64ANDQlock, ssa.OpAMD64ORBlock, ssa.OpAMD64ORLlock, ssa.OpAMD64ORQlock:
  1508  		// Atomic memory operations that don't need to return the old value.
  1509  		s.Prog(x86.ALOCK)
  1510  		p := s.Prog(v.Op.Asm())
  1511  		p.From.Type = obj.TYPE_REG
  1512  		p.From.Reg = v.Args[1].Reg()
  1513  		p.To.Type = obj.TYPE_MEM
  1514  		p.To.Reg = v.Args[0].Reg()
  1515  		ssagen.AddAux(&p.To, v)
  1516  	case ssa.OpAMD64LoweredAtomicAnd64, ssa.OpAMD64LoweredAtomicOr64, ssa.OpAMD64LoweredAtomicAnd32, ssa.OpAMD64LoweredAtomicOr32:
  1517  		// Atomic memory operations that need to return the old value.
  1518  		// We need to do these with compare-and-exchange to get access to the old value.
  1519  		// loop:
  1520  		// MOVQ mask, tmp
  1521  		// MOVQ (addr), AX
  1522  		// ANDQ AX, tmp
  1523  		// LOCK CMPXCHGQ tmp, (addr) : note that AX is implicit old value to compare against
  1524  		// JNE loop
  1525  		// : result in AX
  1526  		mov := x86.AMOVQ
  1527  		op := x86.AANDQ
  1528  		cmpxchg := x86.ACMPXCHGQ
  1529  		switch v.Op {
  1530  		case ssa.OpAMD64LoweredAtomicOr64:
  1531  			op = x86.AORQ
  1532  		case ssa.OpAMD64LoweredAtomicAnd32:
  1533  			mov = x86.AMOVL
  1534  			op = x86.AANDL
  1535  			cmpxchg = x86.ACMPXCHGL
  1536  		case ssa.OpAMD64LoweredAtomicOr32:
  1537  			mov = x86.AMOVL
  1538  			op = x86.AORL
  1539  			cmpxchg = x86.ACMPXCHGL
  1540  		}
  1541  		addr := v.Args[0].Reg()
  1542  		mask := v.Args[1].Reg()
  1543  		tmp := v.RegTmp()
  1544  		p1 := s.Prog(mov)
  1545  		p1.From.Type = obj.TYPE_REG
  1546  		p1.From.Reg = mask
  1547  		p1.To.Type = obj.TYPE_REG
  1548  		p1.To.Reg = tmp
  1549  		p2 := s.Prog(mov)
  1550  		p2.From.Type = obj.TYPE_MEM
  1551  		p2.From.Reg = addr
  1552  		ssagen.AddAux(&p2.From, v)
  1553  		p2.To.Type = obj.TYPE_REG
  1554  		p2.To.Reg = x86.REG_AX
  1555  		p3 := s.Prog(op)
  1556  		p3.From.Type = obj.TYPE_REG
  1557  		p3.From.Reg = x86.REG_AX
  1558  		p3.To.Type = obj.TYPE_REG
  1559  		p3.To.Reg = tmp
  1560  		s.Prog(x86.ALOCK)
  1561  		p5 := s.Prog(cmpxchg)
  1562  		p5.From.Type = obj.TYPE_REG
  1563  		p5.From.Reg = tmp
  1564  		p5.To.Type = obj.TYPE_MEM
  1565  		p5.To.Reg = addr
  1566  		ssagen.AddAux(&p5.To, v)
  1567  		p6 := s.Prog(x86.AJNE)
  1568  		p6.To.Type = obj.TYPE_BRANCH
  1569  		p6.To.SetTarget(p1)
  1570  	case ssa.OpAMD64PrefetchT0, ssa.OpAMD64PrefetchNTA:
  1571  		p := s.Prog(v.Op.Asm())
  1572  		p.From.Type = obj.TYPE_MEM
  1573  		p.From.Reg = v.Args[0].Reg()
  1574  	case ssa.OpClobber:
  1575  		p := s.Prog(x86.AMOVL)
  1576  		p.From.Type = obj.TYPE_CONST
  1577  		p.From.Offset = 0xdeaddead
  1578  		p.To.Type = obj.TYPE_MEM
  1579  		p.To.Reg = x86.REG_SP
  1580  		ssagen.AddAux(&p.To, v)
  1581  		p = s.Prog(x86.AMOVL)
  1582  		p.From.Type = obj.TYPE_CONST
  1583  		p.From.Offset = 0xdeaddead
  1584  		p.To.Type = obj.TYPE_MEM
  1585  		p.To.Reg = x86.REG_SP
  1586  		ssagen.AddAux(&p.To, v)
  1587  		p.To.Offset += 4
  1588  	case ssa.OpClobberReg:
  1589  		x := uint64(0xdeaddeaddeaddead)
  1590  		p := s.Prog(x86.AMOVQ)
  1591  		p.From.Type = obj.TYPE_CONST
  1592  		p.From.Offset = int64(x)
  1593  		p.To.Type = obj.TYPE_REG
  1594  		p.To.Reg = v.Reg()
  1595  	default:
  1596  		v.Fatalf("genValue not implemented: %s", v.LongString())
  1597  	}
  1598  }
  1599  
  1600  var blockJump = [...]struct {
  1601  	asm, invasm obj.As
  1602  }{
  1603  	ssa.BlockAMD64EQ:  {x86.AJEQ, x86.AJNE},
  1604  	ssa.BlockAMD64NE:  {x86.AJNE, x86.AJEQ},
  1605  	ssa.BlockAMD64LT:  {x86.AJLT, x86.AJGE},
  1606  	ssa.BlockAMD64GE:  {x86.AJGE, x86.AJLT},
  1607  	ssa.BlockAMD64LE:  {x86.AJLE, x86.AJGT},
  1608  	ssa.BlockAMD64GT:  {x86.AJGT, x86.AJLE},
  1609  	ssa.BlockAMD64OS:  {x86.AJOS, x86.AJOC},
  1610  	ssa.BlockAMD64OC:  {x86.AJOC, x86.AJOS},
  1611  	ssa.BlockAMD64ULT: {x86.AJCS, x86.AJCC},
  1612  	ssa.BlockAMD64UGE: {x86.AJCC, x86.AJCS},
  1613  	ssa.BlockAMD64UGT: {x86.AJHI, x86.AJLS},
  1614  	ssa.BlockAMD64ULE: {x86.AJLS, x86.AJHI},
  1615  	ssa.BlockAMD64ORD: {x86.AJPC, x86.AJPS},
  1616  	ssa.BlockAMD64NAN: {x86.AJPS, x86.AJPC},
  1617  }
  1618  
  1619  var eqfJumps = [2][2]ssagen.IndexJump{
  1620  	{{Jump: x86.AJNE, Index: 1}, {Jump: x86.AJPS, Index: 1}}, // next == b.Succs[0]
  1621  	{{Jump: x86.AJNE, Index: 1}, {Jump: x86.AJPC, Index: 0}}, // next == b.Succs[1]
  1622  }
  1623  var nefJumps = [2][2]ssagen.IndexJump{
  1624  	{{Jump: x86.AJNE, Index: 0}, {Jump: x86.AJPC, Index: 1}}, // next == b.Succs[0]
  1625  	{{Jump: x86.AJNE, Index: 0}, {Jump: x86.AJPS, Index: 0}}, // next == b.Succs[1]
  1626  }
  1627  
  1628  func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) {
  1629  	switch b.Kind {
  1630  	case ssa.BlockPlain, ssa.BlockDefer:
  1631  		if b.Succs[0].Block() != next {
  1632  			p := s.Prog(obj.AJMP)
  1633  			p.To.Type = obj.TYPE_BRANCH
  1634  			s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()})
  1635  		}
  1636  	case ssa.BlockExit, ssa.BlockRetJmp:
  1637  	case ssa.BlockRet:
  1638  		s.Prog(obj.ARET)
  1639  
  1640  	case ssa.BlockAMD64EQF:
  1641  		s.CombJump(b, next, &eqfJumps)
  1642  
  1643  	case ssa.BlockAMD64NEF:
  1644  		s.CombJump(b, next, &nefJumps)
  1645  
  1646  	case ssa.BlockAMD64EQ, ssa.BlockAMD64NE,
  1647  		ssa.BlockAMD64LT, ssa.BlockAMD64GE,
  1648  		ssa.BlockAMD64LE, ssa.BlockAMD64GT,
  1649  		ssa.BlockAMD64OS, ssa.BlockAMD64OC,
  1650  		ssa.BlockAMD64ULT, ssa.BlockAMD64UGT,
  1651  		ssa.BlockAMD64ULE, ssa.BlockAMD64UGE:
  1652  		jmp := blockJump[b.Kind]
  1653  		switch next {
  1654  		case b.Succs[0].Block():
  1655  			s.Br(jmp.invasm, b.Succs[1].Block())
  1656  		case b.Succs[1].Block():
  1657  			s.Br(jmp.asm, b.Succs[0].Block())
  1658  		default:
  1659  			if b.Likely != ssa.BranchUnlikely {
  1660  				s.Br(jmp.asm, b.Succs[0].Block())
  1661  				s.Br(obj.AJMP, b.Succs[1].Block())
  1662  			} else {
  1663  				s.Br(jmp.invasm, b.Succs[1].Block())
  1664  				s.Br(obj.AJMP, b.Succs[0].Block())
  1665  			}
  1666  		}
  1667  
  1668  	case ssa.BlockAMD64JUMPTABLE:
  1669  		// JMP      *(TABLE)(INDEX*8)
  1670  		p := s.Prog(obj.AJMP)
  1671  		p.To.Type = obj.TYPE_MEM
  1672  		p.To.Reg = b.Controls[1].Reg()
  1673  		p.To.Index = b.Controls[0].Reg()
  1674  		p.To.Scale = 8
  1675  		// Save jump tables for later resolution of the target blocks.
  1676  		s.JumpTables = append(s.JumpTables, b)
  1677  
  1678  	default:
  1679  		b.Fatalf("branch not implemented: %s", b.LongString())
  1680  	}
  1681  }
  1682  
  1683  func loadRegResult(s *ssagen.State, f *ssa.Func, t *types.Type, reg int16, n *ir.Name, off int64) *obj.Prog {
  1684  	p := s.Prog(loadByType(t))
  1685  	p.From.Type = obj.TYPE_MEM
  1686  	p.From.Name = obj.NAME_AUTO
  1687  	p.From.Sym = n.Linksym()
  1688  	p.From.Offset = n.FrameOffset() + off
  1689  	p.To.Type = obj.TYPE_REG
  1690  	p.To.Reg = reg
  1691  	return p
  1692  }
  1693  
  1694  func spillArgReg(pp *objw.Progs, p *obj.Prog, f *ssa.Func, t *types.Type, reg int16, n *ir.Name, off int64) *obj.Prog {
  1695  	p = pp.Append(p, storeByType(t), obj.TYPE_REG, reg, 0, obj.TYPE_MEM, 0, n.FrameOffset()+off)
  1696  	p.To.Name = obj.NAME_PARAM
  1697  	p.To.Sym = n.Linksym()
  1698  	p.Pos = p.Pos.WithNotStmt()
  1699  	return p
  1700  }
  1701  
  1702  // zero 16 bytes at reg+off.
  1703  func zero16(s *ssagen.State, reg int16, off int64) {
  1704  	//   MOVUPS  X15, off(ptrReg)
  1705  	p := s.Prog(x86.AMOVUPS)
  1706  	p.From.Type = obj.TYPE_REG
  1707  	p.From.Reg = x86.REG_X15
  1708  	p.To.Type = obj.TYPE_MEM
  1709  	p.To.Reg = reg
  1710  	p.To.Offset = off
  1711  }
  1712  

View as plain text