Source file src/cmd/compile/internal/ssacompile/decomposeaddr.go

     1  // Copyright 2026 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package ssacompile
     6  
     7  // This file contains the decomposeAddr pass.
     8  // decomposeAddr decomposes aggregate variables
     9  // (structs and arrays) into separate scalar variables. This allows the subsequent
    10  // mem2reg pass to promote these scalars to ssa values, improving
    11  // performance by reducing memory traffic. This is assuming the regalloc
    12  // can make good decisions under the increased pressure of more ssa values.
    13  //
    14  // This pass is also known as SROA (scalar replacement of aggregates) in other compilers.
    15  //
    16  // Overview:
    17  // 1. Candidate identification: find local variables that don't qualify for mem2reg and
    18  //    are not special. Special variables are NonMergeable or OpenDeferSlot.
    19  // 2. Partitioning: determine if the aggregate's fields are accessed in a way that
    20  //    permits decomposition (e.g., aligned with leaf field boundaries).
    21  // 3. Breakdown of block operations: replace OpZero and OpMove on aggregates with
    22  //    field-by-field scalar operations.
    23  // 4. Rewriting: create new individual local variables for each leaf field and
    24  //    rewrite all loads, stores, and VarDefs to use the new variables.
    25  //
    26  // Stability constraints:
    27  // - OpVarDef is only emitted for new scalars if they:
    28  //		- have pointers: the GC needs to know the live ranges of these variables.
    29  //		- are merge candidates: mergelocals need to know the live ranges of these variables.
    30  // - Aggregates that are too large are skipped to prevent compiler hangs or excessive
    31  //   memory usage.
    32  
    33  import (
    34  	"cmd/compile/internal/base"
    35  	"cmd/compile/internal/ir"
    36  	"cmd/compile/internal/ssa"
    37  	"cmd/compile/internal/ssa/ssaop"
    38  	"cmd/compile/internal/types"
    39  	"fmt"
    40  	"slices"
    41  )
    42  
    43  type leafField struct {
    44  	off int64
    45  	typ *types.Type
    46  }
    47  
    48  type decomposeCandidate struct {
    49  	n              *ir.Name
    50  	leaves         []leafField
    51  	leavesByOff    map[int64]leafField
    52  	localAddrs     []*ssa.Value // all the LocalAddrs of this candidate.
    53  	unDecomposable bool
    54  }
    55  
    56  var decomposeAddrDummyValue = &ssa.Value{}
    57  
    58  // zeroValueForType returns a zero-valued SSA constant for the given type.
    59  // Returns nil for unsupported types.
    60  //
    61  // f may be nil, in which case no constant is materialized: the function
    62  // returns the non-nil decomposeAddrDummyValue for supported types (and nil,
    63  // as usual, for unsupported ones). This mode is used during candidate
    64  // selection to ask "could we build a zero value for this type?" without
    65  // mutating the function; the dummy value itself must never be used as an
    66  // SSA value.
    67  func zeroValueForType(f *ssa.Func, t *types.Type) *ssa.Value {
    68  	switch {
    69  	case t.IsBoolean():
    70  		if f == nil {
    71  			return decomposeAddrDummyValue
    72  		}
    73  		return f.ConstBool(t, false)
    74  	case t.IsInteger():
    75  		if f == nil {
    76  			return decomposeAddrDummyValue
    77  		}
    78  		switch t.Size() {
    79  		case 1:
    80  			return f.ConstInt8(t, 0)
    81  		case 2:
    82  			return f.ConstInt16(t, 0)
    83  		case 4:
    84  			return f.ConstInt32(t, 0)
    85  		default:
    86  			return f.ConstInt64(t, 0)
    87  		}
    88  	case t.IsFloat():
    89  		if f == nil {
    90  			return decomposeAddrDummyValue
    91  		}
    92  		switch t.Size() {
    93  		case 4:
    94  			return f.ConstFloat32(t, 0)
    95  		default:
    96  			return f.ConstFloat64(t, 0)
    97  		}
    98  	case t.IsPtrShaped():
    99  		if f == nil {
   100  			return decomposeAddrDummyValue
   101  		}
   102  		return f.ConstNil(t)
   103  	case t.IsString():
   104  		if f == nil {
   105  			return decomposeAddrDummyValue
   106  		}
   107  		return f.ConstEmptyString(t)
   108  	case t.IsSlice():
   109  		if f == nil {
   110  			return decomposeAddrDummyValue
   111  		}
   112  		return f.ConstSlice(t)
   113  	case t.IsInterface():
   114  		if f == nil {
   115  			return decomposeAddrDummyValue
   116  		}
   117  		return f.ConstInterface(t)
   118  	case t.IsSIMD():
   119  		if f == nil {
   120  			return decomposeAddrDummyValue
   121  		}
   122  		return f.ConstSIMD(t)
   123  	default:
   124  		// Unsupported types.
   125  		// TODO: sometimes composite types like [1]uint64
   126  		// can also get passed down here, maybe make this function aware of them as well?
   127  		return nil
   128  	}
   129  }
   130  
   131  // decomposeAddr decomposes aggregate variables
   132  // (structs and arrays) accessed via OffPtr
   133  // into separate scalar variables, allowing mem2reg to promote them.
   134  func decomposeAddr(f *ssa.Func) {
   135  	if base.Flag.N != 0 {
   136  		return
   137  	}
   138  
   139  	// Get demographics up front: for a function with no candidate
   140  	// LocalAddr (the common case) this allocates nothing and the pass
   141  	// exits before setting up any of its machinery below.
   142  	demographics, localAddrs, uses := variableDemographics(f)
   143  	if uses == nil {
   144  		return
   145  	}
   146  	defer func() { uses.free(f) }()
   147  
   148  	st := f.NewStats("decomposeAddr")
   149  
   150  	// maxPartitions limits the number of fields an aggregate can be decomposed into.
   151  	// This prevents OOM and hangs on extremely large arrays or deeply nested structs.
   152  	// Tunable for experiments via -d=ssa/decompose_addr/@maxpartitions=<n>.
   153  	maxPartitions := f.Pass.Val("maxpartitions", 64)
   154  
   155  	// maxActivePartitions limits the number of *active* fields an aggregate
   156  	// can be decomposed into. While maxPartitions bounds how many leaf fields
   157  	// an aggregate may have at all (limiting the work this pass does per
   158  	// variable), maxActivePartitions bounds how many of those leaves are
   159  	// actually read (limiting the register pressure the promotion creates):
   160  	// a leaf is "active" if some Load reads it, and only active leaves become
   161  	// live SSA values after the follow-up mem2reg promotion. For example, for
   162  	//
   163  	//	var a [8]int
   164  	//	a[1] = x
   165  	//	a[2] = y
   166  	//	use(a[1])
   167  	//
   168  	// the aggregate has 8 leaves (all within maxPartitions), but only a[1] is
   169  	// active. Leaves that are only stored to die immediately after promotion,
   170  	// so they add no register pressure.
   171  	// This is a parameter that controls the trade-off of less memory accesses
   172  	// and higher register pressure for mem2reg.
   173  	// If we set this number too high, the register allocation might behave suboptimally
   174  	// and the performance might even regress.
   175  	// Tunable for experiments via -d=ssa/decompose_addr/@maxactivepartitions=<n>.
   176  	maxActivePartitions := f.Pass.Val("maxactivepartitions", 4)
   177  
   178  	type partitionStatus int
   179  	const (
   180  		partSuccess partitionStatus = iota
   181  		partTooMany                 // too many partitions, above maxPartitions limit
   182  		partLarge                   // leaf field too large to be stored in one register
   183  	)
   184  
   185  	// getLeafFields recursively finds all scalar fields of a type with their given offsets.
   186  	var getLeafFields func(t *types.Type, base int64) (leaves []leafField, status partitionStatus)
   187  	getLeafFields = func(t *types.Type, base int64) (leaves []leafField, status partitionStatus) {
   188  		// checkField updates leaves by looking into a composite type field (or element) of
   189  		// type t2 at offset. It returns the status code.
   190  		checkField := func(t2 *types.Type, offset int64) partitionStatus {
   191  			ls, s := getLeafFields(t2, offset)
   192  			if s != partSuccess {
   193  				return s
   194  			}
   195  			if int64(len(leaves)+len(ls)) > maxPartitions {
   196  				return partTooMany
   197  			}
   198  			leaves = append(leaves, ls...)
   199  			return partSuccess
   200  		}
   201  		if t.IsStruct() && !t.IsSIMD() {
   202  			for i := range t.NumFields() {
   203  				fld := t.Field(i)
   204  				if s := checkField(fld.Type, base+fld.Offset); s != partSuccess {
   205  					return nil, s
   206  				}
   207  			}
   208  		} else if t.IsArray() {
   209  			// Limit decomposition of large arrays.
   210  			if t.NumElem() > maxPartitions {
   211  				return nil, partTooMany
   212  			}
   213  			for i := range t.NumElem() {
   214  				if s := checkField(t.Elem(), base+int64(i)*t.Elem().Size()); s != partSuccess {
   215  					return nil, s
   216  				}
   217  			}
   218  		} else if t.IsString() {
   219  			// Strings split into a pointer and a length word, mirroring
   220  			// (*Func).SplitString. By the time this pass runs, decompose
   221  			// builtin has already rewritten string-typed accesses into
   222  			// word-level ones, so accesses line up with these leaves.
   223  			ptrType := f.Config.Types.BytePtr
   224  			lenType := types.Types[types.TINT]
   225  			if s := checkField(ptrType, base); s != partSuccess {
   226  				return nil, s
   227  			}
   228  			if s := checkField(lenType, base+ptrType.Size()); s != partSuccess {
   229  				return nil, s
   230  			}
   231  		} else if t.IsSlice() {
   232  			// Mirrors (*Func).SplitSlice.
   233  			ptrType := types.NewPtr(t.Elem())
   234  			lenType := types.Types[types.TINT]
   235  			if s := checkField(ptrType, base); s != partSuccess {
   236  				return nil, s
   237  			}
   238  			if s := checkField(lenType, base+ptrType.Size()); s != partSuccess {
   239  				return nil, s
   240  			}
   241  			if s := checkField(lenType, base+ptrType.Size()+lenType.Size()); s != partSuccess {
   242  				return nil, s
   243  			}
   244  		} else if t.IsInterface() {
   245  			// Mirrors (*Func).SplitInterface: the itab/type word is a
   246  			// uintptr (see the comment in typebits.Set), the data word is
   247  			// a scannable pointer.
   248  			itabType := types.Types[types.TUINTPTR]
   249  			dataType := f.Config.Types.BytePtr
   250  			if s := checkField(itabType, base); s != partSuccess {
   251  				return nil, s
   252  			}
   253  			if s := checkField(dataType, base+itabType.Size()); s != partSuccess {
   254  				return nil, s
   255  			}
   256  		} else if t.IsComplex() {
   257  			// Mirrors (*Func).SplitComplex.
   258  			elemType := types.Types[types.TFLOAT32]
   259  			if t.Size() == 16 {
   260  				elemType = types.Types[types.TFLOAT64]
   261  			}
   262  			if s := checkField(elemType, base); s != partSuccess {
   263  				return nil, s
   264  			}
   265  			if s := checkField(elemType, base+elemType.Size()); s != partSuccess {
   266  				return nil, s
   267  			}
   268  		} else {
   269  			// Leaf field
   270  			if !t.IsSIMD() && t.Size() > f.Config.RegSize {
   271  				// This leaf field is too large to be stored in one register.
   272  				// This can still trigger for e.g. 8-byte scalars on 32-bit
   273  				// platforms; wide builtin composites (strings, slices,
   274  				// interfaces, complexes) are split into words above.
   275  				return nil, partLarge
   276  			}
   277  			if t.Size() == 0 {
   278  				// The only zero-sized types are (arrays of) empty structs
   279  				// and zero-length arrays, which are both handled by the
   280  				// composite cases above, so a zero-sized leaf is impossible.
   281  				f.Fatalf("zero-sized leaf field of type %v in decomposeAddr", t)
   282  			}
   283  			return []leafField{{base, t}}, partSuccess
   284  		}
   285  		return leaves, partSuccess
   286  	}
   287  
   288  	// 1. Collect initial candidates.
   289  	candidates := make(map[*ir.Name]*decomposeCandidate)
   290  	// Different ir.Name can have the same name string.
   291  	// We need to give them different split names to distinguish them.
   292  	stringToNames := make(map[string][]*ir.Name)
   293  	localAddrsByName := make(map[*ir.Name][]*ssa.Value)
   294  	seenNames := make(map[*ir.Name]bool)
   295  	for _, v := range localAddrs {
   296  		n := v.Aux.(*ir.Name)
   297  		if !seenNames[n] {
   298  			stringToNames[n.Sym().Name] = append(stringToNames[n.Sym().Name], n)
   299  			seenNames[n] = true
   300  		}
   301  		localAddrsByName[n] = append(localAddrsByName[n], v)
   302  	}
   303  
   304  	namesOrdered := []*ir.Name{}
   305  	for n, las := range localAddrsByName {
   306  		namesOrdered = append(namesOrdered, n)
   307  		slices.SortFunc(las, func(a, b *ssa.Value) int { return int(a.ID - b.ID) })
   308  	}
   309  	slices.SortFunc(namesOrdered, func(a, b *ir.Name) int {
   310  		return int(localAddrsByName[a][0].ID - localAddrsByName[b][0].ID)
   311  	})
   312  NameLoop:
   313  	for _, n := range namesOrdered {
   314  		if dg, ok := demographics[n]; !ok || !dg.lszmco {
   315  			st.Record("non qualifying variable", 1)
   316  			continue
   317  		}
   318  		// Skip special variables
   319  		if n.Class != ir.PAUTO {
   320  			st.Record("params", 1)
   321  			continue
   322  		}
   323  		t := n.Type()
   324  		splittable := (t.IsStruct() && !t.IsSIMD()) || t.IsArray() ||
   325  			t.IsString() || t.IsSlice() || t.IsInterface() || t.IsComplex()
   326  		if !splittable {
   327  			// Bare multi-word builtin composites (strings, slices,
   328  			// interfaces, complexes) split into word leaves just like
   329  			// aggregate fields of those types do; everything else that
   330  			// is not a struct or array stays with mem2reg alone.
   331  			st.Record("non composite", 1)
   332  			continue
   333  		}
   334  		if n.NonMergeable() || n.OpenDeferSlot() {
   335  			// Non mergeable variables and open defer slot are special,
   336  			// the compiler and the runtime expects them to be in the memory.
   337  			// Keep them in memory.
   338  			st.Record("non mergeable or open defer slot", 1)
   339  			continue
   340  		}
   341  		leaves, status := getLeafFields(n.Type(), 0)
   342  		if status != partSuccess {
   343  			switch status {
   344  			case partTooMany:
   345  				st.Record("too many partitions", 1)
   346  			case partLarge:
   347  				st.Record("large leaf field", 1)
   348  			default:
   349  				f.Fatalf("unknown leaf status in decomposeAddr")
   350  			}
   351  			continue
   352  		}
   353  		if len(leaves) == 0 {
   354  			st.Record("no partition", 1)
   355  			continue
   356  		}
   357  		leavesByOff := make(map[int64]leafField, len(leaves))
   358  		for _, lf := range leaves {
   359  			if _, ok := leavesByOff[lf.off]; ok {
   360  				// Go does not have union types, is this possible?
   361  				// Should we fatal out here instead?
   362  				st.Record("duplicate leaf field offset", 1)
   363  				continue NameLoop
   364  			}
   365  			leavesByOff[lf.off] = lf
   366  		}
   367  
   368  		ok := true
   369  		for _, lf := range leaves {
   370  			if zeroValueForType(nil, lf.typ) == nil {
   371  				ok = false
   372  				break
   373  			}
   374  		}
   375  		if !ok {
   376  			continue
   377  		}
   378  
   379  		candidates[n] = &decomposeCandidate{n: n, leaves: leaves, localAddrs: localAddrsByName[n], leavesByOff: leavesByOff}
   380  	}
   381  
   382  	if len(candidates) == 0 {
   383  		return
   384  	}
   385  
   386  	// Find entry memory and SP.
   387  	var entryMem, entrySP *ssa.Value
   388  	for _, v := range f.Entry.Values {
   389  		if v.Op == ssaop.OpSP {
   390  			entrySP = v
   391  		}
   392  		if v.Op == ssaop.OpInitMem {
   393  			entryMem = v
   394  		}
   395  		if entryMem != nil && entrySP != nil {
   396  			break
   397  		}
   398  	}
   399  	if entryMem == nil || entrySP == nil {
   400  		// A function only reaches this point with a candidate variable, whose loads,
   401  		// stores and LocalAddrs all take a memory operand, and every memory chain
   402  		// roots at InitMem through the entry block.
   403  		f.Fatalf("entry mem or sp is not found but there are local variables")
   404  	}
   405  
   406  	// isDerivedFrom checks if v is an address that's some offset from n.
   407  	isDerivedFrom := func(v *ssa.Value, n *ir.Name) bool {
   408  		for {
   409  			if v.Op == ssaop.OpLocalAddr && v.Aux == n {
   410  				return true
   411  			}
   412  			if v.Op != ssaop.OpOffPtr && v.Op != ssaop.OpCopy {
   413  				return false
   414  			}
   415  			v = v.Args[0]
   416  		}
   417  	}
   418  
   419  	// localAddrBase returns the LocalAddr that v is derived from through
   420  	// OffPtrs and Copies, or nil if v is not derived from one.
   421  	localAddrBase := func(v *ssa.Value) *ssa.Value {
   422  		for {
   423  			switch v.Op {
   424  			case ssaop.OpLocalAddr:
   425  				return v
   426  			case ssaop.OpOffPtr, ssaop.OpCopy:
   427  				v = v.Args[0]
   428  			default:
   429  				return nil
   430  			}
   431  		}
   432  	}
   433  
   434  	// findOffset returns the offset of v from its LocalAddr base.
   435  	findOffset := func(ptr *ssa.Value) int64 {
   436  		var offset int64
   437  		for {
   438  			switch ptr.Op {
   439  			case ssaop.OpLocalAddr:
   440  				return offset
   441  			case ssaop.OpOffPtr:
   442  				offset += ptr.AuxInt
   443  				ptr = ptr.Args[0]
   444  			case ssaop.OpCopy:
   445  				ptr = ptr.Args[0]
   446  			default:
   447  				// Should not happen.
   448  				f.Fatalf("not derived from local address: %v", ptr)
   449  				return -1
   450  			}
   451  		}
   452  	}
   453  
   454  	matchToLeaf := func(c *decomposeCandidate, v *ssa.Value, offset int64) (leafField, bool) {
   455  		var sz int64
   456  		if v.Op == ssaop.OpLoad {
   457  			sz = v.Type.Size()
   458  		} else if v.Op == ssaop.OpStore {
   459  			sz = ssa.AuxToType(v.Aux).Size()
   460  		} else {
   461  			return leafField{}, false
   462  		}
   463  		if lf, ok := c.leavesByOff[offset]; ok {
   464  			// Access must exactly match a leaf field's offset and size.
   465  			if lf.typ.Size() == sz {
   466  				// found a match
   467  				return lf, true
   468  			}
   469  		}
   470  		return leafField{}, false
   471  	}
   472  
   473  	// movedLeaves returns the leaves of c fully covered by the byte range
   474  	// [off, off+size), with offsets relative to off, in increasing offset
   475  	// order. ok is false if the range partially covers some leaf of c.
   476  	movedLeaves := func(c *decomposeCandidate, off, size int64) (leaves []leafField, ok bool) {
   477  		end := off + size
   478  		for _, lf := range c.leaves {
   479  			lend := lf.off + lf.typ.Size()
   480  			if lf.off >= end || lend <= off {
   481  				continue // not touched
   482  			}
   483  			if lf.off < off || lend > end {
   484  				return nil, false // partially covered
   485  			}
   486  			leaves = append(leaves, leafField{lf.off - off, lf.typ})
   487  		}
   488  		return leaves, true
   489  	}
   490  
   491  	// 2. Check partition access alignment, and
   492  	// 3. Breakdown OpZero and OpMove if check pass.
   493  	// We convert aggregate-level zeros and moves into field-level stores and loads/stores.
   494  	// This exposes the underlying scalar access patterns to the promotion logic.
   495  	// We need to keep track of them to not process them twice, i.e. a Move could be traversed
   496  	// twice from source and destination.
   497  	brokenDown := make(map[*ssa.Value]bool)
   498  	// Note: the values created while breaking down a Zero/Move are not added
   499  	// to the uses map, so when a later candidate reaches the same (already
   500  	// brokenDown) Zero/Move it does not see the replacement scalar accesses
   501  	// either. That is fine for correctness: the replacements are constructed
   502  	// leaf-aligned for both operands, and the recomputation of the uses map
   503  	// after this loop lets the final promotion pick them up.
   504  	// usedLeaf records the leaves that scalar Loads read, it's use to abort decomposition when it's
   505  	// larger than maxActivePartitions.
   506  	changed := false
   507  	var mutations []func()
   508  NamesLoop:
   509  	for _, n := range namesOrdered {
   510  		usedLeaf := make(map[int64]struct{})
   511  		c, ok := candidates[n]
   512  		if !ok || c.unDecomposable {
   513  			// Already ruled out (possibly by an earlier candidate that
   514  			// shares a Move with n): nothing to break down for it, and
   515  			// it must not rule out further partners on its own account.
   516  			continue
   517  		}
   518  		mutations = mutations[:0]
   519  		// Partners of n's Moves whose leaves would be straddled by n's
   520  		// breakdown. They are ruled out together with the mutations, so
   521  		// that if n gives up later in this iteration its partners are
   522  		// left alone; one of them may then break the Move down by its
   523  		// own leaves instead.
   524  		var partnerDisq []*decomposeCandidate
   525  		// This is to keep track of the values that are ready to be processed in a
   526  		// batched way, it will be commited to brokenDown entirely or not at all.
   527  		localBrokenDown := make(map[*ssa.Value]bool)
   528  		derived := []*ssa.Value{} // all the addresses that are derived from the current name.
   529  		var collectDerived func(v *ssa.Value)
   530  		collectDerived = func(v *ssa.Value) {
   531  			derived = append(derived, v)
   532  			for _, use := range uses.of(v) {
   533  				// OffPtr and Copy can both be used to derive a new address
   534  				// from a name. The uses table predates this loop, so a
   535  				// Zero or Move that an earlier candidate already broke down
   536  				// still shows up here, reset to a Copy of memory; only a
   537  				// pointer-shaped Copy is an address.
   538  				if use.Op == ssaop.OpOffPtr || (use.Op == ssaop.OpCopy && use.Type.IsPtrShaped()) {
   539  					collectDerived(use)
   540  				}
   541  			}
   542  		}
   543  		for _, vAddr := range c.localAddrs {
   544  			collectDerived(vAddr)
   545  		}
   546  		slices.SortFunc(derived, func(a, b *ssa.Value) int { return int(a.ID - b.ID) })
   547  
   548  		for _, d := range derived {
   549  			for _, use := range uses.of(d) {
   550  				if brokenDown[use] || localBrokenDown[use] {
   551  					continue
   552  				}
   553  				switch use.Op {
   554  				case ssaop.OpOffPtr, ssaop.OpCopy:
   555  					// Handled by derived.
   556  				case ssaop.OpLoad, ssaop.OpStore:
   557  					leaf, ok := matchToLeaf(c, use, findOffset(use.Args[0]))
   558  					if !ok {
   559  						// we can't find a leaf node that matches
   560  						if f.Pass.Debug > 1 {
   561  							f.Warnl(n.Pos(), "decomposeAddr: %v has a load or store not on leaf level", n)
   562  						}
   563  						st.Record("load or store not on leaf level", 1)
   564  						c.unDecomposable = true
   565  						continue NamesLoop
   566  					}
   567  					if use.Op == ssaop.OpLoad {
   568  						// Only a scalar Load marks a leaf active.
   569  						if _, ok := usedLeaf[leaf.off]; !ok {
   570  							usedLeaf[leaf.off] = struct{}{}
   571  							if int64(len(usedLeaf)) > maxActivePartitions {
   572  								st.Record("too many active partitions", 1)
   573  								c.unDecomposable = true
   574  								continue NamesLoop
   575  							}
   576  						}
   577  					}
   578  				case ssaop.OpZero:
   579  					off := findOffset(use.Args[0])
   580  					size := use.AuxInt
   581  					end := off + size
   582  
   583  					if off < 0 || end > n.Type().Size() {
   584  						st.Record("zero range out of bounds", 1)
   585  						c.unDecomposable = true
   586  						continue NamesLoop
   587  					}
   588  
   589  					// Check if all fields within the zero are fully covered.
   590  					// Partial zeros (where a leaf field is only partly zeroed)
   591  					// are not currently supported.
   592  					for _, lf := range c.leaves {
   593  						lend := lf.off + lf.typ.Size()
   594  						if lf.off < end && lend > off { // touches
   595  							if lf.off < off || lend > end {
   596  								st.Record("partially zeroed", 1)
   597  								c.unDecomposable = true
   598  								continue NamesLoop
   599  							}
   600  						}
   601  					}
   602  
   603  					memIn := use.MemoryArg()
   604  					curM := memIn
   605  					for _, lf := range c.leaves {
   606  						if lf.off >= off && lf.off+lf.typ.Size() <= end {
   607  							mutations = append(mutations, func() {
   608  								newPtr := use.Block.NewValue1I(use.Pos, ssaop.OpOffPtr, lf.typ.PtrTo(), lf.off-off, use.Args[0])
   609  								curM = use.Block.NewValue3A(use.Pos, ssaop.OpStore, types.TypeMem, lf.typ, newPtr, zeroValueForType(f, lf.typ), curM)
   610  							})
   611  						}
   612  					}
   613  					mutations = append(mutations, func() {
   614  						use.Reset(ssaop.OpCopy)
   615  						use.Aux = nil
   616  						use.SetArgs1(curM)
   617  					})
   618  					localBrokenDown[use] = true
   619  				case ssaop.OpMove:
   620  					isDst := isDerivedFrom(use.Args[0], n)
   621  					isSrc := isDerivedFrom(use.Args[1], n)
   622  					if !isDst && !isSrc {
   623  						f.Fatalf("non of move's operands is derived from local address: %v", use)
   624  					}
   625  					// If the Move's other operand is derived from a candidate (either another
   626  					// candidate, or n itself in a self-move), breaking this Move down by
   627  					// n's leaves creates accesses on that operand at n-shaped offsets.
   628  					// That is fine when the covered leaf shapes agree (the common same-type copy),
   629  					// but via an unsafe.Pointer cast the two sides can be shaped differently,
   630  					// and the emitted accesses would straddle the partner's or n's own leaves.
   631  					nSide := use.Args[0]
   632  					oSide := use.Args[1]
   633  					if !isDst {
   634  						nSide = use.Args[1]
   635  						oSide = use.Args[0]
   636  					}
   637  					if base := localAddrBase(oSide); base != nil {
   638  						on := base.Aux.(*ir.Name)
   639  						if oc, ok := candidates[on]; ok && !oc.unDecomposable {
   640  							// The breakdown is only correct for both sides if the
   641  							// Move covers exactly the same set of leaves (same
   642  							// relative offsets and sizes) on both of them, and
   643  							// straddles no leaf on either side. Otherwise, e.g. if
   644  							// one side has padding where the other has a leaf, a
   645  							// breakdown by one side's leaves skips some leaves of
   646  							// the other side.
   647  							nLeaves, nOk := movedLeaves(c, findOffset(nSide), use.AuxInt)
   648  							oLeaves, oOk := movedLeaves(oc, findOffset(oSide), use.AuxInt)
   649  							compatible := nOk && oOk && len(nLeaves) == len(oLeaves)
   650  							for i := 0; compatible && i < len(nLeaves); i++ {
   651  								if nLeaves[i].off != oLeaves[i].off || nLeaves[i].typ.Size() != oLeaves[i].typ.Size() {
   652  									compatible = false
   653  								}
   654  							}
   655  							if !compatible {
   656  								if oc == c {
   657  									st.Record("self move with straddled leaves", 1)
   658  									c.unDecomposable = true
   659  									continue NamesLoop
   660  								}
   661  								partnerDisq = append(partnerDisq, oc)
   662  							}
   663  						}
   664  					}
   665  
   666  					moved := []int64{}
   667  					if isDst {
   668  						moved = append(moved, findOffset(use.Args[0]))
   669  					}
   670  					if isSrc {
   671  						moved = append(moved, findOffset(use.Args[1]))
   672  					}
   673  					if len(moved) == 2 {
   674  						// Overlapping moves are still possible at this point,
   675  						// if below the threshold specified in IsInlinableMemmove.
   676  						// Bailout them here too.
   677  						if moved[1] < moved[0]+use.AuxInt && moved[0] < moved[1]+use.AuxInt {
   678  							st.Record("overlapping move", 1)
   679  							c.unDecomposable = true
   680  							continue NamesLoop
   681  						}
   682  					}
   683  					size := use.AuxInt
   684  
   685  					// Check bounds and partial moves for all operands derived from n.
   686  					for _, m := range moved {
   687  						end := m + size
   688  						if m < 0 || end > n.Type().Size() {
   689  							st.Record("move range out of bounds", 1)
   690  							c.unDecomposable = true
   691  							continue NamesLoop
   692  						}
   693  						for _, lf := range c.leaves {
   694  							lend := lf.off + lf.typ.Size()
   695  							if lf.off < end && lend > m { // touches
   696  								if lf.off < m || lend > end {
   697  									st.Record("partially moved", 1)
   698  									c.unDecomposable = true
   699  									continue NamesLoop
   700  								}
   701  							}
   702  						}
   703  					}
   704  					// The breakdown below emits a Load of every leaf the
   705  					// Move covers, but each of those Loads is consumed at
   706  					// once by the paired Store into the destination: after
   707  					// promotion the value is a register briefly on its
   708  					// way to memory, not a live leaf. So a Move does not
   709  					// mark leaves active; only scalar Loads do (see the
   710  					// OpLoad case). Counting them would rule out every
   711  					// whole-aggregate copy of a variable with more leaves
   712  					// than maxActivePartitions, which is most of the
   713  					// candidates in practice.
   714  
   715  					memIn := use.MemoryArg()
   716  					curM := memIn
   717  					for _, lf := range c.leaves {
   718  						// Take the leader offset from the moved slice.
   719  						offInMove := lf.off - moved[0]
   720  						if offInMove >= 0 && offInMove+lf.typ.Size() <= size {
   721  							mutations = append(mutations, func() {
   722  								// Emit scalar load/store for each field.
   723  								newSrcPtr := use.Block.NewValue1I(use.Pos, ssaop.OpOffPtr, lf.typ.PtrTo(), offInMove, use.Args[1])
   724  								newDstPtr := use.Block.NewValue1I(use.Pos, ssaop.OpOffPtr, lf.typ.PtrTo(), offInMove, use.Args[0])
   725  								val := use.Block.NewValue2(use.Pos, ssaop.OpLoad, lf.typ, newSrcPtr, memIn)
   726  								curM = use.Block.NewValue3A(use.Pos, ssaop.OpStore, types.TypeMem, lf.typ, newDstPtr, val, curM)
   727  							})
   728  						}
   729  					}
   730  					mutations = append(mutations, func() {
   731  						use.Reset(ssaop.OpCopy)
   732  						use.Aux = nil
   733  						use.SetArgs1(curM)
   734  					})
   735  					localBrokenDown[use] = true
   736  				}
   737  			}
   738  		}
   739  		for _, m := range mutations {
   740  			m()
   741  		}
   742  		for _, oc := range partnerDisq {
   743  			if oc.unDecomposable {
   744  				continue
   745  			}
   746  			if f.Pass.Debug > 1 {
   747  				f.Warnl(oc.n.Pos(), "decomposeAddr: %v has leaves straddled by a move of %v; disqualified", oc.n, n)
   748  			}
   749  			st.Record("move partner with straddled leaves", 1)
   750  			oc.unDecomposable = true
   751  		}
   752  		for u := range localBrokenDown {
   753  			brokenDown[u] = true
   754  		}
   755  		if len(mutations) > 0 {
   756  			changed = true
   757  		}
   758  	}
   759  
   760  	if changed {
   761  		// The Zero/Move breakdown above introduced new values (OffPtrs off
   762  		// the candidates' LocalAddrs, and scalar Loads/Stores through them)
   763  		// that are not in the demographics/uses computed at the top of this
   764  		// pass. Recompute them so that the final promotion below sees, and
   765  		// rewrites, those new accesses too.
   766  		uses.free(f)
   767  		demographics, localAddrs, uses = variableDemographics(f)
   768  		if uses == nil {
   769  			// Cannot happen (the candidates' LocalAddrs still exist), but
   770  			// keep the invariant that a nil table means nothing to do.
   771  			return
   772  		}
   773  	}
   774  
   775  	// 4. Final Promotion.
   776  	// For each valid candidate, create new individual variables for each field
   777  	// and update all usages.
   778  	for _, n := range namesOrdered {
   779  		c, ok := candidates[n]
   780  		if !ok || c.unDecomposable {
   781  			continue
   782  		}
   783  		if dg, ok := demographics[n]; !ok || !dg.lszmco {
   784  			// The transformation should not have caused the variable
   785  			// unknown uses.
   786  			f.Fatalf("zero and move breakdown disqualify %v from decomposeAddr", n)
   787  		}
   788  
   789  		type pAccess struct {
   790  			a   *ssa.Value
   791  			off int64
   792  		}
   793  		var accesses []pAccess
   794  		var collectSimpleUses func(v *ssa.Value, offset int64)
   795  		collectSimpleUses = func(v *ssa.Value, offset int64) {
   796  			for _, use := range uses.of(v) {
   797  				switch use.Op {
   798  				case ssaop.OpOffPtr:
   799  					collectSimpleUses(use, offset+use.AuxInt)
   800  				case ssaop.OpLoad, ssaop.OpStore:
   801  					accesses = append(accesses, pAccess{use, offset})
   802  				case ssaop.OpCopy:
   803  					collectSimpleUses(use, offset)
   804  				default:
   805  					f.Fatalf("unhandled uses: %s", use.Op.String())
   806  				}
   807  			}
   808  		}
   809  		for _, vAddr := range c.localAddrs {
   810  			collectSimpleUses(vAddr, 0)
   811  		}
   812  		if len(accesses) == 0 {
   813  			st.Record("no accesses", 1)
   814  			continue
   815  		}
   816  
   817  		type partition struct {
   818  			offset  int64
   819  			typ     *types.Type
   820  			newAddr *ssa.Value
   821  			newName *ir.Name
   822  		}
   823  		partitions := make(map[int64]*partition)
   824  		partitionsValid := true
   825  		for _, a := range accesses {
   826  			if lf, ok := matchToLeaf(c, a.a, a.off); ok {
   827  				partitions[a.off] = &partition{offset: a.off, typ: lf.typ}
   828  				continue
   829  			}
   830  			// If we didn't find a matching leaf field, we cannot promote this variable.
   831  			partitionsValid = false
   832  			break
   833  		}
   834  		if !partitionsValid || len(partitions) == 0 {
   835  			f.Fatalf("not all accesses are on leaf level")
   836  		}
   837  
   838  		if f.Pass.Debug > 0 {
   839  			f.Warnl(n.Pos(), "promoted %v", n)
   840  		}
   841  		st.Record("promoted variable", 1)
   842  
   843  		// Deterministically create new variables for each used partition.
   844  		var partOffsets []int64
   845  		for off := range partitions {
   846  			partOffsets = append(partOffsets, off)
   847  		}
   848  		slices.Sort(partOffsets)
   849  
   850  		// Different variables can have the same names, we need to distinguish them
   851  		// if that's the case.
   852  		nameIdx := -1
   853  		ns := stringToNames[n.Sym().Name]
   854  		if len(ns) > 1 {
   855  			for i, nn := range ns {
   856  				if nn == n {
   857  					nameIdx = i
   858  					break
   859  				}
   860  			}
   861  		}
   862  
   863  		for _, off := range partOffsets {
   864  			p := partitions[off]
   865  			// TODO: Ideally the names should be the name you would use in Go for that part,
   866  			// to the extent that we can do that. Like myVariable.window.menu[3].text, or something.
   867  			// Just using offsets to distinguish is fine for now.
   868  			var newSymName string
   869  			if nameIdx >= 0 {
   870  				newSymName = fmt.Sprintf("%s#%d.part.%d", n.Sym().Name, nameIdx, p.offset)
   871  			} else {
   872  				newSymName = fmt.Sprintf("%s.part.%d", n.Sym().Name, p.offset)
   873  			}
   874  			newSym := &types.Sym{Name: newSymName, Pkg: n.Sym().Pkg}
   875  			// TODO: should we use the existing stack slot for the old variable instead?
   876  			// That might be more work.
   877  			newName := n.Curfn.NewLocal(n.Pos(), newSym, p.typ)
   878  			newName.SetUsed(true)
   879  			// Parts of a compiler-generated temporary are temporaries too.
   880  			newName.SetAutoTemp(n.AutoTemp())
   881  			p.newName = newName
   882  			// New LocalAddr at the entry block for the new scalar variable.
   883  			p.newAddr = f.Entry.NewValue2A(n.Pos(), ssaop.OpLocalAddr, types.NewPtr(p.typ), newName, entrySP, entryMem)
   884  		}
   885  
   886  		// Rewrite loads and stores to use the new scalar addresses.
   887  		for _, a := range accesses {
   888  			p := partitions[a.off]
   889  			a.a.SetArg(0, p.newAddr)
   890  		}
   891  
   892  		// Rewrite VarDef annotations to def the new names.
   893  		// Note: We only emit VarDef if the new variable contains pointers
   894  		// or is large enough to be a merge candidate, satisfying SSA checker.
   895  		dg, ok := demographics[n]
   896  		if !ok {
   897  			f.Fatalf("decomposeAddr candidate variable unseen: %v", dg)
   898  		}
   899  		for _, v := range dg.varDefs {
   900  			curM := v.MemoryArg()
   901  			for _, off := range partOffsets {
   902  				p := partitions[off]
   903  				if p.newName.Type().HasPointers() || ssa.IsMergeCandidate(p.newName) {
   904  					curM = v.Block.NewValue1A(v.Pos, ssaop.OpVarDef, types.TypeMem, p.newName, curM)
   905  				}
   906  			}
   907  			v.Reset(ssaop.OpCopy)
   908  			v.Aux = nil
   909  			v.SetArgs1(curM)
   910  		}
   911  		changed = true
   912  	}
   913  
   914  	if changed {
   915  		deadcode(f)
   916  	}
   917  }
   918  

View as plain text