Source file src/runtime/crash_test.go

     1  // Copyright 2012 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime_test
     6  
     7  import (
     8  	"bytes"
     9  	"context"
    10  	"errors"
    11  	"flag"
    12  	"fmt"
    13  	"internal/asan"
    14  	"internal/msan"
    15  	"internal/profile"
    16  	"internal/race"
    17  	"internal/testenv"
    18  	traceparse "internal/trace"
    19  	"io"
    20  	"log"
    21  	"os"
    22  	"os/exec"
    23  	"path/filepath"
    24  	"regexp"
    25  	"runtime"
    26  	"runtime/trace"
    27  	"strings"
    28  	"sync"
    29  	"testing"
    30  	"time"
    31  )
    32  
    33  var toRemove []string
    34  
    35  const entrypointVar = "RUNTIME_TEST_ENTRYPOINT"
    36  
    37  func TestMain(m *testing.M) {
    38  	switch entrypoint := os.Getenv(entrypointVar); entrypoint {
    39  	case "panic":
    40  		crashViaPanic()
    41  		panic("unreachable")
    42  	case "trap":
    43  		crashViaTrap()
    44  		panic("unreachable")
    45  	default:
    46  		log.Fatalf("invalid %s: %q", entrypointVar, entrypoint)
    47  	case "":
    48  		// fall through to normal behavior
    49  	}
    50  
    51  	_, coreErrBefore := os.Stat("core")
    52  
    53  	status := m.Run()
    54  	for _, file := range toRemove {
    55  		os.RemoveAll(file)
    56  	}
    57  
    58  	_, coreErrAfter := os.Stat("core")
    59  	if coreErrBefore != nil && coreErrAfter == nil {
    60  		fmt.Fprintln(os.Stderr, "runtime.test: some test left a core file behind")
    61  		if status == 0 {
    62  			status = 1
    63  		}
    64  	}
    65  
    66  	os.Exit(status)
    67  }
    68  
    69  var testprog struct {
    70  	sync.Mutex
    71  	dir    string
    72  	target map[string]*buildexe
    73  }
    74  
    75  type buildexe struct {
    76  	once sync.Once
    77  	exe  string
    78  	err  error
    79  }
    80  
    81  func runTestProg(t *testing.T, binary, name string, env ...string) string {
    82  	if *flagQuick {
    83  		t.Skip("-quick")
    84  	}
    85  
    86  	testenv.MustHaveGoBuild(t)
    87  	t.Helper()
    88  
    89  	exe, err := buildTestProg(t, binary)
    90  	if err != nil {
    91  		t.Fatal(err)
    92  	}
    93  
    94  	return runBuiltTestProg(t, exe, name, env...)
    95  }
    96  
    97  func runBuiltTestProg(t *testing.T, exe, name string, env ...string) string {
    98  	t.Helper()
    99  
   100  	if *flagQuick {
   101  		t.Skip("-quick")
   102  	}
   103  
   104  	start := time.Now()
   105  
   106  	cmd := testenv.CleanCmdEnv(testenv.Command(t, exe, name))
   107  	cmd.Env = append(cmd.Env, env...)
   108  	if testing.Short() {
   109  		cmd.Env = append(cmd.Env, "RUNTIME_TEST_SHORT=1")
   110  	}
   111  	out, err := cmd.CombinedOutput()
   112  	if err == nil {
   113  		t.Logf("%v (%v): ok", cmd, time.Since(start))
   114  	} else {
   115  		if _, ok := err.(*exec.ExitError); ok {
   116  			t.Logf("%v: %v", cmd, err)
   117  		} else if errors.Is(err, exec.ErrWaitDelay) {
   118  			t.Fatalf("%v: %v", cmd, err)
   119  		} else {
   120  			t.Fatalf("%v failed to start: %v", cmd, err)
   121  		}
   122  	}
   123  	return string(out)
   124  }
   125  
   126  var serializeBuild = make(chan bool, 2)
   127  
   128  func buildTestProg(t *testing.T, binary string, flags ...string) (string, error) {
   129  	if *flagQuick {
   130  		t.Skip("-quick")
   131  	}
   132  	testenv.MustHaveGoBuild(t)
   133  
   134  	testprog.Lock()
   135  	if testprog.dir == "" {
   136  		dir, err := os.MkdirTemp("", "go-build")
   137  		if err != nil {
   138  			t.Fatalf("failed to create temp directory: %v", err)
   139  		}
   140  		testprog.dir = dir
   141  		toRemove = append(toRemove, dir)
   142  	}
   143  
   144  	if testprog.target == nil {
   145  		testprog.target = make(map[string]*buildexe)
   146  	}
   147  	name := binary
   148  	if len(flags) > 0 {
   149  		name += "_" + strings.Join(flags, "_")
   150  	}
   151  	target, ok := testprog.target[name]
   152  	if !ok {
   153  		target = &buildexe{}
   154  		testprog.target[name] = target
   155  	}
   156  
   157  	dir := testprog.dir
   158  
   159  	// Unlock testprog while actually building, so that other
   160  	// tests can look up executables that were already built.
   161  	testprog.Unlock()
   162  
   163  	target.once.Do(func() {
   164  		// Only do two "go build"'s at a time,
   165  		// to keep load from getting too high.
   166  		serializeBuild <- true
   167  		defer func() { <-serializeBuild }()
   168  
   169  		// Don't get confused if testenv.GoToolPath calls t.Skip.
   170  		target.err = errors.New("building test called t.Skip")
   171  
   172  		if asan.Enabled {
   173  			flags = append(flags, "-asan")
   174  		}
   175  		if msan.Enabled {
   176  			flags = append(flags, "-msan")
   177  		}
   178  		if race.Enabled {
   179  			flags = append(flags, "-race")
   180  		}
   181  
   182  		exe := filepath.Join(dir, name+".exe")
   183  
   184  		start := time.Now()
   185  		cmd := exec.Command(testenv.GoToolPath(t), append([]string{"build", "-o", exe}, flags...)...)
   186  		t.Logf("running %v", cmd)
   187  		cmd.Dir = "testdata/" + binary
   188  		cmd = testenv.CleanCmdEnv(cmd)
   189  		out, err := cmd.CombinedOutput()
   190  		if err != nil {
   191  			target.err = fmt.Errorf("building %s %v: %v\n%s", binary, flags, err, out)
   192  		} else {
   193  			t.Logf("built %v in %v", name, time.Since(start))
   194  			target.exe = exe
   195  			target.err = nil
   196  		}
   197  	})
   198  
   199  	return target.exe, target.err
   200  }
   201  
   202  func TestVDSO(t *testing.T) {
   203  	t.Parallel()
   204  	output := runTestProg(t, "testprog", "SignalInVDSO")
   205  	want := "success\n"
   206  	if output != want {
   207  		t.Fatalf("output:\n%s\n\nwanted:\n%s", output, want)
   208  	}
   209  }
   210  
   211  func testCrashHandler(t *testing.T, cgo bool) {
   212  	type crashTest struct {
   213  		Cgo bool
   214  	}
   215  	var output string
   216  	if cgo {
   217  		if runtime.GOOS == "freebsd" && race.Enabled {
   218  			t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.")
   219  		}
   220  		output = runTestProg(t, "testprogcgo", "Crash")
   221  	} else {
   222  		output = runTestProg(t, "testprog", "Crash")
   223  	}
   224  	want := "main: recovered done\nnew-thread: recovered done\nsecond-new-thread: recovered done\nmain-again: recovered done\n"
   225  	if output != want {
   226  		t.Fatalf("output:\n%s\n\nwanted:\n%s", output, want)
   227  	}
   228  }
   229  
   230  func TestCrashHandler(t *testing.T) {
   231  	testCrashHandler(t, false)
   232  }
   233  
   234  var deadlockBuildTypes = testenv.SpecialBuildTypes{
   235  	// External linking brings in cgo, causing deadlock detection not working.
   236  	Cgo:  false,
   237  	Asan: asan.Enabled,
   238  	Msan: msan.Enabled,
   239  	Race: race.Enabled,
   240  }
   241  
   242  func testDeadlock(t *testing.T, name string) {
   243  	// External linking brings in cgo, causing deadlock detection not working.
   244  	testenv.MustInternalLink(t, deadlockBuildTypes)
   245  
   246  	output := runTestProg(t, "testprog", name)
   247  	want := "fatal error: all goroutines are asleep - deadlock!\n"
   248  	if !strings.HasPrefix(output, want) {
   249  		t.Fatalf("output does not start with %q:\n%s", want, output)
   250  	}
   251  }
   252  
   253  func TestSimpleDeadlock(t *testing.T) {
   254  	testDeadlock(t, "SimpleDeadlock")
   255  }
   256  
   257  func TestInitDeadlock(t *testing.T) {
   258  	testDeadlock(t, "InitDeadlock")
   259  }
   260  
   261  func TestLockedDeadlock(t *testing.T) {
   262  	testDeadlock(t, "LockedDeadlock")
   263  }
   264  
   265  func TestLockedDeadlock2(t *testing.T) {
   266  	testDeadlock(t, "LockedDeadlock2")
   267  }
   268  
   269  func TestGoexitDeadlock(t *testing.T) {
   270  	// External linking brings in cgo, causing deadlock detection not working.
   271  	testenv.MustInternalLink(t, deadlockBuildTypes)
   272  
   273  	output := runTestProg(t, "testprog", "GoexitDeadlock")
   274  	want := "no goroutines (main called runtime.Goexit) - deadlock!"
   275  	if !strings.Contains(output, want) {
   276  		t.Fatalf("output:\n%s\n\nwant output containing: %s", output, want)
   277  	}
   278  }
   279  
   280  func TestStackOverflow(t *testing.T) {
   281  	output := runTestProg(t, "testprog", "StackOverflow")
   282  	want := []string{
   283  		"runtime: goroutine stack exceeds 1474560-byte limit\n",
   284  		"fatal error: stack overflow",
   285  		// information about the current SP and stack bounds
   286  		"runtime: sp=",
   287  		"stack=[",
   288  	}
   289  	if !strings.HasPrefix(output, want[0]) {
   290  		t.Errorf("output does not start with %q", want[0])
   291  	}
   292  	for _, s := range want[1:] {
   293  		if !strings.Contains(output, s) {
   294  			t.Errorf("output does not contain %q", s)
   295  		}
   296  	}
   297  	if t.Failed() {
   298  		t.Logf("output:\n%s", output)
   299  	}
   300  }
   301  
   302  func TestThreadExhaustion(t *testing.T) {
   303  	output := runTestProg(t, "testprog", "ThreadExhaustion")
   304  	want := "runtime: program exceeds 10-thread limit\nfatal error: thread exhaustion"
   305  	if !strings.HasPrefix(output, want) {
   306  		t.Fatalf("output does not start with %q:\n%s", want, output)
   307  	}
   308  }
   309  
   310  func TestRecursivePanic(t *testing.T) {
   311  	output := runTestProg(t, "testprog", "RecursivePanic")
   312  	want := `wrap: bad
   313  panic: again
   314  
   315  `
   316  	if !strings.HasPrefix(output, want) {
   317  		t.Fatalf("output does not start with %q:\n%s", want, output)
   318  	}
   319  
   320  }
   321  
   322  func TestRecursivePanic2(t *testing.T) {
   323  	output := runTestProg(t, "testprog", "RecursivePanic2")
   324  	want := `first panic
   325  second panic
   326  panic: third panic
   327  
   328  `
   329  	if !strings.HasPrefix(output, want) {
   330  		t.Fatalf("output does not start with %q:\n%s", want, output)
   331  	}
   332  
   333  }
   334  
   335  func TestRecursivePanic3(t *testing.T) {
   336  	output := runTestProg(t, "testprog", "RecursivePanic3")
   337  	want := `panic: first panic
   338  
   339  `
   340  	if !strings.HasPrefix(output, want) {
   341  		t.Fatalf("output does not start with %q:\n%s", want, output)
   342  	}
   343  
   344  }
   345  
   346  func TestRecursivePanic4(t *testing.T) {
   347  	output := runTestProg(t, "testprog", "RecursivePanic4")
   348  	want := `panic: first panic [recovered]
   349  	panic: second panic
   350  `
   351  	if !strings.HasPrefix(output, want) {
   352  		t.Fatalf("output does not start with %q:\n%s", want, output)
   353  	}
   354  
   355  }
   356  
   357  func TestRecursivePanic5(t *testing.T) {
   358  	output := runTestProg(t, "testprog", "RecursivePanic5")
   359  	want := `first panic
   360  second panic
   361  panic: third panic
   362  `
   363  	if !strings.HasPrefix(output, want) {
   364  		t.Fatalf("output does not start with %q:\n%s", want, output)
   365  	}
   366  
   367  }
   368  
   369  func TestRepanickedPanic(t *testing.T) {
   370  	output := runTestProg(t, "testprog", "RepanickedPanic")
   371  	want := `panic: message [recovered, repanicked]
   372  `
   373  	if !strings.HasPrefix(output, want) {
   374  		t.Fatalf("output does not start with %q:\n%s", want, output)
   375  	}
   376  }
   377  
   378  func TestRepanickedMiddlePanic(t *testing.T) {
   379  	output := runTestProg(t, "testprog", "RepanickedMiddlePanic")
   380  	want := `panic: inner [recovered]
   381  	panic: middle [recovered, repanicked]
   382  	panic: outer
   383  `
   384  	if !strings.HasPrefix(output, want) {
   385  		t.Fatalf("output does not start with %q:\n%s", want, output)
   386  	}
   387  }
   388  
   389  func TestRepanickedPanicSandwich(t *testing.T) {
   390  	output := runTestProg(t, "testprog", "RepanickedPanicSandwich")
   391  	want := `panic: outer [recovered]
   392  	panic: inner [recovered]
   393  	panic: outer
   394  `
   395  	if !strings.HasPrefix(output, want) {
   396  		t.Fatalf("output does not start with %q:\n%s", want, output)
   397  	}
   398  }
   399  
   400  func TestGoexitCrash(t *testing.T) {
   401  	// External linking brings in cgo, causing deadlock detection not working.
   402  	testenv.MustInternalLink(t, deadlockBuildTypes)
   403  
   404  	output := runTestProg(t, "testprog", "GoexitExit")
   405  	want := "no goroutines (main called runtime.Goexit) - deadlock!"
   406  	if !strings.Contains(output, want) {
   407  		t.Fatalf("output:\n%s\n\nwant output containing: %s", output, want)
   408  	}
   409  }
   410  
   411  func TestGoexitDefer(t *testing.T) {
   412  	c := make(chan struct{})
   413  	go func() {
   414  		defer func() {
   415  			r := recover()
   416  			if r != nil {
   417  				t.Errorf("non-nil recover during Goexit")
   418  			}
   419  			c <- struct{}{}
   420  		}()
   421  		runtime.Goexit()
   422  	}()
   423  	// Note: if the defer fails to run, we will get a deadlock here
   424  	<-c
   425  }
   426  
   427  func TestGoNil(t *testing.T) {
   428  	output := runTestProg(t, "testprog", "GoNil")
   429  	want := "go of nil func value"
   430  	if !strings.Contains(output, want) {
   431  		t.Fatalf("output:\n%s\n\nwant output containing: %s", output, want)
   432  	}
   433  }
   434  
   435  func TestMainGoroutineID(t *testing.T) {
   436  	output := runTestProg(t, "testprog", "MainGoroutineID")
   437  	want := "panic: test\n\ngoroutine 1 [running]:\n"
   438  	if !strings.HasPrefix(output, want) {
   439  		t.Fatalf("output does not start with %q:\n%s", want, output)
   440  	}
   441  }
   442  
   443  func TestNoHelperGoroutines(t *testing.T) {
   444  	output := runTestProg(t, "testprog", "NoHelperGoroutines")
   445  	matches := regexp.MustCompile(`goroutine [0-9]+ \[`).FindAllStringSubmatch(output, -1)
   446  	if len(matches) != 1 || matches[0][0] != "goroutine 1 [" {
   447  		t.Fatalf("want to see only goroutine 1, see:\n%s", output)
   448  	}
   449  }
   450  
   451  func TestBreakpoint(t *testing.T) {
   452  	output := runTestProg(t, "testprog", "Breakpoint")
   453  	// If runtime.Breakpoint() is inlined, then the stack trace prints
   454  	// "runtime.Breakpoint(...)" instead of "runtime.Breakpoint()".
   455  	want := "runtime.Breakpoint("
   456  	if !strings.Contains(output, want) {
   457  		t.Fatalf("output:\n%s\n\nwant output containing: %s", output, want)
   458  	}
   459  }
   460  
   461  func TestGoexitInPanic(t *testing.T) {
   462  	// External linking brings in cgo, causing deadlock detection not working.
   463  	testenv.MustInternalLink(t, deadlockBuildTypes)
   464  
   465  	// see issue 8774: this code used to trigger an infinite recursion
   466  	output := runTestProg(t, "testprog", "GoexitInPanic")
   467  	want := "fatal error: no goroutines (main called runtime.Goexit) - deadlock!"
   468  	if !strings.HasPrefix(output, want) {
   469  		t.Fatalf("output does not start with %q:\n%s", want, output)
   470  	}
   471  }
   472  
   473  // Issue 14965: Runtime panics should be of type runtime.Error
   474  func TestRuntimePanicWithRuntimeError(t *testing.T) {
   475  	testCases := [...]func(){
   476  		0: func() {
   477  			var m map[uint64]bool
   478  			m[1234] = true
   479  		},
   480  		1: func() {
   481  			ch := make(chan struct{})
   482  			close(ch)
   483  			close(ch)
   484  		},
   485  		2: func() {
   486  			var ch = make(chan struct{})
   487  			close(ch)
   488  			ch <- struct{}{}
   489  		},
   490  		3: func() {
   491  			var s = make([]int, 2)
   492  			_ = s[2]
   493  		},
   494  		4: func() {
   495  			n := -1
   496  			_ = make(chan bool, n)
   497  		},
   498  		5: func() {
   499  			close((chan bool)(nil))
   500  		},
   501  	}
   502  
   503  	for i, fn := range testCases {
   504  		got := panicValue(fn)
   505  		if _, ok := got.(runtime.Error); !ok {
   506  			t.Errorf("test #%d: recovered value %v(type %T) does not implement runtime.Error", i, got, got)
   507  		}
   508  	}
   509  }
   510  
   511  func panicValue(fn func()) (recovered any) {
   512  	defer func() {
   513  		recovered = recover()
   514  	}()
   515  	fn()
   516  	return
   517  }
   518  
   519  func TestPanicAfterGoexit(t *testing.T) {
   520  	// an uncaught panic should still work after goexit
   521  	output := runTestProg(t, "testprog", "PanicAfterGoexit")
   522  	want := "panic: hello"
   523  	if !strings.HasPrefix(output, want) {
   524  		t.Fatalf("output does not start with %q:\n%s", want, output)
   525  	}
   526  }
   527  
   528  func TestRecoveredPanicAfterGoexit(t *testing.T) {
   529  	// External linking brings in cgo, causing deadlock detection not working.
   530  	testenv.MustInternalLink(t, deadlockBuildTypes)
   531  
   532  	output := runTestProg(t, "testprog", "RecoveredPanicAfterGoexit")
   533  	want := "fatal error: no goroutines (main called runtime.Goexit) - deadlock!"
   534  	if !strings.HasPrefix(output, want) {
   535  		t.Fatalf("output does not start with %q:\n%s", want, output)
   536  	}
   537  }
   538  
   539  func TestRecoverBeforePanicAfterGoexit(t *testing.T) {
   540  	// External linking brings in cgo, causing deadlock detection not working.
   541  	testenv.MustInternalLink(t, deadlockBuildTypes)
   542  
   543  	t.Parallel()
   544  	output := runTestProg(t, "testprog", "RecoverBeforePanicAfterGoexit")
   545  	want := "fatal error: no goroutines (main called runtime.Goexit) - deadlock!"
   546  	if !strings.HasPrefix(output, want) {
   547  		t.Fatalf("output does not start with %q:\n%s", want, output)
   548  	}
   549  }
   550  
   551  func TestRecoverBeforePanicAfterGoexit2(t *testing.T) {
   552  	// External linking brings in cgo, causing deadlock detection not working.
   553  	testenv.MustInternalLink(t, deadlockBuildTypes)
   554  
   555  	t.Parallel()
   556  	output := runTestProg(t, "testprog", "RecoverBeforePanicAfterGoexit2")
   557  	want := "fatal error: no goroutines (main called runtime.Goexit) - deadlock!"
   558  	if !strings.HasPrefix(output, want) {
   559  		t.Fatalf("output does not start with %q:\n%s", want, output)
   560  	}
   561  }
   562  
   563  func TestNetpollDeadlock(t *testing.T) {
   564  	t.Parallel()
   565  	output := runTestProg(t, "testprognet", "NetpollDeadlock")
   566  	want := "done\n"
   567  	if !strings.HasSuffix(output, want) {
   568  		t.Fatalf("output does not start with %q:\n%s", want, output)
   569  	}
   570  }
   571  
   572  func TestPanicTraceback(t *testing.T) {
   573  	t.Parallel()
   574  	output := runTestProg(t, "testprog", "PanicTraceback")
   575  	want := "panic: hello\n\tpanic: panic pt2\n\tpanic: panic pt1\n"
   576  	if !strings.HasPrefix(output, want) {
   577  		t.Fatalf("output does not start with %q:\n%s", want, output)
   578  	}
   579  
   580  	// Check functions in the traceback.
   581  	fns := []string{"main.pt1.func1", "panic", "main.pt2.func1", "panic", "main.pt2", "main.pt1"}
   582  	for _, fn := range fns {
   583  		re := regexp.MustCompile(`(?m)^` + regexp.QuoteMeta(fn) + `\(.*\n`)
   584  		idx := re.FindStringIndex(output)
   585  		if idx == nil {
   586  			t.Fatalf("expected %q function in traceback:\n%s", fn, output)
   587  		}
   588  		output = output[idx[1]:]
   589  	}
   590  }
   591  
   592  func testPanicDeadlock(t *testing.T, name string, want string) {
   593  	// test issue 14432
   594  	output := runTestProg(t, "testprog", name)
   595  	if !strings.HasPrefix(output, want) {
   596  		t.Fatalf("output does not start with %q:\n%s", want, output)
   597  	}
   598  }
   599  
   600  func TestPanicDeadlockGosched(t *testing.T) {
   601  	testPanicDeadlock(t, "GoschedInPanic", "panic: errorThatGosched\n\n")
   602  }
   603  
   604  func TestPanicDeadlockSyscall(t *testing.T) {
   605  	testPanicDeadlock(t, "SyscallInPanic", "1\n2\npanic: 3\n\n")
   606  }
   607  
   608  func TestPanicLoop(t *testing.T) {
   609  	output := runTestProg(t, "testprog", "PanicLoop")
   610  	if want := "panic while printing panic value"; !strings.Contains(output, want) {
   611  		t.Errorf("output does not contain %q:\n%s", want, output)
   612  	}
   613  }
   614  
   615  func TestMemPprof(t *testing.T) {
   616  	testenv.MustHaveGoRun(t)
   617  
   618  	exe, err := buildTestProg(t, "testprog")
   619  	if err != nil {
   620  		t.Fatal(err)
   621  	}
   622  
   623  	got, err := testenv.CleanCmdEnv(exec.Command(exe, "MemProf")).CombinedOutput()
   624  	if err != nil {
   625  		t.Fatalf("testprog failed: %s, output:\n%s", err, got)
   626  	}
   627  	fn := strings.TrimSpace(string(got))
   628  	defer os.Remove(fn)
   629  
   630  	for try := 0; try < 2; try++ {
   631  		cmd := testenv.CleanCmdEnv(exec.Command(testenv.GoToolPath(t), "tool", "pprof", "-alloc_space", "-top"))
   632  		// Check that pprof works both with and without explicit executable on command line.
   633  		if try == 0 {
   634  			cmd.Args = append(cmd.Args, exe, fn)
   635  		} else {
   636  			cmd.Args = append(cmd.Args, fn)
   637  		}
   638  		found := false
   639  		for i, e := range cmd.Env {
   640  			if strings.HasPrefix(e, "PPROF_TMPDIR=") {
   641  				cmd.Env[i] = "PPROF_TMPDIR=" + os.TempDir()
   642  				found = true
   643  				break
   644  			}
   645  		}
   646  		if !found {
   647  			cmd.Env = append(cmd.Env, "PPROF_TMPDIR="+os.TempDir())
   648  		}
   649  
   650  		top, err := cmd.CombinedOutput()
   651  		t.Logf("%s:\n%s", cmd.Args, top)
   652  		if err != nil {
   653  			t.Error(err)
   654  		} else if !bytes.Contains(top, []byte("MemProf")) {
   655  			t.Error("missing MemProf in pprof output")
   656  		}
   657  	}
   658  }
   659  
   660  var concurrentMapTest = flag.Bool("run_concurrent_map_tests", false, "also run flaky concurrent map tests")
   661  
   662  func TestConcurrentMapWrites(t *testing.T) {
   663  	if !*concurrentMapTest {
   664  		t.Skip("skipping without -run_concurrent_map_tests")
   665  	}
   666  	if race.Enabled {
   667  		t.Skip("skipping test: -race will catch the race, this test is for the built-in race detection")
   668  	}
   669  	testenv.MustHaveGoRun(t)
   670  	output := runTestProg(t, "testprog", "concurrentMapWrites")
   671  	want := "fatal error: concurrent map writes\n"
   672  	// Concurrent writes can corrupt the map in a way that we
   673  	// detect with a separate throw.
   674  	want2 := "fatal error: small map with no empty slot (concurrent map writes?)\n"
   675  	if !strings.HasPrefix(output, want) && !strings.HasPrefix(output, want2) {
   676  		t.Fatalf("output does not start with %q:\n%s", want, output)
   677  	}
   678  }
   679  func TestConcurrentMapReadWrite(t *testing.T) {
   680  	if !*concurrentMapTest {
   681  		t.Skip("skipping without -run_concurrent_map_tests")
   682  	}
   683  	if race.Enabled {
   684  		t.Skip("skipping test: -race will catch the race, this test is for the built-in race detection")
   685  	}
   686  	testenv.MustHaveGoRun(t)
   687  	output := runTestProg(t, "testprog", "concurrentMapReadWrite")
   688  	want := "fatal error: concurrent map read and map write\n"
   689  	// Concurrent writes can corrupt the map in a way that we
   690  	// detect with a separate throw.
   691  	want2 := "fatal error: small map with no empty slot (concurrent map writes?)\n"
   692  	if !strings.HasPrefix(output, want) && !strings.HasPrefix(output, want2) {
   693  		t.Fatalf("output does not start with %q:\n%s", want, output)
   694  	}
   695  }
   696  func TestConcurrentMapIterateWrite(t *testing.T) {
   697  	if !*concurrentMapTest {
   698  		t.Skip("skipping without -run_concurrent_map_tests")
   699  	}
   700  	if race.Enabled {
   701  		t.Skip("skipping test: -race will catch the race, this test is for the built-in race detection")
   702  	}
   703  	testenv.MustHaveGoRun(t)
   704  	output := runTestProg(t, "testprog", "concurrentMapIterateWrite")
   705  	want := "fatal error: concurrent map iteration and map write\n"
   706  	// Concurrent writes can corrupt the map in a way that we
   707  	// detect with a separate throw.
   708  	want2 := "fatal error: small map with no empty slot (concurrent map writes?)\n"
   709  	if !strings.HasPrefix(output, want) && !strings.HasPrefix(output, want2) {
   710  		t.Fatalf("output does not start with %q:\n%s", want, output)
   711  	}
   712  }
   713  
   714  func TestConcurrentMapWritesIssue69447(t *testing.T) {
   715  	testenv.MustHaveGoRun(t)
   716  	if race.Enabled {
   717  		t.Skip("skipping test: -race will catch the race, this test is for the built-in race detection")
   718  	}
   719  	exe, err := buildTestProg(t, "testprog")
   720  	if err != nil {
   721  		t.Fatal(err)
   722  	}
   723  	for i := 0; i < 200; i++ {
   724  		output := runBuiltTestProg(t, exe, "concurrentMapWrites")
   725  		if output == "" {
   726  			// If we didn't detect an error, that's ok.
   727  			// This case makes this test not flaky like
   728  			// the other ones above.
   729  			// (More correctly, this case makes this test flaky
   730  			// in the other direction, in that it might not
   731  			// detect a problem even if there is one.)
   732  			continue
   733  		}
   734  		want := "fatal error: concurrent map writes\n"
   735  		// Concurrent writes can corrupt the map in a way that we
   736  		// detect with a separate throw.
   737  		want2 := "fatal error: small map with no empty slot (concurrent map writes?)\n"
   738  		if !strings.HasPrefix(output, want) && !strings.HasPrefix(output, want2) {
   739  			t.Fatalf("output does not start with %q:\n%s", want, output)
   740  		}
   741  	}
   742  }
   743  
   744  type point struct {
   745  	x, y *int
   746  }
   747  
   748  func (p *point) negate() {
   749  	*p.x = *p.x * -1
   750  	*p.y = *p.y * -1
   751  }
   752  
   753  // Test for issue #10152.
   754  func TestPanicInlined(t *testing.T) {
   755  	defer func() {
   756  		r := recover()
   757  		if r == nil {
   758  			t.Fatalf("recover failed")
   759  		}
   760  		buf := make([]byte, 2048)
   761  		n := runtime.Stack(buf, false)
   762  		buf = buf[:n]
   763  		if !bytes.Contains(buf, []byte("(*point).negate(")) {
   764  			t.Fatalf("expecting stack trace to contain call to (*point).negate()")
   765  		}
   766  	}()
   767  
   768  	pt := new(point)
   769  	pt.negate()
   770  }
   771  
   772  // Test for issues #3934 and #20018.
   773  // We want to delay exiting until a panic print is complete.
   774  func TestPanicRace(t *testing.T) {
   775  	testenv.MustHaveGoRun(t)
   776  
   777  	exe, err := buildTestProg(t, "testprog")
   778  	if err != nil {
   779  		t.Fatal(err)
   780  	}
   781  
   782  	// The test is intentionally racy, and in my testing does not
   783  	// produce the expected output about 0.05% of the time.
   784  	// So run the program in a loop and only fail the test if we
   785  	// get the wrong output ten times in a row.
   786  	const tries = 10
   787  retry:
   788  	for i := 0; i < tries; i++ {
   789  		got, err := testenv.CleanCmdEnv(exec.Command(exe, "PanicRace")).CombinedOutput()
   790  		if err == nil {
   791  			t.Logf("try %d: program exited successfully, should have failed", i+1)
   792  			continue
   793  		}
   794  
   795  		if i > 0 {
   796  			t.Logf("try %d:\n", i+1)
   797  		}
   798  		t.Logf("%s\n", got)
   799  
   800  		wants := []string{
   801  			"panic: crash",
   802  			"PanicRace",
   803  			"created by ",
   804  		}
   805  		for _, want := range wants {
   806  			if !bytes.Contains(got, []byte(want)) {
   807  				t.Logf("did not find expected string %q", want)
   808  				continue retry
   809  			}
   810  		}
   811  
   812  		// Test generated expected output.
   813  		return
   814  	}
   815  	t.Errorf("test ran %d times without producing expected output", tries)
   816  }
   817  
   818  func TestBadTraceback(t *testing.T) {
   819  	if asan.Enabled || msan.Enabled || race.Enabled {
   820  		t.Skip("skipped test: checkptr mode catches the corruption")
   821  	}
   822  	output := runTestProg(t, "testprog", "BadTraceback")
   823  	for _, want := range []string{
   824  		"unexpected return pc",
   825  		"called from 0xbad",
   826  		"00000bad",    // Smashed LR in hex dump
   827  		"<main.badLR", // Symbolization in hex dump (badLR1 or badLR2)
   828  	} {
   829  		if !strings.Contains(output, want) {
   830  			t.Errorf("output does not contain %q:\n%s", want, output)
   831  		}
   832  	}
   833  }
   834  
   835  func TestTimePprof(t *testing.T) {
   836  	// This test is unreliable on any system in which nanotime
   837  	// calls into libc.
   838  	switch runtime.GOOS {
   839  	case "aix", "darwin", "illumos", "openbsd", "solaris":
   840  		t.Skipf("skipping on %s because nanotime calls libc", runtime.GOOS)
   841  	}
   842  	if race.Enabled || asan.Enabled || msan.Enabled {
   843  		t.Skip("skipping on sanitizers because the sanitizer runtime is external code")
   844  	}
   845  
   846  	// Pass GOTRACEBACK for issue #41120 to try to get more
   847  	// information on timeout.
   848  	fn := runTestProg(t, "testprog", "TimeProf", "GOTRACEBACK=crash")
   849  	fn = strings.TrimSpace(fn)
   850  	defer os.Remove(fn)
   851  
   852  	cmd := testenv.CleanCmdEnv(exec.Command(testenv.GoToolPath(t), "tool", "pprof", "-top", "-nodecount=1", fn))
   853  	cmd.Env = append(cmd.Env, "PPROF_TMPDIR="+os.TempDir())
   854  	top, err := cmd.CombinedOutput()
   855  	t.Logf("%s", top)
   856  	if err != nil {
   857  		t.Error(err)
   858  	} else if bytes.Contains(top, []byte("ExternalCode")) {
   859  		t.Error("profiler refers to ExternalCode")
   860  	}
   861  }
   862  
   863  // Test that runtime.abort does so.
   864  func TestAbort(t *testing.T) {
   865  	// Pass GOTRACEBACK to ensure we get runtime frames.
   866  	output := runTestProg(t, "testprog", "Abort", "GOTRACEBACK=system")
   867  	if want := "runtime.abort"; !strings.Contains(output, want) {
   868  		t.Errorf("output does not contain %q:\n%s", want, output)
   869  	}
   870  	if strings.Contains(output, "BAD") {
   871  		t.Errorf("output contains BAD:\n%s", output)
   872  	}
   873  	// Check that it's a signal traceback.
   874  	want := "PC="
   875  	// For systems that use a breakpoint, check specifically for that.
   876  	switch runtime.GOARCH {
   877  	case "386", "amd64":
   878  		switch runtime.GOOS {
   879  		case "plan9":
   880  			want = "sys: breakpoint"
   881  		case "windows":
   882  			want = "Exception 0x80000003"
   883  		default:
   884  			want = "SIGTRAP"
   885  		}
   886  	}
   887  	if !strings.Contains(output, want) {
   888  		t.Errorf("output does not contain %q:\n%s", want, output)
   889  	}
   890  }
   891  
   892  // For TestRuntimePanic: test a panic in the runtime package without
   893  // involving the testing harness.
   894  func init() {
   895  	if os.Getenv("GO_TEST_RUNTIME_PANIC") == "1" {
   896  		defer func() {
   897  			if r := recover(); r != nil {
   898  				// We expect to crash, so exit 0
   899  				// to indicate failure.
   900  				os.Exit(0)
   901  			}
   902  		}()
   903  		runtime.PanicForTesting(nil, 1)
   904  		// We expect to crash, so exit 0 to indicate failure.
   905  		os.Exit(0)
   906  	}
   907  	if os.Getenv("GO_TEST_RUNTIME_NPE_READMEMSTATS") == "1" {
   908  		runtime.ReadMemStats(nil)
   909  		os.Exit(0)
   910  	}
   911  	if os.Getenv("GO_TEST_RUNTIME_NPE_FUNCMETHOD") == "1" {
   912  		var f *runtime.Func
   913  		_ = f.Entry()
   914  		os.Exit(0)
   915  	}
   916  
   917  }
   918  
   919  func TestRuntimePanic(t *testing.T) {
   920  	cmd := testenv.CleanCmdEnv(exec.Command(testenv.Executable(t), "-test.run=^TestRuntimePanic$"))
   921  	cmd.Env = append(cmd.Env, "GO_TEST_RUNTIME_PANIC=1")
   922  	out, err := cmd.CombinedOutput()
   923  	t.Logf("%s", out)
   924  	if err == nil {
   925  		t.Error("child process did not fail")
   926  	} else if want := "runtime.unexportedPanicForTesting"; !bytes.Contains(out, []byte(want)) {
   927  		t.Errorf("output did not contain expected string %q", want)
   928  	}
   929  }
   930  
   931  func TestTracebackRuntimeFunction(t *testing.T) {
   932  	cmd := testenv.CleanCmdEnv(exec.Command(testenv.Executable(t), "-test.run=^TestTracebackRuntimeFunction$"))
   933  	cmd.Env = append(cmd.Env, "GO_TEST_RUNTIME_NPE_READMEMSTATS=1")
   934  	out, err := cmd.CombinedOutput()
   935  	t.Logf("%s", out)
   936  	if err == nil {
   937  		t.Error("child process did not fail")
   938  	} else if want := "runtime.ReadMemStats"; !bytes.Contains(out, []byte(want)) {
   939  		t.Errorf("output did not contain expected string %q", want)
   940  	}
   941  }
   942  
   943  func TestTracebackRuntimeMethod(t *testing.T) {
   944  	cmd := testenv.CleanCmdEnv(exec.Command(testenv.Executable(t), "-test.run=^TestTracebackRuntimeMethod$"))
   945  	cmd.Env = append(cmd.Env, "GO_TEST_RUNTIME_NPE_FUNCMETHOD=1")
   946  	out, err := cmd.CombinedOutput()
   947  	t.Logf("%s", out)
   948  	if err == nil {
   949  		t.Error("child process did not fail")
   950  	} else if want := "runtime.(*Func).Entry"; !bytes.Contains(out, []byte(want)) {
   951  		t.Errorf("output did not contain expected string %q", want)
   952  	}
   953  }
   954  
   955  // Test that g0 stack overflows are handled gracefully.
   956  func TestG0StackOverflow(t *testing.T) {
   957  	if runtime.GOOS == "ios" {
   958  		testenv.SkipFlaky(t, 62671)
   959  	}
   960  
   961  	if os.Getenv("TEST_G0_STACK_OVERFLOW") != "1" {
   962  		cmd := testenv.CleanCmdEnv(testenv.Command(t, testenv.Executable(t), "-test.run=^TestG0StackOverflow$", "-test.v"))
   963  		cmd.Env = append(cmd.Env, "TEST_G0_STACK_OVERFLOW=1")
   964  		out, err := cmd.CombinedOutput()
   965  		t.Logf("output:\n%s", out)
   966  		// Don't check err since it's expected to crash.
   967  		if n := strings.Count(string(out), "morestack on g0\n"); n != 1 {
   968  			t.Fatalf("%s\n(exit status %v)", out, err)
   969  		}
   970  		if runtime.CrashStackImplemented {
   971  			// check for a stack trace
   972  			want := "runtime.stackOverflow"
   973  			if n := strings.Count(string(out), want); n < 5 {
   974  				t.Errorf("output does not contain %q at least 5 times:\n%s", want, out)
   975  			}
   976  			return // it's not a signal-style traceback
   977  		}
   978  		// Check that it's a signal-style traceback.
   979  		if runtime.GOOS != "windows" {
   980  			if want := "PC="; !strings.Contains(string(out), want) {
   981  				t.Errorf("output does not contain %q:\n%s", want, out)
   982  			}
   983  		}
   984  		return
   985  	}
   986  
   987  	runtime.G0StackOverflow()
   988  }
   989  
   990  // For TestCrashWhileTracing: test a panic without involving the testing
   991  // harness, as we rely on stdout only containing trace output.
   992  func init() {
   993  	if os.Getenv("TEST_CRASH_WHILE_TRACING") == "1" {
   994  		trace.Start(os.Stdout)
   995  		trace.Log(context.Background(), "xyzzy-cat", "xyzzy-msg")
   996  		panic("yzzyx")
   997  	}
   998  }
   999  
  1000  func TestCrashWhileTracing(t *testing.T) {
  1001  	testenv.MustHaveExec(t)
  1002  
  1003  	cmd := testenv.CleanCmdEnv(testenv.Command(t, testenv.Executable(t)))
  1004  	cmd.Env = append(cmd.Env, "TEST_CRASH_WHILE_TRACING=1")
  1005  	stdOut, err := cmd.StdoutPipe()
  1006  	var errOut bytes.Buffer
  1007  	cmd.Stderr = &errOut
  1008  
  1009  	if err := cmd.Start(); err != nil {
  1010  		t.Fatalf("could not start subprocess: %v", err)
  1011  	}
  1012  	r, err := traceparse.NewReader(stdOut)
  1013  	if err != nil {
  1014  		t.Fatalf("could not create trace.NewReader: %v", err)
  1015  	}
  1016  	var seen bool
  1017  	nSync := 0
  1018  	i := 1
  1019  loop:
  1020  	for ; ; i++ {
  1021  		ev, err := r.ReadEvent()
  1022  		if err != nil {
  1023  			// We may have a broken tail to the trace -- that's OK.
  1024  			// We'll make sure we saw at least one complete generation.
  1025  			if err != io.EOF {
  1026  				t.Logf("error at event %d: %v", i, err)
  1027  			}
  1028  			break loop
  1029  		}
  1030  		switch ev.Kind() {
  1031  		case traceparse.EventSync:
  1032  			nSync = ev.Sync().N
  1033  		case traceparse.EventLog:
  1034  			v := ev.Log()
  1035  			if v.Category == "xyzzy-cat" && v.Message == "xyzzy-msg" {
  1036  				// Should we already stop reading here? More events may come, but
  1037  				// we're not guaranteeing a fully unbroken trace until the last
  1038  				// byte...
  1039  				seen = true
  1040  			}
  1041  		}
  1042  	}
  1043  	if err := cmd.Wait(); err == nil {
  1044  		t.Error("the process should have panicked")
  1045  	}
  1046  	if nSync <= 1 {
  1047  		t.Errorf("expected at least one full generation to have been emitted before the trace was considered broken")
  1048  	}
  1049  	if !seen {
  1050  		t.Errorf("expected one matching log event matching, but none of the %d received trace events match", i)
  1051  	}
  1052  	t.Logf("stderr output:\n%s", errOut.String())
  1053  	needle := "yzzyx\n"
  1054  	if n := strings.Count(errOut.String(), needle); n != 1 {
  1055  		t.Fatalf("did not find expected panic message %q\n(exit status %v)", needle, err)
  1056  	}
  1057  }
  1058  
  1059  // Test that panic message is not clobbered.
  1060  // See issue 30150.
  1061  func TestDoublePanic(t *testing.T) {
  1062  	output := runTestProg(t, "testprog", "DoublePanic", "GODEBUG=clobberfree=1")
  1063  	wants := []string{"panic: XXX", "panic: YYY"}
  1064  	for _, want := range wants {
  1065  		if !strings.Contains(output, want) {
  1066  			t.Errorf("output:\n%s\n\nwant output containing: %s", output, want)
  1067  		}
  1068  	}
  1069  }
  1070  
  1071  // Test that panic while panicking discards error message
  1072  // See issue 52257
  1073  func TestPanicWhilePanicking(t *testing.T) {
  1074  	tests := []struct {
  1075  		Want string
  1076  		Func string
  1077  	}{
  1078  		{
  1079  			"panic while printing panic value: important multi-line\n\terror message",
  1080  			"ErrorPanic",
  1081  		},
  1082  		{
  1083  			"panic while printing panic value: important multi-line\n\tstringer message",
  1084  			"StringerPanic",
  1085  		},
  1086  		{
  1087  			"panic while printing panic value: type",
  1088  			"DoubleErrorPanic",
  1089  		},
  1090  		{
  1091  			"panic while printing panic value: type",
  1092  			"DoubleStringerPanic",
  1093  		},
  1094  		{
  1095  			"panic while printing panic value: type",
  1096  			"CircularPanic",
  1097  		},
  1098  		{
  1099  			"important multi-line\n\tstring message",
  1100  			"StringPanic",
  1101  		},
  1102  		{
  1103  			"nil",
  1104  			"NilPanic",
  1105  		},
  1106  	}
  1107  	for _, x := range tests {
  1108  		output := runTestProg(t, "testprog", x.Func)
  1109  		if !strings.Contains(output, x.Want) {
  1110  			t.Errorf("output does not contain %q:\n%s", x.Want, output)
  1111  		}
  1112  	}
  1113  }
  1114  
  1115  func TestPanicOnUnsafeSlice(t *testing.T) {
  1116  	output := runTestProg(t, "testprog", "panicOnNilAndEleSizeIsZero")
  1117  	// Note: This is normally a panic, but is a throw when checkptr is
  1118  	// enabled.
  1119  	want := "unsafe.Slice: ptr is nil and len is not zero"
  1120  	if !strings.Contains(output, want) {
  1121  		t.Errorf("output does not contain %q:\n%s", want, output)
  1122  	}
  1123  }
  1124  
  1125  func TestNetpollWaiters(t *testing.T) {
  1126  	t.Parallel()
  1127  	output := runTestProg(t, "testprognet", "NetpollWaiters")
  1128  	want := "OK\n"
  1129  	if output != want {
  1130  		t.Fatalf("output is not %q\n%s", want, output)
  1131  	}
  1132  }
  1133  
  1134  func TestFinalizerOrCleanupDeadlock(t *testing.T) {
  1135  	t.Parallel()
  1136  
  1137  	for _, useCleanup := range []bool{false, true} {
  1138  		progName := "Finalizer"
  1139  		want := "runtime.runFinalizers"
  1140  		if useCleanup {
  1141  			progName = "Cleanup"
  1142  			want = "runtime.runCleanups"
  1143  		}
  1144  
  1145  		// The runtime.runFinalizers/runtime.runCleanups frame should appear in panics, even if
  1146  		// runtime frames are normally hidden (GOTRACEBACK=all).
  1147  		t.Run("Panic", func(t *testing.T) {
  1148  			t.Parallel()
  1149  			output := runTestProg(t, "testprog", progName+"Deadlock", "GOTRACEBACK=all", "GO_TEST_FINALIZER_DEADLOCK=panic")
  1150  			want := want + "()"
  1151  			if !strings.Contains(output, want) {
  1152  				t.Errorf("output does not contain %q:\n%s", want, output)
  1153  			}
  1154  		})
  1155  
  1156  		// The runtime.runFinalizers/runtime.Cleanups frame should appear in runtime.Stack,
  1157  		// even though runtime frames are normally hidden.
  1158  		t.Run("Stack", func(t *testing.T) {
  1159  			t.Parallel()
  1160  			output := runTestProg(t, "testprog", progName+"Deadlock", "GO_TEST_FINALIZER_DEADLOCK=stack")
  1161  			want := want + "()"
  1162  			if !strings.Contains(output, want) {
  1163  				t.Errorf("output does not contain %q:\n%s", want, output)
  1164  			}
  1165  		})
  1166  
  1167  		// The runtime.runFinalizers/runtime.Cleanups frame should appear in goroutine
  1168  		// profiles.
  1169  		t.Run("PprofProto", func(t *testing.T) {
  1170  			t.Parallel()
  1171  			output := runTestProg(t, "testprog", progName+"Deadlock", "GO_TEST_FINALIZER_DEADLOCK=pprof_proto")
  1172  
  1173  			p, err := profile.Parse(strings.NewReader(output))
  1174  			if err != nil {
  1175  				// Logging the binary proto data is not very nice, but it might
  1176  				// be a text error message instead.
  1177  				t.Logf("Output: %s", output)
  1178  				t.Fatalf("Error parsing proto output: %v", err)
  1179  			}
  1180  			for _, s := range p.Sample {
  1181  				for _, loc := range s.Location {
  1182  					for _, line := range loc.Line {
  1183  						if line.Function.Name == want {
  1184  							// Done!
  1185  							return
  1186  						}
  1187  					}
  1188  				}
  1189  			}
  1190  			t.Errorf("Profile does not contain %q:\n%s", want, p)
  1191  		})
  1192  
  1193  		// The runtime.runFinalizers/runtime.runCleanups frame should appear in goroutine
  1194  		// profiles (debug=1).
  1195  		t.Run("PprofDebug1", func(t *testing.T) {
  1196  			t.Parallel()
  1197  			output := runTestProg(t, "testprog", progName+"Deadlock", "GO_TEST_FINALIZER_DEADLOCK=pprof_debug1")
  1198  			want := want + "+"
  1199  			if !strings.Contains(output, want) {
  1200  				t.Errorf("output does not contain %q:\n%s", want, output)
  1201  			}
  1202  		})
  1203  
  1204  		// The runtime.runFinalizers/runtime.runCleanups frame should appear in goroutine
  1205  		// profiles (debug=2).
  1206  		t.Run("PprofDebug2", func(t *testing.T) {
  1207  			t.Parallel()
  1208  			output := runTestProg(t, "testprog", progName+"Deadlock", "GO_TEST_FINALIZER_DEADLOCK=pprof_debug2")
  1209  			want := want + "()"
  1210  			if !strings.Contains(output, want) {
  1211  				t.Errorf("output does not contain %q:\n%s", want, output)
  1212  			}
  1213  		})
  1214  	}
  1215  }
  1216  
  1217  func TestSynctestCondSignalFromNoBubble(t *testing.T) {
  1218  	for _, test := range []string{
  1219  		"SynctestCond/signal/no_bubble",
  1220  		"SynctestCond/broadcast/no_bubble",
  1221  		"SynctestCond/signal/other_bubble",
  1222  		"SynctestCond/broadcast/other_bubble",
  1223  	} {
  1224  		t.Run(test, func(t *testing.T) {
  1225  			output := runTestProg(t, "testprog", test)
  1226  			want := "fatal error: semaphore wake of synctest goroutine from outside bubble"
  1227  			if !strings.Contains(output, want) {
  1228  				t.Fatalf("output:\n%s\n\nwant output containing: %s", output, want)
  1229  			}
  1230  		})
  1231  	}
  1232  }
  1233  

View as plain text