1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 package driver
16
17 import (
18 "bytes"
19 "fmt"
20 "io"
21 "net/http"
22 "net/url"
23 "os"
24 "os/exec"
25 "path/filepath"
26 "runtime"
27 "strconv"
28 "strings"
29 "sync"
30 "time"
31
32 "github.com/google/pprof/internal/measurement"
33 "github.com/google/pprof/internal/plugin"
34 "github.com/google/pprof/profile"
35 )
36
37
38
39
40
41 func fetchProfiles(s *source, o *plugin.Options) (*profile.Profile, error) {
42 sources := make([]profileSource, 0, len(s.Sources))
43 for _, src := range s.Sources {
44 sources = append(sources, profileSource{
45 addr: src,
46 source: s,
47 })
48 }
49
50 bases := make([]profileSource, 0, len(s.Base))
51 for _, src := range s.Base {
52 bases = append(bases, profileSource{
53 addr: src,
54 source: s,
55 })
56 }
57
58 p, pbase, m, mbase, save, err := grabSourcesAndBases(sources, bases, o.Fetch, o.Obj, o.UI, o.HTTPTransport)
59 if err != nil {
60 return nil, err
61 }
62
63 if pbase != nil {
64 if s.DiffBase {
65 pbase.SetLabel("pprof::base", []string{"true"})
66 }
67 if s.Normalize {
68 err := p.Normalize(pbase)
69 if err != nil {
70 return nil, err
71 }
72 }
73 pbase.Scale(-1)
74 p, m, err = combineProfiles([]*profile.Profile{p, pbase}, []plugin.MappingSources{m, mbase})
75 if err != nil {
76 return nil, err
77 }
78 }
79
80
81 if err := o.Sym.Symbolize(s.Symbolize, m, p); err != nil {
82 return nil, err
83 }
84 p.RemoveUninteresting()
85 unsourceMappings(p)
86
87 if s.Comment != "" {
88 p.Comments = append(p.Comments, s.Comment)
89 }
90
91
92 if save {
93 dir, err := setTmpDir(o.UI)
94 if err != nil {
95 return nil, err
96 }
97
98 prefix := "pprof."
99 if len(p.Mapping) > 0 && p.Mapping[0].File != "" {
100 prefix += filepath.Base(p.Mapping[0].File) + "."
101 }
102 for _, s := range p.SampleType {
103 prefix += s.Type + "."
104 }
105
106 tempFile, err := newTempFile(dir, prefix, ".pb.gz")
107 if err == nil {
108 if err = p.Write(tempFile); err == nil {
109 o.UI.PrintErr("Saved profile in ", tempFile.Name())
110 }
111 }
112 if err != nil {
113 o.UI.PrintErr("Could not save profile: ", err)
114 }
115 }
116
117 if err := p.CheckValid(); err != nil {
118 return nil, err
119 }
120
121 return p, nil
122 }
123
124 func grabSourcesAndBases(sources, bases []profileSource, fetch plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI, tr http.RoundTripper) (*profile.Profile, *profile.Profile, plugin.MappingSources, plugin.MappingSources, bool, error) {
125 wg := sync.WaitGroup{}
126 wg.Add(2)
127 var psrc, pbase *profile.Profile
128 var msrc, mbase plugin.MappingSources
129 var savesrc, savebase bool
130 var errsrc, errbase error
131 var countsrc, countbase int
132 go func() {
133 defer wg.Done()
134 psrc, msrc, savesrc, countsrc, errsrc = chunkedGrab(sources, fetch, obj, ui, tr)
135 }()
136 go func() {
137 defer wg.Done()
138 pbase, mbase, savebase, countbase, errbase = chunkedGrab(bases, fetch, obj, ui, tr)
139 }()
140 wg.Wait()
141 save := savesrc || savebase
142
143 if errsrc != nil {
144 return nil, nil, nil, nil, false, fmt.Errorf("problem fetching source profiles: %v", errsrc)
145 }
146 if errbase != nil {
147 return nil, nil, nil, nil, false, fmt.Errorf("problem fetching base profiles: %v,", errbase)
148 }
149 if countsrc == 0 {
150 return nil, nil, nil, nil, false, fmt.Errorf("failed to fetch any source profiles")
151 }
152 if countbase == 0 && len(bases) > 0 {
153 return nil, nil, nil, nil, false, fmt.Errorf("failed to fetch any base profiles")
154 }
155 if want, got := len(sources), countsrc; want != got {
156 ui.PrintErr(fmt.Sprintf("Fetched %d source profiles out of %d", got, want))
157 }
158 if want, got := len(bases), countbase; want != got {
159 ui.PrintErr(fmt.Sprintf("Fetched %d base profiles out of %d", got, want))
160 }
161
162 return psrc, pbase, msrc, mbase, save, nil
163 }
164
165
166
167
168 func chunkedGrab(sources []profileSource, fetch plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI, tr http.RoundTripper) (*profile.Profile, plugin.MappingSources, bool, int, error) {
169 const chunkSize = 128
170
171 var p *profile.Profile
172 var msrc plugin.MappingSources
173 var save bool
174 var count int
175
176 for start := 0; start < len(sources); start += chunkSize {
177 end := min(start+chunkSize, len(sources))
178 chunkP, chunkMsrc, chunkSave, chunkCount, chunkErr := concurrentGrab(sources[start:end], fetch, obj, ui, tr)
179 switch {
180 case chunkErr != nil:
181 return nil, nil, false, 0, chunkErr
182 case chunkP == nil:
183 continue
184 case p == nil:
185 p, msrc, save, count = chunkP, chunkMsrc, chunkSave, chunkCount
186 default:
187 p, msrc, chunkErr = combineProfiles([]*profile.Profile{p, chunkP}, []plugin.MappingSources{msrc, chunkMsrc})
188 if chunkErr != nil {
189 return nil, nil, false, 0, chunkErr
190 }
191 if chunkSave {
192 save = true
193 }
194 count += chunkCount
195 }
196 }
197
198 return p, msrc, save, count, nil
199 }
200
201
202 func concurrentGrab(sources []profileSource, fetch plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI, tr http.RoundTripper) (*profile.Profile, plugin.MappingSources, bool, int, error) {
203 wg := sync.WaitGroup{}
204 wg.Add(len(sources))
205 for i := range sources {
206 go func(s *profileSource) {
207 defer wg.Done()
208 s.p, s.msrc, s.remote, s.err = grabProfile(s.source, s.addr, fetch, obj, ui, tr)
209 }(&sources[i])
210 }
211 wg.Wait()
212
213 var save bool
214 profiles := make([]*profile.Profile, 0, len(sources))
215 msrcs := make([]plugin.MappingSources, 0, len(sources))
216 for i := range sources {
217 s := &sources[i]
218 if err := s.err; err != nil {
219 ui.PrintErr(s.addr + ": " + err.Error())
220 continue
221 }
222 save = save || s.remote
223 profiles = append(profiles, s.p)
224 msrcs = append(msrcs, s.msrc)
225 *s = profileSource{}
226 }
227
228 if len(profiles) == 0 {
229 return nil, nil, false, 0, nil
230 }
231
232 p, msrc, err := combineProfiles(profiles, msrcs)
233 if err != nil {
234 return nil, nil, false, 0, err
235 }
236 return p, msrc, save, len(profiles), nil
237 }
238
239 func combineProfiles(profiles []*profile.Profile, msrcs []plugin.MappingSources) (*profile.Profile, plugin.MappingSources, error) {
240
241
242
243
244
245 if err := profile.CompatibilizeSampleTypes(profiles); err != nil {
246 return nil, nil, err
247 }
248 if err := measurement.ScaleProfiles(profiles); err != nil {
249 return nil, nil, err
250 }
251
252
253 if len(profiles) == 1 && len(msrcs) == 1 {
254 return profiles[0], msrcs[0], nil
255 }
256
257 p, err := profile.Merge(profiles)
258 if err != nil {
259 return nil, nil, err
260 }
261
262
263 msrc := make(plugin.MappingSources)
264 for _, ms := range msrcs {
265 for m, s := range ms {
266 msrc[m] = append(msrc[m], s...)
267 }
268 }
269 return p, msrc, nil
270 }
271
272 type profileSource struct {
273 addr string
274 source *source
275
276 p *profile.Profile
277 msrc plugin.MappingSources
278 remote bool
279 err error
280 }
281
282 func homeEnv() string {
283 switch runtime.GOOS {
284 case "windows":
285 return "USERPROFILE"
286 case "plan9":
287 return "home"
288 default:
289 return "HOME"
290 }
291 }
292
293
294
295
296 func setTmpDir(ui plugin.UI) (string, error) {
297 var dirs []string
298 if profileDir := os.Getenv("PPROF_TMPDIR"); profileDir != "" {
299 dirs = append(dirs, profileDir)
300 }
301 if homeDir := os.Getenv(homeEnv()); homeDir != "" {
302 dirs = append(dirs, filepath.Join(homeDir, "pprof"))
303 }
304 dirs = append(dirs, os.TempDir())
305 for _, tmpDir := range dirs {
306 if err := os.MkdirAll(tmpDir, 0755); err != nil {
307 ui.PrintErr("Could not use temp dir ", tmpDir, ": ", err.Error())
308 continue
309 }
310 return tmpDir, nil
311 }
312 return "", fmt.Errorf("failed to identify temp dir")
313 }
314
315 const testSourceAddress = "pproftest.local"
316
317
318
319
320 func grabProfile(s *source, source string, fetcher plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI, tr http.RoundTripper) (p *profile.Profile, msrc plugin.MappingSources, remote bool, err error) {
321 var src string
322 duration, timeout := time.Duration(s.Seconds)*time.Second, time.Duration(s.Timeout)*time.Second
323 if fetcher != nil {
324 p, src, err = fetcher.Fetch(source, duration, timeout)
325 if err != nil {
326 return
327 }
328 }
329 if err != nil || p == nil {
330
331 p, src, err = fetch(source, duration, timeout, ui, tr)
332 if err != nil {
333 return
334 }
335 }
336
337 if err = p.CheckValid(); err != nil {
338 return
339 }
340
341
342 locateBinaries(p, s, obj, ui)
343
344
345 if src != "" {
346 msrc = collectMappingSources(p, src)
347 remote = true
348 if strings.HasPrefix(src, "http://"+testSourceAddress) {
349
350
351 remote = false
352 }
353 }
354 return
355 }
356
357
358 func collectMappingSources(p *profile.Profile, source string) plugin.MappingSources {
359 ms := plugin.MappingSources{}
360 for _, m := range p.Mapping {
361 src := struct {
362 Source string
363 Start uint64
364 }{
365 source, m.Start,
366 }
367 key := m.BuildID
368 if key == "" {
369 key = m.File
370 }
371 if key == "" {
372
373
374
375
376
377 m.File = source
378 key = source
379 }
380 ms[key] = append(ms[key], src)
381 }
382 return ms
383 }
384
385
386
387 func unsourceMappings(p *profile.Profile) {
388 for _, m := range p.Mapping {
389 if m.BuildID == "" && filepath.VolumeName(m.File) == "" {
390 if u, err := url.Parse(m.File); err == nil && u.IsAbs() {
391 m.File = ""
392 }
393 }
394 }
395 }
396
397
398
399 func locateBinaries(p *profile.Profile, s *source, obj plugin.ObjTool, ui plugin.UI) {
400
401 searchPath := os.Getenv("PPROF_BINARY_PATH")
402 if searchPath == "" {
403
404 searchPath = filepath.Join(os.Getenv(homeEnv()), "pprof", "binaries")
405 }
406 mapping:
407 for _, m := range p.Mapping {
408 var noVolumeFile string
409 var baseName string
410 var dirName string
411 if m.File != "" {
412 noVolumeFile = strings.TrimPrefix(m.File, filepath.VolumeName(m.File))
413 baseName = filepath.Base(m.File)
414 dirName = filepath.Dir(noVolumeFile)
415 }
416
417 for _, path := range filepath.SplitList(searchPath) {
418 var fileNames []string
419 if m.BuildID != "" {
420 fileNames = []string{filepath.Join(path, m.BuildID, baseName)}
421 if matches, err := filepath.Glob(filepath.Join(path, m.BuildID, "*")); err == nil {
422 fileNames = append(fileNames, matches...)
423 }
424 fileNames = append(fileNames, filepath.Join(path, noVolumeFile, m.BuildID))
425
426
427
428 fileNames = append(fileNames, filepath.Join(path, m.BuildID[:2], m.BuildID[2:]+".debug"))
429 }
430 if m.File != "" {
431
432
433 fileNames = append(fileNames, filepath.Join(path, baseName))
434 fileNames = append(fileNames, filepath.Join(path, noVolumeFile))
435
436
437 fileNames = append(fileNames, filepath.Join(path, noVolumeFile+".debug"))
438 fileNames = append(fileNames, filepath.Join(path, dirName, ".debug", baseName+".debug"))
439 fileNames = append(fileNames, filepath.Join(path, "usr", "lib", "debug", dirName, baseName+".debug"))
440 }
441 for _, name := range fileNames {
442 if f, err := obj.Open(name, m.Start, m.Limit, m.Offset, m.KernelRelocationSymbol); err == nil {
443 defer f.Close()
444 fileBuildID := f.BuildID()
445 if m.BuildID != "" && m.BuildID != fileBuildID {
446 ui.PrintErr("Ignoring local file " + name + ": build-id mismatch (" + m.BuildID + " != " + fileBuildID + ")")
447 } else {
448
449
450 m.File = name
451 continue mapping
452 }
453 }
454 }
455 }
456 }
457 if len(p.Mapping) == 0 {
458
459
460
461
462 m := &profile.Mapping{ID: 1}
463 p.Mapping = []*profile.Mapping{m}
464 for _, l := range p.Location {
465 l.Mapping = m
466 }
467 }
468
469
470 if execName, buildID := s.ExecName, s.BuildID; execName != "" || buildID != "" {
471 m := p.Mapping[0]
472 if execName != "" {
473
474
475 m.File = execName
476 }
477
478
479
480 if buildID != "" && m.BuildID == "" {
481 m.BuildID = buildID
482 }
483 }
484 }
485
486
487
488
489 func fetch(source string, duration, timeout time.Duration, ui plugin.UI, tr http.RoundTripper) (p *profile.Profile, src string, err error) {
490 var f io.ReadCloser
491
492
493 if _, err = os.Stat(source); err == nil {
494 if isPerfFile(source) {
495 f, err = convertPerfData(source, ui)
496 } else {
497 f, err = os.Open(source)
498 }
499 } else {
500 sourceURL, timeout := adjustURL(source, duration, timeout)
501 if sourceURL != "" {
502 ui.Print("Fetching profile over HTTP from " + sourceURL)
503 if duration > 0 {
504 ui.Print(fmt.Sprintf("Please wait... (%v)", duration))
505 }
506 f, err = fetchURL(sourceURL, timeout, tr)
507 src = sourceURL
508 }
509 }
510 if err == nil {
511 defer f.Close()
512 p, err = profile.Parse(f)
513 }
514 return
515 }
516
517
518 func fetchURL(source string, timeout time.Duration, tr http.RoundTripper) (io.ReadCloser, error) {
519 client := &http.Client{
520 Transport: tr,
521 Timeout: timeout + 5*time.Second,
522 }
523 resp, err := client.Get(source)
524 if err != nil {
525 return nil, fmt.Errorf("http fetch: %v", err)
526 }
527 if resp.StatusCode != http.StatusOK {
528 defer resp.Body.Close()
529 return nil, statusCodeError(resp)
530 }
531
532 return resp.Body, nil
533 }
534
535 func statusCodeError(resp *http.Response) error {
536 if resp.Header.Get("X-Go-Pprof") != "" && strings.Contains(resp.Header.Get("Content-Type"), "text/plain") {
537
538 if body, err := io.ReadAll(resp.Body); err == nil {
539 return fmt.Errorf("server response: %s - %s", resp.Status, body)
540 }
541 }
542 return fmt.Errorf("server response: %s", resp.Status)
543 }
544
545
546
547 func isPerfFile(path string) bool {
548 sourceFile, openErr := os.Open(path)
549 if openErr != nil {
550 return false
551 }
552 defer sourceFile.Close()
553
554
555
556 perfHeader := []byte("PERFILE2")
557 actualHeader := make([]byte, len(perfHeader))
558 if _, readErr := sourceFile.Read(actualHeader); readErr != nil {
559 return false
560 }
561 return bytes.Equal(actualHeader, perfHeader)
562 }
563
564
565
566
567 func convertPerfData(perfPath string, ui plugin.UI) (*os.File, error) {
568 ui.Print(fmt.Sprintf(
569 "Converting %s to a profile.proto... (May take a few minutes)",
570 perfPath))
571 profile, err := newTempFile(os.TempDir(), "pprof_", ".pb.gz")
572 if err != nil {
573 return nil, err
574 }
575 deferDeleteTempFile(profile.Name())
576 cmd := exec.Command("perf_to_profile", "-i", perfPath, "-o", profile.Name(), "-f")
577 cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
578 if err := cmd.Run(); err != nil {
579 profile.Close()
580 return nil, fmt.Errorf("failed to convert perf.data file. Try github.com/google/perf_data_converter: %v", err)
581 }
582 return profile, nil
583 }
584
585
586
587
588 func adjustURL(source string, duration, timeout time.Duration) (string, time.Duration) {
589 u, err := url.Parse(source)
590 if err != nil || (u.Host == "" && u.Scheme != "" && u.Scheme != "file") {
591
592
593 u, err = url.Parse("http://" + source)
594 }
595 if err != nil || u.Host == "" {
596 return "", 0
597 }
598
599
600 values := u.Query()
601 if duration > 0 {
602 values.Set("seconds", fmt.Sprint(int(duration.Seconds())))
603 } else {
604 if urlSeconds := values.Get("seconds"); urlSeconds != "" {
605 if us, err := strconv.ParseInt(urlSeconds, 10, 32); err == nil {
606 duration = time.Duration(us) * time.Second
607 }
608 }
609 }
610 if timeout <= 0 {
611 if duration > 0 {
612 timeout = duration + duration/2
613 } else {
614 timeout = 60 * time.Second
615 }
616 }
617 u.RawQuery = values.Encode()
618 return u.String(), timeout
619 }
620
View as plain text