runtime: mark several types go:notinheap

[gostls13.git] / src / runtime / cpuprof.go

// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// CPU profiling.
// Based on algorithms and data structures used in
// https://github.com/google/pprof.
//
// The main difference between this code and the google-perftools
// code is that this code is written to allow copying the profile data
// to an arbitrary io.Writer, while the google-perftools code always
// writes to an operating system file.
//
// The signal handler for the profiling clock tick adds a new stack trace
// to a hash table tracking counts for recent traces. Most clock ticks
// hit in the cache. In the event of a cache miss, an entry must be
// evicted from the hash table, copied to a log that will eventually be
// written as profile data. The google-perftools code flushed the
// log itself during the signal handler. This code cannot do that, because
// the io.Writer might block or need system calls or locks that are not
// safe to use from within the signal handler. Instead, we split the log
// into two halves and let the signal handler fill one half while a goroutine
// is writing out the other half. When the signal handler fills its half, it
// offers to swap with the goroutine. If the writer is not done with its half,
// we lose the stack trace for this clock tick (and record that loss).
// The goroutine interacts with the signal handler by calling getprofile() to
// get the next log piece to write, implicitly handing back the last log
// piece it obtained.
//
// The state of this dance between the signal handler and the goroutine
// is encoded in the Profile.handoff field. If handoff == 0, then the goroutine
// is not using either log half and is waiting (or will soon be waiting) for
// a new piece by calling notesleep(&p.wait).  If the signal handler
// changes handoff from 0 to non-zero, it must call notewakeup(&p.wait)
// to wake the goroutine. The value indicates the number of entries in the
// log half being handed off. The goroutine leaves the non-zero value in
// place until it has finished processing the log half and then flips the number
// back to zero. Setting the high bit in handoff means that the profiling is over,
// and the goroutine is now in charge of flushing the data left in the hash table
// to the log and returning that data.
//
// The handoff field is manipulated using atomic operations.
// For the most part, the manipulation of handoff is orderly: if handoff == 0
// then the signal handler owns it and can change it to non-zero.
// If handoff != 0 then the goroutine owns it and can change it to zero.
// If that were the end of the story then we would not need to manipulate
// handoff using atomic operations. The operations are needed, however,
// in order to let the log closer set the high bit to indicate "EOF" safely
// in the situation when normally the goroutine "owns" handoff.

package runtime

import (
        "runtime/internal/atomic"
        "unsafe"
)

const (
        numBuckets      = 1 << 10
        logSize         = 1 << 17
        assoc           = 4
        maxCPUProfStack = 64
)

type cpuprofEntry struct {
        count uintptr
        depth int
        stack [maxCPUProfStack]uintptr
}

//go:notinheap
type cpuProfile struct {
        on     bool    // profiling is on
        wait   note    // goroutine waits here
        count  uintptr // tick count
        evicts uintptr // eviction count
        lost   uintptr // lost ticks that need to be logged

        // Active recent stack traces.
        hash [numBuckets]struct {
                entry [assoc]cpuprofEntry
        }

        // Log of traces evicted from hash.
        // Signal handler has filled log[toggle][:nlog].
        // Goroutine is writing log[1-toggle][:handoff].
        log     [2][logSize / 2]uintptr
        nlog    int
        toggle  int32
        handoff uint32

        // Writer state.
        // Writer maintains its own toggle to avoid races
        // looking at signal handler's toggle.
        wtoggle  uint32
        wholding bool // holding & need to release a log half
        flushing bool // flushing hash table - profile is over
        eodSent  bool // special end-of-data record sent; => flushing
}

var (
        cpuprofLock mutex
        cpuprof     *cpuProfile

        eod = [3]uintptr{0, 1, 0}
)

func setcpuprofilerate(hz int32) {
        systemstack(func() {
                setcpuprofilerate_m(hz)
        })
}

// lostProfileData is a no-op function used in profiles
// to mark the number of profiling stack traces that were
// discarded due to slow data writers.
func lostProfileData() {}

// SetCPUProfileRate sets the CPU profiling rate to hz samples per second.
// If hz <= 0, SetCPUProfileRate turns off profiling.
// If the profiler is on, the rate cannot be changed without first turning it off.
//
// Most clients should use the runtime/pprof package or
// the testing package's -test.cpuprofile flag instead of calling
// SetCPUProfileRate directly.
func SetCPUProfileRate(hz int) {
        // Clamp hz to something reasonable.
        if hz < 0 {
                hz = 0
        }
        if hz > 1000000 {
                hz = 1000000
        }

        lock(&cpuprofLock)
        if hz > 0 {
                if cpuprof == nil {
                        cpuprof = (*cpuProfile)(sysAlloc(unsafe.Sizeof(cpuProfile{}), &memstats.other_sys))
                        if cpuprof == nil {
                                print("runtime: cpu profiling cannot allocate memory\n")
                                unlock(&cpuprofLock)
                                return
                        }
                }
                if cpuprof.on || cpuprof.handoff != 0 {
                        print("runtime: cannot set cpu profile rate until previous profile has finished.\n")
                        unlock(&cpuprofLock)
                        return
                }

                cpuprof.on = true
                // pprof binary header format.
                // https://github.com/gperftools/gperftools/blob/master/src/profiledata.cc#L119
                p := &cpuprof.log[0]
                p[0] = 0                 // count for header
                p[1] = 3                 // depth for header
                p[2] = 0                 // version number
                p[3] = uintptr(1e6 / hz) // period (microseconds)
                p[4] = 0
                cpuprof.nlog = 5
                cpuprof.toggle = 0
                cpuprof.wholding = false
                cpuprof.wtoggle = 0
                cpuprof.flushing = false
                cpuprof.eodSent = false
                noteclear(&cpuprof.wait)

                setcpuprofilerate(int32(hz))
        } else if cpuprof != nil && cpuprof.on {
                setcpuprofilerate(0)
                cpuprof.on = false

                // Now add is not running anymore, and getprofile owns the entire log.
                // Set the high bit in cpuprof.handoff to tell getprofile.
                for {
                        n := cpuprof.handoff
                        if n&0x80000000 != 0 {
                                print("runtime: setcpuprofile(off) twice\n")
                        }
                        if atomic.Cas(&cpuprof.handoff, n, n|0x80000000) {
                                if n == 0 {
                                        // we did the transition from 0 -> nonzero so we wake getprofile
                                        notewakeup(&cpuprof.wait)
                                }
                                break
                        }
                }
        }
        unlock(&cpuprofLock)
}

// add adds the stack trace to the profile.
// It is called from signal handlers and other limited environments
// and cannot allocate memory or acquire locks that might be
// held at the time of the signal, nor can it use substantial amounts
// of stack. It is allowed to call evict.
//go:nowritebarrierrec
func (p *cpuProfile) add(pc []uintptr) {
        p.addWithFlushlog(pc, p.flushlog)
}

// addWithFlushlog implements add and addNonGo.
// It is called from signal handlers and other limited environments
// and cannot allocate memory or acquire locks that might be
// held at the time of the signal, nor can it use substantial amounts
// of stack. It may be called by a signal handler with no g or m.
// It is allowed to call evict, passing the flushlog parameter.
//go:nosplit
//go:nowritebarrierrec
func (p *cpuProfile) addWithFlushlog(pc []uintptr, flushlog func() bool) {
        if len(pc) > maxCPUProfStack {
                pc = pc[:maxCPUProfStack]
        }

        // Compute hash.
        h := uintptr(0)
        for _, x := range pc {
                h = h<<8 | (h >> (8 * (unsafe.Sizeof(h) - 1)))
                h += x * 41
        }
        p.count++

        // Add to entry count if already present in table.
        b := &p.hash[h%numBuckets]
Assoc:
        for i := range b.entry {
                e := &b.entry[i]
                if e.depth != len(pc) {
                        continue
                }
                for j := range pc {
                        if e.stack[j] != pc[j] {
                                continue Assoc
                        }
                }
                e.count++
                return
        }

        // Evict entry with smallest count.
        var e *cpuprofEntry
        for i := range b.entry {
                if e == nil || b.entry[i].count < e.count {
                        e = &b.entry[i]
                }
        }
        if e.count > 0 {
                if !p.evict(e, flushlog) {
                        // Could not evict entry. Record lost stack.
                        p.lost++
                        return
                }
                p.evicts++
        }

        // Reuse the newly evicted entry.
        e.depth = len(pc)
        e.count = 1
        copy(e.stack[:], pc)
}

// evict copies the given entry's data into the log, so that
// the entry can be reused.  evict is called from add, which
// is called from the profiling signal handler, so it must not
// allocate memory or block, and it may be called with no g or m.
// It is safe to call flushlog. evict returns true if the entry was
// copied to the log, false if there was no room available.
//go:nosplit
//go:nowritebarrierrec
func (p *cpuProfile) evict(e *cpuprofEntry, flushlog func() bool) bool {
        d := e.depth
        nslot := d + 2
        log := &p.log[p.toggle]
        if p.nlog+nslot > len(log) {
                if !flushlog() {
                        return false
                }
                log = &p.log[p.toggle]
        }

        q := p.nlog
        log[q] = e.count
        q++
        log[q] = uintptr(d)
        q++
        copy(log[q:], e.stack[:d])
        q += d
        p.nlog = q
        e.count = 0
        return true
}

// flushlog tries to flush the current log and switch to the other one.
// flushlog is called from evict, called from add, called from the signal handler,
// so it cannot allocate memory or block. It can try to swap logs with
// the writing goroutine, as explained in the comment at the top of this file.
//go:nowritebarrierrec
func (p *cpuProfile) flushlog() bool {
        if !atomic.Cas(&p.handoff, 0, uint32(p.nlog)) {
                return false
        }
        notewakeup(&p.wait)

        p.toggle = 1 - p.toggle
        log := &p.log[p.toggle]
        q := 0
        if p.lost > 0 {
                lostPC := funcPC(lostProfileData)
                log[0] = p.lost
                log[1] = 1
                log[2] = lostPC
                q = 3
                p.lost = 0
        }
        p.nlog = q
        return true
}

// addNonGo is like add, but runs on a non-Go thread.
// It can't do anything that might need a g or an m.
// With this entry point, we don't try to flush the log when evicting an
// old entry. Instead, we just drop the stack trace if we're out of space.
//go:nosplit
//go:nowritebarrierrec
func (p *cpuProfile) addNonGo(pc []uintptr) {
        p.addWithFlushlog(pc, func() bool { return false })
}

// getprofile blocks until the next block of profiling data is available
// and returns it as a []byte. It is called from the writing goroutine.
func (p *cpuProfile) getprofile() []byte {
        if p == nil {
                return nil
        }

        if p.wholding {
                // Release previous log to signal handling side.
                // Loop because we are racing against SetCPUProfileRate(0).
                for {
                        n := p.handoff
                        if n == 0 {
                                print("runtime: phase error during cpu profile handoff\n")
                                return nil
                        }
                        if n&0x80000000 != 0 {
                                p.wtoggle = 1 - p.wtoggle
                                p.wholding = false
                                p.flushing = true
                                goto Flush
                        }
                        if atomic.Cas(&p.handoff, n, 0) {
                                break
                        }
                }
                p.wtoggle = 1 - p.wtoggle
                p.wholding = false
        }

        if p.flushing {
                goto Flush
        }

        if !p.on && p.handoff == 0 {
                return nil
        }

        // Wait for new log.
        notetsleepg(&p.wait, -1)
        noteclear(&p.wait)

        switch n := p.handoff; {
        case n == 0:
                print("runtime: phase error during cpu profile wait\n")
                return nil
        case n == 0x80000000:
                p.flushing = true
                goto Flush
        default:
                n &^= 0x80000000

                // Return new log to caller.
                p.wholding = true

                return uintptrBytes(p.log[p.wtoggle][:n])
        }

        // In flush mode.
        // Add is no longer being called. We own the log.
        // Also, p.handoff is non-zero, so flushlog will return false.
        // Evict the hash table into the log and return it.
Flush:
        for i := range p.hash {
                b := &p.hash[i]
                for j := range b.entry {
                        e := &b.entry[j]
                        if e.count > 0 && !p.evict(e, p.flushlog) {
                                // Filled the log. Stop the loop and return what we've got.
                                break Flush
                        }
                }
        }

        // Return pending log data.
        if p.nlog > 0 {
                // Note that we're using toggle now, not wtoggle,
                // because we're working on the log directly.
                n := p.nlog
                p.nlog = 0
                return uintptrBytes(p.log[p.toggle][:n])
        }

        // Made it through the table without finding anything to log.
        if !p.eodSent {
                // We may not have space to append this to the partial log buf,
                // so we always return a new slice for the end-of-data marker.
                p.eodSent = true
                return uintptrBytes(eod[:])
        }

        // Finally done. Clean up and return nil.
        p.flushing = false
        if !atomic.Cas(&p.handoff, p.handoff, 0) {
                print("runtime: profile flush racing with something\n")
        }
        return nil
}

func uintptrBytes(p []uintptr) (ret []byte) {
        pp := (*slice)(unsafe.Pointer(&p))
        rp := (*slice)(unsafe.Pointer(&ret))

        rp.array = pp.array
        rp.len = pp.len * int(unsafe.Sizeof(p[0]))
        rp.cap = rp.len

        return
}

// CPUProfile returns the next chunk of binary CPU profiling stack trace data,
// blocking until data is available. If profiling is turned off and all the profile
// data accumulated while it was on has been returned, CPUProfile returns nil.
// The caller must save the returned data before calling CPUProfile again.
//
// Most clients should use the runtime/pprof package or
// the testing package's -test.cpuprofile flag instead of calling
// CPUProfile directly.
func CPUProfile() []byte {
        return cpuprof.getprofile()
}

//go:linkname runtime_pprof_runtime_cyclesPerSecond runtime/pprof.runtime_cyclesPerSecond
func runtime_pprof_runtime_cyclesPerSecond() int64 {
        return tickspersecond()
}