1 // Copyright 2019 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
7 // The page allocator manages mapped pages (defined by pageSize, NOT
8 // physPageSize) for allocation and re-use. It is embedded into mheap.
10 // Pages are managed using a bitmap that is sharded into chunks.
11 // In the bitmap, 1 means in-use, and 0 means free. The bitmap spans the
12 // process's address space. Chunks are managed in a sparse-array-style structure
13 // similar to mheap.arenas, since the bitmap may be large on some systems.
15 // The bitmap is efficiently searched by using a radix tree in combination
16 // with fast bit-wise intrinsics. Allocation is performed using an address-ordered
17 // first-fit approach.
19 // Each entry in the radix tree is a summary that describes three properties of
20 // a particular region of the address space: the number of contiguous free pages
21 // at the start and end of the region it represents, and the maximum number of
22 // contiguous free pages found anywhere in that region.
24 // Each level of the radix tree is stored as one contiguous array, which represents
25 // a different granularity of subdivision of the processes' address space. Thus, this
26 // radix tree is actually implicit in these large arrays, as opposed to having explicit
27 // dynamically-allocated pointer-based node structures. Naturally, these arrays may be
28 // quite large for system with large address spaces, so in these cases they are mapped
29 // into memory as needed. The leaf summaries of the tree correspond to a bitmap chunk.
31 // The root level (referred to as L0 and index 0 in pageAlloc.summary) has each
32 // summary represent the largest section of address space (16 GiB on 64-bit systems),
33 // with each subsequent level representing successively smaller subsections until we
34 // reach the finest granularity at the leaves, a chunk.
36 // More specifically, each summary in each level (except for leaf summaries)
37 // represents some number of entries in the following level. For example, each
38 // summary in the root level may represent a 16 GiB region of address space,
39 // and in the next level there could be 8 corresponding entries which represent 2
40 // GiB subsections of that 16 GiB region, each of which could correspond to 8
41 // entries in the next level which each represent 256 MiB regions, and so on.
43 // Thus, this design only scales to heaps so large, but can always be extended to
44 // larger heaps by simply adding levels to the radix tree, which mostly costs
45 // additional virtual address space. The choice of managing large arrays also means
46 // that a large amount of virtual address space may be reserved by the runtime.
51 "runtime/internal/atomic"
56 // The size of a bitmap chunk, i.e. the amount of bits (that is, pages) to consider
57 // in the bitmap at once.
58 pallocChunkPages = 1 << logPallocChunkPages
59 pallocChunkBytes = pallocChunkPages * pageSize
60 logPallocChunkPages = 9
61 logPallocChunkBytes = logPallocChunkPages + pageShift
63 // The number of radix bits for each level.
65 // The value of 3 is chosen such that the block of summaries we need to scan at
66 // each level fits in 64 bytes (2^3 summaries * 8 bytes per summary), which is
67 // close to the L1 cache line width on many systems. Also, a value of 3 fits 4 tree
68 // levels perfectly into the 21-bit pallocBits summary field at the root level.
70 // The following equation explains how each of the constants relate:
71 // summaryL0Bits + (summaryLevels-1)*summaryLevelBits + logPallocChunkBytes = heapAddrBits
73 // summaryLevels is an architecture-dependent value defined in mpagealloc_*.go.
75 summaryL0Bits = heapAddrBits - logPallocChunkBytes - (summaryLevels-1)*summaryLevelBits
77 // pallocChunksL2Bits is the number of bits of the chunk index number
78 // covered by the second level of the chunks map.
80 // See (*pageAlloc).chunks for more details. Update the documentation
81 // there should this change.
82 pallocChunksL2Bits = heapAddrBits - logPallocChunkBytes - pallocChunksL1Bits
83 pallocChunksL1Shift = pallocChunksL2Bits
86 // maxSearchAddr returns the maximum searchAddr value, which indicates
87 // that the heap has no free space.
89 // This function exists just to make it clear that this is the maximum address
90 // for the page allocator's search space. See maxOffAddr for details.
92 // It's a function (rather than a variable) because it needs to be
93 // usable before package runtime's dynamic initialization is complete.
94 // See #51913 for details.
95 func maxSearchAddr() offAddr { return maxOffAddr }
97 // Global chunk index.
99 // Represents an index into the leaf level of the radix tree.
100 // Similar to arenaIndex, except instead of arenas, it divides the address
101 // space into chunks.
104 // chunkIndex returns the global index of the palloc chunk containing the
106 func chunkIndex(p uintptr) chunkIdx {
107 return chunkIdx((p - arenaBaseOffset) / pallocChunkBytes)
110 // chunkBase returns the base address of the palloc chunk at index ci.
111 func chunkBase(ci chunkIdx) uintptr {
112 return uintptr(ci)*pallocChunkBytes + arenaBaseOffset
115 // chunkPageIndex computes the index of the page that contains p,
116 // relative to the chunk which contains p.
117 func chunkPageIndex(p uintptr) uint {
118 return uint(p % pallocChunkBytes / pageSize)
121 // l1 returns the index into the first level of (*pageAlloc).chunks.
122 func (i chunkIdx) l1() uint {
123 if pallocChunksL1Bits == 0 {
124 // Let the compiler optimize this away if there's no
128 return uint(i) >> pallocChunksL1Shift
132 // l2 returns the index into the second level of (*pageAlloc).chunks.
133 func (i chunkIdx) l2() uint {
134 if pallocChunksL1Bits == 0 {
137 return uint(i) & (1<<pallocChunksL2Bits - 1)
141 // offAddrToLevelIndex converts an address in the offset address space
142 // to the index into summary[level] containing addr.
143 func offAddrToLevelIndex(level int, addr offAddr) int {
144 return int((addr.a - arenaBaseOffset) >> levelShift[level])
147 // levelIndexToOffAddr converts an index into summary[level] into
148 // the corresponding address in the offset address space.
149 func levelIndexToOffAddr(level, idx int) offAddr {
150 return offAddr{(uintptr(idx) << levelShift[level]) + arenaBaseOffset}
153 // addrsToSummaryRange converts base and limit pointers into a range
154 // of entries for the given summary level.
156 // The returned range is inclusive on the lower bound and exclusive on
158 func addrsToSummaryRange(level int, base, limit uintptr) (lo int, hi int) {
159 // This is slightly more nuanced than just a shift for the exclusive
160 // upper-bound. Note that the exclusive upper bound may be within a
161 // summary at this level, meaning if we just do the obvious computation
162 // hi will end up being an inclusive upper bound. Unfortunately, just
163 // adding 1 to that is too broad since we might be on the very edge
164 // of a summary's max page count boundary for this level
165 // (1 << levelLogPages[level]). So, make limit an inclusive upper bound
166 // then shift, then add 1, so we get an exclusive upper bound at the end.
167 lo = int((base - arenaBaseOffset) >> levelShift[level])
168 hi = int(((limit-1)-arenaBaseOffset)>>levelShift[level]) + 1
172 // blockAlignSummaryRange aligns indices into the given level to that
173 // level's block width (1 << levelBits[level]). It assumes lo is inclusive
174 // and hi is exclusive, and so aligns them down and up respectively.
175 func blockAlignSummaryRange(level int, lo, hi int) (int, int) {
176 e := uintptr(1) << levelBits[level]
177 return int(alignDown(uintptr(lo), e)), int(alignUp(uintptr(hi), e))
180 type pageAlloc struct {
181 // Radix tree of summaries.
183 // Each slice's cap represents the whole memory reservation.
184 // Each slice's len reflects the allocator's maximum known
185 // mapped heap address for that level.
187 // The backing store of each summary level is reserved in init
188 // and may or may not be committed in grow (small address spaces
189 // may commit all the memory in init).
191 // The purpose of keeping len <= cap is to enforce bounds checks
192 // on the top end of the slice so that instead of an unknown
193 // runtime segmentation fault, we get a much friendlier out-of-bounds
196 // To iterate over a summary level, use inUse to determine which ranges
197 // are currently available. Otherwise one might try to access
198 // memory which is only Reserved which may result in a hard fault.
200 // We may still get segmentation faults < len since some of that
201 // memory may not be committed yet.
202 summary [summaryLevels][]pallocSum
204 // chunks is a slice of bitmap chunks.
206 // The total size of chunks is quite large on most 64-bit platforms
207 // (O(GiB) or more) if flattened, so rather than making one large mapping
208 // (which has problems on some platforms, even when PROT_NONE) we use a
209 // two-level sparse array approach similar to the arena index in mheap.
211 // To find the chunk containing a memory address `a`, do:
212 // chunkOf(chunkIndex(a))
214 // Below is a table describing the configuration for chunks for various
215 // heapAddrBits supported by the runtime.
217 // heapAddrBits | L1 Bits | L2 Bits | L2 Entry Size
218 // ------------------------------------------------
219 // 32 | 0 | 10 | 128 KiB
220 // 33 (iOS) | 0 | 11 | 256 KiB
221 // 48 | 13 | 13 | 1 MiB
223 // There's no reason to use the L1 part of chunks on 32-bit, the
224 // address space is small so the L2 is small. For platforms with a
225 // 48-bit address space, we pick the L1 such that the L2 is 1 MiB
226 // in size, which is a good balance between low granularity without
227 // making the impact on BSS too high (note the L1 is stored directly
230 // To iterate over the bitmap, use inUse to determine which ranges
231 // are currently available. Otherwise one might iterate over unused
234 // Protected by mheapLock.
236 // TODO(mknyszek): Consider changing the definition of the bitmap
237 // such that 1 means free and 0 means in-use so that summaries and
238 // the bitmaps align better on zero-values.
239 chunks [1 << pallocChunksL1Bits]*[1 << pallocChunksL2Bits]pallocData
241 // The address to start an allocation search with. It must never
242 // point to any memory that is not contained in inUse, i.e.
243 // inUse.contains(searchAddr.addr()) must always be true. The one
244 // exception to this rule is that it may take on the value of
245 // maxOffAddr to indicate that the heap is exhausted.
247 // We guarantee that all valid heap addresses below this value
248 // are allocated and not worth searching.
251 // start and end represent the chunk indices
252 // which pageAlloc knows about. It assumes
253 // chunks in the range [start, end) are
254 // currently ready to use.
257 // inUse is a slice of ranges of address space which are
258 // known by the page allocator to be currently in-use (passed
261 // We care much more about having a contiguous heap in these cases
262 // and take additional measures to ensure that, so in nearly all
263 // cases this should have just 1 element.
265 // All access is protected by the mheapLock.
268 // scav stores the scavenger state.
270 // index is an efficient index of chunks that have pages available to
274 // releasedBg is the amount of memory released in the background this
276 releasedBg atomic.Uintptr
278 // releasedEager is the amount of memory released eagerly this scavenge
280 releasedEager atomic.Uintptr
283 // mheap_.lock. This level of indirection makes it possible
284 // to test pageAlloc independently of the runtime allocator.
287 // sysStat is the runtime memstat to update when new system
288 // memory is committed by the pageAlloc for allocation metadata.
291 // summaryMappedReady is the number of bytes mapped in the Ready state
292 // in the summary structure. Used only for testing currently.
294 // Protected by mheapLock.
295 summaryMappedReady uintptr
297 // chunkHugePages indicates whether page bitmap chunks should be backed
301 // Whether or not this struct is being used in tests.
305 func (p *pageAlloc) init(mheapLock *mutex, sysStat *sysMemStat, test bool) {
306 if levelLogPages[0] > logMaxPackedValue {
307 // We can't represent 1<<levelLogPages[0] pages, the maximum number
308 // of pages we need to represent at the root level, in a summary, which
309 // is a big problem. Throw.
310 print("runtime: root level max pages = ", 1<<levelLogPages[0], "\n")
311 print("runtime: summary max pages = ", maxPackedValue, "\n")
312 throw("root level max pages doesn't fit in summary")
316 // Initialize p.inUse.
317 p.inUse.init(sysStat)
319 // System-dependent initialization.
322 // Start with the searchAddr in a state indicating there's no free memory.
323 p.searchAddr = maxSearchAddr()
325 // Set the mheapLock.
326 p.mheapLock = mheapLock
328 // Initialize the scavenge index.
329 p.summaryMappedReady += p.scav.index.init(test, sysStat)
331 // Set if we're in a test.
335 // tryChunkOf returns the bitmap data for the given chunk.
337 // Returns nil if the chunk data has not been mapped.
338 func (p *pageAlloc) tryChunkOf(ci chunkIdx) *pallocData {
339 l2 := p.chunks[ci.l1()]
346 // chunkOf returns the chunk at the given chunk index.
348 // The chunk index must be valid or this method may throw.
349 func (p *pageAlloc) chunkOf(ci chunkIdx) *pallocData {
350 return &p.chunks[ci.l1()][ci.l2()]
353 // grow sets up the metadata for the address range [base, base+size).
354 // It may allocate metadata, in which case *p.sysStat will be updated.
356 // p.mheapLock must be held.
357 func (p *pageAlloc) grow(base, size uintptr) {
358 assertLockHeld(p.mheapLock)
360 // Round up to chunks, since we can't deal with increments smaller
361 // than chunks. Also, sysGrow expects aligned values.
362 limit := alignUp(base+size, pallocChunkBytes)
363 base = alignDown(base, pallocChunkBytes)
365 // Grow the summary levels in a system-dependent manner.
366 // We just update a bunch of additional metadata here.
367 p.sysGrow(base, limit)
369 // Grow the scavenge index.
370 p.summaryMappedReady += p.scav.index.grow(base, limit, p.sysStat)
372 // Update p.start and p.end.
373 // If no growth happened yet, start == 0. This is generally
374 // safe since the zero page is unmapped.
375 firstGrowth := p.start == 0
376 start, end := chunkIndex(base), chunkIndex(limit)
377 if firstGrowth || start < p.start {
383 // Note that [base, limit) will never overlap with any existing
384 // range inUse because grow only ever adds never-used memory
385 // regions to the page allocator.
386 p.inUse.add(makeAddrRange(base, limit))
388 // A grow operation is a lot like a free operation, so if our
389 // chunk ends up below p.searchAddr, update p.searchAddr to the
390 // new address, just like in free.
391 if b := (offAddr{base}); b.lessThan(p.searchAddr) {
395 // Add entries into chunks, which is sparse, if needed. Then,
396 // initialize the bitmap.
398 // Newly-grown memory is always considered scavenged.
399 // Set all the bits in the scavenged bitmaps high.
400 for c := chunkIndex(base); c < chunkIndex(limit); c++ {
401 if p.chunks[c.l1()] == nil {
402 // Create the necessary l2 entry.
403 const l2Size = unsafe.Sizeof(*p.chunks[0])
404 r := sysAlloc(l2Size, p.sysStat)
406 throw("pageAlloc: out of memory")
409 // Make the chunk mapping eligible or ineligible
410 // for huge pages, depending on what our current
412 if p.chunkHugePages {
413 sysHugePage(r, l2Size)
415 sysNoHugePage(r, l2Size)
418 // Store the new chunk block but avoid a write barrier.
419 // grow is used in call chains that disallow write barriers.
420 *(*uintptr)(unsafe.Pointer(&p.chunks[c.l1()])) = uintptr(r)
422 p.chunkOf(c).scavenged.setRange(0, pallocChunkPages)
425 // Update summaries accordingly. The grow acts like a free, so
426 // we need to ensure this newly-free memory is visible in the
428 p.update(base, size/pageSize, true, false)
431 // enableChunkHugePages enables huge pages for the chunk bitmap mappings (disabled by default).
433 // This function is idempotent.
435 // A note on latency: for sufficiently small heaps (<10s of GiB) this function will take constant
436 // time, but may take time proportional to the size of the mapped heap beyond that.
438 // The heap lock must not be held over this operation, since it will briefly acquire
440 func (p *pageAlloc) enableChunkHugePages() {
441 // Grab the heap lock to turn on huge pages for new chunks and clone the current
442 // heap address space ranges.
444 // After the lock is released, we can be sure that bitmaps for any new chunks may
445 // be backed with huge pages, and we have the address space for the rest of the
446 // chunks. At the end of this function, all chunk metadata should be backed by huge
449 if p.chunkHugePages {
453 p.chunkHugePages = true
455 inUse.sysStat = p.sysStat
456 p.inUse.cloneInto(&inUse)
459 // This might seem like a lot of work, but all these loops are for generality.
461 // For a 1 GiB contiguous heap, a 48-bit address space, 13 L1 bits, a palloc chunk size
462 // of 4 MiB, and adherence to the default set of heap address hints, this will result in
463 // exactly 1 call to sysHugePage.
464 for _, r := range p.inUse.ranges {
465 for i := chunkIndex(r.base.addr()).l1(); i < chunkIndex(r.limit.addr()-1).l1(); i++ {
466 // N.B. We can assume that p.chunks[i] is non-nil and in a mapped part of p.chunks
467 // because it's derived from inUse, which never shrinks.
468 sysHugePage(unsafe.Pointer(p.chunks[i]), unsafe.Sizeof(*p.chunks[0]))
473 // update updates heap metadata. It must be called each time the bitmap
476 // If contig is true, update does some optimizations assuming that there was
477 // a contiguous allocation or free between addr and addr+npages. alloc indicates
478 // whether the operation performed was an allocation or a free.
480 // p.mheapLock must be held.
481 func (p *pageAlloc) update(base, npages uintptr, contig, alloc bool) {
482 assertLockHeld(p.mheapLock)
484 // base, limit, start, and end are inclusive.
485 limit := base + npages*pageSize - 1
486 sc, ec := chunkIndex(base), chunkIndex(limit)
488 // Handle updating the lowest level first.
490 // Fast path: the allocation doesn't span more than one chunk,
491 // so update this one and if the summary didn't change, return.
492 x := p.summary[len(p.summary)-1][sc]
493 y := p.chunkOf(sc).summarize()
497 p.summary[len(p.summary)-1][sc] = y
499 // Slow contiguous path: the allocation spans more than one chunk
500 // and at least one summary is guaranteed to change.
501 summary := p.summary[len(p.summary)-1]
503 // Update the summary for chunk sc.
504 summary[sc] = p.chunkOf(sc).summarize()
506 // Update the summaries for chunks in between, which are
507 // either totally allocated or freed.
508 whole := p.summary[len(p.summary)-1][sc+1 : ec]
510 // Should optimize into a memclr.
511 for i := range whole {
515 for i := range whole {
516 whole[i] = freeChunkSum
520 // Update the summary for chunk ec.
521 summary[ec] = p.chunkOf(ec).summarize()
523 // Slow general path: the allocation spans more than one chunk
524 // and at least one summary is guaranteed to change.
526 // We can't assume a contiguous allocation happened, so walk over
527 // every chunk in the range and manually recompute the summary.
528 summary := p.summary[len(p.summary)-1]
529 for c := sc; c <= ec; c++ {
530 summary[c] = p.chunkOf(c).summarize()
534 // Walk up the radix tree and update the summaries appropriately.
536 for l := len(p.summary) - 2; l >= 0 && changed; l-- {
537 // Update summaries at level l from summaries at level l+1.
540 // "Constants" for the previous level which we
541 // need to compute the summary from that level.
542 logEntriesPerBlock := levelBits[l+1]
543 logMaxPages := levelLogPages[l+1]
545 // lo and hi describe all the parts of the level we need to look at.
546 lo, hi := addrsToSummaryRange(l, base, limit+1)
548 // Iterate over each block, updating the corresponding summary in the less-granular level.
549 for i := lo; i < hi; i++ {
550 children := p.summary[l+1][i<<logEntriesPerBlock : (i+1)<<logEntriesPerBlock]
551 sum := mergeSummaries(children, logMaxPages)
552 old := p.summary[l][i]
555 p.summary[l][i] = sum
561 // allocRange marks the range of memory [base, base+npages*pageSize) as
562 // allocated. It also updates the summaries to reflect the newly-updated
565 // Returns the amount of scavenged memory in bytes present in the
568 // p.mheapLock must be held.
569 func (p *pageAlloc) allocRange(base, npages uintptr) uintptr {
570 assertLockHeld(p.mheapLock)
572 limit := base + npages*pageSize - 1
573 sc, ec := chunkIndex(base), chunkIndex(limit)
574 si, ei := chunkPageIndex(base), chunkPageIndex(limit)
578 // The range doesn't cross any chunk boundaries.
579 chunk := p.chunkOf(sc)
580 scav += chunk.scavenged.popcntRange(si, ei+1-si)
581 chunk.allocRange(si, ei+1-si)
582 p.scav.index.alloc(sc, ei+1-si)
584 // The range crosses at least one chunk boundary.
585 chunk := p.chunkOf(sc)
586 scav += chunk.scavenged.popcntRange(si, pallocChunkPages-si)
587 chunk.allocRange(si, pallocChunkPages-si)
588 p.scav.index.alloc(sc, pallocChunkPages-si)
589 for c := sc + 1; c < ec; c++ {
590 chunk := p.chunkOf(c)
591 scav += chunk.scavenged.popcntRange(0, pallocChunkPages)
593 p.scav.index.alloc(c, pallocChunkPages)
595 chunk = p.chunkOf(ec)
596 scav += chunk.scavenged.popcntRange(0, ei+1)
597 chunk.allocRange(0, ei+1)
598 p.scav.index.alloc(ec, ei+1)
600 p.update(base, npages, true, true)
601 return uintptr(scav) * pageSize
604 // findMappedAddr returns the smallest mapped offAddr that is
605 // >= addr. That is, if addr refers to mapped memory, then it is
606 // returned. If addr is higher than any mapped region, then
607 // it returns maxOffAddr.
609 // p.mheapLock must be held.
610 func (p *pageAlloc) findMappedAddr(addr offAddr) offAddr {
611 assertLockHeld(p.mheapLock)
613 // If we're not in a test, validate first by checking mheap_.arenas.
614 // This is a fast path which is only safe to use outside of testing.
615 ai := arenaIndex(addr.addr())
616 if p.test || mheap_.arenas[ai.l1()] == nil || mheap_.arenas[ai.l1()][ai.l2()] == nil {
617 vAddr, ok := p.inUse.findAddrGreaterEqual(addr.addr())
619 return offAddr{vAddr}
621 // The candidate search address is greater than any
622 // known address, which means we definitely have no
630 // find searches for the first (address-ordered) contiguous free region of
631 // npages in size and returns a base address for that region.
633 // It uses p.searchAddr to prune its search and assumes that no palloc chunks
634 // below chunkIndex(p.searchAddr) contain any free memory at all.
636 // find also computes and returns a candidate p.searchAddr, which may or
637 // may not prune more of the address space than p.searchAddr already does.
638 // This candidate is always a valid p.searchAddr.
640 // find represents the slow path and the full radix tree search.
642 // Returns a base address of 0 on failure, in which case the candidate
643 // searchAddr returned is invalid and must be ignored.
645 // p.mheapLock must be held.
646 func (p *pageAlloc) find(npages uintptr) (uintptr, offAddr) {
647 assertLockHeld(p.mheapLock)
651 // This algorithm walks each level l of the radix tree from the root level
652 // to the leaf level. It iterates over at most 1 << levelBits[l] of entries
653 // in a given level in the radix tree, and uses the summary information to
655 // 1) That a given subtree contains a large enough contiguous region, at
656 // which point it continues iterating on the next level, or
657 // 2) That there are enough contiguous boundary-crossing bits to satisfy
658 // the allocation, at which point it knows exactly where to start
661 // i tracks the index into the current level l's structure for the
662 // contiguous 1 << levelBits[l] entries we're actually interested in.
664 // NOTE: Technically this search could allocate a region which crosses
665 // the arenaBaseOffset boundary, which when arenaBaseOffset != 0, is
666 // a discontinuity. However, the only way this could happen is if the
667 // page at the zero address is mapped, and this is impossible on
668 // every system we support where arenaBaseOffset != 0. So, the
669 // discontinuity is already encoded in the fact that the OS will never
670 // map the zero page for us, and this function doesn't try to handle
671 // this case in any way.
673 // i is the beginning of the block of entries we're searching at the
677 // firstFree is the region of address space that we are certain to
678 // find the first free page in the heap. base and bound are the inclusive
679 // bounds of this window, and both are addresses in the linearized, contiguous
680 // view of the address space (with arenaBaseOffset pre-added). At each level,
681 // this window is narrowed as we find the memory region containing the
682 // first free page of memory. To begin with, the range reflects the
683 // full process address space.
685 // firstFree is updated by calling foundFree each time free space in the
686 // heap is discovered.
688 // At the end of the search, base.addr() is the best new
689 // searchAddr we could deduce in this search.
690 firstFree := struct {
696 // foundFree takes the given address range [addr, addr+size) and
697 // updates firstFree if it is a narrower range. The input range must
698 // either be fully contained within firstFree or not overlap with it
701 // This way, we'll record the first summary we find with any free
702 // pages on the root level and narrow that down if we descend into
703 // that summary. But as soon as we need to iterate beyond that summary
704 // in a level to find a large enough range, we'll stop narrowing.
705 foundFree := func(addr offAddr, size uintptr) {
706 if firstFree.base.lessEqual(addr) && addr.add(size-1).lessEqual(firstFree.bound) {
707 // This range fits within the current firstFree window, so narrow
708 // down the firstFree window to the base and bound of this range.
709 firstFree.base = addr
710 firstFree.bound = addr.add(size - 1)
711 } else if !(addr.add(size-1).lessThan(firstFree.base) || firstFree.bound.lessThan(addr)) {
712 // This range only partially overlaps with the firstFree range,
714 print("runtime: addr = ", hex(addr.addr()), ", size = ", size, "\n")
715 print("runtime: base = ", hex(firstFree.base.addr()), ", bound = ", hex(firstFree.bound.addr()), "\n")
716 throw("range partially overlaps")
720 // lastSum is the summary which we saw on the previous level that made us
721 // move on to the next level. Used to print additional information in the
722 // case of a catastrophic failure.
723 // lastSumIdx is that summary's index in the previous level.
724 lastSum := packPallocSum(0, 0, 0)
728 for l := 0; l < len(p.summary); l++ {
729 // For the root level, entriesPerBlock is the whole level.
730 entriesPerBlock := 1 << levelBits[l]
731 logMaxPages := levelLogPages[l]
733 // We've moved into a new level, so let's update i to our new
734 // starting index. This is a no-op for level 0.
737 // Slice out the block of entries we care about.
738 entries := p.summary[l][i : i+entriesPerBlock]
740 // Determine j0, the first index we should start iterating from.
741 // The searchAddr may help us eliminate iterations if we followed the
742 // searchAddr on the previous level or we're on the root level, in which
743 // case the searchAddr should be the same as i after levelShift.
745 if searchIdx := offAddrToLevelIndex(l, p.searchAddr); searchIdx&^(entriesPerBlock-1) == i {
746 j0 = searchIdx & (entriesPerBlock - 1)
749 // Run over the level entries looking for
750 // a contiguous run of at least npages either
751 // within an entry or across entries.
753 // base contains the page index (relative to
754 // the first entry's first page) of the currently
755 // considered run of consecutive pages.
757 // size contains the size of the currently considered
758 // run of consecutive pages.
760 for j := j0; j < len(entries); j++ {
763 // A full entry means we broke any streak and
764 // that we should skip it altogether.
769 // We've encountered a non-zero summary which means
770 // free memory, so update firstFree.
771 foundFree(levelIndexToOffAddr(l, i+j), (uintptr(1)<<logMaxPages)*pageSize)
774 if size+s >= uint(npages) {
775 // If size == 0 we don't have a run yet,
776 // which means base isn't valid. So, set
777 // base to the first page in this block.
779 base = uint(j) << logMaxPages
781 // We hit npages; we're done!
785 if sum.max() >= uint(npages) {
786 // The entry itself contains npages contiguous
787 // free pages, so continue on the next level
794 if size == 0 || s < 1<<logMaxPages {
795 // We either don't have a current run started, or this entry
796 // isn't totally free (meaning we can't continue the current
797 // one), so try to begin a new run by setting size and base
800 base = uint(j+1)<<logMaxPages - size
803 // The entry is completely free, so continue the run.
804 size += 1 << logMaxPages
806 if size >= uint(npages) {
807 // We found a sufficiently large run of free pages straddling
808 // some boundary, so compute the address and return it.
809 addr := levelIndexToOffAddr(l, i).add(uintptr(base) * pageSize).addr()
810 return addr, p.findMappedAddr(firstFree.base)
813 // We're at level zero, so that means we've exhausted our search.
814 return 0, maxSearchAddr()
817 // We're not at level zero, and we exhausted the level we were looking in.
818 // This means that either our calculations were wrong or the level above
819 // lied to us. In either case, dump some useful state and throw.
820 print("runtime: summary[", l-1, "][", lastSumIdx, "] = ", lastSum.start(), ", ", lastSum.max(), ", ", lastSum.end(), "\n")
821 print("runtime: level = ", l, ", npages = ", npages, ", j0 = ", j0, "\n")
822 print("runtime: p.searchAddr = ", hex(p.searchAddr.addr()), ", i = ", i, "\n")
823 print("runtime: levelShift[level] = ", levelShift[l], ", levelBits[level] = ", levelBits[l], "\n")
824 for j := 0; j < len(entries); j++ {
826 print("runtime: summary[", l, "][", i+j, "] = (", sum.start(), ", ", sum.max(), ", ", sum.end(), ")\n")
828 throw("bad summary data")
831 // Since we've gotten to this point, that means we haven't found a
832 // sufficiently-sized free region straddling some boundary (chunk or larger).
833 // This means the last summary we inspected must have had a large enough "max"
834 // value, so look inside the chunk to find a suitable run.
836 // After iterating over all levels, i must contain a chunk index which
837 // is what the final level represents.
839 j, searchIdx := p.chunkOf(ci).find(npages, 0)
841 // We couldn't find any space in this chunk despite the summaries telling
842 // us it should be there. There's likely a bug, so dump some state and throw.
843 sum := p.summary[len(p.summary)-1][i]
844 print("runtime: summary[", len(p.summary)-1, "][", i, "] = (", sum.start(), ", ", sum.max(), ", ", sum.end(), ")\n")
845 print("runtime: npages = ", npages, "\n")
846 throw("bad summary data")
849 // Compute the address at which the free space starts.
850 addr := chunkBase(ci) + uintptr(j)*pageSize
852 // Since we actually searched the chunk, we may have
853 // found an even narrower free window.
854 searchAddr := chunkBase(ci) + uintptr(searchIdx)*pageSize
855 foundFree(offAddr{searchAddr}, chunkBase(ci+1)-searchAddr)
856 return addr, p.findMappedAddr(firstFree.base)
859 // alloc allocates npages worth of memory from the page heap, returning the base
860 // address for the allocation and the amount of scavenged memory in bytes
861 // contained in the region [base address, base address + npages*pageSize).
863 // Returns a 0 base address on failure, in which case other returned values
864 // should be ignored.
866 // p.mheapLock must be held.
868 // Must run on the system stack because p.mheapLock must be held.
871 func (p *pageAlloc) alloc(npages uintptr) (addr uintptr, scav uintptr) {
872 assertLockHeld(p.mheapLock)
874 // If the searchAddr refers to a region which has a higher address than
875 // any known chunk, then we know we're out of memory.
876 if chunkIndex(p.searchAddr.addr()) >= p.end {
880 // If npages has a chance of fitting in the chunk where the searchAddr is,
881 // search it directly.
882 searchAddr := minOffAddr
883 if pallocChunkPages-chunkPageIndex(p.searchAddr.addr()) >= uint(npages) {
884 // npages is guaranteed to be no greater than pallocChunkPages here.
885 i := chunkIndex(p.searchAddr.addr())
886 if max := p.summary[len(p.summary)-1][i].max(); max >= uint(npages) {
887 j, searchIdx := p.chunkOf(i).find(npages, chunkPageIndex(p.searchAddr.addr()))
889 print("runtime: max = ", max, ", npages = ", npages, "\n")
890 print("runtime: searchIdx = ", chunkPageIndex(p.searchAddr.addr()), ", p.searchAddr = ", hex(p.searchAddr.addr()), "\n")
891 throw("bad summary data")
893 addr = chunkBase(i) + uintptr(j)*pageSize
894 searchAddr = offAddr{chunkBase(i) + uintptr(searchIdx)*pageSize}
898 // We failed to use a searchAddr for one reason or another, so try
900 addr, searchAddr = p.find(npages)
903 // We failed to find a single free page, the smallest unit
904 // of allocation. This means we know the heap is completely
905 // exhausted. Otherwise, the heap still might have free
906 // space in it, just not enough contiguous space to
907 // accommodate npages.
908 p.searchAddr = maxSearchAddr()
913 // Go ahead and actually mark the bits now that we have an address.
914 scav = p.allocRange(addr, npages)
916 // If we found a higher searchAddr, we know that all the
917 // heap memory before that searchAddr in an offset address space is
918 // allocated, so bump p.searchAddr up to the new one.
919 if p.searchAddr.lessThan(searchAddr) {
920 p.searchAddr = searchAddr
925 // free returns npages worth of memory starting at base back to the page heap.
927 // p.mheapLock must be held.
929 // Must run on the system stack because p.mheapLock must be held.
932 func (p *pageAlloc) free(base, npages uintptr) {
933 assertLockHeld(p.mheapLock)
935 // If we're freeing pages below the p.searchAddr, update searchAddr.
936 if b := (offAddr{base}); b.lessThan(p.searchAddr) {
939 limit := base + npages*pageSize - 1
941 // Fast path: we're clearing a single bit, and we know exactly
942 // where it is, so mark it directly.
943 i := chunkIndex(base)
944 pi := chunkPageIndex(base)
945 p.chunkOf(i).free1(pi)
946 p.scav.index.free(i, pi, 1)
948 // Slow path: we're clearing more bits so we may need to iterate.
949 sc, ec := chunkIndex(base), chunkIndex(limit)
950 si, ei := chunkPageIndex(base), chunkPageIndex(limit)
953 // The range doesn't cross any chunk boundaries.
954 p.chunkOf(sc).free(si, ei+1-si)
955 p.scav.index.free(sc, si, ei+1-si)
957 // The range crosses at least one chunk boundary.
958 p.chunkOf(sc).free(si, pallocChunkPages-si)
959 p.scav.index.free(sc, si, pallocChunkPages-si)
960 for c := sc + 1; c < ec; c++ {
961 p.chunkOf(c).freeAll()
962 p.scav.index.free(c, 0, pallocChunkPages)
964 p.chunkOf(ec).free(0, ei+1)
965 p.scav.index.free(ec, 0, ei+1)
968 p.update(base, npages, true, false)
972 pallocSumBytes = unsafe.Sizeof(pallocSum(0))
974 // maxPackedValue is the maximum value that any of the three fields in
975 // the pallocSum may take on.
976 maxPackedValue = 1 << logMaxPackedValue
977 logMaxPackedValue = logPallocChunkPages + (summaryLevels-1)*summaryLevelBits
979 freeChunkSum = pallocSum(uint64(pallocChunkPages) |
980 uint64(pallocChunkPages<<logMaxPackedValue) |
981 uint64(pallocChunkPages<<(2*logMaxPackedValue)))
984 // pallocSum is a packed summary type which packs three numbers: start, max,
985 // and end into a single 8-byte value. Each of these values are a summary of
986 // a bitmap and are thus counts, each of which may have a maximum value of
987 // 2^21 - 1, or all three may be equal to 2^21. The latter case is represented
988 // by just setting the 64th bit.
989 type pallocSum uint64
991 // packPallocSum takes a start, max, and end value and produces a pallocSum.
992 func packPallocSum(start, max, end uint) pallocSum {
993 if max == maxPackedValue {
994 return pallocSum(uint64(1 << 63))
996 return pallocSum((uint64(start) & (maxPackedValue - 1)) |
997 ((uint64(max) & (maxPackedValue - 1)) << logMaxPackedValue) |
998 ((uint64(end) & (maxPackedValue - 1)) << (2 * logMaxPackedValue)))
1001 // start extracts the start value from a packed sum.
1002 func (p pallocSum) start() uint {
1003 if uint64(p)&uint64(1<<63) != 0 {
1004 return maxPackedValue
1006 return uint(uint64(p) & (maxPackedValue - 1))
1009 // max extracts the max value from a packed sum.
1010 func (p pallocSum) max() uint {
1011 if uint64(p)&uint64(1<<63) != 0 {
1012 return maxPackedValue
1014 return uint((uint64(p) >> logMaxPackedValue) & (maxPackedValue - 1))
1017 // end extracts the end value from a packed sum.
1018 func (p pallocSum) end() uint {
1019 if uint64(p)&uint64(1<<63) != 0 {
1020 return maxPackedValue
1022 return uint((uint64(p) >> (2 * logMaxPackedValue)) & (maxPackedValue - 1))
1025 // unpack unpacks all three values from the summary.
1026 func (p pallocSum) unpack() (uint, uint, uint) {
1027 if uint64(p)&uint64(1<<63) != 0 {
1028 return maxPackedValue, maxPackedValue, maxPackedValue
1030 return uint(uint64(p) & (maxPackedValue - 1)),
1031 uint((uint64(p) >> logMaxPackedValue) & (maxPackedValue - 1)),
1032 uint((uint64(p) >> (2 * logMaxPackedValue)) & (maxPackedValue - 1))
1035 // mergeSummaries merges consecutive summaries which may each represent at
1036 // most 1 << logMaxPagesPerSum pages each together into one.
1037 func mergeSummaries(sums []pallocSum, logMaxPagesPerSum uint) pallocSum {
1038 // Merge the summaries in sums into one.
1040 // We do this by keeping a running summary representing the merged
1041 // summaries of sums[:i] in start, most, and end.
1042 start, most, end := sums[0].unpack()
1043 for i := 1; i < len(sums); i++ {
1044 // Merge in sums[i].
1045 si, mi, ei := sums[i].unpack()
1047 // Merge in sums[i].start only if the running summary is
1048 // completely free, otherwise this summary's start
1049 // plays no role in the combined sum.
1050 if start == uint(i)<<logMaxPagesPerSum {
1054 // Recompute the max value of the running sum by looking
1055 // across the boundary between the running sum and sums[i]
1056 // and at the max sums[i], taking the greatest of those two
1057 // and the max of the running sum.
1058 most = max(most, end+si, mi)
1060 // Merge in end by checking if this new summary is totally
1061 // free. If it is, then we want to extend the running sum's
1062 // end by the new summary. If not, then we have some alloc'd
1063 // pages in there and we just want to take the end value in
1065 if ei == 1<<logMaxPagesPerSum {
1066 end += 1 << logMaxPagesPerSum
1071 return packPallocSum(start, most, end)