mem_sbrk.go

     1  // Copyright 2023 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build plan9 || wasm
     6  
     7  package runtime
     8  
     9  import "unsafe"
    10  
    11  const isSbrkPlatform = true
    12  
    13  const memDebug = false
    14  
    15  // Memory management on sbrk systems (including the linear memory
    16  // on Wasm).
    17  
    18  // bloc is the runtime's sense of the break, which can go up or
    19  // down. blocMax is the system's break, also the high water mark
    20  // of bloc. The runtime uses memory up to bloc. The memory
    21  // between bloc and blocMax is allocated by the OS but not used
    22  // by the runtime.
    23  //
    24  // When the runtime needs to grow the heap address range, it
    25  // increases bloc. When it needs to grow beyond blocMax, it calls
    26  // the system sbrk to allocate more memory (and therefore
    27  // increase blocMax).
    28  //
    29  // When the runtime frees memory at the end of the address space,
    30  // it decreases bloc, but does not reduces the system break (as
    31  // the OS doesn't support it). When the runtime frees memory in
    32  // the middle of the address space, the memory goes to a free
    33  // list.
    34  
    35  var bloc uintptr    // The runtime's sense of break. Can go up or down.
    36  var blocMax uintptr // The break of the OS. Only increase.
    37  var memlock mutex
    38  
    39  type memHdr struct {
    40  	next memHdrPtr
    41  	size uintptr
    42  }
    43  
    44  var memFreelist memHdrPtr // sorted in ascending order
    45  
    46  type memHdrPtr uintptr
    47  
    48  func (p memHdrPtr) ptr() *memHdr   { return (*memHdr)(unsafe.Pointer(p)) }
    49  func (p *memHdrPtr) set(x *memHdr) { *p = memHdrPtr(unsafe.Pointer(x)) }
    50  
    51  func memAlloc(n uintptr) unsafe.Pointer {
    52  	if p := memAllocNoGrow(n); p != nil {
    53  		return p
    54  	}
    55  	return sbrk(n)
    56  }
    57  
    58  func memAllocNoGrow(n uintptr) unsafe.Pointer {
    59  	n = memRound(n)
    60  	var prevp *memHdr
    61  	for p := memFreelist.ptr(); p != nil; p = p.next.ptr() {
    62  		if p.size >= n {
    63  			if p.size == n {
    64  				if prevp != nil {
    65  					prevp.next = p.next
    66  				} else {
    67  					memFreelist = p.next
    68  				}
    69  			} else {
    70  				p.size -= n
    71  				p = (*memHdr)(add(unsafe.Pointer(p), p.size))
    72  			}
    73  			*p = memHdr{}
    74  			return unsafe.Pointer(p)
    75  		}
    76  		prevp = p
    77  	}
    78  	return nil
    79  }
    80  
    81  func memFree(ap unsafe.Pointer, n uintptr) {
    82  	n = memRound(n)
    83  	memclrNoHeapPointers(ap, n)
    84  	bp := (*memHdr)(ap)
    85  	bp.size = n
    86  	bpn := uintptr(ap)
    87  	if memFreelist == 0 {
    88  		bp.next = 0
    89  		memFreelist.set(bp)
    90  		return
    91  	}
    92  	p := memFreelist.ptr()
    93  	if bpn < uintptr(unsafe.Pointer(p)) {
    94  		memFreelist.set(bp)
    95  		if bpn+bp.size == uintptr(unsafe.Pointer(p)) {
    96  			bp.size += p.size
    97  			bp.next = p.next
    98  			*p = memHdr{}
    99  		} else {
   100  			bp.next.set(p)
   101  		}
   102  		return
   103  	}
   104  	for ; p.next != 0; p = p.next.ptr() {
   105  		if bpn > uintptr(unsafe.Pointer(p)) && bpn < uintptr(unsafe.Pointer(p.next)) {
   106  			break
   107  		}
   108  	}
   109  	if bpn+bp.size == uintptr(unsafe.Pointer(p.next)) {
   110  		bp.size += p.next.ptr().size
   111  		bp.next = p.next.ptr().next
   112  		*p.next.ptr() = memHdr{}
   113  	} else {
   114  		bp.next = p.next
   115  	}
   116  	if uintptr(unsafe.Pointer(p))+p.size == bpn {
   117  		p.size += bp.size
   118  		p.next = bp.next
   119  		*bp = memHdr{}
   120  	} else {
   121  		p.next.set(bp)
   122  	}
   123  }
   124  
   125  func memCheck() {
   126  	if !memDebug {
   127  		return
   128  	}
   129  	for p := memFreelist.ptr(); p != nil && p.next != 0; p = p.next.ptr() {
   130  		if uintptr(unsafe.Pointer(p)) == uintptr(unsafe.Pointer(p.next)) {
   131  			print("runtime: ", unsafe.Pointer(p), " == ", unsafe.Pointer(p.next), "\n")
   132  			throw("mem: infinite loop")
   133  		}
   134  		if uintptr(unsafe.Pointer(p)) > uintptr(unsafe.Pointer(p.next)) {
   135  			print("runtime: ", unsafe.Pointer(p), " > ", unsafe.Pointer(p.next), "\n")
   136  			throw("mem: unordered list")
   137  		}
   138  		if uintptr(unsafe.Pointer(p))+p.size > uintptr(unsafe.Pointer(p.next)) {
   139  			print("runtime: ", unsafe.Pointer(p), "+", p.size, " > ", unsafe.Pointer(p.next), "\n")
   140  			throw("mem: overlapping blocks")
   141  		}
   142  		for b := add(unsafe.Pointer(p), unsafe.Sizeof(memHdr{})); uintptr(b) < uintptr(unsafe.Pointer(p))+p.size; b = add(b, 1) {
   143  			if *(*byte)(b) != 0 {
   144  				print("runtime: value at addr ", b, " with offset ", uintptr(b)-uintptr(unsafe.Pointer(p)), " in block ", p, " of size ", p.size, " is not zero\n")
   145  				throw("mem: uninitialised memory")
   146  			}
   147  		}
   148  	}
   149  }
   150  
   151  func memRound(p uintptr) uintptr {
   152  	return alignUp(p, physPageSize)
   153  }
   154  
   155  func initBloc() {
   156  	bloc = memRound(firstmoduledata.end)
   157  	blocMax = bloc
   158  }
   159  
   160  func sysAllocOS(n uintptr) unsafe.Pointer {
   161  	lock(&memlock)
   162  	p := memAlloc(n)
   163  	memCheck()
   164  	unlock(&memlock)
   165  	return p
   166  }
   167  
   168  func sysFreeOS(v unsafe.Pointer, n uintptr) {
   169  	lock(&memlock)
   170  	if uintptr(v)+n == bloc {
   171  		// Address range being freed is at the end of memory,
   172  		// so record a new lower value for end of memory.
   173  		// Can't actually shrink address space because segment is shared.
   174  		memclrNoHeapPointers(v, n)
   175  		bloc -= n
   176  	} else {
   177  		memFree(v, n)
   178  		memCheck()
   179  	}
   180  	unlock(&memlock)
   181  }
   182  
   183  func sysUnusedOS(v unsafe.Pointer, n uintptr) {
   184  }
   185  
   186  func sysUsedOS(v unsafe.Pointer, n uintptr) {
   187  }
   188  
   189  func sysHugePageOS(v unsafe.Pointer, n uintptr) {
   190  }
   191  
   192  func sysNoHugePageOS(v unsafe.Pointer, n uintptr) {
   193  }
   194  
   195  func sysHugePageCollapseOS(v unsafe.Pointer, n uintptr) {
   196  }
   197  
   198  func sysMapOS(v unsafe.Pointer, n uintptr) {
   199  }
   200  
   201  func sysFaultOS(v unsafe.Pointer, n uintptr) {
   202  }
   203  
   204  func sysReserveOS(v unsafe.Pointer, n uintptr) unsafe.Pointer {
   205  	lock(&memlock)
   206  	var p unsafe.Pointer
   207  	if uintptr(v) == bloc {
   208  		// Address hint is the current end of memory,
   209  		// so try to extend the address space.
   210  		p = sbrk(n)
   211  	}
   212  	if p == nil && v == nil {
   213  		p = memAlloc(n)
   214  		memCheck()
   215  	}
   216  	unlock(&memlock)
   217  	return p
   218  }
   219  
   220  func sysReserveAlignedSbrk(size, align uintptr) (unsafe.Pointer, uintptr) {
   221  	lock(&memlock)
   222  	if p := memAllocNoGrow(size + align); p != nil {
   223  		// We can satisfy the reservation from the free list.
   224  		// Trim off the unaligned parts.
   225  		pAligned := alignUp(uintptr(p), align)
   226  		if startLen := pAligned - uintptr(p); startLen > 0 {
   227  			memFree(p, startLen)
   228  		}
   229  		end := pAligned + size
   230  		if endLen := (uintptr(p) + size + align) - end; endLen > 0 {
   231  			memFree(unsafe.Pointer(end), endLen)
   232  		}
   233  		memCheck()
   234  		return unsafe.Pointer(pAligned), size
   235  	}
   236  
   237  	// Round up bloc to align, then allocate size.
   238  	p := alignUp(bloc, align)
   239  	r := sbrk(p + size - bloc)
   240  	if r == nil {
   241  		p, size = 0, 0
   242  	} else if l := p - uintptr(r); l > 0 {
   243  		// Free the area we skipped over for alignment.
   244  		memFree(r, l)
   245  		memCheck()
   246  	}
   247  	unlock(&memlock)
   248  	return unsafe.Pointer(p), size
   249  }
   250
View as plain text