github.com/NVIDIA/aistore@v1.3.23-0.20240517131212-7df6609be51d/cmn/cos/oom.go (about)

     1  // Package cos provides common low-level types and utilities for all aistore projects.
     2  /*
     3   * Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
     4   */
     5  package cos
     6  
     7  import (
     8  	rdebug "runtime/debug"
     9  	ratomic "sync/atomic"
    10  	"time"
    11  
    12  	"github.com/NVIDIA/aistore/cmn/mono"
    13  	"github.com/NVIDIA/aistore/cmn/nlog"
    14  )
    15  
    16  const (
    17  	minFreeMem   = 30 * time.Minute
    18  	forceFreeMem = 10 * time.Minute
    19  )
    20  
    21  var (
    22  	lastTrigOOM int64
    23  	runningOOM  int64
    24  )
    25  
    26  // FreeMemToOS executes in a separate goroutine with at least so-many minutes (above)
    27  // between the runs. It calls GC and returns allocated memory to the operating system.
    28  // Notes:
    29  //   - `forceFreeMemTime` is expected to be significantly greater than the time spent
    30  //     in the goroutine
    31  //   - still, an unlikely case of overlapping runs is treated via `runningOOM`
    32  //   - for somewhat related OOS handling - esp. constants - see ais/tgtspace.go
    33  func FreeMemToOS(force bool) bool {
    34  	prev := ratomic.LoadInt64(&lastTrigOOM)
    35  	ival := minFreeMem
    36  	if force {
    37  		ival = forceFreeMem
    38  	}
    39  
    40  	now := mono.NanoTime()
    41  	if elapsed := time.Duration(now - prev); elapsed < ival {
    42  		nlog.Infoln("not running - only " + elapsed.String() + " passed since the previous run")
    43  		return false
    44  	}
    45  	if ratomic.CompareAndSwapInt64(&runningOOM, 0, now) {
    46  		nlog.Infoln("not running - running now")
    47  		return false
    48  	}
    49  
    50  	go _freeMem(now)
    51  	return true
    52  }
    53  
    54  func _freeMem(started int64) {
    55  	rdebug.FreeOSMemory()
    56  
    57  	now := mono.NanoTime()
    58  	if elapsed := time.Duration(now - started); elapsed > forceFreeMem/2 {
    59  		nlog.Errorln("Warning: spent " + elapsed.String() + " freeing memory")
    60  	}
    61  	ratomic.StoreInt64(&lastTrigOOM, now)
    62  	ratomic.StoreInt64(&runningOOM, 0)
    63  }