github.com/matrixorigin/matrixone@v1.2.0/pkg/vm/engine/tae/db/dbutils/mem.go (about)

     1  // Copyright 2021 Matrix Origin
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package dbutils
    16  
    17  import (
    18  	"bytes"
    19  	"encoding/base64"
    20  	"fmt"
    21  	"runtime"
    22  	"runtime/debug"
    23  	"runtime/pprof"
    24  
    25  	"github.com/matrixorigin/matrixone/pkg/common/mpool"
    26  	"github.com/matrixorigin/matrixone/pkg/logutil"
    27  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/common"
    28  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/containers"
    29  	"github.com/shirou/gopsutil/v3/mem"
    30  	"go.uber.org/zap"
    31  )
    32  
    33  func MakeDefaultSmallPool(name string) *containers.VectorPool {
    34  	var (
    35  		limit            int
    36  		memtableCapacity int
    37  	)
    38  	memStats, err := mem.VirtualMemory()
    39  	if err != nil {
    40  		panic(err)
    41  	}
    42  	if memStats.Total > mpool.GB*20 {
    43  		limit = mpool.KB * 64
    44  		memtableCapacity = 10240
    45  	} else if memStats.Total > mpool.GB*10 {
    46  		limit = mpool.KB * 32
    47  		memtableCapacity = 10240
    48  	} else if memStats.Total > mpool.GB*5 {
    49  		limit = mpool.KB * 16
    50  		memtableCapacity = 10240
    51  	} else {
    52  		limit = mpool.KB * 8
    53  		memtableCapacity = 10240
    54  	}
    55  
    56  	return containers.NewVectorPool(
    57  		name,
    58  		memtableCapacity,
    59  		containers.WithAllocationLimit(limit),
    60  		containers.WithMPool(common.SmallAllocator),
    61  	)
    62  }
    63  
    64  func MakeDefaultTransientPool(name string) *containers.VectorPool {
    65  	var (
    66  		limit            int
    67  		trasientCapacity int
    68  	)
    69  	memStats, err := mem.VirtualMemory()
    70  	if err != nil {
    71  		panic(err)
    72  	}
    73  	if memStats.Total > mpool.GB*20 {
    74  		limit = mpool.MB
    75  		trasientCapacity = 512
    76  	} else if memStats.Total > mpool.GB*10 {
    77  		limit = mpool.KB * 512
    78  		trasientCapacity = 512
    79  	} else if memStats.Total > mpool.GB*5 {
    80  		limit = mpool.KB * 256
    81  		trasientCapacity = 512
    82  	} else {
    83  		limit = mpool.KB * 256
    84  		trasientCapacity = 256
    85  	}
    86  
    87  	return containers.NewVectorPool(
    88  		name,
    89  		trasientCapacity,
    90  		containers.WithAllocationLimit(limit),
    91  	)
    92  }
    93  func FormatMemStats(memstats runtime.MemStats) string {
    94  	return fmt.Sprintf(
    95  		"TotalAlloc:%dMB Sys:%dMB HeapAlloc:%dMB HeapSys:%dMB HeapIdle:%dMB HeapReleased:%dMB HeapInuse:%dMB NextGC:%dMB NumGC:%d PauseNs:%d",
    96  		memstats.TotalAlloc/mpool.MB,
    97  		memstats.Sys/mpool.MB,
    98  		memstats.HeapAlloc/mpool.MB,
    99  		memstats.HeapSys/mpool.MB,
   100  		memstats.HeapIdle/mpool.MB,
   101  		memstats.HeapReleased/mpool.MB,
   102  		memstats.HeapInuse/mpool.MB,
   103  		memstats.NextGC/mpool.MB,
   104  		memstats.NumGC,
   105  		memstats.PauseTotalNs,
   106  	)
   107  }
   108  
   109  var prevHeapInuse uint64
   110  
   111  func PrintMemStats() {
   112  	var memstats runtime.MemStats
   113  	runtime.ReadMemStats(&memstats)
   114  
   115  	// found a spike in heapInuse
   116  	if prevHeapInuse > 0 && memstats.HeapInuse > prevHeapInuse &&
   117  		memstats.HeapInuse-prevHeapInuse > common.Const1GBytes*10 {
   118  		heapp := pprof.Lookup("heap")
   119  		buf := &bytes.Buffer{}
   120  		heapp.WriteTo(buf, 0)
   121  		mlimit := debug.SetMemoryLimit(-1)
   122  		log := buf.Bytes()
   123  		for len(log) > 200*1024 {
   124  			logutil.Info(base64.RawStdEncoding.EncodeToString(log[:200*1024]))
   125  			log = log[200*1024:]
   126  		}
   127  		logutil.Info(
   128  			base64.RawStdEncoding.EncodeToString(log),
   129  			zap.String("mlimit", common.HumanReadableBytes(int(mlimit))))
   130  	}
   131  
   132  	prevHeapInuse = memstats.HeapInuse
   133  	logutil.Infof("HeapInfo:%s", FormatMemStats(memstats))
   134  }