github.com/matrixorigin/matrixone@v0.7.0/pkg/vm/engine/tae/samples/sample2/main.go (about)

     1  // Copyright 2021 Matrix Origin
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package main
    16  
    17  import (
    18  	"bytes"
    19  	"os"
    20  	"runtime/pprof"
    21  	"sync"
    22  	"time"
    23  
    24  	"github.com/matrixorigin/matrixone/pkg/logutil"
    25  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/catalog"
    26  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/common"
    27  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/containers"
    28  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/db"
    29  	"github.com/panjf2000/ants/v2"
    30  )
    31  
    32  var sampleDir = "/tmp/sample2"
    33  var dbName = "db"
    34  var cpuprofile = "/tmp/sample2/cpuprofile"
    35  var memprofile = "/tmp/sample2/memprofile"
    36  
    37  func init() {
    38  	os.RemoveAll(sampleDir)
    39  }
    40  
    41  func startProfile() {
    42  	f, _ := os.Create(cpuprofile)
    43  	if err := pprof.StartCPUProfile(f); err != nil {
    44  		panic(err)
    45  	}
    46  }
    47  
    48  func stopProfile() {
    49  	pprof.StopCPUProfile()
    50  	memf, _ := os.Create(memprofile)
    51  	defer memf.Close()
    52  	_ = pprof.Lookup("heap").WriteTo(memf, 0)
    53  }
    54  
    55  func main() {
    56  	tae, _ := db.Open(sampleDir, nil)
    57  	defer tae.Close()
    58  
    59  	schema := catalog.MockSchemaAll(10, 3)
    60  	schema.BlockMaxRows = 1000
    61  	schema.SegmentMaxBlocks = 10
    62  	batchCnt := uint32(10)
    63  	batchRows := schema.BlockMaxRows * 1 / 2 * batchCnt
    64  	{
    65  		txn, _ := tae.StartTxn(nil)
    66  		db, _ := txn.CreateDatabase(dbName, "")
    67  		_, _ = db.CreateRelation(schema)
    68  		if err := txn.Commit(); err != nil {
    69  			panic(err)
    70  		}
    71  	}
    72  	bat := catalog.MockBatch(schema, int(batchRows))
    73  	defer bat.Close()
    74  	bats := bat.Split(int(batchCnt))
    75  	var wg sync.WaitGroup
    76  	doAppend := func(b *containers.Batch) func() {
    77  		return func() {
    78  			defer wg.Done()
    79  			txn, _ := tae.StartTxn(nil)
    80  			db, err := txn.GetDatabase(dbName)
    81  			if err != nil {
    82  				panic(err)
    83  			}
    84  			rel, err := db.GetRelationByName(schema.Name)
    85  			if err != nil {
    86  				panic(err)
    87  			}
    88  			if err := rel.Append(b); err != nil {
    89  				panic(err)
    90  			}
    91  			if err := txn.Commit(); err != nil {
    92  				panic(err)
    93  			}
    94  		}
    95  	}
    96  	p, _ := ants.NewPool(4)
    97  	now := time.Now()
    98  	startProfile()
    99  	for _, b := range bats {
   100  		wg.Add(1)
   101  		_ = p.Submit(doAppend(b))
   102  	}
   103  	wg.Wait()
   104  	stopProfile()
   105  	logutil.Infof("Append takes: %s", time.Since(now))
   106  
   107  	{
   108  		txn, _ := tae.StartTxn(nil)
   109  		db, err := txn.GetDatabase(dbName)
   110  		if err != nil {
   111  			panic(err)
   112  		}
   113  		rel, err := db.GetRelationByName(schema.Name)
   114  		if err != nil {
   115  			panic(err)
   116  		}
   117  		var buffer bytes.Buffer
   118  		segIt := rel.MakeSegmentIt()
   119  		for segIt.Valid() {
   120  			seg := segIt.GetSegment()
   121  			logutil.Info(seg.String())
   122  			blkIt := seg.MakeBlockIt()
   123  			for blkIt.Valid() {
   124  				blk := blkIt.GetBlock()
   125  				logutil.Info(blk.String())
   126  				buffer.Reset()
   127  				view, err := blk.GetColumnDataById(0, &buffer)
   128  				logutil.Infof("Block %s Rows %d", blk.Fingerprint().BlockString(), view.Length())
   129  				if err != nil {
   130  					panic(err)
   131  				}
   132  				defer view.Close()
   133  				blkIt.Next()
   134  			}
   135  			segIt.Next()
   136  		}
   137  		if err = txn.Commit(); err != nil {
   138  			panic(err)
   139  		}
   140  	}
   141  	logutil.Info(tae.Opts.Catalog.SimplePPString(common.PPL1))
   142  }