github.com/matrixorigin/matrixone@v1.2.0/pkg/vm/engine/tae/samples/sample2/main.go (about)

     1  // Copyright 2021 Matrix Origin
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package main
    16  
    17  import (
    18  	"context"
    19  	"os"
    20  	"runtime/pprof"
    21  	"sync"
    22  	"time"
    23  
    24  	"github.com/matrixorigin/matrixone/pkg/logutil"
    25  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/catalog"
    26  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/common"
    27  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/containers"
    28  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/db"
    29  	"github.com/panjf2000/ants/v2"
    30  )
    31  
    32  var sampleDir = "/tmp/sample2"
    33  var dbName = "db"
    34  var cpuprofile = "/tmp/sample2/cpuprofile"
    35  var memprofile = "/tmp/sample2/memprofile"
    36  
    37  func init() {
    38  	os.RemoveAll(sampleDir)
    39  }
    40  
    41  func startProfile() {
    42  	f, _ := os.Create(cpuprofile)
    43  	if err := pprof.StartCPUProfile(f); err != nil {
    44  		panic(err)
    45  	}
    46  }
    47  
    48  func stopProfile() {
    49  	pprof.StopCPUProfile()
    50  	memf, _ := os.Create(memprofile)
    51  	defer memf.Close()
    52  	_ = pprof.Lookup("heap").WriteTo(memf, 0)
    53  }
    54  
    55  func main() {
    56  	ctx := context.Background()
    57  
    58  	tae, _ := db.Open(ctx, sampleDir, nil)
    59  	defer tae.Close()
    60  
    61  	schema := catalog.MockSchemaAll(10, 3)
    62  	schema.BlockMaxRows = 1000
    63  	schema.ObjectMaxBlocks = 10
    64  	batchCnt := uint32(10)
    65  	batchRows := schema.BlockMaxRows * 1 / 2 * batchCnt
    66  	{
    67  		txn, _ := tae.StartTxn(nil)
    68  		db, _ := txn.CreateDatabase(dbName, "", "")
    69  		_, _ = db.CreateRelation(schema)
    70  		if err := txn.Commit(context.Background()); err != nil {
    71  			panic(err)
    72  		}
    73  	}
    74  	bat := catalog.MockBatch(schema, int(batchRows))
    75  	defer bat.Close()
    76  	bats := bat.Split(int(batchCnt))
    77  	var wg sync.WaitGroup
    78  	doAppend := func(b *containers.Batch) func() {
    79  		return func() {
    80  			defer wg.Done()
    81  			txn, _ := tae.StartTxn(nil)
    82  			db, err := txn.GetDatabase(dbName)
    83  			if err != nil {
    84  				panic(err)
    85  			}
    86  			rel, err := db.GetRelationByName(schema.Name)
    87  			if err != nil {
    88  				panic(err)
    89  			}
    90  			if err := rel.Append(context.Background(), b); err != nil {
    91  				panic(err)
    92  			}
    93  			if err := txn.Commit(context.Background()); err != nil {
    94  				panic(err)
    95  			}
    96  		}
    97  	}
    98  	p, _ := ants.NewPool(4)
    99  	now := time.Now()
   100  	startProfile()
   101  	for _, b := range bats {
   102  		wg.Add(1)
   103  		_ = p.Submit(doAppend(b))
   104  	}
   105  	wg.Wait()
   106  	stopProfile()
   107  	logutil.Infof("Append takes: %s", time.Since(now))
   108  
   109  	{
   110  		txn, _ := tae.StartTxn(nil)
   111  		db, err := txn.GetDatabase(dbName)
   112  		if err != nil {
   113  			panic(err)
   114  		}
   115  		rel, err := db.GetRelationByName(schema.Name)
   116  		if err != nil {
   117  			panic(err)
   118  		}
   119  		objIt := rel.MakeObjectIt()
   120  		for objIt.Valid() {
   121  			obj := objIt.GetObject()
   122  			logutil.Info(obj.String())
   123  			for i := 0; i < obj.BlkCnt(); i++ {
   124  				view, err := obj.GetColumnDataById(context.Background(), uint16(i), 0, common.DefaultAllocator)
   125  				logutil.Infof("Block %s Rows %d", obj.Fingerprint().BlockString(), view.Length())
   126  				if err != nil {
   127  					panic(err)
   128  				}
   129  				defer view.Close()
   130  			}
   131  			objIt.Next()
   132  		}
   133  		if err = txn.Commit(context.Background()); err != nil {
   134  			panic(err)
   135  		}
   136  	}
   137  	logutil.Info(tae.Catalog.SimplePPString(common.PPL1))
   138  }