github.com/matrixorigin/matrixone@v1.2.0/pkg/vm/engine/tae/blockio/writer_test.go (about) 1 // Copyright 2021 Matrix Origin 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package blockio 16 17 import ( 18 "context" 19 "path" 20 "testing" 21 22 "github.com/matrixorigin/matrixone/pkg/container/types" 23 "github.com/matrixorigin/matrixone/pkg/objectio" 24 "github.com/matrixorigin/matrixone/pkg/pb/api" 25 26 "github.com/matrixorigin/matrixone/pkg/common/mpool" 27 "github.com/matrixorigin/matrixone/pkg/defines" 28 "github.com/matrixorigin/matrixone/pkg/fileservice" 29 "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/catalog" 30 "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/containers" 31 "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/index" 32 "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/testutils" 33 "github.com/stretchr/testify/assert" 34 "github.com/stretchr/testify/require" 35 ) 36 37 const ( 38 ModuleName = "BlockIO" 39 ) 40 41 func TestWriter_WriteBlockAndZoneMap(t *testing.T) { 42 defer testutils.AfterTest(t)() 43 ctx := context.Background() 44 45 dir := testutils.InitTestEnv(ModuleName, t) 46 dir = path.Join(dir, "/local") 47 name := objectio.BuildObjectName(objectio.NewSegmentid(), 0) 48 c := fileservice.Config{ 49 Name: defines.LocalFileServiceName, 50 Backend: "DISK", 51 DataDir: dir, 52 } 53 service, err := fileservice.NewFileService(ctx, c, nil) 54 assert.Nil(t, err) 55 writer, _ := NewBlockWriterNew(service, name, 0, nil) 56 57 schema := catalog.MockSchemaAll(13, 2) 58 bats := catalog.MockBatch(schema, 40000*2).Split(2) 59 60 _, err = writer.WriteBatch(containers.ToCNBatch(bats[0])) 61 assert.Nil(t, err) 62 _, err = writer.WriteBatch(containers.ToCNBatch(bats[1])) 63 assert.Nil(t, err) 64 blocks, _, err := writer.Sync(context.Background()) 65 assert.Nil(t, err) 66 assert.Equal(t, 2, len(blocks)) 67 fd := blocks[0] 68 col := fd.MustGetColumn(2) 69 colZoneMap := col.ZoneMap() 70 zm := index.DecodeZM(colZoneMap) 71 72 require.NoError(t, err) 73 res := zm.Contains(int32(500)) 74 require.True(t, res) 75 res = zm.Contains(int32(39999)) 76 require.True(t, res) 77 res = zm.Contains(int32(40000)) 78 require.False(t, res) 79 require.Equal(t, int32(0), zm.GetMin()) 80 require.Equal(t, int32(39999), zm.GetMax()) 81 sum := int64(0) 82 for i := int64(0); i < 40000; i++ { 83 sum += i 84 } 85 require.Equal(t, sum, zm.GetSum()) 86 87 mp := mpool.MustNewZero() 88 metaloc := EncodeLocation(writer.GetName(), blocks[0].GetExtent(), 40000, blocks[0].GetID()) 89 require.NoError(t, err) 90 reader, err := NewObjectReader(service, metaloc) 91 require.NoError(t, err) 92 meta, err := reader.LoadObjectMeta(context.TODO(), mp) 93 require.NoError(t, err) 94 blkMeta1 := meta.GetBlockMeta(0) 95 blkMeta2 := meta.GetBlockMeta(1) 96 for i := uint16(0); i < meta.BlockHeader().ColumnCount(); i++ { 97 offset := blkMeta1.ColumnMeta(i).Location().Offset() 98 length := blkMeta1.ColumnMeta(i).Location().Length() + blkMeta2.ColumnMeta(i).Location().Length() 99 oSize := blkMeta1.ColumnMeta(i).Location().OriginSize() + blkMeta2.ColumnMeta(i).Location().OriginSize() 100 assert.Equal(t, offset, meta.MustGetColumn(i).Location().Offset()) 101 assert.Equal(t, length, meta.MustGetColumn(i).Location().Length()) 102 assert.Equal(t, oSize, meta.MustGetColumn(i).Location().OriginSize()) 103 } 104 header := meta.BlockHeader() 105 require.Equal(t, uint32(80000), header.Rows()) 106 t.Log(meta.MustGetColumn(0).Ndv(), meta.MustGetColumn(1).Ndv(), meta.MustGetColumn(2).Ndv()) 107 zm = meta.MustGetColumn(2).ZoneMap() 108 require.True(t, zm.Contains(int32(40000))) 109 require.False(t, zm.Contains(int32(100000))) 110 zm = meta.GetColumnMeta(0, 2).ZoneMap() 111 require.True(t, zm.Contains(int32(39999))) 112 require.False(t, zm.Contains(int32(40000))) 113 zm = meta.GetColumnMeta(1, 2).ZoneMap() 114 require.True(t, zm.Contains(int32(40000))) 115 require.True(t, zm.Contains(int32(79999))) 116 require.False(t, zm.Contains(int32(80000))) 117 sum = int64(0) 118 for i := int64(40000); i < 80000; i++ { 119 sum += i 120 if sum < 0 { 121 sum = 0 122 break 123 } 124 } 125 require.Equal(t, int32(40000), zm.GetMin()) 126 require.Equal(t, int32(79999), zm.GetMax()) 127 require.Equal(t, sum, zm.GetSum()) 128 } 129 130 func TestWriter_WriteBlockAfterAlter(t *testing.T) { 131 defer testutils.AfterTest(t)() 132 ctx := context.Background() 133 134 dir := testutils.InitTestEnv(ModuleName, t) 135 dir = path.Join(dir, "/local") 136 name := objectio.BuildObjectName(objectio.NewSegmentid(), 0) 137 c := fileservice.Config{ 138 Name: defines.LocalFileServiceName, 139 Backend: "DISK", 140 DataDir: dir, 141 } 142 service, err := fileservice.NewFileService(ctx, c, nil) 143 assert.Nil(t, err) 144 145 schema := catalog.MockSchemaAll(13, 2) 146 schema.ApplyAlterTable(api.NewAddColumnReq(0, 0, "xyz", types.NewProtoType(types.T_int32), 1)) 147 schema.ApplyAlterTable(api.NewRemoveColumnReq(0, 0, 6, 5)) 148 149 seqnums := make([]uint16, 0, len(schema.ColDefs)) 150 for _, col := range schema.ColDefs { 151 seqnums = append(seqnums, col.SeqNum) 152 } 153 t.Log(seqnums) 154 bats := containers.MockBatchWithAttrs( 155 schema.AllTypes(), 156 schema.AllNames(), 157 40000*2, 158 schema.GetSingleSortKey().Idx, nil).Split(2) 159 160 writer, _ := NewBlockWriterNew(service, name, 1, seqnums) 161 _, err = writer.WriteBatch(containers.ToCNBatch(bats[0])) 162 assert.Nil(t, err) 163 _, err = writer.WriteBatch(containers.ToCNBatch(bats[1])) 164 assert.Nil(t, err) 165 blocks, _, err := writer.Sync(context.Background()) 166 assert.Nil(t, err) 167 assert.Equal(t, 2, len(blocks)) 168 fd := blocks[0] 169 colx := fd.MustGetColumn(13 /* xyz seqnum*/) 170 assert.Equal(t, uint8(types.T_int32), colx.DataType()) 171 assert.Equal(t, uint16(1), colx.Idx()) 172 colx = fd.MustGetColumn(14 /* rowid seqnum*/) 173 assert.Equal(t, uint8(types.T_Rowid), colx.DataType()) 174 assert.Equal(t, uint16(13), colx.Idx()) 175 assert.NoError(t, err) 176 177 col := fd.MustGetColumn(2 /*pk seqnum*/) 178 assert.Nil(t, err) 179 colZoneMap := col.ZoneMap() 180 zm := index.DecodeZM(colZoneMap) 181 182 require.NoError(t, err) 183 res := zm.Contains(int32(500)) 184 require.True(t, res) 185 res = zm.Contains(int32(39999)) 186 require.True(t, res) 187 res = zm.Contains(int32(40000)) 188 require.False(t, res) 189 190 mp := mpool.MustNewZero() 191 metaloc := EncodeLocation(writer.GetName(), blocks[0].GetExtent(), 40000, blocks[0].GetID()) 192 require.NoError(t, err) 193 reader, err := NewObjectReader(service, metaloc) 194 require.NoError(t, err) 195 meta, err := reader.LoadObjectMeta(context.TODO(), mp) 196 require.Equal(t, uint16(15), meta.BlockHeader().MetaColumnCount()) 197 require.Equal(t, uint16(14), meta.BlockHeader().ColumnCount()) 198 require.NoError(t, err) 199 header := meta.BlockHeader() 200 require.Equal(t, uint32(80000), header.Rows()) 201 t.Log(meta.MustGetColumn(0).Ndv(), meta.MustGetColumn(1).Ndv(), meta.MustGetColumn(2).Ndv()) 202 zm = meta.MustGetColumn(2).ZoneMap() 203 require.True(t, zm.Contains(int32(40000))) 204 require.False(t, zm.Contains(int32(100000))) 205 zm = meta.GetColumnMeta(0, 2).ZoneMap() 206 require.True(t, zm.Contains(int32(39999))) 207 require.False(t, zm.Contains(int32(40000))) 208 zm = meta.GetColumnMeta(1, 2).ZoneMap() 209 require.True(t, zm.Contains(int32(40000))) 210 require.True(t, zm.Contains(int32(79999))) 211 require.False(t, zm.Contains(int32(80000))) 212 }