github.com/matrixorigin/matrixone@v1.2.0/pkg/vm/engine/tae/blockio/prefetch.go (about)

     1  // Copyright 2021 Matrix Origin
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package blockio
    16  
    17  import (
    18  	"github.com/matrixorigin/matrixone/pkg/fileservice"
    19  	"github.com/matrixorigin/matrixone/pkg/objectio"
    20  )
    21  
    22  // PrefetchParams is the parameter of the executed IoPipeline.PrefetchParams, which
    23  // provides the merge function, which can merge the PrefetchParams requests of
    24  // multiple blocks in an object/file
    25  type PrefetchParams struct {
    26  	ids          map[uint16]*objectio.ReadBlockOptions
    27  	fs           fileservice.FileService
    28  	key          objectio.Location
    29  	reader       *objectio.ObjectReader
    30  	prefetchFile bool
    31  }
    32  
    33  func BuildPrefetchParams(service fileservice.FileService, key objectio.Location) (PrefetchParams, error) {
    34  	pp := buildPrefetchParams(service, key)
    35  	return pp, nil
    36  }
    37  
    38  func buildPrefetchParams(service fileservice.FileService, key objectio.Location) PrefetchParams {
    39  	ids := make(map[uint16]*objectio.ReadBlockOptions)
    40  	return PrefetchParams{
    41  		ids: ids,
    42  		fs:  service,
    43  		key: key,
    44  	}
    45  }
    46  
    47  func buildPrefetchParamsByReader(reader *BlockReader) PrefetchParams {
    48  	ids := make(map[uint16]*objectio.ReadBlockOptions)
    49  	return PrefetchParams{
    50  		ids:    ids,
    51  		reader: reader.GetObjectReader(),
    52  	}
    53  }
    54  
    55  func (p *PrefetchParams) AddBlock(idxes []uint16, ids []uint16) {
    56  	blocks := make(map[uint16]*objectio.ReadBlockOptions)
    57  	columns := make(map[uint16]bool)
    58  	for _, idx := range idxes {
    59  		columns[idx] = true
    60  	}
    61  	for _, id := range ids {
    62  		blocks[id] = &objectio.ReadBlockOptions{
    63  			Id:       id,
    64  			Idxes:    columns,
    65  			DataType: uint16(objectio.SchemaData),
    66  		}
    67  	}
    68  	p.mergeIds(blocks)
    69  }
    70  func (p *PrefetchParams) AddBlockWithType(idxes []uint16, ids []uint16, dataType uint16) {
    71  	blocks := make(map[uint16]*objectio.ReadBlockOptions)
    72  	columns := make(map[uint16]bool)
    73  	for _, idx := range idxes {
    74  		columns[idx] = true
    75  	}
    76  	for _, id := range ids {
    77  		blocks[id] = &objectio.ReadBlockOptions{
    78  			Id:       id,
    79  			DataType: dataType,
    80  			Idxes:    columns,
    81  		}
    82  	}
    83  	p.mergeIds(blocks)
    84  }
    85  
    86  func (p *PrefetchParams) mergeIds(ids2 map[uint16]*objectio.ReadBlockOptions) {
    87  	for id, block := range ids2 {
    88  		if p.ids[id] == nil {
    89  			p.ids[id] = block
    90  			continue
    91  		}
    92  		for index := range block.Idxes {
    93  			p.ids[id].Idxes[index] = true
    94  		}
    95  	}
    96  }
    97  
    98  func mergePrefetch(processes []PrefetchParams) map[string]PrefetchParams {
    99  	pc := make(map[string]PrefetchParams)
   100  	for _, p := range processes {
   101  		var name string
   102  		if p.reader != nil {
   103  			name = p.reader.GetName()
   104  		} else {
   105  			name = p.key.Name().String()
   106  		}
   107  		old, ok := pc[name]
   108  		if !ok {
   109  			pc[name] = p
   110  			continue
   111  		}
   112  		old.mergeIds(p.ids)
   113  		pc[name] = old
   114  	}
   115  	return pc
   116  }