github.com/weaviate/weaviate@v1.24.6/adapters/repos/db/shard_cyclecallbacks.go (about)

     1  //                           _       _
     2  // __      _____  __ ___   ___  __ _| |_ ___
     3  // \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
     4  //  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
     5  //   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
     6  //
     7  //  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
     8  //
     9  //  CONTACT: hello@weaviate.io
    10  //
    11  
    12  package db
    13  
    14  import (
    15  	"strings"
    16  
    17  	"github.com/weaviate/weaviate/entities/cyclemanager"
    18  )
    19  
    20  type shardCycleCallbacks struct {
    21  	compactionCallbacks     cyclemanager.CycleCallbackGroup
    22  	compactionCallbacksCtrl cyclemanager.CycleCallbackCtrl
    23  
    24  	flushCallbacks     cyclemanager.CycleCallbackGroup
    25  	flushCallbacksCtrl cyclemanager.CycleCallbackCtrl
    26  
    27  	vectorCommitLoggerCallbacks     cyclemanager.CycleCallbackGroup
    28  	vectorTombstoneCleanupCallbacks cyclemanager.CycleCallbackGroup
    29  	vectorCombinedCallbacksCtrl     cyclemanager.CycleCallbackCtrl
    30  
    31  	geoPropsCommitLoggerCallbacks     cyclemanager.CycleCallbackGroup
    32  	geoPropsTombstoneCleanupCallbacks cyclemanager.CycleCallbackGroup
    33  	geoPropsCombinedCallbacksCtrl     cyclemanager.CycleCallbackCtrl
    34  }
    35  
    36  func (s *Shard) initCycleCallbacks() {
    37  	id := func(elems ...string) string {
    38  		elems = append([]string{"shard", s.index.ID(), s.name}, elems...)
    39  		return strings.Join(elems, "/")
    40  	}
    41  
    42  	compactionId := id("compaction")
    43  	compactionCallbacks := cyclemanager.NewCallbackGroup(compactionId, s.index.logger, 1)
    44  	compactionCallbacksCtrl := s.index.cycleCallbacks.compactionCallbacks.Register(
    45  		compactionId, compactionCallbacks.CycleCallback,
    46  		cyclemanager.WithIntervals(cyclemanager.CompactionCycleIntervals()))
    47  
    48  	flushId := id("flush")
    49  	flushCallbacks := cyclemanager.NewCallbackGroup(flushId, s.index.logger, 1)
    50  	flushCallbacksCtrl := s.index.cycleCallbacks.flushCallbacks.Register(
    51  		flushId, flushCallbacks.CycleCallback,
    52  		cyclemanager.WithIntervals(cyclemanager.MemtableFlushCycleIntervals()))
    53  
    54  	vectorCommitLoggerId := id("vector", "commit_logger")
    55  	vectorCommitLoggerCallbacks := cyclemanager.NewCallbackGroup(vectorCommitLoggerId, s.index.logger, 1)
    56  	vectorCommitLoggerCallbacksCtrl := s.index.cycleCallbacks.vectorCommitLoggerCallbacks.Register(
    57  		vectorCommitLoggerId, vectorCommitLoggerCallbacks.CycleCallback,
    58  		cyclemanager.WithIntervals(cyclemanager.HnswCommitLoggerCycleIntervals()))
    59  
    60  	vectorTombstoneCleanupId := id("vector", "tombstone_cleanup")
    61  	vectorTombstoneCleanupCallbacks := cyclemanager.NewCallbackGroup(vectorTombstoneCleanupId, s.index.logger, 1)
    62  	// fixed interval on class level, no need to specify separate on shard level
    63  	vectorTombstoneCleanupCallbacksCtrl := s.index.cycleCallbacks.vectorTombstoneCleanupCallbacks.Register(
    64  		vectorTombstoneCleanupId, vectorTombstoneCleanupCallbacks.CycleCallback)
    65  
    66  	vectorCombinedCallbacksCtrl := cyclemanager.NewCombinedCallbackCtrl(2, s.index.logger,
    67  		vectorCommitLoggerCallbacksCtrl, vectorTombstoneCleanupCallbacksCtrl)
    68  
    69  	geoPropsCommitLoggerId := id("geo_props", "commit_logger")
    70  	geoPropsCommitLoggerCallbacks := cyclemanager.NewCallbackGroup(geoPropsCommitLoggerId, s.index.logger, 1)
    71  	geoPropsCommitLoggerCallbacksCtrl := s.index.cycleCallbacks.geoPropsCommitLoggerCallbacks.Register(
    72  		geoPropsCommitLoggerId, geoPropsCommitLoggerCallbacks.CycleCallback,
    73  		cyclemanager.WithIntervals(cyclemanager.GeoCommitLoggerCycleIntervals()))
    74  
    75  	geoPropsTombstoneCleanupId := id("geoProps", "tombstone_cleanup")
    76  	geoPropsTombstoneCleanupCallbacks := cyclemanager.NewCallbackGroup(geoPropsTombstoneCleanupId, s.index.logger, 1)
    77  	// fixed interval on class level, no need to specify separate on shard level
    78  	geoPropsTombstoneCleanupCallbacksCtrl := s.index.cycleCallbacks.geoPropsTombstoneCleanupCallbacks.Register(
    79  		geoPropsTombstoneCleanupId, geoPropsTombstoneCleanupCallbacks.CycleCallback)
    80  
    81  	geoPropsCombinedCallbacksCtrl := cyclemanager.NewCombinedCallbackCtrl(2, s.index.logger,
    82  		geoPropsCommitLoggerCallbacksCtrl, geoPropsTombstoneCleanupCallbacksCtrl)
    83  
    84  	s.cycleCallbacks = &shardCycleCallbacks{
    85  		compactionCallbacks:     compactionCallbacks,
    86  		compactionCallbacksCtrl: compactionCallbacksCtrl,
    87  
    88  		flushCallbacks:     flushCallbacks,
    89  		flushCallbacksCtrl: flushCallbacksCtrl,
    90  
    91  		vectorCommitLoggerCallbacks:     vectorCommitLoggerCallbacks,
    92  		vectorTombstoneCleanupCallbacks: vectorTombstoneCleanupCallbacks,
    93  		vectorCombinedCallbacksCtrl:     vectorCombinedCallbacksCtrl,
    94  
    95  		geoPropsCommitLoggerCallbacks:     geoPropsCommitLoggerCallbacks,
    96  		geoPropsTombstoneCleanupCallbacks: geoPropsTombstoneCleanupCallbacks,
    97  		geoPropsCombinedCallbacksCtrl:     geoPropsCombinedCallbacksCtrl,
    98  	}
    99  }