github.com/weaviate/weaviate@v1.24.6/entities/vectorindex/flat/config.go (about)

     1  //                           _       _
     2  // __      _____  __ ___   ___  __ _| |_ ___
     3  // \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
     4  //  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
     5  //   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
     6  //
     7  //  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
     8  //
     9  //  CONTACT: hello@weaviate.io
    10  //
    11  
    12  package flat
    13  
    14  import (
    15  	"errors"
    16  	"fmt"
    17  
    18  	"github.com/weaviate/weaviate/entities/schema"
    19  	vectorindexcommon "github.com/weaviate/weaviate/entities/vectorindex/common"
    20  )
    21  
    22  const (
    23  	DefaultVectorCache           = false
    24  	DefaultVectorCacheMaxObjects = 1e12
    25  	DefaultCompressionEnabled    = false
    26  	DefaultCompressionRescore    = -1 // indicates "let Weaviate pick"
    27  )
    28  
    29  type CompressionUserConfig struct {
    30  	Enabled      bool `json:"enabled"`
    31  	RescoreLimit int  `json:"rescoreLimit"`
    32  	Cache        bool `json:"cache"`
    33  }
    34  
    35  type UserConfig struct {
    36  	Distance              string                `json:"distance"`
    37  	VectorCacheMaxObjects int                   `json:"vectorCacheMaxObjects"`
    38  	PQ                    CompressionUserConfig `json:"pq"`
    39  	BQ                    CompressionUserConfig `json:"bq"`
    40  }
    41  
    42  // IndexType returns the type of the underlying vector index, thus making sure
    43  // the schema.VectorIndexConfig interface is implemented
    44  func (u UserConfig) IndexType() string {
    45  	return "flat"
    46  }
    47  
    48  func (u UserConfig) DistanceName() string {
    49  	return u.Distance
    50  }
    51  
    52  // SetDefaults in the user-specifyable part of the config
    53  func (u *UserConfig) SetDefaults() {
    54  	u.PQ.Cache = DefaultVectorCache
    55  	u.BQ.Cache = DefaultVectorCache
    56  	u.VectorCacheMaxObjects = DefaultVectorCacheMaxObjects
    57  	u.Distance = vectorindexcommon.DefaultDistanceMetric
    58  	u.PQ.Enabled = DefaultCompressionEnabled
    59  	u.PQ.RescoreLimit = DefaultCompressionRescore
    60  	u.BQ.Enabled = DefaultCompressionEnabled
    61  	u.BQ.RescoreLimit = DefaultCompressionRescore
    62  }
    63  
    64  // ParseAndValidateConfig from an unknown input value, as this is not further
    65  // specified in the API to allow of exchanging the index type
    66  func ParseAndValidateConfig(input interface{}) (schema.VectorIndexConfig, error) {
    67  	uc := UserConfig{}
    68  	uc.SetDefaults()
    69  
    70  	if input == nil {
    71  		return uc, nil
    72  	}
    73  
    74  	asMap, ok := input.(map[string]interface{})
    75  	if !ok || asMap == nil {
    76  		return uc, fmt.Errorf("input must be a non-nil map")
    77  	}
    78  
    79  	if err := vectorindexcommon.OptionalStringFromMap(asMap, "distance", func(v string) {
    80  		uc.Distance = v
    81  	}); err != nil {
    82  		return uc, err
    83  	}
    84  
    85  	if err := vectorindexcommon.OptionalIntFromMap(asMap, "vectorCacheMaxObjects", func(v int) {
    86  		uc.VectorCacheMaxObjects = v
    87  	}); err != nil {
    88  		return uc, err
    89  	}
    90  
    91  	if err := parseCompressionMap(asMap, &uc); err != nil {
    92  		return uc, err
    93  	}
    94  
    95  	return uc, nil
    96  }
    97  
    98  func parseCompressionMap(in map[string]interface{}, uc *UserConfig) error {
    99  	pqConfigValue, pqOk := in["pq"]
   100  	bqConfigValue, bqOk := in["bq"]
   101  	if !pqOk && !bqOk {
   102  		return nil
   103  	}
   104  
   105  	if pqOk {
   106  		pqConfigMap, ok := pqConfigValue.(map[string]interface{})
   107  		if ok {
   108  			if err := vectorindexcommon.OptionalBoolFromMap(pqConfigMap, "enabled", func(v bool) {
   109  				uc.PQ.Enabled = v
   110  			}); err != nil {
   111  				return err
   112  			}
   113  
   114  			if err := vectorindexcommon.OptionalBoolFromMap(pqConfigMap, "cache", func(v bool) {
   115  				uc.PQ.Cache = v
   116  			}); err != nil {
   117  				return err
   118  			}
   119  
   120  			if err := vectorindexcommon.OptionalIntFromMap(pqConfigMap, "rescoreLimit", func(v int) {
   121  				uc.PQ.RescoreLimit = v
   122  			}); err != nil {
   123  				return err
   124  			}
   125  		}
   126  	}
   127  
   128  	if bqOk {
   129  		bqConfigMap, ok := bqConfigValue.(map[string]interface{})
   130  		if !ok {
   131  			return nil
   132  		}
   133  
   134  		if err := vectorindexcommon.OptionalBoolFromMap(bqConfigMap, "enabled", func(v bool) {
   135  			uc.BQ.Enabled = v
   136  		}); err != nil {
   137  			return err
   138  		}
   139  
   140  		if err := vectorindexcommon.OptionalBoolFromMap(bqConfigMap, "cache", func(v bool) {
   141  			uc.BQ.Cache = v
   142  		}); err != nil {
   143  			return err
   144  		}
   145  
   146  		if err := vectorindexcommon.OptionalIntFromMap(bqConfigMap, "rescoreLimit", func(v int) {
   147  			uc.BQ.RescoreLimit = v
   148  		}); err != nil {
   149  			return err
   150  		}
   151  
   152  	}
   153  	// TODO: remove once PQ is supported
   154  	if uc.PQ.Enabled {
   155  		return errors.New("PQ is not currently supported for flat indices")
   156  	}
   157  	if uc.PQ.Cache && !uc.PQ.Enabled {
   158  		return errors.New("not possible to use the cache without compression")
   159  	}
   160  	if uc.BQ.Cache && !uc.BQ.Enabled {
   161  		return errors.New("not possible to use the cache without compression")
   162  	}
   163  	if uc.PQ.Enabled && uc.BQ.Enabled {
   164  		return errors.New("cannot activate dual compression. Select either PQ or BQ please")
   165  	}
   166  	return nil
   167  }
   168  
   169  func NewDefaultUserConfig() UserConfig {
   170  	uc := UserConfig{}
   171  	uc.SetDefaults()
   172  	return uc
   173  }