github.com/weaviate/weaviate@v1.24.6/adapters/repos/db/vector/hnsw/flat_search.go (about)

     1  //                           _       _
     2  // __      _____  __ ___   ___  __ _| |_ ___
     3  // \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
     4  //  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
     5  //   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
     6  //
     7  //  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
     8  //
     9  //  CONTACT: hello@weaviate.io
    10  //
    11  
    12  package hnsw
    13  
    14  import (
    15  	"github.com/weaviate/weaviate/adapters/repos/db/helpers"
    16  	"github.com/weaviate/weaviate/adapters/repos/db/priorityqueue"
    17  )
    18  
    19  func (h *hnsw) flatSearch(queryVector []float32, limit int,
    20  	allowList helpers.AllowList,
    21  ) ([]uint64, []float32, error) {
    22  	results := priorityqueue.NewMax[any](limit)
    23  
    24  	it := allowList.Iterator()
    25  	for candidate, ok := it.Next(); ok; candidate, ok = it.Next() {
    26  		h.RLock()
    27  		// Hot fix for https://github.com/weaviate/weaviate/issues/1937
    28  		// this if statement mitigates the problem but it doesn't resolve the issue
    29  		if candidate >= uint64(len(h.nodes)) {
    30  			h.logger.WithField("action", "flatSearch").
    31  				Warnf("trying to get candidate: %v but we only have: %v elements.",
    32  					candidate, len(h.nodes))
    33  			h.RUnlock()
    34  			continue
    35  		}
    36  		if len(h.nodes) <= int(candidate) { // if index hasn't grown yet for a newly inserted node
    37  			continue
    38  		}
    39  
    40  		h.shardedNodeLocks.RLock(candidate)
    41  		c := h.nodes[candidate]
    42  		h.shardedNodeLocks.RUnlock(candidate)
    43  
    44  		if c == nil || h.hasTombstone(candidate) {
    45  			h.RUnlock()
    46  			continue
    47  		}
    48  		h.RUnlock()
    49  		dist, ok, err := h.distBetweenNodeAndVec(candidate, queryVector)
    50  		if err != nil {
    51  			return nil, nil, err
    52  		}
    53  
    54  		if !ok {
    55  			// deleted node, ignore
    56  			continue
    57  		}
    58  
    59  		if results.Len() < limit {
    60  			results.Insert(candidate, dist)
    61  		} else if results.Top().Dist > dist {
    62  			results.Pop()
    63  			results.Insert(candidate, dist)
    64  		}
    65  	}
    66  
    67  	ids := make([]uint64, results.Len())
    68  	dists := make([]float32, results.Len())
    69  
    70  	// results is ordered in reverse, we need to flip the order before presenting
    71  	// to the user!
    72  	i := len(ids) - 1
    73  	for results.Len() > 0 {
    74  		res := results.Pop()
    75  		ids[i] = res.ID
    76  		dists[i] = res.Dist
    77  		i--
    78  	}
    79  
    80  	return ids, dists, nil
    81  }