github.com/balzaczyy/golucene@v0.0.0-20151210033525-d0be9ee89713/core/index/docValuesWriter.go (about)

     1  package index
     2  
     3  import (
     4  	. "github.com/balzaczyy/golucene/core/codec/spi"
     5  	. "github.com/balzaczyy/golucene/core/index/model"
     6  	"github.com/balzaczyy/golucene/core/util"
     7  	"github.com/balzaczyy/golucene/core/util/packed"
     8  )
     9  
    10  type DocValuesWriter interface {
    11  	finish(int)
    12  	flush(*SegmentWriteState, DocValuesConsumer) error
    13  }
    14  
    15  // index/NumericDocValuesWriter.java
    16  
    17  const MISSING int64 = 0
    18  
    19  /* Buffers up pending long per doc, then flushes when segment flushes. */
    20  type NumericDocValuesWriter struct {
    21  	pending       packed.PackedLongValuesBuilder
    22  	iwBytesUsed   util.Counter
    23  	bytesUsed     int64
    24  	docsWithField *util.FixedBitSet
    25  	fieldInfo     *FieldInfo
    26  }
    27  
    28  func newNumericDocValuesWriter(fieldInfo *FieldInfo,
    29  	iwBytesUsed util.Counter, trackDocsWithField bool) *NumericDocValuesWriter {
    30  	ans := &NumericDocValuesWriter{
    31  		fieldInfo:   fieldInfo,
    32  		iwBytesUsed: iwBytesUsed,
    33  	}
    34  	if trackDocsWithField {
    35  		ans.docsWithField = util.NewFixedBitSetOf(64)
    36  	}
    37  	ans.pending = packed.DeltaPackedBuilder(packed.PackedInts.COMPACT)
    38  	ans.bytesUsed = ans.pending.RamBytesUsed() + ans.docsWithFieldBytesUsed()
    39  	ans.iwBytesUsed.AddAndGet(ans.bytesUsed)
    40  	return ans
    41  }
    42  
    43  func (w *NumericDocValuesWriter) addValue(docId int, value int64) {
    44  	assert2(int64(docId) >= w.pending.Size(),
    45  		"DocValuesField '%v' appears more than once in this document (only one value is allowed per field)",
    46  		w.fieldInfo.Name)
    47  
    48  	// Fill in any holes
    49  	for i := int(w.pending.Size()); i < docId; i++ {
    50  		w.pending.Add(MISSING)
    51  	}
    52  
    53  	w.pending.Add(value)
    54  	if w.docsWithField != nil {
    55  		w.docsWithField = util.EnsureFixedBitSet(w.docsWithField, docId)
    56  		w.docsWithField.Set(docId)
    57  	}
    58  
    59  	w.updateBytesUsed()
    60  }
    61  
    62  func (w *NumericDocValuesWriter) docsWithFieldBytesUsed() int64 {
    63  	// size of the []int64 + some overhead
    64  	if w.docsWithField == nil {
    65  		return 0
    66  	}
    67  	return util.SizeOf(w.docsWithField.Bits()) + 64
    68  }
    69  
    70  func (w *NumericDocValuesWriter) updateBytesUsed() {
    71  	newBytesUsed := w.pending.RamBytesUsed() + w.docsWithFieldBytesUsed()
    72  	w.iwBytesUsed.AddAndGet(newBytesUsed - w.bytesUsed)
    73  	w.bytesUsed = newBytesUsed
    74  }
    75  
    76  func (w *NumericDocValuesWriter) finish(numDoc int) {}
    77  
    78  func (w *NumericDocValuesWriter) flush(state *SegmentWriteState,
    79  	dvConsumer DocValuesConsumer) error {
    80  
    81  	maxDoc := state.SegmentInfo.DocCount()
    82  	values := w.pending.Build()
    83  	dvConsumer.AddNumericField(w.fieldInfo, func() func() (interface{}, bool) {
    84  		return newNumericIterator(maxDoc, values, w.docsWithField)
    85  	})
    86  	return nil
    87  }
    88  
    89  /* Iterates over the values we have in ram */
    90  type NumericIterator struct{}
    91  
    92  func newNumericIterator(maxDoc int,
    93  	values packed.PackedLongValues,
    94  	docsWithFields *util.FixedBitSet) func() (interface{}, bool) {
    95  
    96  	upto, size := 0, int(values.Size())
    97  	iter := values.Iterator()
    98  	return func() (interface{}, bool) {
    99  		if upto >= maxDoc {
   100  			return nil, false
   101  		}
   102  		var value interface{}
   103  		if upto < size {
   104  			v, _ := iter()
   105  			if docsWithFields == nil || docsWithFields.At(upto) {
   106  				value = v
   107  			} else {
   108  				value = nil
   109  			}
   110  		} else if docsWithFields != nil {
   111  			value = nil
   112  		} else {
   113  			value = MISSING
   114  		}
   115  		upto++
   116  		return value, true
   117  	}
   118  }