github.com/balzaczyy/golucene@v0.0.0-20151210033525-d0be9ee89713/analysis/util/filtering.go (about)

     1  package util
     2  
     3  import (
     4  	. "github.com/balzaczyy/golucene/core/analysis"
     5  	. "github.com/balzaczyy/golucene/core/analysis/tokenattributes"
     6  	"github.com/balzaczyy/golucene/core/util"
     7  )
     8  
     9  // util/FilteringTokenFilter.java
    10  
    11  type FilteringTokenFilterSPI interface {
    12  	// Override this method and return if the current input token
    13  	// should be returned by IncrementToken().
    14  	Accept() bool
    15  }
    16  
    17  /*
    18  Abstract base class for TokenFilters that may remove tokens. You have
    19  to implement Accept() and return a boolean if the current token
    20  should be preserved. IncrementToken() uses this method to decide if a
    21  token should be passed to the caller.
    22  
    23  As for Lucene 4.4, GoLucene would panic when trying to disable
    24  position increments when filtering terms.
    25  */
    26  type FilteringTokenFilter struct {
    27  	*TokenFilter
    28  	spi                      FilteringTokenFilterSPI
    29  	input                    TokenStream
    30  	version                  util.Version
    31  	posIncrAtt               PositionIncrementAttribute
    32  	enablePositionIncrements bool
    33  	first                    bool
    34  	skippedPositions         int
    35  }
    36  
    37  /* Creates a new FilteringTokenFilter. */
    38  func NewFilteringTokenFilter(spi FilteringTokenFilterSPI,
    39  	version util.Version, in TokenStream) *FilteringTokenFilter {
    40  	ans := &FilteringTokenFilter{
    41  		spi:         spi,
    42  		input:       in,
    43  		TokenFilter: NewTokenFilter(in),
    44  		version:     version,
    45  		first:       true,
    46  	}
    47  	ans.posIncrAtt = ans.Attributes().Add("PositionIncrementAttribute").(PositionIncrementAttribute)
    48  	ans.enablePositionIncrements = true
    49  	return ans
    50  }
    51  
    52  func (f *FilteringTokenFilter) IncrementToken() (bool, error) {
    53  	if f.enablePositionIncrements {
    54  		f.skippedPositions = 0
    55  		ok, err := f.input.IncrementToken()
    56  		if err != nil {
    57  			return false, err
    58  		}
    59  		for ok {
    60  			if f.spi.Accept() {
    61  				if f.skippedPositions != 0 {
    62  					f.posIncrAtt.SetPositionIncrement(f.posIncrAtt.PositionIncrement() + f.skippedPositions)
    63  				}
    64  				return true, nil
    65  			}
    66  			f.skippedPositions += f.posIncrAtt.PositionIncrement()
    67  
    68  			ok, err = f.input.IncrementToken()
    69  			if err != nil {
    70  				return false, err
    71  			}
    72  		}
    73  	} else {
    74  		ok, err := f.input.IncrementToken()
    75  		if err != nil {
    76  			return false, err
    77  		}
    78  		for ok {
    79  			if f.spi.Accept() {
    80  				if f.first {
    81  					// first token having posinc=0 is illegal.
    82  					if f.posIncrAtt.PositionIncrement() == 0 {
    83  						f.posIncrAtt.SetPositionIncrement(1)
    84  					}
    85  					f.first = false
    86  				}
    87  				return true, nil
    88  			}
    89  
    90  			ok, err = f.input.IncrementToken()
    91  			if err != nil {
    92  				return false, err
    93  			}
    94  		}
    95  	}
    96  	// reached EOS -- return false
    97  	return false, nil
    98  }
    99  
   100  func (f *FilteringTokenFilter) Reset() error {
   101  	err := f.TokenFilter.Reset()
   102  	if err == nil {
   103  		f.first = true
   104  		f.skippedPositions = 0
   105  	}
   106  	return err
   107  }
   108  
   109  func (f *FilteringTokenFilter) End() error {
   110  	err := f.TokenFilter.End()
   111  	if err == nil && f.enablePositionIncrements {
   112  		f.posIncrAtt.SetPositionIncrement(f.posIncrAtt.PositionIncrement() + f.skippedPositions)
   113  	}
   114  	return err
   115  }