gitlab.com/thomasboni/go-enry/v2@v2.8.3-0.20220418031202-30b0d7a3de98/internal/tokenizer/common.go (about)

     1  // Package tokenizer implements file tokenization used by the enry content
     2  // classifier. This package is an implementation detail of enry and should not
     3  // be imported by other packages.
     4  package tokenizer
     5  
     6  // ByteLimit defines the maximum prefix of an input text that will be tokenized.
     7  const ByteLimit = 100000