github.com/weaviate/weaviate@v1.24.6/modules/generative-openai/clients/openai_tokens.go (about)

     1  //                           _       _
     2  // __      _____  __ ___   ___  __ _| |_ ___
     3  // \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
     4  //  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
     5  //   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
     6  //
     7  //  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
     8  //
     9  //  CONTACT: hello@weaviate.io
    10  //
    11  
    12  package clients
    13  
    14  import (
    15  	"fmt"
    16  	"strings"
    17  
    18  	"github.com/pkoukk/tiktoken-go"
    19  )
    20  
    21  func getTokensCount(model string, messages []message) (int, error) {
    22  	tke, err := tiktoken.EncodingForModel(model)
    23  	if err != nil {
    24  		return 0, fmt.Errorf("encoding for model %s: %w", model, err)
    25  	}
    26  
    27  	tokensPerMessage := 3
    28  	if strings.HasPrefix(model, "gpt-3.5-turbo") {
    29  		tokensPerMessage = 4
    30  	}
    31  
    32  	tokensPerName := 1
    33  	if strings.HasPrefix(model, "gpt-3.5-turbo") {
    34  		tokensPerName = -1
    35  	}
    36  
    37  	tokensCount := 3
    38  	for _, message := range messages {
    39  		tokensCount += tokensPerMessage
    40  		tokensCount += len(tke.Encode(message.Role, nil, nil))
    41  		tokensCount += len(tke.Encode(message.Content, nil, nil))
    42  		if message.Name != "" {
    43  			tokensCount += tokensPerName
    44  			tokensCount += len(tke.Encode(message.Name, nil, nil))
    45  		}
    46  	}
    47  	return tokensCount, nil
    48  }