github.com/instill-ai/component@v0.16.0-beta/pkg/operator/text/v0/split_test.go (about)

     1  package text
     2  
     3  import (
     4  	"testing"
     5  
     6  	"google.golang.org/protobuf/types/known/structpb"
     7  )
     8  
     9  // TestSplitByToken tests the split by token task
    10  func TestSplitByToken(t *testing.T) {
    11  	input := &structpb.Struct{
    12  		Fields: map[string]*structpb.Value{
    13  			"text":  {Kind: &structpb.Value_StringValue{StringValue: "Hello world. This is a test."}},
    14  			"model": {Kind: &structpb.Value_StringValue{StringValue: "gpt-3.5-turbo"}},
    15  		},
    16  	}
    17  	inputs := []*structpb.Struct{
    18  		input,
    19  	}
    20  
    21  	e := &execution{}
    22  	e.Task = "TASK_SPLIT_BY_TOKEN"
    23  
    24  	if _, err := e.Execute(inputs); err != nil {
    25  		t.Fatalf("splitByToken returned an error: %v", err)
    26  	}
    27  }