github.com/instill-ai/component@v0.16.0-beta/pkg/connector/website/v0/main.go (about)

     1  //go:generate compogen readme --connector ./config ./README.mdx
     2  package website
     3  
     4  import (
     5  	_ "embed"
     6  	"fmt"
     7  	"sync"
     8  
     9  	"go.uber.org/zap"
    10  	"google.golang.org/protobuf/types/known/structpb"
    11  
    12  	"github.com/instill-ai/component/pkg/base"
    13  )
    14  
    15  const (
    16  	taskScrapeWebsite = "TASK_SCRAPE_WEBSITE"
    17  )
    18  
    19  var (
    20  	//go:embed config/definition.json
    21  	definitionJSON []byte
    22  	//go:embed config/tasks.json
    23  	tasksJSON []byte
    24  
    25  	once sync.Once
    26  	con  *connector
    27  )
    28  
    29  type connector struct {
    30  	base.BaseConnector
    31  }
    32  
    33  type execution struct {
    34  	base.BaseConnectorExecution
    35  }
    36  
    37  func Init(l *zap.Logger, u base.UsageHandler) *connector {
    38  	once.Do(func() {
    39  		con = &connector{
    40  			BaseConnector: base.BaseConnector{
    41  				Logger:       l,
    42  				UsageHandler: u,
    43  			},
    44  		}
    45  		err := con.LoadConnectorDefinition(definitionJSON, tasksJSON, nil)
    46  		if err != nil {
    47  			panic(err)
    48  		}
    49  	})
    50  	return con
    51  }
    52  
    53  func (c *connector) CreateExecution(sysVars map[string]any, connection *structpb.Struct, task string) (*base.ExecutionWrapper, error) {
    54  	return &base.ExecutionWrapper{Execution: &execution{
    55  		BaseConnectorExecution: base.BaseConnectorExecution{Connector: c, SystemVariables: sysVars, Connection: connection, Task: task},
    56  	}}, nil
    57  }
    58  
    59  func (e *execution) Execute(inputs []*structpb.Struct) ([]*structpb.Struct, error) {
    60  	outputs := []*structpb.Struct{}
    61  
    62  	for _, input := range inputs {
    63  		switch e.Task {
    64  		case taskScrapeWebsite:
    65  			inputStruct := ScrapeWebsiteInput{}
    66  			err := base.ConvertFromStructpb(input, &inputStruct)
    67  			if err != nil {
    68  				return nil, err
    69  			}
    70  
    71  			outputStruct, err := Scrape(inputStruct)
    72  			if err != nil {
    73  				return nil, err
    74  			}
    75  			output, err := base.ConvertToStructpb(outputStruct)
    76  			if err != nil {
    77  				return nil, err
    78  			}
    79  			outputs = append(outputs, output)
    80  		default:
    81  			return nil, fmt.Errorf("not supported task: %s", e.Task)
    82  		}
    83  	}
    84  
    85  	return outputs, nil
    86  }
    87  
    88  func (c *connector) Test(sysVars map[string]any, connection *structpb.Struct) error {
    89  	return nil
    90  }