go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/analysis/internal/changepoints/bqexporter/client.go (about)

     1  // Copyright 2023 The LUCI Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package bqexporter
    16  
    17  import (
    18  	"context"
    19  	"fmt"
    20  
    21  	"cloud.google.com/go/bigquery"
    22  	"cloud.google.com/go/bigquery/storage/managedwriter"
    23  	"google.golang.org/protobuf/proto"
    24  
    25  	"go.chromium.org/luci/common/errors"
    26  
    27  	"go.chromium.org/luci/analysis/internal/bqutil"
    28  	bqpb "go.chromium.org/luci/analysis/proto/bq"
    29  )
    30  
    31  // NewClient creates a new client for exporting test variant branches
    32  // via the BigQuery Write API.
    33  // projectID is the project ID of the GCP project.
    34  func NewClient(ctx context.Context, projectID string) (s *Client, reterr error) {
    35  	if projectID == "" {
    36  		return nil, errors.New("GCP Project must be specified")
    37  	}
    38  
    39  	bqClient, err := bqutil.Client(ctx, projectID)
    40  	if err != nil {
    41  		return nil, errors.Annotate(err, "creating BQ client").Err()
    42  	}
    43  	defer func() {
    44  		if reterr != nil {
    45  			bqClient.Close()
    46  		}
    47  	}()
    48  
    49  	mwClient, err := bqutil.NewWriterClient(ctx, projectID)
    50  	if err != nil {
    51  		return nil, errors.Annotate(err, "create managed writer client").Err()
    52  	}
    53  	return &Client{
    54  		projectID: projectID,
    55  		bqClient:  bqClient,
    56  		mwClient:  mwClient,
    57  	}, nil
    58  }
    59  
    60  // Close releases resources held by the client.
    61  func (s *Client) Close() (reterr error) {
    62  	// Ensure both bqClient and mwClient Close() methods
    63  	// are called, even if one panics or fails.
    64  	defer func() {
    65  		err := s.mwClient.Close()
    66  		if reterr == nil {
    67  			reterr = err
    68  		}
    69  	}()
    70  	return s.bqClient.Close()
    71  }
    72  
    73  // Client provides methods to export clustered failures to BigQuery
    74  // via the BigQuery Write API.
    75  type Client struct {
    76  	// projectID is the name of the GCP project that contains LUCI Analysis datasets.
    77  	projectID string
    78  	bqClient  *bigquery.Client
    79  	mwClient  *managedwriter.Client
    80  }
    81  
    82  func (s *Client) ensureSchema(ctx context.Context) error {
    83  	table := s.bqClient.Dataset(bqutil.InternalDatasetID).Table(updatesTableName)
    84  	if err := schemaApplyer.EnsureTable(ctx, table, tableMetadata); err != nil {
    85  		return errors.Annotate(err, "ensuring test_variant_segment_updates table").Err()
    86  	}
    87  	return nil
    88  }
    89  
    90  // Insert inserts the given rows in BigQuery.
    91  func (s *Client) Insert(ctx context.Context, rows []*bqpb.TestVariantBranchRow) error {
    92  	if err := s.ensureSchema(ctx); err != nil {
    93  		return errors.Annotate(err, "ensure schema").Err()
    94  	}
    95  	tableName := fmt.Sprintf("projects/%s/datasets/%s/tables/%s", s.projectID, bqutil.InternalDatasetID, updatesTableName)
    96  	writer := bqutil.NewWriter(s.mwClient, tableName, tableSchemaDescriptor)
    97  	payload := make([]proto.Message, len(rows))
    98  	for i, r := range rows {
    99  		payload[i] = r
   100  	}
   101  	return writer.AppendRowsWithDefaultStream(ctx, payload)
   102  }