go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/bisection/bqexporter/ensure_views.go (about)

     1  // Copyright 2023 The LUCI Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package bqexporter
    16  
    17  import (
    18  	"context"
    19  	"strings"
    20  	"time"
    21  
    22  	"cloud.google.com/go/bigquery"
    23  	"google.golang.org/api/iterator"
    24  
    25  	"go.chromium.org/luci/bisection/util/bqutil"
    26  	"go.chromium.org/luci/common/bq"
    27  	"go.chromium.org/luci/common/errors"
    28  	"go.chromium.org/luci/common/logging"
    29  	"go.chromium.org/luci/gae/service/info"
    30  )
    31  
    32  type makeTableMetadata func(luciProject string) *bigquery.TableMetadata
    33  
    34  var luciProjectViewQueries = map[string]makeTableMetadata{
    35  	"test_failure_analyses": func(luciProject string) *bigquery.TableMetadata {
    36  		return &bigquery.TableMetadata{
    37  			ViewQuery: `SELECT * FROM internal.test_failure_analyses WHERE project = "` + luciProject + `"`,
    38  			Labels:    map[string]string{bq.MetadataVersionKey: "1"},
    39  		}
    40  	},
    41  }
    42  
    43  func EnsureViews(ctx context.Context) error {
    44  	if !isEnabled(ctx) {
    45  		logging.Warningf(ctx, "ensure view is not enabled")
    46  	}
    47  
    48  	client, err := bqutil.Client(ctx, info.AppID(ctx))
    49  	if err != nil {
    50  		return errors.Annotate(err, "create bq client").Err()
    51  	}
    52  	defer client.Close()
    53  
    54  	if err := ensureViews(ctx, client); err != nil {
    55  		return errors.Annotate(err, "ensure view").Err()
    56  	}
    57  	return nil
    58  }
    59  
    60  func ensureViews(ctx context.Context, bqClient *bigquery.Client) error {
    61  	// Get datasets for LUCI projects.
    62  	datasetIDs, err := projectDatasets(ctx, bqClient)
    63  	if err != nil {
    64  		return errors.Annotate(err, "get LUCI project datasets").Err()
    65  	}
    66  	// Create views that is common to each LUCI project's dataset.
    67  	for _, projectDatasetID := range datasetIDs {
    68  		if err := createViewsForLUCIDataset(ctx, bqClient, projectDatasetID); err != nil {
    69  			return errors.Annotate(err, "ensure view for LUCI project dataset %s", projectDatasetID).Err()
    70  		}
    71  	}
    72  	return nil
    73  }
    74  
    75  // createViewsForLUCIDataset creates views with the given tableSpecs under the given datasetID
    76  func createViewsForLUCIDataset(ctx context.Context, bqClient *bigquery.Client, datasetID string) error {
    77  	luciProject, err := bqutil.ProjectForDataset(datasetID)
    78  	if err != nil {
    79  		return errors.Annotate(err, "get LUCI project with dataset name %s", datasetID).Err()
    80  	}
    81  	for tableName, specFunc := range luciProjectViewQueries {
    82  		table := bqClient.Dataset(datasetID).Table(tableName)
    83  		spec := specFunc(luciProject)
    84  		if err := bq.EnsureTable(ctx, table, spec, bq.UpdateMetadata(), bq.RefreshViewInterval(time.Hour)); err != nil {
    85  			return errors.Annotate(err, "ensure view %s", tableName).Err()
    86  		}
    87  	}
    88  	return nil
    89  }
    90  
    91  // projectDatasets returns all project datasets in the GCP Project.
    92  // E.g. "chromium", "chrome", ....
    93  func projectDatasets(ctx context.Context, bqClient *bigquery.Client) ([]string, error) {
    94  	var datasets []string
    95  	di := bqClient.Datasets(ctx)
    96  	for {
    97  		d, err := di.Next()
    98  		if err == iterator.Done {
    99  			break
   100  		} else if err != nil {
   101  			return nil, err
   102  		}
   103  		// The internal dataset is a special dataset that does
   104  		// not belong to a LUCI project.
   105  		if strings.EqualFold(d.DatasetID, bqutil.InternalDatasetID) {
   106  			continue
   107  		}
   108  		datasets = append(datasets, d.DatasetID)
   109  	}
   110  	return datasets, nil
   111  }