go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/analysis/internal/testverdicts/schema.go (about)

     1  // Copyright 2023 The LUCI Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package testverdicts
    16  
    17  import (
    18  	"time"
    19  
    20  	"cloud.google.com/go/bigquery"
    21  	"cloud.google.com/go/bigquery/storage/managedwriter/adapt"
    22  	"github.com/golang/protobuf/descriptor"
    23  	desc "github.com/golang/protobuf/protoc-gen-go/descriptor"
    24  	"google.golang.org/protobuf/types/descriptorpb"
    25  
    26  	"go.chromium.org/luci/analysis/internal/bqutil"
    27  	bqpb "go.chromium.org/luci/analysis/proto/bq"
    28  	pb "go.chromium.org/luci/analysis/proto/v1"
    29  )
    30  
    31  // tableName is the name of the exported BigQuery table.
    32  const tableName = "test_verdicts"
    33  
    34  const partitionExpirationTime = 540 * 24 * time.Hour // 540 days (~1.5 years).
    35  
    36  const rowMessage = "luci.analysis.bq.TestVerdictRow"
    37  
    38  var tableMetadata *bigquery.TableMetadata
    39  
    40  // tableSchemaDescriptor is a self-contained DescriptorProto for describing
    41  // row protocol buffers sent to the BigQuery Write API.
    42  var tableSchemaDescriptor *descriptorpb.DescriptorProto
    43  
    44  func init() {
    45  	var err error
    46  	var schema bigquery.Schema
    47  	if schema, err = generateRowSchema(); err != nil {
    48  		panic(err)
    49  	}
    50  	if tableSchemaDescriptor, err = generateRowSchemaDescriptor(); err != nil {
    51  		panic(err)
    52  	}
    53  
    54  	tableMetadata = &bigquery.TableMetadata{
    55  		TimePartitioning: &bigquery.TimePartitioning{
    56  			Type:       bigquery.DayPartitioningType,
    57  			Expiration: partitionExpirationTime,
    58  			Field:      "partition_time",
    59  		},
    60  		Clustering: &bigquery.Clustering{
    61  			Fields: []string{"project", "test_id"},
    62  		},
    63  		// Relax ensures no fields are marked "required".
    64  		Schema: schema.Relax(),
    65  	}
    66  }
    67  
    68  func generateRowSchema() (schema bigquery.Schema, err error) {
    69  	fd, _ := descriptor.MessageDescriptorProto(&bqpb.TestVerdictRow{})
    70  	// We also need to get FileDescriptorProto for StringPair, FailureReason,
    71  	// Sources, GitilesCommit, Changelist, TestMetadata, TestLocation and
    72  	// BugTrackingComponent.
    73  	fdsp, _ := descriptor.MessageDescriptorProto(&pb.StringPair{})
    74  	fdfr, _ := descriptor.MessageDescriptorProto(&pb.FailureReason{})
    75  	fdsrc, _ := descriptor.MessageDescriptorProto(&pb.Sources{})
    76  	fdgc, _ := descriptor.MessageDescriptorProto(&pb.GitilesCommit{})
    77  	fdcl, _ := descriptor.MessageDescriptorProto(&pb.Changelist{})
    78  	fdtmd, _ := descriptor.MessageDescriptorProto(&pb.TestMetadata{})
    79  	fdtl, _ := descriptor.MessageDescriptorProto(&pb.TestLocation{})
    80  	fdbtc, _ := descriptor.MessageDescriptorProto(&pb.BugTrackingComponent{})
    81  	fdset := &desc.FileDescriptorSet{File: []*desc.FileDescriptorProto{
    82  		fd, fdsp, fdfr, fdsrc, fdgc, fdcl, fdtmd, fdtl, fdbtc,
    83  	}}
    84  	return bqutil.GenerateSchema(fdset, rowMessage)
    85  }
    86  
    87  func generateRowSchemaDescriptor() (*desc.DescriptorProto, error) {
    88  	m := &bqpb.TestVerdictRow{}
    89  	descriptorProto, err := adapt.NormalizeDescriptor(m.ProtoReflect().Descriptor())
    90  	if err != nil {
    91  		return nil, err
    92  	}
    93  	return descriptorProto, nil
    94  }