go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/analysis/app/buildbucket.go (about)

     1  // Copyright 2023 The LUCI Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package app
    16  
    17  import (
    18  	"context"
    19  	"encoding/json"
    20  	"net/http"
    21  
    22  	"google.golang.org/protobuf/encoding/protojson"
    23  
    24  	buildbucketpb "go.chromium.org/luci/buildbucket/proto"
    25  	"go.chromium.org/luci/common/errors"
    26  	"go.chromium.org/luci/common/logging"
    27  	"go.chromium.org/luci/common/tsmon/field"
    28  	"go.chromium.org/luci/common/tsmon/metric"
    29  	"go.chromium.org/luci/server/router"
    30  
    31  	"go.chromium.org/luci/analysis/internal/services/buildjoiner"
    32  	"go.chromium.org/luci/analysis/internal/tasks/taskspb"
    33  )
    34  
    35  var (
    36  	buildCounter = metric.NewCounter(
    37  		"analysis/ingestion/pubsub/buildbucket_builds",
    38  		"The number of buildbucket builds received by LUCI Analysis from PubSub.",
    39  		nil,
    40  		// The LUCI Project.
    41  		field.String("project"),
    42  		// "success", "ignored", "transient-failure" or "permanent-failure".
    43  		field.String("status"))
    44  )
    45  
    46  // BuildbucketPubSubHandler accepts and process buildbucket v2 Pub/Sub messages.
    47  // LUCI Analysis ingests buildbucket builds upon completion, with the
    48  // caveat that for builds related to CV runs, we also wait for the
    49  // CV run to complete (via CV Pub/Sub).
    50  func BuildbucketPubSubHandler(ctx *router.Context) {
    51  	project := "unknown"
    52  	status := "unknown"
    53  	defer func() {
    54  		// Closure for late binding.
    55  		buildCounter.Add(ctx.Request.Context(), 1, project, status)
    56  	}()
    57  
    58  	project, processed, err := bbPubSubHandlerImpl(ctx.Request.Context(), ctx.Request)
    59  	if err != nil {
    60  		errors.Log(ctx.Request.Context(), errors.Annotate(err, "handling buildbucket pubsub event").Err())
    61  		status = processErr(ctx, err)
    62  		return
    63  	}
    64  	if processed {
    65  		status = "success"
    66  		// Use subtly different "success" response codes to surface in
    67  		// standard GAE logs whether an ingestion was ignored or not,
    68  		// while still acknowledging the pub/sub.
    69  		// See https://cloud.google.com/pubsub/docs/push#receiving_messages.
    70  		ctx.Writer.WriteHeader(http.StatusOK)
    71  	} else {
    72  		status = "ignored"
    73  		ctx.Writer.WriteHeader(http.StatusNoContent) // 204
    74  	}
    75  }
    76  
    77  func bbPubSubHandlerImpl(ctx context.Context, request *http.Request) (project string, processed bool, err error) {
    78  	var psMsg pubsubMessage
    79  	if err := json.NewDecoder(request.Body).Decode(&psMsg); err != nil {
    80  		return "unknown", false, errors.Annotate(err, "could not decode buildbucket pubsub message").Err()
    81  	}
    82  	// Handle message from the builds (v2) topic.
    83  	msg, err := parseBBV2Message(ctx, psMsg)
    84  	if err != nil {
    85  		return "unknown", false, errors.Annotate(err, "unmarshal buildbucket v2 pub/sub message").Err()
    86  	}
    87  	processed, err = processBBV2Message(ctx, msg)
    88  	if err != nil {
    89  		return msg.Build.Builder.Project, false, errors.Annotate(err, "process buildbucket v2 build").Err()
    90  	}
    91  	return msg.Build.Builder.Project, processed, nil
    92  }
    93  
    94  func parseBBV2Message(ctx context.Context, pbMsg pubsubMessage) (*buildbucketpb.BuildsV2PubSub, error) {
    95  	buildsV2Msg := &buildbucketpb.BuildsV2PubSub{}
    96  	opts := protojson.UnmarshalOptions{AllowPartial: true, DiscardUnknown: true}
    97  	if err := opts.Unmarshal(pbMsg.Message.Data, buildsV2Msg); err != nil {
    98  		return nil, err
    99  	}
   100  	// Optional: implement decompression of large fields. As we don't need build.input,
   101  	// build.output or build.steps here, we omit it.
   102  	// See https://source.chromium.org/chromium/infra/infra/+/main:go/src/go.chromium.org/luci/luci_notify/notify/pubsub.go;l=442;drc=2ed735a67ecfe6a824076d231a4c7268b84e8e95;bpv=0
   103  	// for an example.
   104  	return buildsV2Msg, nil
   105  }
   106  
   107  func processBBV2Message(ctx context.Context, message *buildbucketpb.BuildsV2PubSub) (processed bool, err error) {
   108  	if message.Build.Status&buildbucketpb.Status_ENDED_MASK != buildbucketpb.Status_ENDED_MASK {
   109  		// Received build that hasn't completed yet, ignore it.
   110  		return false, nil
   111  	}
   112  	if message.Build.Infra.GetBuildbucket().GetHostname() == "" {
   113  		// Invalid build. Ignore.
   114  		logging.Warningf(ctx, "Build %v did not specify buildbucket hostname, ignoring.", message.Build.Id)
   115  		return false, nil
   116  	}
   117  
   118  	project := message.Build.Builder.Project
   119  	task := &taskspb.JoinBuild{
   120  		Project: project,
   121  		Id:      message.Build.Id,
   122  		Host:    message.Build.Infra.Buildbucket.Hostname,
   123  	}
   124  	err = buildjoiner.Schedule(ctx, task)
   125  	if err != nil {
   126  		return false, err
   127  	}
   128  	return true, nil
   129  }