go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/led/ledcmd/consolidate_isolateds.go (about)

     1  // Copyright 2020 The LUCI Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package ledcmd
    16  
    17  import (
    18  	"context"
    19  	"fmt"
    20  	"io/ioutil"
    21  	"os"
    22  
    23  	"go.chromium.org/luci/auth"
    24  	"go.chromium.org/luci/client/casclient"
    25  	"go.chromium.org/luci/common/errors"
    26  	"go.chromium.org/luci/common/logging"
    27  	"go.chromium.org/luci/led/job"
    28  	swarmingpb "go.chromium.org/luci/swarming/proto/api_v2"
    29  )
    30  
    31  // ConsolidateRbeCasSources combines RBE-CAS inputs in slice.Properties.CasInputRoot
    32  // and CasUserPayload for swarming tasks. For the same file, the one in CasUserPayload
    33  // will replace the one in slice.Properties.CasInputRoot.
    34  func ConsolidateRbeCasSources(ctx context.Context, authOpts auth.Options, jd *job.Definition) error {
    35  	if jd.GetSwarming() == nil || (jd.GetSwarming().CasUserPayload.GetDigest().GetHash() == "") {
    36  		return nil
    37  	}
    38  	logging.Infof(ctx, "consolidating RBE-CAS sources...")
    39  	tdir, err := ioutil.TempDir("", "led-consolidate-rbe-cas")
    40  	if err != nil {
    41  		return errors.Annotate(err, "failed to create tempdir in consolidation step").Err()
    42  	}
    43  	defer func() {
    44  		if err = os.RemoveAll(tdir); err != nil {
    45  			logging.Errorf(ctx, "failed to cleanup temp dir %q: %s", tdir, err)
    46  		}
    47  	}()
    48  	casClient, err := casclient.NewLegacy(ctx, casclient.AddrProd, jd.GetSwarming().CasUserPayload.CasInstance, authOpts, false)
    49  	if err != nil {
    50  		return err
    51  	}
    52  	defer casClient.Close()
    53  
    54  	for i, slc := range jd.GetSwarming().GetTask().GetTaskSlices() {
    55  		if slc.Properties == nil {
    56  			slc.Properties = &swarmingpb.TaskProperties{}
    57  		}
    58  		props := slc.Properties
    59  		if props.CasInputRoot == nil || props.CasInputRoot.Digest.GetHash() == jd.GetSwarming().CasUserPayload.Digest.Hash {
    60  			continue
    61  		}
    62  
    63  		subDir := fmt.Sprintf("%s/%d", tdir, i)
    64  		if err := downloadFromCas(ctx, props.CasInputRoot, casClient, subDir); err != nil {
    65  			return errors.Annotate(err, "consolidation").Err()
    66  		}
    67  		if err := downloadFromCas(ctx, jd.GetSwarming().CasUserPayload, casClient, subDir); err != nil {
    68  			return errors.Annotate(err, "consolidation").Err()
    69  		}
    70  		casRef, err := uploadToCas(ctx, casClient, subDir)
    71  		if err != nil {
    72  			return errors.Annotate(err, "consolidation").Err()
    73  		}
    74  		props.CasInputRoot.Digest = casRef.GetDigest()
    75  	}
    76  	if jd.GetSwarming().CasUserPayload != nil {
    77  		jd.GetSwarming().CasUserPayload.Digest = nil
    78  	}
    79  	return nil
    80  }