go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/cipkg/base/generators/url.go (about)

     1  // Copyright 2023 The LUCI Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //	http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package generators
    16  
    17  import (
    18  	"context"
    19  	"crypto/sha256"
    20  	"encoding/base32"
    21  	"fmt"
    22  	"io/fs"
    23  	"strings"
    24  
    25  	"go.chromium.org/luci/cipkg/core"
    26  	luciproto "go.chromium.org/luci/common/proto"
    27  
    28  	"google.golang.org/protobuf/proto"
    29  )
    30  
    31  type FetchURL struct {
    32  	URL           string
    33  	Mode          fs.FileMode
    34  	HashAlgorithm core.HashAlgorithm
    35  	HashValue     string
    36  }
    37  
    38  // FetchURLs downloads files from servers based on the path-url pairs of URLs.
    39  type FetchURLs struct {
    40  	Name     string
    41  	Metadata *core.Action_Metadata
    42  	URLs     map[string]FetchURL
    43  }
    44  
    45  func (f *FetchURLs) Generate(ctx context.Context, plats Platforms) (*core.Action, error) {
    46  	var deps []*core.Action
    47  
    48  	// Generate separate action for every url.
    49  	files := make(map[string]*core.ActionFilesCopy_Source)
    50  	for k, v := range f.URLs {
    51  		spec := &core.ActionURLFetch{
    52  			Url:           v.URL,
    53  			HashAlgorithm: v.HashAlgorithm,
    54  			HashValue:     v.HashValue,
    55  		}
    56  
    57  		// Truncate the id to save some characters for windows because this id will
    58  		// be used as part of the path. 32^6 = 2^30 should be good enough.
    59  		id, err := stableID(spec, 6)
    60  		if err != nil {
    61  			return nil, err
    62  		}
    63  		name := fmt.Sprintf("%s_%s", f.Name, id)
    64  
    65  		deps = append(deps, &core.Action{
    66  			Name: name,
    67  			Spec: &core.Action_Url{Url: spec},
    68  		})
    69  
    70  		m := v.Mode
    71  		if m == 0 {
    72  			m = 0o666
    73  		}
    74  		files[k] = &core.ActionFilesCopy_Source{
    75  			Content: &core.ActionFilesCopy_Source_Output_{
    76  				Output: &core.ActionFilesCopy_Source_Output{Name: name, Path: "file"},
    77  			},
    78  			Mode: uint32(m),
    79  		}
    80  	}
    81  
    82  	// TODO: Sort deps
    83  	return &core.Action{
    84  		Name:     f.Name,
    85  		Metadata: f.Metadata,
    86  		Deps:     deps,
    87  		Spec: &core.Action_Copy{
    88  			Copy: &core.ActionFilesCopy{
    89  				Files: files,
    90  			},
    91  		},
    92  	}, nil
    93  }
    94  
    95  func stableID(m proto.Message, n uint) (string, error) {
    96  	h := sha256.New()
    97  	if err := luciproto.StableHash(h, m); err != nil {
    98  		return "", err
    99  	}
   100  	enc := base32.HexEncoding.WithPadding(base32.NoPadding)
   101  	return strings.ToLower(enc.EncodeToString(h.Sum(nil)[:n])), nil
   102  }