go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/scheduler/appengine/task/gitiles/state_test.go (about)

     1  // Copyright 2017 The LUCI Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package gitiles
    16  
    17  import (
    18  	"context"
    19  	"crypto/sha256"
    20  	"fmt"
    21  	"strconv"
    22  	"testing"
    23  	"time"
    24  
    25  	"go.chromium.org/luci/common/tsmon"
    26  	"go.chromium.org/luci/common/tsmon/store"
    27  	"go.chromium.org/luci/common/tsmon/target"
    28  	"go.chromium.org/luci/common/tsmon/types"
    29  	"go.chromium.org/luci/gae/impl/memory"
    30  	ds "go.chromium.org/luci/gae/service/datastore"
    31  
    32  	. "github.com/smartystreets/goconvey/convey"
    33  )
    34  
    35  func TestLoadSave(t *testing.T) {
    36  	t.Parallel()
    37  
    38  	Convey("storeState/loadState work", t, func() {
    39  		c := memory.Use(context.Background())
    40  		repo := "https://example.googlesource.com/what/ever.git"
    41  		jobID := "job"
    42  
    43  		loadNoError := func(repo string) map[string]string {
    44  			r, err := loadState(c, jobID, repo)
    45  			if err != nil {
    46  				panic(err)
    47  			}
    48  			return r
    49  		}
    50  
    51  		Convey("load first time ever", func() {
    52  			So(loadNoError(repo), ShouldResemble, map[string]string{})
    53  			So(loadNoError(repo), ShouldNotBeNil)
    54  		})
    55  
    56  		Convey("save/load/save/load", func() {
    57  			So(saveState(c, jobID, repo, map[string]string{"refs/heads/master": "beefcafe"}), ShouldBeNil)
    58  			So(loadNoError(repo), ShouldResemble, map[string]string{"refs/heads/master": "beefcafe"})
    59  			So(saveState(c, jobID, repo, map[string]string{"refs/tails/master": "efacfeeb"}), ShouldBeNil)
    60  			So(loadNoError(repo), ShouldResemble, map[string]string{"refs/tails/master": "efacfeeb"})
    61  		})
    62  
    63  		Convey("save/change repo name/load", func() {
    64  			So(saveState(c, jobID, repo, map[string]string{"refs/heads/master": "beefcafe"}), ShouldBeNil)
    65  			So(loadNoError("https://some-other.googlesource.com/repo"), ShouldResemble, map[string]string{})
    66  		})
    67  
    68  		Convey("save/load with deeply nested refs", func() {
    69  			nested := map[string]string{
    70  				"refs/weirdo":                  "00",
    71  				"refs/heads/master":            "11",
    72  				"refs/heads/branch":            "22",
    73  				"refs/heads/infra/config":      "33",
    74  				"refs/heads/infra/deploy":      "44",
    75  				"refs/heads/infra/configs/why": "55",
    76  				"refs/heads/infra/configs/not": "66",
    77  			}
    78  			So(saveState(c, jobID, repo, nested), ShouldBeNil)
    79  			So(loadNoError(repo), ShouldResemble, nested)
    80  		})
    81  	})
    82  }
    83  
    84  func TestLoadSaveCompression(t *testing.T) {
    85  	t.Parallel()
    86  
    87  	Convey("Compress lots of similar refs", t, func() {
    88  		c := memory.Use(context.Background())
    89  		c, _, _ = tsmon.WithFakes(c)
    90  		tsmon.GetState(c).SetStore(store.NewInMemory(&target.Task{}))
    91  
    92  		repo := "https://example.googlesource.com/what/ever.git"
    93  		jobID := "job"
    94  
    95  		many16Ki := 16 * 1024
    96  		tags := make(map[string]string, many16Ki)
    97  		for i := 0; i < many16Ki; i++ {
    98  			ref := "refs/tags/" + strconv.FormatInt(int64(i), 20)
    99  			hsh := fmt.Sprintf("%x", sha256.Sum256([]byte(ref)))[:40]
   100  			tags[ref] = hsh
   101  		}
   102  
   103  		So(saveState(c, jobID, repo, tags), ShouldBeNil)
   104  		id, err := repositoryID(jobID, repo)
   105  		So(err, ShouldBeNil)
   106  		stored := Repository{ID: id}
   107  		So(ds.Get(c, &stored), ShouldBeNil)
   108  		// Given that SHA1 must have high entropy and hence shouldn't be
   109  		// compressible. Thus, we can't go below 20 bytes (len of SHA1) per ref,
   110  		// hence 20*many16Ki = 320 KiB.
   111  		So(len(stored.CompressedState), ShouldBeGreaterThan, 20*many16Ki)
   112  		// But refs themselves should be quite compressible.
   113  		So(len(stored.CompressedState), ShouldBeLessThan, 20*many16Ki*5/4)
   114  
   115  		So(getSentMetric(c, metricTaskGitilesStoredRefs, jobID), ShouldEqual, many16Ki)
   116  		So(getSentMetric(c, metricTaskGitilesStoredSize, jobID), ShouldEqual, len(stored.CompressedState))
   117  	})
   118  }
   119  
   120  // getSentMetric returns sent value or nil if value wasn't sent.
   121  func getSentMetric(c context.Context, m types.Metric, fieldVals ...any) any {
   122  	return tsmon.GetState(c).Store().Get(c, m, time.Time{}, fieldVals)
   123  }