github.com/GoogleCloudPlatform/testgrid@v0.0.174/config/snapshot/config_snapshot.go (about)

     1  /*
     2  Copyright 2022 The TestGrid Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package snapshot
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"time"
    23  
    24  	"cloud.google.com/go/storage"
    25  	"github.com/GoogleCloudPlatform/testgrid/config"
    26  	configpb "github.com/GoogleCloudPlatform/testgrid/pb/config"
    27  	"github.com/GoogleCloudPlatform/testgrid/util/gcs"
    28  	"github.com/sethvargo/go-retry"
    29  	"github.com/sirupsen/logrus"
    30  )
    31  
    32  // Config is a mapped representation of the config at a particular point in time.
    33  // Not concurrency-safe; meant for reading only
    34  type Config struct {
    35  	DashboardGroups map[string]*configpb.DashboardGroup
    36  	Dashboards      map[string]*configpb.Dashboard
    37  	Groups          map[string]*configpb.TestGroup
    38  	Attrs           storage.ReaderObjectAttrs
    39  }
    40  
    41  // Observe reads the config at configPath and return a ConfigSnapshot initially and, if a ticker is supplied, when the config changes
    42  // Returns an error instead if there is no file at configPath
    43  func Observe(ctx context.Context, log logrus.FieldLogger, client gcs.ConditionalClient, configPath gcs.Path, ticker <-chan time.Time) (<-chan *Config, error) {
    44  	ch := make(chan *Config)
    45  	if log == nil {
    46  		log = logrus.New()
    47  	}
    48  	log = log.WithField("observed-path", configPath.String())
    49  
    50  	initialSnap, err := updateHash(ctx, client, configPath)
    51  	if err != nil {
    52  		return nil, fmt.Errorf("can't read %q: %w", configPath.String(), err)
    53  	}
    54  	cond := storage.Conditions{
    55  		GenerationNotMatch: initialSnap.Attrs.Generation,
    56  	}
    57  	client = client.If(&cond, nil)
    58  
    59  	go func() {
    60  		defer close(ch)
    61  		ch <- initialSnap
    62  		if ticker == nil {
    63  			return
    64  		}
    65  	nextTick:
    66  		for {
    67  			select {
    68  			case <-ctx.Done():
    69  				return
    70  			case <-ticker:
    71  				snap, err := updateHash(ctx, client, configPath)
    72  				if err != nil {
    73  					if !gcs.IsPreconditionFailed(err) {
    74  						log.WithError(err).Warning("Error fetching updated config")
    75  					}
    76  					continue nextTick
    77  				}
    78  				// Configuration changed
    79  				select {
    80  				case <-ctx.Done():
    81  					return
    82  				case ch <- snap:
    83  					cond.GenerationNotMatch = snap.Attrs.Generation
    84  				}
    85  			}
    86  
    87  		}
    88  	}()
    89  
    90  	return ch, nil
    91  }
    92  
    93  func updateHash(ctx context.Context, client gcs.Opener, configPath gcs.Path) (*Config, error) {
    94  	var cs Config
    95  	cfg, attrs, err := fetchConfig(ctx, client, configPath)
    96  	if err != nil {
    97  		return nil, err
    98  	}
    99  
   100  	namedDashboardGroups := make(map[string]*configpb.DashboardGroup, len(cfg.DashboardGroups))
   101  	for _, dg := range cfg.DashboardGroups {
   102  		namedDashboardGroups[dg.Name] = dg
   103  	}
   104  	namedDashboards := make(map[string]*configpb.Dashboard, len(cfg.Dashboards))
   105  	for _, d := range cfg.Dashboards {
   106  		namedDashboards[d.Name] = d
   107  	}
   108  	namedGroups := make(map[string]*configpb.TestGroup, len(cfg.TestGroups))
   109  	for _, tg := range cfg.TestGroups {
   110  		namedGroups[tg.Name] = tg
   111  	}
   112  
   113  	cs.DashboardGroups = namedDashboardGroups
   114  	cs.Dashboards = namedDashboards
   115  	cs.Groups = namedGroups
   116  	if attrs != nil {
   117  		cs.Attrs = *attrs
   118  	}
   119  	return &cs, nil
   120  }
   121  
   122  func fetchConfig(ctx context.Context, client gcs.Opener, configPath gcs.Path) (*configpb.Configuration, *storage.ReaderObjectAttrs, error) {
   123  	backoff := retry.WithMaxRetries(2, retry.NewExponential(5*time.Second))
   124  
   125  	var cfg *configpb.Configuration
   126  	var attrs *storage.ReaderObjectAttrs
   127  	err := retry.Do(ctx, backoff, func(innerCtx context.Context) error {
   128  		var onceErr error
   129  		cfg, attrs, onceErr = fetchConfigOnce(innerCtx, client, configPath)
   130  		if onceErr != nil {
   131  			return retry.RetryableError(onceErr)
   132  		}
   133  		return nil
   134  	})
   135  	return cfg, attrs, err
   136  }
   137  
   138  func fetchConfigOnce(ctx context.Context, client gcs.Opener, configPath gcs.Path) (*configpb.Configuration, *storage.ReaderObjectAttrs, error) {
   139  	r, attrs, err := client.Open(ctx, configPath)
   140  	if err != nil {
   141  		return nil, nil, fmt.Errorf("open: %w", err)
   142  	}
   143  
   144  	cfg, err := config.Unmarshal(r)
   145  	if err != nil {
   146  		return nil, nil, fmt.Errorf("unmarshal: %v", err)
   147  	}
   148  	return cfg, attrs, nil
   149  }