github.com/GoogleCloudPlatform/testgrid@v0.0.174/pkg/api/v1/config_cache.go (about)

     1  /*
     2  Copyright 2022 The TestGrid Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  // Package v1 (api/v1) is the first versioned implementation of the API
    18  package v1
    19  
    20  import (
    21  	"context"
    22  	"errors"
    23  	"fmt"
    24  	"sync"
    25  	"time"
    26  
    27  	"github.com/sirupsen/logrus"
    28  
    29  	"github.com/GoogleCloudPlatform/testgrid/config"
    30  	"github.com/GoogleCloudPlatform/testgrid/config/snapshot"
    31  	"github.com/GoogleCloudPlatform/testgrid/util/gcs"
    32  )
    33  
    34  const (
    35  	reobservationTime = 2 * time.Minute
    36  	configFileName    = "config"
    37  )
    38  
    39  var (
    40  	errScopeNotProvided = errors.New("scope is not provided")
    41  )
    42  
    43  // Cached contains a config and a mapping of "normalized" names to actual resources.
    44  // Avoid using normalized names: normalization is costly and the raw name itself works better as a key.
    45  // Used within the API to sanitize inputs: ie. "dashboards/mytab" matches "My Tab"
    46  type cachedConfig struct {
    47  	Config               *snapshot.Config
    48  	NormalDashboardGroup map[string]string
    49  	NormalDashboard      map[string]string
    50  	NormalDashboardTab   map[string]map[string]string // Normal dashboard name AND normal tab name
    51  	NormalTestGroup      map[string]string
    52  	Mutex                sync.RWMutex
    53  }
    54  
    55  func (c *cachedConfig) generateNormalCache() {
    56  	if c == nil || c.Config == nil {
    57  		return
    58  	}
    59  
    60  	c.NormalDashboardGroup = make(map[string]string, len(c.Config.DashboardGroups))
    61  	c.NormalDashboard = make(map[string]string, len(c.Config.Dashboards))
    62  	c.NormalTestGroup = make(map[string]string, len(c.Config.Groups))
    63  	c.NormalDashboardTab = make(map[string]map[string]string, len(c.Config.Dashboards))
    64  
    65  	for name := range c.Config.DashboardGroups {
    66  		c.NormalDashboardGroup[config.Normalize(name)] = name
    67  	}
    68  
    69  	for name := range c.Config.Dashboards {
    70  		normalName := config.Normalize((name))
    71  		c.NormalDashboard[normalName] = name
    72  		c.NormalDashboardTab[normalName] = make(map[string]string, len(c.Config.Dashboards[name].DashboardTab))
    73  		for _, tab := range c.Config.Dashboards[name].DashboardTab {
    74  			normalTabName := config.Normalize((tab.Name))
    75  			c.NormalDashboardTab[normalName][normalTabName] = tab.Name
    76  		}
    77  	}
    78  
    79  	for name := range c.Config.Groups {
    80  		c.NormalTestGroup[config.Normalize(name)] = name
    81  	}
    82  }
    83  
    84  func (s *Server) configPath(scope string) (path *gcs.Path, isDefault bool, err error) {
    85  	if scope != "" {
    86  		path, err = gcs.NewPath(fmt.Sprintf("%s/%s", scope, configFileName))
    87  		return path, false, err
    88  	}
    89  	if s.DefaultBucket != "" {
    90  		path, err = gcs.NewPath(fmt.Sprintf("%s/%s", s.DefaultBucket, configFileName))
    91  		return path, true, err
    92  	}
    93  	return nil, false, errScopeNotProvided
    94  }
    95  
    96  // getConfig will return a config or an error. The config contains a mutex that you should RLock before reading.
    97  // Does not expose wrapped errors to the user, instead logging them to the console.
    98  func (s *Server) getConfig(ctx context.Context, log *logrus.Entry, scope string) (*cachedConfig, error) {
    99  	configPath, isDefault, err := s.configPath(scope)
   100  	if err != nil || configPath == nil {
   101  		return nil, errScopeNotProvided
   102  	}
   103  
   104  	if isDefault {
   105  		if s.defaultCache == nil || s.defaultCache.Config == nil {
   106  			log = log.WithField("config-path", configPath.String())
   107  			configChanged, err := snapshot.Observe(ctx, log, s.Client, *configPath, time.NewTicker(reobservationTime).C)
   108  			if err != nil {
   109  				log.WithError(err).Errorf("Can't read default config; check permissions")
   110  				return nil, fmt.Errorf("Could not read config at %q", configPath.String())
   111  			}
   112  
   113  			s.defaultCache = &cachedConfig{
   114  				Config: <-configChanged,
   115  			}
   116  			s.defaultCache.generateNormalCache()
   117  
   118  			log.Info("Observing default config")
   119  			go func(ctx context.Context) {
   120  				for {
   121  					select {
   122  					case newCfg := <-configChanged:
   123  						s.defaultCache.Mutex.Lock()
   124  						s.defaultCache.Config = newCfg
   125  						s.defaultCache.generateNormalCache()
   126  						log.Info("Observed config updated")
   127  						s.defaultCache.Mutex.Unlock()
   128  					case <-ctx.Done():
   129  						return
   130  					}
   131  				}
   132  			}(ctx)
   133  
   134  		}
   135  		return s.defaultCache, nil
   136  	}
   137  
   138  	cfgChan, err := snapshot.Observe(ctx, log, s.Client, *configPath, nil)
   139  	if err != nil {
   140  		// Do not log; invalid requests will write useless logs.
   141  		return nil, fmt.Errorf("Could not read config at %q", configPath.String())
   142  	}
   143  
   144  	result := cachedConfig{
   145  		Config: <-cfgChan,
   146  	}
   147  	result.generateNormalCache()
   148  	return &result, nil
   149  }