github.com/GoogleCloudPlatform/testgrid@v0.0.174/cmd/config_merger/main.go (about)

     1  /*
     2  Copyright 2021 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package main
    18  
    19  import (
    20  	"context"
    21  	"flag"
    22  	"io/ioutil"
    23  	"net/http"
    24  	"time"
    25  
    26  	"github.com/GoogleCloudPlatform/testgrid/pkg/merger"
    27  	"github.com/GoogleCloudPlatform/testgrid/util/gcs"
    28  	"github.com/GoogleCloudPlatform/testgrid/util/metrics/prometheus"
    29  
    30  	"github.com/sirupsen/logrus"
    31  )
    32  
    33  const componentName = "config-merger"
    34  
    35  type options struct {
    36  	listPath     string
    37  	listURL      string
    38  	creds        string
    39  	confirm      bool
    40  	wait         time.Duration
    41  	skipValidate bool
    42  }
    43  
    44  func (o *options) validate(log logrus.FieldLogger) {
    45  	if o.listPath == "" && o.listURL == "" {
    46  		log.Fatal("List of configurations to merge required (--config-list or --config-url)")
    47  	}
    48  	if !o.confirm {
    49  		log.Info("--confirm=false (DRY-RUN): will not write to gcs")
    50  	}
    51  	if o.skipValidate {
    52  		log.Info("--allow-invalid-configs: result may not validate either")
    53  	}
    54  }
    55  
    56  func gatherOptions() options {
    57  	var o options
    58  	flag.StringVar(&o.listPath, "config-list", "", "List of configurations to merge (at file)")
    59  	flag.StringVar(&o.listURL, "config-url", "", "List of configurations to merge (at web URL)")
    60  	flag.StringVar(&o.creds, "gcp-service-account", "", "/path/to/gcp/creds (use local creds if empty)")
    61  	flag.BoolVar(&o.confirm, "confirm", false, "Upload data if set")
    62  	flag.DurationVar(&o.wait, "wait", 0, "Ensure at least this much time ahs passed since the last loop. (Run only once if zero)")
    63  	flag.BoolVar(&o.skipValidate, "allow-invalid-configs", false, "Allows merging of configs that don't validate. Usually skips invalid configs")
    64  	flag.Parse()
    65  	return o
    66  }
    67  
    68  func main() {
    69  	log := logrus.WithField("component", componentName)
    70  	opt := gatherOptions()
    71  	opt.validate(log)
    72  
    73  	var file []byte
    74  
    75  	if opt.listPath != "" {
    76  		var err error
    77  		file, err = ioutil.ReadFile(opt.listPath)
    78  		if err != nil {
    79  			log.WithField("--config-list", opt.listPath).WithError(err).Fatalf("Can't find --config-list")
    80  		}
    81  	}
    82  
    83  	if opt.listURL != "" {
    84  		resp, err := http.Get(opt.listURL)
    85  		if err != nil {
    86  			log.WithField("--config-url", opt.listURL).WithError(err).Fatalf("Can't GET --config-url")
    87  		}
    88  		defer resp.Body.Close()
    89  		file, err = ioutil.ReadAll(resp.Body)
    90  		if err != nil {
    91  			log.WithField("--config-url", opt.listURL).WithError(err).Fatalf("Can't read contents at --config-url")
    92  		}
    93  	}
    94  
    95  	list, err := merger.ParseAndCheck(file)
    96  	if err != nil {
    97  		log.WithError(err).Fatal("Can't parse YAML merge config")
    98  	}
    99  
   100  	log.WithField("merge-list", list).Debug("YAML mergelist read successful")
   101  
   102  	ctx, cancel := context.WithCancel(context.Background())
   103  	defer cancel()
   104  	storageClient, err := gcs.ClientWithCreds(ctx, opt.creds)
   105  	if err != nil {
   106  		log.WithError(err).Fatalf("Can't make storage client")
   107  	}
   108  
   109  	client := gcs.NewClient(storageClient)
   110  
   111  	mets := merger.CreateMetrics(prometheus.NewFactory())
   112  
   113  	updateOnce := func(ctx context.Context) {
   114  		ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
   115  		defer cancel()
   116  		log.Info("Starting MergeAndUpdate")
   117  		_, err := merger.MergeAndUpdate(ctx, client, mets, list, opt.skipValidate, opt.confirm)
   118  		if err != nil {
   119  			log.WithError(err).Error("Update failed")
   120  			return
   121  		}
   122  	}
   123  
   124  	updateOnce(ctx)
   125  	if opt.wait == 0 {
   126  		return
   127  	}
   128  	timer := time.NewTimer(opt.wait)
   129  	defer timer.Stop()
   130  	for range timer.C {
   131  		timer.Reset(opt.wait)
   132  		updateOnce(ctx)
   133  		log.WithField("--wait", opt.wait).Info("Sleeping")
   134  	}
   135  }