go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/lucicfg/configset.go

// Copyright 2018 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lucicfg

import (
	"bufio"
	"bytes"
	"context"
	"crypto/sha256"
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"

	"github.com/dustin/go-humanize"
	"github.com/klauspost/compress/gzip"
	"github.com/klauspost/compress/zlib"
	"golang.org/x/sync/errgroup"
	"google.golang.org/api/googleapi"
	"google.golang.org/grpc"
	"google.golang.org/grpc/status"

	legacy_config "go.chromium.org/luci/common/api/luci_config/config/v1"
	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/data/stringset"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/proto/config"
	"go.chromium.org/luci/common/sync/parallel"
	configpb "go.chromium.org/luci/config_service/proto"
)

// ConfigSet is an in-memory representation of a single config set.
type ConfigSet struct {
	// Name is a name of this config set, e.g. "projects/something".
	//
	// It is used by LUCI Config to figure out how to validate files in the set.
	Name string

	// Data is files belonging to the config set.
	//
	//  Keys are slash-separated filenames, values are corresponding file bodies.
	Data map[string][]byte
}

// AsOutput converts this config set into Output that have it at the given root
// path (usually ".").
func (cs ConfigSet) AsOutput(root string) Output {
	data := make(map[string]Datum, len(cs.Data))
	for k, v := range cs.Data {
		data[k] = BlobDatum(v)
	}
	return Output{
		Data:  data,
		Roots: map[string]string{cs.Name: root},
	}
}

// ValidationResult is what we get after validating a config set.
type ValidationResult struct {
	ConfigSet string                             `json:"config_set"`          // a config set being validated
	Failed    bool                               `json:"failed"`              // true if the config is bad
	Messages  []*config.ValidationResult_Message `json:"messages"`            // errors, warnings, infos, etc.
	RPCError  string                             `json:"rpc_error,omitempty"` // set if the RPC itself failed
}

// ConfigSetValidator is primarily implemented through config.Service, but can
// also be mocked in tests.
type ConfigSetValidator interface {
	// Validate sends the validation request to the service.
	//
	// Returns errors only on RPC errors. Actual validation errors are
	// communicated through []*config.ValidationResult_Message.
	Validate(ctx context.Context, cs ConfigSet) ([]*config.ValidationResult_Message, error)
}

type remoteValidator struct {
	cfgClient configpb.ConfigsClient
}

func NewRemoteValidator(conn *grpc.ClientConn) ConfigSetValidator {
	return &remoteValidator{
		cfgClient: configpb.NewConfigsClient(conn),
	}
}

// Validate implements ConfigSetValidator
func (r *remoteValidator) Validate(ctx context.Context, cs ConfigSet) ([]*config.ValidationResult_Message, error) {
	if len(cs.Data) == 0 {
		return nil, nil
	}
	validateReq := &configpb.ValidateConfigsRequest{
		ConfigSet:  cs.Name,
		FileHashes: make([]*configpb.ValidateConfigsRequest_FileHash, len(cs.Data)),
	}
	for i, file := range cs.Files() {
		content := cs.Data[file]
		h := sha256.New()
		h.Write(content)
		validateReq.FileHashes[i] = &configpb.ValidateConfigsRequest_FileHash{
			Path:   file,
			Sha256: hex.EncodeToString(h.Sum(nil)),
		}
	}
	res, err := r.cfgClient.ValidateConfigs(ctx, validateReq)
	switch fixInfo := findBadRequestFixInfo(err); {
	case fixInfo != nil:
		if err := uploadMissingFiles(ctx, cs, fixInfo.GetUploadFiles()); err != nil {
			return nil, err
		}
		validateReq.FileHashes = filterOutUnvalidatableFiles(ctx, validateReq.GetFileHashes(), fixInfo.GetUnvalidatableFiles())
		if len(validateReq.FileHashes) == 0 {
			logging.Debugf(ctx, "No config file need to be validated by LUCI Config")
			return nil, nil
		}
		switch res, err := r.cfgClient.ValidateConfigs(ctx, validateReq); { // now try again
		case err != nil:
			return nil, errors.Annotate(err, "failed to call LUCI Config").Err()
		default:
			return res.GetMessages(), nil
		}
	case err != nil:
		return nil, errors.Annotate(err, "failed to call LUCI Config").Err()
	default:
		return res.GetMessages(), nil
	}
}

func findBadRequestFixInfo(err error) *configpb.BadValidationRequestFixInfo {
	for _, detail := range status.Convert(err).Details() {
		switch t := detail.(type) {
		case *configpb.BadValidationRequestFixInfo:
			return t
		}
	}
	return nil
}

func filterOutUnvalidatableFiles(ctx context.Context,
	fileHashes []*configpb.ValidateConfigsRequest_FileHash,
	unvalidatableFiles []string) []*configpb.ValidateConfigsRequest_FileHash {
	if len(unvalidatableFiles) == 0 {
		return fileHashes
	}
	logging.Debugf(ctx, "No services can validate following files:\n  - %s", strings.Join(unvalidatableFiles, "\n  - "))
	unvalidatableFileSet := stringset.NewFromSlice(unvalidatableFiles...)
	ret := make([]*configpb.ValidateConfigsRequest_FileHash, 0, len(fileHashes))
	for _, fh := range fileHashes {
		if !unvalidatableFileSet.Has(fh.Path) {
			ret = append(ret, fh)
		}
	}
	return ret
}

func uploadMissingFiles(ctx context.Context, cs ConfigSet, uploadFiles []*configpb.BadValidationRequestFixInfo_UploadFile) error {
	if len(uploadFiles) == 0 {
		return nil
	}
	eg, ectx := errgroup.WithContext(ctx)
	for _, uf := range uploadFiles {
		uf := uf
		eg.Go(func() error {
			logging.Debugf(ectx, "Uploading file %q for validation", uf.GetPath())
			start := clock.Now(ctx)

			pr, pw := io.Pipe()

			// Read and gzip in background, writing the compressed data into the pipe.
			// Buffer writes, since gzip writer outputs often and writes to a pipe are
			// slow-ish if unbuffered.
			done := make(chan struct{})
			go func() (err error) {
				defer func() {
					_ = pw.CloseWithError(err)
					close(done)
				}()
				bw := bufio.NewWriterSize(pw, 1024*512)
				zw := gzip.NewWriter(bw)
				if _, err := zw.Write(cs.Data[uf.GetPath()]); err != nil {
					_ = zw.Close()
					return errors.Annotate(err, "failed to write gzip data").Err()
				}
				if err := zw.Close(); err != nil {
					return errors.Annotate(err, "failed to close gzip writer").Err()
				}
				if err := bw.Flush(); err != nil {
					return errors.Annotate(err, "failed to flush writer").Err()
				}
				return nil
			}()
			defer func() {
				_ = pr.Close() // unblocks writes in the goroutine, if still blocked
				<-done         // waits for the goroutine to finish running
			}()

			// Read from the pipe and upload.
			req, err := http.NewRequestWithContext(ectx, http.MethodPut, uf.GetSignedUrl(), pr)
			if err != nil {
				return errors.Annotate(err, "failed to create http request to upload file %q", uf.GetPath()).Err()
			}
			req.Header.Add("Content-Encoding", "gzip")
			req.Header.Add("x-goog-content-length-range", fmt.Sprintf("0,%d", uf.GetMaxConfigSize()))

			switch res, err := http.DefaultClient.Do(req); {
			case err != nil:
				return errors.Annotate(err, "failed to execute http request to upload file %q", uf.GetPath()).Err()

			case res.StatusCode != http.StatusOK:
				defer func() { _ = res.Body.Close() }()
				body, err := io.ReadAll(res.Body)
				if err != nil {
					return errors.Annotate(err, "failed to read response body").Err()
				}
				return errors.Reason("failed to upload file %q;  got http response code: %d, body: %s", uf.GetPath(), res.StatusCode, string(body)).Err()

			default:
				defer func() { _ = res.Body.Close() }()
				logging.Debugf(ectx, "Successfully uploaded file %q for validation in %s", uf.GetPath(), clock.Since(ctx, start))
				return nil
			}
		})
	}
	return eg.Wait()
}

type legacyRemoteValidator struct {
	validateConfig        func(context.Context, *legacy_config.LuciConfigValidateConfigRequestMessage) (*legacy_config.LuciConfigValidateConfigResponseMessage, error)
	requestSizeLimitBytes int64
}

func (r *legacyRemoteValidator) Validate(ctx context.Context, cs ConfigSet) ([]*config.ValidationResult_Message, error) {
	// Sort by size, smaller first, to group small files in a single request.
	files := make([]*legacy_config.LuciConfigValidateConfigRequestMessageFile, 0, len(cs.Data))
	for path, content := range cs.Data {
		files = append(files, &legacy_config.LuciConfigValidateConfigRequestMessageFile{
			Path:    path,
			Content: base64.StdEncoding.EncodeToString(content),
		})
	}
	sort.Slice(files, func(i, j int) bool {
		if len(files[i].Content) == len(files[j].Content) {
			return strings.Compare(files[i].Path, files[j].Path) < 0
		}
		return len(files[i].Content) < len(files[j].Content)
	})

	// Split all files into a bunch of smallish validation requests to avoid
	// hitting 32MB request size limit.
	var (
		requests []*legacy_config.LuciConfigValidateConfigRequestMessage
		curFiles []*legacy_config.LuciConfigValidateConfigRequestMessageFile
		curSize  int64
	)
	flush := func() {
		if len(curFiles) > 0 {
			requests = append(requests, &legacy_config.LuciConfigValidateConfigRequestMessage{
				ConfigSet: cs.Name,
				Files:     curFiles,
			})
		}
		curFiles = nil
		curSize = 0
	}
	for _, f := range files {
		switch contentSize := int64(len(f.Content)); {
		case contentSize > r.requestSizeLimitBytes:
			return nil, errors.Reason("the size of file %q is %s that is exceeding the limit of %s", f.Path, humanize.Bytes(uint64(contentSize)), humanize.Bytes(uint64(r.requestSizeLimitBytes))).Err()
		case curSize+contentSize > r.requestSizeLimitBytes:
			flush()
			fallthrough
		default:
			curFiles = append(curFiles, f)
			curSize += int64(len(f.Content))
		}
	}
	flush()

	var (
		lock     sync.Mutex
		messages []*config.ValidationResult_Message
	)

	// Execute all requests in parallel.
	err := parallel.FanOutIn(func(gen chan<- func() error) {
		for _, req := range requests {
			req := req
			gen <- func() error {
				resp, err := r.validateConfig(ctx, req)
				if resp != nil {
					lock.Lock()
					for _, msg := range resp.Messages {
						if val, ok := config.ValidationResult_Severity_value[strings.ToUpper(msg.Severity)]; ok {
							messages = append(messages, &config.ValidationResult_Message{
								Path:     msg.Path,
								Severity: config.ValidationResult_Severity(val),
								Text:     msg.Text,
							})
						} else {
							logging.Warningf(ctx, "unknown severity %q; full msg: %+v", msg.Severity, msg)
						}
					}
					lock.Unlock()
				}
				return err
			}
		}
	})

	// Sort messages by path for determinism.
	sort.Slice(messages, func(i, j int) bool {
		return messages[i].Path < messages[j].Path
	})

	return messages, err
}

// LegacyRemoteValidator returns ConfigSetValidator that makes RPCs to legacy
// LUCI Config service.
func LegacyRemoteValidator(client *http.Client, host string) ConfigSetValidator {
	validateURL := fmt.Sprintf("https://%s/_ah/api/config/v1/validate-config", host)
	return &legacyRemoteValidator{
		// 160 MiB is picked because compression is done before sending the final
		// request and the real request size limit is 32 MiB. Since config is
		// highly repetitive content, it should easily achieve 5:1 compression
		// ratio.
		requestSizeLimitBytes: 160 * 1024 * 1024,
		validateConfig: func(ctx context.Context, req *legacy_config.LuciConfigValidateConfigRequestMessage) (*legacy_config.LuciConfigValidateConfigResponseMessage, error) {

			debug := make([]string, len(req.Files))
			for i, f := range req.Files {
				debug[i] = fmt.Sprintf("%s (%s)", f.Path, humanize.Bytes(uint64(len(f.Content))))
			}
			logging.Debugf(ctx, "Sending request to %s to validate %d files: %s",
				validateURL,
				len(req.Files),
				strings.Join(debug, ", "),
			)

			var body bytes.Buffer
			zlibWriter := zlib.NewWriter(&body)
			if err := json.NewEncoder(zlibWriter).Encode(req); err != nil {
				return nil, errors.Annotate(err, "failed to encode the request").Err()
			}
			if err := zlibWriter.Close(); err != nil {
				return nil, errors.Annotate(err, "failed to close the zlib stream").Err()
			}
			httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, validateURL, &body)
			if err != nil {
				return nil, errors.Annotate(err, "failed to create a new request").Err()
			}
			httpReq.Header.Add("Content-Type", `application/json-zlib`)
			httpReq.Header.Add("User-Agent", UserAgent)

			res, err := client.Do(httpReq)
			if err != nil {
				return nil, errors.Annotate(err, "failed to execute HTTP request").Err()
			}
			defer func() { _ = res.Body.Close() }()
			if res.StatusCode < 200 || res.StatusCode > 299 {
				return nil, googleapi.CheckResponse(res)
			}
			ret := &legacy_config.LuciConfigValidateConfigResponseMessage{
				ServerResponse: googleapi.ServerResponse{
					Header:         res.Header,
					HTTPStatusCode: res.StatusCode,
				},
			}
			if err := json.NewDecoder(res.Body).Decode(&ret); err != nil {
				return nil, err
			}
			return ret, nil
		},
	}
}

// ReadConfigSet reads all regular files in the given directory (recursively)
// and returns them as a ConfigSet with given name.
func ReadConfigSet(dir, name string) (ConfigSet, error) {
	configs := map[string][]byte{}
	err := filepath.Walk(dir, func(p string, info os.FileInfo, err error) error {
		if err != nil || !info.Mode().IsRegular() {
			return err
		}
		content, err := os.ReadFile(p)
		if err != nil {
			return err
		}
		relPath, err := filepath.Rel(dir, p)
		if err != nil {
			return err
		}
		configs[filepath.ToSlash(relPath)] = content
		return nil
	})
	if err != nil {
		return ConfigSet{}, errors.Annotate(err, "failed to read config files").Err()
	}
	return ConfigSet{
		Name: name,
		Data: configs,
	}, nil
}

// Files returns a sorted list of file names in the config set.
func (cs ConfigSet) Files() []string {
	f := make([]string, 0, len(cs.Data))
	for k := range cs.Data {
		f = append(f, k)
	}
	sort.Strings(f)
	return f
}

// Validate sends the config set for validation to LUCI Config service.
//
// Returns ValidationResult with a list of validation message (errors, warnings,
// etc). The list of messages may be empty if the config set is 100% valid.
//
// If the RPC call itself failed, ValidationResult is still returned, but it has
// only ConfigSet and RPCError fields populated.
func (cs ConfigSet) Validate(ctx context.Context, val ConfigSetValidator) *ValidationResult {
	logging.Infof(ctx, "Sending to LUCI Config for validation as config set %q:", cs.Name)
	for _, f := range cs.Files() {
		logging.Infof(ctx, "  %s (%s)", f, humanize.Bytes(uint64(len(cs.Data[f]))))
	}

	messages, err := val.Validate(ctx, cs)
	res := &ValidationResult{
		ConfigSet: cs.Name,
		Messages:  messages,
	}
	if err != nil {
		res.RPCError = err.Error()
		res.Failed = true
	}
	return res
}

// Format formats the validation result as a multi-line string
func (vr *ValidationResult) Format() string {
	buf := bytes.Buffer{}
	for _, msg := range vr.Messages {
		fmt.Fprintf(&buf, "%s: %s: %s: %s\n", msg.Severity, vr.ConfigSet, msg.Path, msg.Text)
	}
	return buf.String()
}

// OverallError is nil if the validation succeeded or non-nil if failed.
//
// Beware: mutates Failed field accordingly.
func (vr *ValidationResult) OverallError(failOnWarnings bool) error {
	errs, warns := 0, 0
	for _, msg := range vr.Messages {
		switch msg.Severity {
		case config.ValidationResult_WARNING:
			warns++
		case config.ValidationResult_ERROR, config.ValidationResult_CRITICAL:
			errs++
		}
	}

	switch {
	case errs > 0:
		vr.Failed = true
		return errors.Reason("some files were invalid").Err()
	case warns > 0 && failOnWarnings:
		vr.Failed = true
		return errors.Reason("some files had validation warnings and -fail-on-warnings is set").Err()
	case vr.RPCError != "":
		vr.Failed = true
		return errors.Reason("failed to send RPC to LUCI Config - %s", vr.RPCError).Err()
	}

	vr.Failed = false
	return nil
}