istio.io/istio@v0.0.0-20240520182934-d79c90f27776/pkg/test/framework/components/echo/deployment/builder.go (about)

     1  // Copyright Istio Authors
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package deployment
    16  
    17  import (
    18  	"context"
    19  	"fmt"
    20  	"strings"
    21  	"sync"
    22  	"time"
    23  
    24  	"github.com/google/go-cmp/cmp"
    25  	"github.com/hashicorp/go-multierror"
    26  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    27  
    28  	"istio.io/api/annotation"
    29  	"istio.io/istio/pkg/kube/inject"
    30  	"istio.io/istio/pkg/test"
    31  	"istio.io/istio/pkg/test/framework/components/cluster"
    32  	"istio.io/istio/pkg/test/framework/components/echo"
    33  	"istio.io/istio/pkg/test/framework/components/echo/kube"
    34  	_ "istio.io/istio/pkg/test/framework/components/echo/staticvm" // force registraton of factory func
    35  	"istio.io/istio/pkg/test/framework/components/istio"
    36  	"istio.io/istio/pkg/test/framework/components/namespace"
    37  	"istio.io/istio/pkg/test/framework/resource"
    38  	"istio.io/istio/pkg/test/framework/resource/config/apply"
    39  	"istio.io/istio/pkg/test/scopes"
    40  	"istio.io/istio/pkg/util/sets"
    41  )
    42  
    43  // Builder for a group of collaborating Echo Instances. Once built, all Instances in the
    44  // group:
    45  //
    46  //  1. Are ready to receive traffic, and
    47  //  2. Can call every other Instance in the group (i.e. have received Envoy config
    48  //     from Pilot).
    49  //
    50  // If a test needs to verify that one Instance is NOT reachable from another, there are
    51  // a couple of options:
    52  //
    53  //  1. Build a group while all Instances ARE reachable. Then apply a policy
    54  //     disallowing the communication.
    55  //  2. Build the source and destination Instances in separate groups and then
    56  //     call `source.WaitUntilCallable(destination)`.
    57  type Builder interface {
    58  	// With adds a new Echo configuration to the Builder. Once built, the instance
    59  	// pointer will be updated to point at the new Instance.
    60  	With(i *echo.Instance, cfg echo.Config) Builder
    61  
    62  	// WithConfig mimics the behavior of With, but does not allow passing a reference
    63  	// and returns an echoboot builder rather than a generic echo builder.
    64  	// TODO rename this to With, and the old method to WithInstance
    65  	WithConfig(cfg echo.Config) Builder
    66  
    67  	// WithClusters will cause subsequent With or WithConfig calls to be applied to the given clusters.
    68  	WithClusters(...cluster.Cluster) Builder
    69  
    70  	// Build and initialize all Echo Instances. Upon returning, the Instance pointers
    71  	// are assigned and all Instances are ready to communicate with each other.
    72  	Build() (echo.Instances, error)
    73  	BuildOrFail(t test.Failer) echo.Instances
    74  }
    75  
    76  var _ Builder = builder{}
    77  
    78  // New builder for echo deployments.
    79  func New(ctx resource.Context, clusters ...cluster.Cluster) Builder {
    80  	// use all workload clusters unless otherwise specified
    81  	if len(clusters) == 0 {
    82  		clusters = ctx.Clusters()
    83  	}
    84  	b := builder{
    85  		ctx:        ctx,
    86  		configs:    map[cluster.Kind][]echo.Config{},
    87  		refs:       map[cluster.Kind][]*echo.Instance{},
    88  		namespaces: map[string]namespace.Instance{},
    89  	}
    90  	templates, err := b.injectionTemplates()
    91  	if err != nil {
    92  		// deal with this when we call Build() to avoid making the New signature unwieldy
    93  		b.errs = multierror.Append(b.errs, fmt.Errorf("failed finding injection templates on clusters %v", err))
    94  	}
    95  	b.templates = templates
    96  
    97  	return b.WithClusters(clusters...)
    98  }
    99  
   100  type builder struct {
   101  	ctx resource.Context
   102  
   103  	// clusters contains the current set of clusters that subsequent With calls will be applied to,
   104  	// if the Config passed to With does not explicitly choose a cluster.
   105  	clusters cluster.Clusters
   106  
   107  	// configs contains configurations to be built, expanded per-cluster and grouped by cluster Kind.
   108  	configs map[cluster.Kind][]echo.Config
   109  	// refs contains the references to assign built Instances to.
   110  	// The length of each refs slice should match the length of the corresponding cluster slice.
   111  	// Only the first per-cluster entry for a given config should have a non-nil ref.
   112  	refs map[cluster.Kind][]*echo.Instance
   113  	// namespaces caches namespaces by their prefix; used for converting Static namespace from configs into actual
   114  	// namespaces
   115  	namespaces map[string]namespace.Instance
   116  	// the set of injection templates for each cluster
   117  	templates map[string]sets.String
   118  	// errs contains a multierror for failed validation during With calls
   119  	errs error
   120  }
   121  
   122  func (b builder) WithConfig(cfg echo.Config) Builder {
   123  	return b.With(nil, cfg).(builder)
   124  }
   125  
   126  // With adds a new Echo configuration to the Builder. When a cluster is provided in the Config, it will only be applied
   127  // to that cluster, otherwise the Config is applied to all WithClusters. Once built, if being built for a single cluster,
   128  // the instance pointer will be updated to point at the new Instance.
   129  func (b builder) With(i *echo.Instance, cfg echo.Config) Builder {
   130  	if b.ctx.Settings().SkipWorkloadClassesAsSet().Contains(cfg.WorkloadClass()) {
   131  		return b
   132  	}
   133  
   134  	cfg = cfg.DeepCopy()
   135  	if err := cfg.FillDefaults(b.ctx); err != nil {
   136  		b.errs = multierror.Append(b.errs, err)
   137  		return b
   138  	}
   139  
   140  	shouldSkip := b.ctx.Settings().Skip(cfg.WorkloadClass())
   141  	if shouldSkip {
   142  		return b
   143  	}
   144  
   145  	// cache the namespace, so manually added echo.Configs can be a part of it
   146  	b.namespaces[cfg.Namespace.Prefix()] = cfg.Namespace
   147  
   148  	targetClusters := b.clusters
   149  	if cfg.Cluster != nil {
   150  		targetClusters = cluster.Clusters{cfg.Cluster}
   151  	}
   152  
   153  	deployedTo := 0
   154  	for idx, c := range targetClusters {
   155  		ec, ok := c.(echo.Cluster)
   156  		if !ok {
   157  			b.errs = multierror.Append(b.errs, fmt.Errorf("attempted to deploy to %s but it does not implement echo.Cluster", c.Name()))
   158  			continue
   159  		}
   160  		perClusterConfig, ok := ec.CanDeploy(cfg)
   161  		if !ok {
   162  			continue
   163  		}
   164  		if !b.validateTemplates(perClusterConfig, c) {
   165  			if c.Kind() == cluster.Kubernetes {
   166  				scopes.Framework.Warnf("%s does not contain injection templates for %s; skipping deployment", c.Name(), perClusterConfig.ClusterLocalFQDN())
   167  			}
   168  			// Don't error out when injection template missing.
   169  			shouldSkip = true
   170  			continue
   171  		}
   172  
   173  		var ref *echo.Instance
   174  		if idx == 0 {
   175  			// ref only applies to the first cluster deployed to
   176  			// refs shouldn't be used when deploying to multiple targetClusters
   177  			// TODO: should we just panic if a ref is passed in a multi-cluster context?
   178  			ref = i
   179  		}
   180  		perClusterConfig = perClusterConfig.DeepCopy()
   181  		k := ec.Kind()
   182  		perClusterConfig.Cluster = ec
   183  		b.configs[k] = append(b.configs[k], perClusterConfig)
   184  		b.refs[k] = append(b.refs[k], ref)
   185  		deployedTo++
   186  	}
   187  
   188  	if deployedTo == 0 && !shouldSkip {
   189  		b.errs = multierror.Append(b.errs, fmt.Errorf("no clusters were eligible for app %s", cfg.Service))
   190  	}
   191  
   192  	return b
   193  }
   194  
   195  // WithClusters will cause subsequent With calls to be applied to the given clusters.
   196  func (b builder) WithClusters(clusters ...cluster.Cluster) Builder {
   197  	next := b
   198  	next.clusters = clusters
   199  	return next
   200  }
   201  
   202  func (b builder) Build() (out echo.Instances, err error) {
   203  	return build(b)
   204  }
   205  
   206  // injectionTemplates lists the set of templates for each Kube cluster
   207  func (b builder) injectionTemplates() (map[string]sets.String, error) {
   208  	ns := "istio-system"
   209  	i, err := istio.Get(b.ctx)
   210  	if err != nil {
   211  		scopes.Framework.Infof("defaulting to istio-system namespace for injection template discovery: %v", err)
   212  	} else {
   213  		ns = i.Settings().SystemNamespace
   214  	}
   215  
   216  	out := map[string]sets.String{}
   217  	for _, c := range b.ctx.Clusters().Kube() {
   218  		out[c.Name()] = sets.New[string]()
   219  		// TODO find a place to read revision(s) and avoid listing
   220  		cms, err := c.Kube().CoreV1().ConfigMaps(ns).List(context.TODO(), metav1.ListOptions{})
   221  		if err != nil {
   222  			return nil, err
   223  		}
   224  
   225  		// take the intersection of the templates available from each revision in this cluster
   226  		intersection := sets.New[string]()
   227  		for _, item := range cms.Items {
   228  			if !strings.HasPrefix(item.Name, "istio-sidecar-injector") {
   229  				continue
   230  			}
   231  			data, err := inject.UnmarshalConfig([]byte(item.Data["config"]))
   232  			if err != nil {
   233  				return nil, fmt.Errorf("failed parsing injection cm in %s: %v", c.Name(), err)
   234  			}
   235  			if data.RawTemplates != nil {
   236  				t := sets.New[string]()
   237  				for name := range data.RawTemplates {
   238  					t.Insert(name)
   239  				}
   240  				// either intersection has not been set or we intersect these templates
   241  				// with the current set.
   242  				if intersection.IsEmpty() {
   243  					intersection = t
   244  				} else {
   245  					intersection = intersection.Intersection(t)
   246  				}
   247  			}
   248  		}
   249  		for name := range intersection {
   250  			out[c.Name()].Insert(name)
   251  		}
   252  	}
   253  
   254  	return out, nil
   255  }
   256  
   257  // build inner allows assigning to b (assignment to receiver would be ineffective)
   258  func build(b builder) (out echo.Instances, err error) {
   259  	start := time.Now()
   260  	scopes.Framework.Info("=== BEGIN: Deploy echo instances ===")
   261  	defer func() {
   262  		if err != nil {
   263  			scopes.Framework.Error("=== FAILED: Deploy echo instances ===")
   264  			scopes.Framework.Error(err)
   265  		} else {
   266  			scopes.Framework.Infof("=== SUCCEEDED: Deploy echo instances in %v ===", time.Since(start))
   267  		}
   268  	}()
   269  
   270  	// load additional configs
   271  	for _, cfg := range *additionalConfigs {
   272  		// swap the namespace.Static for a namespace.kube
   273  		b, cfg.Namespace = b.getOrCreateNamespace(cfg.Namespace.Prefix())
   274  		// register the extra config
   275  		b = b.WithConfig(cfg).(builder)
   276  	}
   277  
   278  	// bail early if there were issues during the configuration stage
   279  	if b.errs != nil {
   280  		return nil, b.errs
   281  	}
   282  
   283  	if err = b.deployServices(); err != nil {
   284  		return
   285  	}
   286  	if out, err = b.deployInstances(); err != nil {
   287  		return
   288  	}
   289  	return
   290  }
   291  
   292  func (b builder) getOrCreateNamespace(prefix string) (builder, namespace.Instance) {
   293  	ns, ok := b.namespaces[prefix]
   294  	if ok {
   295  		return b, ns
   296  	}
   297  	ns, err := namespace.New(b.ctx, namespace.Config{Prefix: prefix, Inject: true})
   298  	if err != nil {
   299  		b.errs = multierror.Append(b.errs, err)
   300  	}
   301  	b.namespaces[prefix] = ns
   302  	return b, ns
   303  }
   304  
   305  // deployServices deploys the kubernetes Service to all clusters. Multicluster meshes should have "sameness"
   306  // per cluster. This avoids concurrent writes later.
   307  func (b builder) deployServices() (err error) {
   308  	services := make(map[string]string)
   309  	for _, cfgs := range b.configs {
   310  		for _, cfg := range cfgs {
   311  			svc, err := kube.GenerateService(cfg)
   312  			if err != nil {
   313  				return err
   314  			}
   315  			if existing, ok := services[cfg.ClusterLocalFQDN()]; ok {
   316  				// we've already run the generation for another echo instance's config, make sure things are the same
   317  				if existing != svc {
   318  					return fmt.Errorf("inconsistency in %s Service definition:\n%s", cfg.Service, cmp.Diff(existing, svc))
   319  				}
   320  			}
   321  			services[cfg.ClusterLocalFQDN()] = svc
   322  		}
   323  	}
   324  
   325  	// Deploy the services to all clusters.
   326  	cfg := b.ctx.ConfigKube().New()
   327  	for svcNs, svcYaml := range services {
   328  		ns := strings.Split(svcNs, ".")[1]
   329  		cfg.YAML(ns, svcYaml)
   330  	}
   331  
   332  	return cfg.Apply(apply.NoCleanup)
   333  }
   334  
   335  func (b builder) deployInstances() (instances echo.Instances, err error) {
   336  	m := sync.Mutex{}
   337  	out := echo.Instances{}
   338  	g := multierror.Group{}
   339  	// run the builder func for each kind of config in parallel
   340  	for kind, configs := range b.configs {
   341  		kind := kind
   342  		configs := configs
   343  		g.Go(func() error {
   344  			buildFunc, err := echo.GetBuilder(kind)
   345  			if err != nil {
   346  				return err
   347  			}
   348  			instances, err := buildFunc(b.ctx, configs)
   349  			if err != nil {
   350  				return err
   351  			}
   352  
   353  			// link reference pointers
   354  			if err := assignRefs(b.refs[kind], instances); err != nil {
   355  				return err
   356  			}
   357  
   358  			// safely merge instances from all kinds of cluster into one list
   359  			m.Lock()
   360  			defer m.Unlock()
   361  			out = append(out, instances...)
   362  			return nil
   363  		})
   364  	}
   365  	if err := g.Wait().ErrorOrNil(); err != nil {
   366  		return nil, err
   367  	}
   368  	return out, nil
   369  }
   370  
   371  func assignRefs(refs []*echo.Instance, instances echo.Instances) error {
   372  	if len(refs) != len(instances) {
   373  		return fmt.Errorf("cannot set %d references, only %d instances were built", len(refs), len(instances))
   374  	}
   375  	for i, ref := range refs {
   376  		if ref != nil {
   377  			*ref = instances[i]
   378  		}
   379  	}
   380  	return nil
   381  }
   382  
   383  func (b builder) BuildOrFail(t test.Failer) echo.Instances {
   384  	t.Helper()
   385  	out, err := b.Build()
   386  	if err != nil {
   387  		t.Fatal(err)
   388  	}
   389  	return out
   390  }
   391  
   392  // validateTemplates returns true if the templates specified by inject.istio.io/templates on the config exist on c
   393  func (b builder) validateTemplates(config echo.Config, c cluster.Cluster) bool {
   394  	expected := sets.New[string]()
   395  	for _, subset := range config.Subsets {
   396  		expected.InsertAll(parseList(subset.Annotations[annotation.InjectTemplates.Name])...)
   397  	}
   398  	if b.templates == nil || b.templates[c.Name()] == nil {
   399  		return expected.IsEmpty()
   400  	}
   401  
   402  	return b.templates[c.Name()].SupersetOf(expected)
   403  }
   404  
   405  func parseList(s string) []string {
   406  	if len(strings.TrimSpace(s)) == 0 {
   407  		return nil
   408  	}
   409  	items := strings.Split(s, ",")
   410  	for i := range items {
   411  		items[i] = strings.TrimSpace(items[i])
   412  	}
   413  	return items
   414  }