istio.io/istio@v0.0.0-20240520182934-d79c90f27776/pkg/test/framework/components/echo/deployment/builder.go (about) 1 // Copyright Istio Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package deployment 16 17 import ( 18 "context" 19 "fmt" 20 "strings" 21 "sync" 22 "time" 23 24 "github.com/google/go-cmp/cmp" 25 "github.com/hashicorp/go-multierror" 26 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 27 28 "istio.io/api/annotation" 29 "istio.io/istio/pkg/kube/inject" 30 "istio.io/istio/pkg/test" 31 "istio.io/istio/pkg/test/framework/components/cluster" 32 "istio.io/istio/pkg/test/framework/components/echo" 33 "istio.io/istio/pkg/test/framework/components/echo/kube" 34 _ "istio.io/istio/pkg/test/framework/components/echo/staticvm" // force registraton of factory func 35 "istio.io/istio/pkg/test/framework/components/istio" 36 "istio.io/istio/pkg/test/framework/components/namespace" 37 "istio.io/istio/pkg/test/framework/resource" 38 "istio.io/istio/pkg/test/framework/resource/config/apply" 39 "istio.io/istio/pkg/test/scopes" 40 "istio.io/istio/pkg/util/sets" 41 ) 42 43 // Builder for a group of collaborating Echo Instances. Once built, all Instances in the 44 // group: 45 // 46 // 1. Are ready to receive traffic, and 47 // 2. Can call every other Instance in the group (i.e. have received Envoy config 48 // from Pilot). 49 // 50 // If a test needs to verify that one Instance is NOT reachable from another, there are 51 // a couple of options: 52 // 53 // 1. Build a group while all Instances ARE reachable. Then apply a policy 54 // disallowing the communication. 55 // 2. Build the source and destination Instances in separate groups and then 56 // call `source.WaitUntilCallable(destination)`. 57 type Builder interface { 58 // With adds a new Echo configuration to the Builder. Once built, the instance 59 // pointer will be updated to point at the new Instance. 60 With(i *echo.Instance, cfg echo.Config) Builder 61 62 // WithConfig mimics the behavior of With, but does not allow passing a reference 63 // and returns an echoboot builder rather than a generic echo builder. 64 // TODO rename this to With, and the old method to WithInstance 65 WithConfig(cfg echo.Config) Builder 66 67 // WithClusters will cause subsequent With or WithConfig calls to be applied to the given clusters. 68 WithClusters(...cluster.Cluster) Builder 69 70 // Build and initialize all Echo Instances. Upon returning, the Instance pointers 71 // are assigned and all Instances are ready to communicate with each other. 72 Build() (echo.Instances, error) 73 BuildOrFail(t test.Failer) echo.Instances 74 } 75 76 var _ Builder = builder{} 77 78 // New builder for echo deployments. 79 func New(ctx resource.Context, clusters ...cluster.Cluster) Builder { 80 // use all workload clusters unless otherwise specified 81 if len(clusters) == 0 { 82 clusters = ctx.Clusters() 83 } 84 b := builder{ 85 ctx: ctx, 86 configs: map[cluster.Kind][]echo.Config{}, 87 refs: map[cluster.Kind][]*echo.Instance{}, 88 namespaces: map[string]namespace.Instance{}, 89 } 90 templates, err := b.injectionTemplates() 91 if err != nil { 92 // deal with this when we call Build() to avoid making the New signature unwieldy 93 b.errs = multierror.Append(b.errs, fmt.Errorf("failed finding injection templates on clusters %v", err)) 94 } 95 b.templates = templates 96 97 return b.WithClusters(clusters...) 98 } 99 100 type builder struct { 101 ctx resource.Context 102 103 // clusters contains the current set of clusters that subsequent With calls will be applied to, 104 // if the Config passed to With does not explicitly choose a cluster. 105 clusters cluster.Clusters 106 107 // configs contains configurations to be built, expanded per-cluster and grouped by cluster Kind. 108 configs map[cluster.Kind][]echo.Config 109 // refs contains the references to assign built Instances to. 110 // The length of each refs slice should match the length of the corresponding cluster slice. 111 // Only the first per-cluster entry for a given config should have a non-nil ref. 112 refs map[cluster.Kind][]*echo.Instance 113 // namespaces caches namespaces by their prefix; used for converting Static namespace from configs into actual 114 // namespaces 115 namespaces map[string]namespace.Instance 116 // the set of injection templates for each cluster 117 templates map[string]sets.String 118 // errs contains a multierror for failed validation during With calls 119 errs error 120 } 121 122 func (b builder) WithConfig(cfg echo.Config) Builder { 123 return b.With(nil, cfg).(builder) 124 } 125 126 // With adds a new Echo configuration to the Builder. When a cluster is provided in the Config, it will only be applied 127 // to that cluster, otherwise the Config is applied to all WithClusters. Once built, if being built for a single cluster, 128 // the instance pointer will be updated to point at the new Instance. 129 func (b builder) With(i *echo.Instance, cfg echo.Config) Builder { 130 if b.ctx.Settings().SkipWorkloadClassesAsSet().Contains(cfg.WorkloadClass()) { 131 return b 132 } 133 134 cfg = cfg.DeepCopy() 135 if err := cfg.FillDefaults(b.ctx); err != nil { 136 b.errs = multierror.Append(b.errs, err) 137 return b 138 } 139 140 shouldSkip := b.ctx.Settings().Skip(cfg.WorkloadClass()) 141 if shouldSkip { 142 return b 143 } 144 145 // cache the namespace, so manually added echo.Configs can be a part of it 146 b.namespaces[cfg.Namespace.Prefix()] = cfg.Namespace 147 148 targetClusters := b.clusters 149 if cfg.Cluster != nil { 150 targetClusters = cluster.Clusters{cfg.Cluster} 151 } 152 153 deployedTo := 0 154 for idx, c := range targetClusters { 155 ec, ok := c.(echo.Cluster) 156 if !ok { 157 b.errs = multierror.Append(b.errs, fmt.Errorf("attempted to deploy to %s but it does not implement echo.Cluster", c.Name())) 158 continue 159 } 160 perClusterConfig, ok := ec.CanDeploy(cfg) 161 if !ok { 162 continue 163 } 164 if !b.validateTemplates(perClusterConfig, c) { 165 if c.Kind() == cluster.Kubernetes { 166 scopes.Framework.Warnf("%s does not contain injection templates for %s; skipping deployment", c.Name(), perClusterConfig.ClusterLocalFQDN()) 167 } 168 // Don't error out when injection template missing. 169 shouldSkip = true 170 continue 171 } 172 173 var ref *echo.Instance 174 if idx == 0 { 175 // ref only applies to the first cluster deployed to 176 // refs shouldn't be used when deploying to multiple targetClusters 177 // TODO: should we just panic if a ref is passed in a multi-cluster context? 178 ref = i 179 } 180 perClusterConfig = perClusterConfig.DeepCopy() 181 k := ec.Kind() 182 perClusterConfig.Cluster = ec 183 b.configs[k] = append(b.configs[k], perClusterConfig) 184 b.refs[k] = append(b.refs[k], ref) 185 deployedTo++ 186 } 187 188 if deployedTo == 0 && !shouldSkip { 189 b.errs = multierror.Append(b.errs, fmt.Errorf("no clusters were eligible for app %s", cfg.Service)) 190 } 191 192 return b 193 } 194 195 // WithClusters will cause subsequent With calls to be applied to the given clusters. 196 func (b builder) WithClusters(clusters ...cluster.Cluster) Builder { 197 next := b 198 next.clusters = clusters 199 return next 200 } 201 202 func (b builder) Build() (out echo.Instances, err error) { 203 return build(b) 204 } 205 206 // injectionTemplates lists the set of templates for each Kube cluster 207 func (b builder) injectionTemplates() (map[string]sets.String, error) { 208 ns := "istio-system" 209 i, err := istio.Get(b.ctx) 210 if err != nil { 211 scopes.Framework.Infof("defaulting to istio-system namespace for injection template discovery: %v", err) 212 } else { 213 ns = i.Settings().SystemNamespace 214 } 215 216 out := map[string]sets.String{} 217 for _, c := range b.ctx.Clusters().Kube() { 218 out[c.Name()] = sets.New[string]() 219 // TODO find a place to read revision(s) and avoid listing 220 cms, err := c.Kube().CoreV1().ConfigMaps(ns).List(context.TODO(), metav1.ListOptions{}) 221 if err != nil { 222 return nil, err 223 } 224 225 // take the intersection of the templates available from each revision in this cluster 226 intersection := sets.New[string]() 227 for _, item := range cms.Items { 228 if !strings.HasPrefix(item.Name, "istio-sidecar-injector") { 229 continue 230 } 231 data, err := inject.UnmarshalConfig([]byte(item.Data["config"])) 232 if err != nil { 233 return nil, fmt.Errorf("failed parsing injection cm in %s: %v", c.Name(), err) 234 } 235 if data.RawTemplates != nil { 236 t := sets.New[string]() 237 for name := range data.RawTemplates { 238 t.Insert(name) 239 } 240 // either intersection has not been set or we intersect these templates 241 // with the current set. 242 if intersection.IsEmpty() { 243 intersection = t 244 } else { 245 intersection = intersection.Intersection(t) 246 } 247 } 248 } 249 for name := range intersection { 250 out[c.Name()].Insert(name) 251 } 252 } 253 254 return out, nil 255 } 256 257 // build inner allows assigning to b (assignment to receiver would be ineffective) 258 func build(b builder) (out echo.Instances, err error) { 259 start := time.Now() 260 scopes.Framework.Info("=== BEGIN: Deploy echo instances ===") 261 defer func() { 262 if err != nil { 263 scopes.Framework.Error("=== FAILED: Deploy echo instances ===") 264 scopes.Framework.Error(err) 265 } else { 266 scopes.Framework.Infof("=== SUCCEEDED: Deploy echo instances in %v ===", time.Since(start)) 267 } 268 }() 269 270 // load additional configs 271 for _, cfg := range *additionalConfigs { 272 // swap the namespace.Static for a namespace.kube 273 b, cfg.Namespace = b.getOrCreateNamespace(cfg.Namespace.Prefix()) 274 // register the extra config 275 b = b.WithConfig(cfg).(builder) 276 } 277 278 // bail early if there were issues during the configuration stage 279 if b.errs != nil { 280 return nil, b.errs 281 } 282 283 if err = b.deployServices(); err != nil { 284 return 285 } 286 if out, err = b.deployInstances(); err != nil { 287 return 288 } 289 return 290 } 291 292 func (b builder) getOrCreateNamespace(prefix string) (builder, namespace.Instance) { 293 ns, ok := b.namespaces[prefix] 294 if ok { 295 return b, ns 296 } 297 ns, err := namespace.New(b.ctx, namespace.Config{Prefix: prefix, Inject: true}) 298 if err != nil { 299 b.errs = multierror.Append(b.errs, err) 300 } 301 b.namespaces[prefix] = ns 302 return b, ns 303 } 304 305 // deployServices deploys the kubernetes Service to all clusters. Multicluster meshes should have "sameness" 306 // per cluster. This avoids concurrent writes later. 307 func (b builder) deployServices() (err error) { 308 services := make(map[string]string) 309 for _, cfgs := range b.configs { 310 for _, cfg := range cfgs { 311 svc, err := kube.GenerateService(cfg) 312 if err != nil { 313 return err 314 } 315 if existing, ok := services[cfg.ClusterLocalFQDN()]; ok { 316 // we've already run the generation for another echo instance's config, make sure things are the same 317 if existing != svc { 318 return fmt.Errorf("inconsistency in %s Service definition:\n%s", cfg.Service, cmp.Diff(existing, svc)) 319 } 320 } 321 services[cfg.ClusterLocalFQDN()] = svc 322 } 323 } 324 325 // Deploy the services to all clusters. 326 cfg := b.ctx.ConfigKube().New() 327 for svcNs, svcYaml := range services { 328 ns := strings.Split(svcNs, ".")[1] 329 cfg.YAML(ns, svcYaml) 330 } 331 332 return cfg.Apply(apply.NoCleanup) 333 } 334 335 func (b builder) deployInstances() (instances echo.Instances, err error) { 336 m := sync.Mutex{} 337 out := echo.Instances{} 338 g := multierror.Group{} 339 // run the builder func for each kind of config in parallel 340 for kind, configs := range b.configs { 341 kind := kind 342 configs := configs 343 g.Go(func() error { 344 buildFunc, err := echo.GetBuilder(kind) 345 if err != nil { 346 return err 347 } 348 instances, err := buildFunc(b.ctx, configs) 349 if err != nil { 350 return err 351 } 352 353 // link reference pointers 354 if err := assignRefs(b.refs[kind], instances); err != nil { 355 return err 356 } 357 358 // safely merge instances from all kinds of cluster into one list 359 m.Lock() 360 defer m.Unlock() 361 out = append(out, instances...) 362 return nil 363 }) 364 } 365 if err := g.Wait().ErrorOrNil(); err != nil { 366 return nil, err 367 } 368 return out, nil 369 } 370 371 func assignRefs(refs []*echo.Instance, instances echo.Instances) error { 372 if len(refs) != len(instances) { 373 return fmt.Errorf("cannot set %d references, only %d instances were built", len(refs), len(instances)) 374 } 375 for i, ref := range refs { 376 if ref != nil { 377 *ref = instances[i] 378 } 379 } 380 return nil 381 } 382 383 func (b builder) BuildOrFail(t test.Failer) echo.Instances { 384 t.Helper() 385 out, err := b.Build() 386 if err != nil { 387 t.Fatal(err) 388 } 389 return out 390 } 391 392 // validateTemplates returns true if the templates specified by inject.istio.io/templates on the config exist on c 393 func (b builder) validateTemplates(config echo.Config, c cluster.Cluster) bool { 394 expected := sets.New[string]() 395 for _, subset := range config.Subsets { 396 expected.InsertAll(parseList(subset.Annotations[annotation.InjectTemplates.Name])...) 397 } 398 if b.templates == nil || b.templates[c.Name()] == nil { 399 return expected.IsEmpty() 400 } 401 402 return b.templates[c.Name()].SupersetOf(expected) 403 } 404 405 func parseList(s string) []string { 406 if len(strings.TrimSpace(s)) == 0 { 407 return nil 408 } 409 items := strings.Split(s, ",") 410 for i := range items { 411 items[i] = strings.TrimSpace(items[i]) 412 } 413 return items 414 }