istio.io/istio@v0.0.0-20240520182934-d79c90f27776/pkg/test/framework/components/istio/configmap.go (about) 1 // Copyright Istio Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package istio 16 17 import ( 18 "context" 19 "crypto/md5" 20 "encoding/hex" 21 "fmt" 22 "io" 23 "sync" 24 25 "github.com/hashicorp/go-multierror" 26 corev1 "k8s.io/api/core/v1" 27 kerrors "k8s.io/apimachinery/pkg/api/errors" 28 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 29 "k8s.io/apimachinery/pkg/types" 30 "sigs.k8s.io/yaml" 31 32 meshconfig "istio.io/api/mesh/v1alpha1" 33 "istio.io/istio/pkg/kube/inject" 34 "istio.io/istio/pkg/test" 35 "istio.io/istio/pkg/test/framework/components/cluster" 36 "istio.io/istio/pkg/test/framework/resource" 37 "istio.io/istio/pkg/test/framework/resource/config/cleanup" 38 "istio.io/istio/pkg/test/scopes" 39 "istio.io/istio/pkg/util/protomarshal" 40 ) 41 42 type configMap struct { 43 ctx resource.Context 44 namespace string 45 mu sync.Mutex 46 revisions resource.RevVerMap 47 } 48 49 type meshConfig struct { 50 configMap 51 meshConfig *meshconfig.MeshConfig 52 } 53 54 type injectConfig struct { 55 configMap 56 injectConfig *inject.Config 57 values *inject.ValuesConfig 58 } 59 60 func newConfigMap(ctx resource.Context, namespace string, revisions resource.RevVerMap) *configMap { 61 return &configMap{ 62 ctx: ctx, 63 namespace: namespace, 64 revisions: revisions, 65 } 66 } 67 68 func (ic *injectConfig) InjectConfig() (*inject.Config, error) { 69 ic.mu.Lock() 70 myic := ic.injectConfig 71 ic.mu.Unlock() 72 73 if myic == nil { 74 c := ic.ctx.AllClusters().Configs()[0] 75 76 cfgMap, err := ic.getConfigMap(c, ic.configMapName()) 77 if err != nil { 78 return nil, err 79 } 80 81 // Get the MeshConfig yaml from the config map. 82 icYAML, err := getInjectConfigYaml(cfgMap, "config") 83 if err != nil { 84 return nil, err 85 } 86 87 // Parse the YAML. 88 myic, err = yamlToInjectConfig(icYAML) 89 if err != nil { 90 return nil, err 91 } 92 93 // Save the updated mesh config. 94 ic.mu.Lock() 95 ic.injectConfig = myic 96 ic.mu.Unlock() 97 } 98 99 return myic, nil 100 } 101 102 func (ic *injectConfig) UpdateInjectionConfig(t resource.Context, update func(*inject.Config) error, cleanupStrategy cleanup.Strategy) error { 103 // Invalidate the member variable. The next time it's requested, it will get a fresh value. 104 ic.mu.Lock() 105 ic.injectConfig = nil 106 ic.mu.Unlock() 107 108 errG := multierror.Group{} 109 origCfg := map[string]string{} 110 mu := sync.RWMutex{} 111 112 for _, c := range ic.ctx.AllClusters().Kube() { 113 c := c 114 errG.Go(func() error { 115 cfgMap, err := ic.getConfigMap(c, ic.configMapName()) 116 if err != nil { 117 return err 118 } 119 120 // Get the MeshConfig yaml from the config map. 121 mcYAML, err := getInjectConfigYaml(cfgMap, "config") 122 if err != nil { 123 return err 124 } 125 126 // Store the original YAML so we can restore later. 127 mu.Lock() 128 origCfg[c.Name()] = mcYAML 129 mu.Unlock() 130 131 // Parse the YAML. 132 mc, err := yamlToInjectConfig(mcYAML) 133 if err != nil { 134 return err 135 } 136 137 // Apply the change. 138 if err := update(mc); err != nil { 139 return err 140 } 141 142 // Store the updated MeshConfig back into the config map. 143 newYAML, err := injectConfigToYaml(mc) 144 if err != nil { 145 return err 146 } 147 cfgMap.Data["config"] = newYAML 148 149 // Write the config map back to the cluster. 150 if err := ic.updateConfigMap(c, cfgMap); err != nil { 151 return err 152 } 153 scopes.Framework.Debugf("patched %s injection configmap:\n%s", c.Name(), cfgMap.Data["config"]) 154 return nil 155 }) 156 } 157 158 // Restore the original value of the MeshConfig when the context completes. 159 t.CleanupStrategy(cleanupStrategy, func() { 160 // Invalidate the member mesh config again, since we're rolling back the changes. 161 ic.mu.Lock() 162 ic.injectConfig = nil 163 ic.mu.Unlock() 164 165 errG := multierror.Group{} 166 mu.RLock() 167 defer mu.RUnlock() 168 for cn, mcYAML := range origCfg { 169 cn, mcYAML := cn, mcYAML 170 c := ic.ctx.AllClusters().GetByName(cn) 171 errG.Go(func() error { 172 cfgMap, err := ic.getConfigMap(c, ic.configMapName()) 173 if err != nil { 174 return err 175 } 176 177 cfgMap.Data["config"] = mcYAML 178 if err := ic.updateConfigMap(c, cfgMap); err != nil { 179 return err 180 } 181 scopes.Framework.Debugf("cleanup patched %s injection configmap:\n%s", c.Name(), cfgMap.Data["config"]) 182 return nil 183 }) 184 } 185 if err := errG.Wait().ErrorOrNil(); err != nil { 186 scopes.Framework.Errorf("failed cleaning up cluster-local config: %v", err) 187 } 188 }) 189 return errG.Wait().ErrorOrNil() 190 } 191 192 func (ic *injectConfig) ValuesConfig() (*inject.ValuesConfig, error) { 193 ic.mu.Lock() 194 myic := ic.values 195 ic.mu.Unlock() 196 197 if myic == nil { 198 c := ic.ctx.AllClusters().Configs()[0] 199 200 cfgMap, err := ic.getConfigMap(c, ic.configMapName()) 201 if err != nil { 202 return nil, err 203 } 204 205 // Get the MeshConfig yaml from the config map. 206 icYAML, err := getInjectConfigYaml(cfgMap, "values") 207 if err != nil { 208 return nil, err 209 } 210 211 // Parse the YAML. 212 s, err := inject.NewValuesConfig(icYAML) 213 if err != nil { 214 return nil, err 215 } 216 myic = &s 217 218 // Save the updated mesh config. 219 ic.mu.Lock() 220 ic.values = myic 221 ic.mu.Unlock() 222 } 223 224 return myic, nil 225 } 226 227 func (ic *injectConfig) configMapName() string { 228 cmName := "istio-sidecar-injector" 229 if rev := ic.revisions.Default(); rev != "default" && rev != "" { 230 cmName += "-" + rev 231 } 232 return cmName 233 } 234 235 func (mc *meshConfig) MeshConfig() (*meshconfig.MeshConfig, error) { 236 mc.mu.Lock() 237 mymc := mc.meshConfig 238 mc.mu.Unlock() 239 240 if mymc == nil { 241 c := mc.ctx.AllClusters().Configs()[0] 242 243 cfgMapName, err := mc.configMapName() 244 if err != nil { 245 return nil, err 246 } 247 248 cfgMap, err := mc.getConfigMap(c, cfgMapName) 249 if err != nil { 250 return nil, err 251 } 252 253 // Get the MeshConfig yaml from the config map. 254 mcYAML, err := getMeshConfigData(c, cfgMap) 255 if err != nil { 256 return nil, err 257 } 258 259 // Parse the YAML. 260 mymc, err = yamlToMeshConfig(mcYAML) 261 if err != nil { 262 return nil, err 263 } 264 265 // Save the updated mesh config. 266 mc.mu.Lock() 267 mc.meshConfig = mymc 268 mc.mu.Unlock() 269 } 270 271 return mymc, nil 272 } 273 274 func (mc *meshConfig) MeshConfigOrFail(t test.Failer) *meshconfig.MeshConfig { 275 t.Helper() 276 out, err := mc.MeshConfig() 277 if err != nil { 278 t.Fatal(err) 279 } 280 return out 281 } 282 283 func (mc *meshConfig) UpdateMeshConfig(t resource.Context, update func(*meshconfig.MeshConfig) error, cleanupStrategy cleanup.Strategy) error { 284 // Invalidate the member variable. The next time it's requested, it will get a fresh value. 285 mc.mu.Lock() 286 mc.meshConfig = nil 287 mc.mu.Unlock() 288 289 errG := multierror.Group{} 290 origCfg := map[string]string{} 291 mu := sync.RWMutex{} 292 293 for _, c := range mc.ctx.AllClusters().Kube() { 294 c := c 295 errG.Go(func() error { 296 cfgMapName, err := mc.configMapName() 297 if err != nil { 298 return err 299 } 300 301 cfgMap, err := mc.getConfigMap(c, cfgMapName) 302 if err != nil { 303 // Remote clusters typically don't have mesh config, allow it to skip 304 if c.IsRemote() && kerrors.IsNotFound(err) { 305 scopes.Framework.Infof("skipped %s meshconfig patch, as it is a remote", c.Name()) 306 return nil 307 } 308 return err 309 } 310 311 // Get the MeshConfig yaml from the config map. 312 mcYAML, err := getMeshConfigData(c, cfgMap) 313 if err != nil { 314 return err 315 } 316 // Store the original YAML so we can restore later. 317 mu.Lock() 318 origCfg[c.Name()] = mcYAML 319 mu.Unlock() 320 321 // Parse the YAML. 322 ymc, err := yamlToMeshConfig(mcYAML) 323 if err != nil { 324 return err 325 } 326 327 // Apply the change. 328 if err := update(ymc); err != nil { 329 return err 330 } 331 332 // Store the updated MeshConfig back into the config map. 333 newYAML, err := meshConfigToYAML(ymc) 334 if err != nil { 335 return err 336 } 337 setMeshConfigData(cfgMap, newYAML) 338 339 // Write the config map back to the cluster. 340 if err := mc.updateConfigMap(c, cfgMap); err != nil { 341 return err 342 } 343 scopes.Framework.Infof("patched %s meshconfig:\n%s", c.Name(), cfgMap.Data["mesh"]) 344 return nil 345 }) 346 } 347 348 // Restore the original value of the MeshConfig when the context completes. 349 t.CleanupStrategy(cleanupStrategy, func() { 350 // Invalidate the member mesh config again, since we're rolling back the changes. 351 mc.mu.Lock() 352 mc.meshConfig = nil 353 mc.mu.Unlock() 354 355 errG := multierror.Group{} 356 mu.RLock() 357 defer mu.RUnlock() 358 for cn, mcYAML := range origCfg { 359 cn, mcYAML := cn, mcYAML 360 c := mc.ctx.AllClusters().GetByName(cn) 361 errG.Go(func() error { 362 cfgMapName, err := mc.configMapName() 363 if err != nil { 364 return err 365 } 366 cfgMap, err := mc.getConfigMap(c, cfgMapName) 367 if err != nil { 368 return err 369 } 370 setMeshConfigData(cfgMap, mcYAML) 371 if err := mc.updateConfigMap(c, cfgMap); err != nil { 372 return err 373 } 374 scopes.Framework.Infof("cleanup patched %s meshconfig:\n%s", c.Name(), cfgMap.Data["mesh"]) 375 return nil 376 }) 377 } 378 if err := errG.Wait().ErrorOrNil(); err != nil { 379 scopes.Framework.Errorf("failed cleaning up cluster-local config: %v", err) 380 } 381 }) 382 return errG.Wait().ErrorOrNil() 383 } 384 385 func (mc *meshConfig) UpdateMeshConfigOrFail(ctx resource.Context, t test.Failer, update func(*meshconfig.MeshConfig) error, cleanupStrategy cleanup.Strategy) { 386 t.Helper() 387 if err := mc.UpdateMeshConfig(ctx, update, cleanupStrategy); err != nil { 388 t.Fatal(err) 389 } 390 } 391 392 func (mc *meshConfig) PatchMeshConfig(t resource.Context, patch string) error { 393 return mc.UpdateMeshConfig(t, func(mc *meshconfig.MeshConfig) error { 394 return protomarshal.ApplyYAML(patch, mc) 395 }, cleanup.Always) 396 } 397 398 func (mc *meshConfig) PatchMeshConfigOrFail(ctx resource.Context, t test.Failer, patch string) { 399 t.Helper() 400 if err := mc.PatchMeshConfig(ctx, patch); err != nil { 401 t.Fatal(err) 402 } 403 } 404 405 func (mc *meshConfig) configMapName() (string, error) { 406 i, err := Get(mc.ctx) 407 if err != nil { 408 return "", err 409 } 410 411 sharedCfgMapName := i.Settings().SharedMeshConfigName 412 if sharedCfgMapName != "" { 413 return sharedCfgMapName, nil 414 } 415 416 cmName := "istio" 417 if rev := mc.revisions.Default(); rev != "default" && rev != "" { 418 cmName += "-" + rev 419 } 420 return cmName, nil 421 } 422 423 func (cm *configMap) getConfigMap(c cluster.Cluster, name string) (*corev1.ConfigMap, error) { 424 return c.Kube().CoreV1().ConfigMaps(cm.namespace).Get(context.TODO(), name, metav1.GetOptions{}) 425 } 426 427 func (cm *configMap) updateConfigMap(c cluster.Cluster, cfgMap *corev1.ConfigMap) error { 428 _, err := c.Kube().CoreV1().ConfigMaps(cm.namespace).Update(context.TODO(), cfgMap, metav1.UpdateOptions{}) 429 if err != nil { 430 return err 431 } 432 if c.IsExternalControlPlane() { 433 // Normal control plane uses ConfigMap informers to load mesh config. This is ~instant. 434 // The external config uses a file mounted ConfigMap. This is super slow, but we can trigger it explicitly: 435 // https://github.com/kubernetes/kubernetes/issues/30189 436 pl, err := c.Kube().CoreV1().Pods(cm.namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: "app=istiod"}) 437 if err != nil { 438 return err 439 } 440 for _, pod := range pl.Items { 441 patchBytes := fmt.Sprintf(`{ "metadata": {"annotations": { "test.istio.io/mesh-config-hash": "%s" } } }`, hash(cfgMap.Data["mesh"])) 442 _, err := c.Kube().CoreV1().Pods(cm.namespace).Patch(context.TODO(), pod.Name, 443 types.MergePatchType, []byte(patchBytes), metav1.PatchOptions{FieldManager: "istio-ci"}) 444 if err != nil { 445 return fmt.Errorf("patch %v: %v", patchBytes, err) 446 } 447 } 448 } 449 return nil 450 } 451 452 func hash(s string) string { 453 // nolint: gosec 454 // Test only code 455 h := md5.New() 456 _, _ = io.WriteString(h, s) 457 return hex.EncodeToString(h.Sum(nil)) 458 } 459 460 func getMeshConfigData(c cluster.Cluster, cm *corev1.ConfigMap) (string, error) { 461 // Get the MeshConfig yaml from the config map. 462 mcYAML, ok := cm.Data["mesh"] 463 if !ok { 464 return "", fmt.Errorf("mesh config was missing in istio config map for %s", c.Name()) 465 } 466 return mcYAML, nil 467 } 468 469 func setMeshConfigData(cm *corev1.ConfigMap, mcYAML string) { 470 cm.Data["mesh"] = mcYAML 471 } 472 473 func yamlToMeshConfig(mcYAML string) (*meshconfig.MeshConfig, error) { 474 // Parse the YAML. 475 mc := &meshconfig.MeshConfig{} 476 if err := protomarshal.ApplyYAML(mcYAML, mc); err != nil { 477 return nil, err 478 } 479 return mc, nil 480 } 481 482 func meshConfigToYAML(mc *meshconfig.MeshConfig) (string, error) { 483 return protomarshal.ToYAML(mc) 484 } 485 486 func getInjectConfigYaml(cm *corev1.ConfigMap, configKey string) (string, error) { 487 if cm == nil { 488 return "", fmt.Errorf("no ConfigMap found") 489 } 490 491 configYaml, exists := cm.Data[configKey] 492 if !exists { 493 return "", fmt.Errorf("missing ConfigMap config key %q", configKey) 494 } 495 return configYaml, nil 496 } 497 498 func injectConfigToYaml(config *inject.Config) (string, error) { 499 bres, err := yaml.Marshal(config) 500 return string(bres), err 501 } 502 503 func yamlToInjectConfig(configYaml string) (*inject.Config, error) { 504 // Parse the YAML. 505 c, err := inject.UnmarshalConfig([]byte(configYaml)) 506 if err != nil { 507 return nil, err 508 } 509 return &c, nil 510 }