istio.io/istio@v0.0.0-20240520182934-d79c90f27776/pkg/test/framework/components/istio/configmap.go (about)

     1  // Copyright Istio Authors
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package istio
    16  
    17  import (
    18  	"context"
    19  	"crypto/md5"
    20  	"encoding/hex"
    21  	"fmt"
    22  	"io"
    23  	"sync"
    24  
    25  	"github.com/hashicorp/go-multierror"
    26  	corev1 "k8s.io/api/core/v1"
    27  	kerrors "k8s.io/apimachinery/pkg/api/errors"
    28  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    29  	"k8s.io/apimachinery/pkg/types"
    30  	"sigs.k8s.io/yaml"
    31  
    32  	meshconfig "istio.io/api/mesh/v1alpha1"
    33  	"istio.io/istio/pkg/kube/inject"
    34  	"istio.io/istio/pkg/test"
    35  	"istio.io/istio/pkg/test/framework/components/cluster"
    36  	"istio.io/istio/pkg/test/framework/resource"
    37  	"istio.io/istio/pkg/test/framework/resource/config/cleanup"
    38  	"istio.io/istio/pkg/test/scopes"
    39  	"istio.io/istio/pkg/util/protomarshal"
    40  )
    41  
    42  type configMap struct {
    43  	ctx       resource.Context
    44  	namespace string
    45  	mu        sync.Mutex
    46  	revisions resource.RevVerMap
    47  }
    48  
    49  type meshConfig struct {
    50  	configMap
    51  	meshConfig *meshconfig.MeshConfig
    52  }
    53  
    54  type injectConfig struct {
    55  	configMap
    56  	injectConfig *inject.Config
    57  	values       *inject.ValuesConfig
    58  }
    59  
    60  func newConfigMap(ctx resource.Context, namespace string, revisions resource.RevVerMap) *configMap {
    61  	return &configMap{
    62  		ctx:       ctx,
    63  		namespace: namespace,
    64  		revisions: revisions,
    65  	}
    66  }
    67  
    68  func (ic *injectConfig) InjectConfig() (*inject.Config, error) {
    69  	ic.mu.Lock()
    70  	myic := ic.injectConfig
    71  	ic.mu.Unlock()
    72  
    73  	if myic == nil {
    74  		c := ic.ctx.AllClusters().Configs()[0]
    75  
    76  		cfgMap, err := ic.getConfigMap(c, ic.configMapName())
    77  		if err != nil {
    78  			return nil, err
    79  		}
    80  
    81  		// Get the MeshConfig yaml from the config map.
    82  		icYAML, err := getInjectConfigYaml(cfgMap, "config")
    83  		if err != nil {
    84  			return nil, err
    85  		}
    86  
    87  		// Parse the YAML.
    88  		myic, err = yamlToInjectConfig(icYAML)
    89  		if err != nil {
    90  			return nil, err
    91  		}
    92  
    93  		// Save the updated mesh config.
    94  		ic.mu.Lock()
    95  		ic.injectConfig = myic
    96  		ic.mu.Unlock()
    97  	}
    98  
    99  	return myic, nil
   100  }
   101  
   102  func (ic *injectConfig) UpdateInjectionConfig(t resource.Context, update func(*inject.Config) error, cleanupStrategy cleanup.Strategy) error {
   103  	// Invalidate the member variable. The next time it's requested, it will get a fresh value.
   104  	ic.mu.Lock()
   105  	ic.injectConfig = nil
   106  	ic.mu.Unlock()
   107  
   108  	errG := multierror.Group{}
   109  	origCfg := map[string]string{}
   110  	mu := sync.RWMutex{}
   111  
   112  	for _, c := range ic.ctx.AllClusters().Kube() {
   113  		c := c
   114  		errG.Go(func() error {
   115  			cfgMap, err := ic.getConfigMap(c, ic.configMapName())
   116  			if err != nil {
   117  				return err
   118  			}
   119  
   120  			// Get the MeshConfig yaml from the config map.
   121  			mcYAML, err := getInjectConfigYaml(cfgMap, "config")
   122  			if err != nil {
   123  				return err
   124  			}
   125  
   126  			// Store the original YAML so we can restore later.
   127  			mu.Lock()
   128  			origCfg[c.Name()] = mcYAML
   129  			mu.Unlock()
   130  
   131  			// Parse the YAML.
   132  			mc, err := yamlToInjectConfig(mcYAML)
   133  			if err != nil {
   134  				return err
   135  			}
   136  
   137  			// Apply the change.
   138  			if err := update(mc); err != nil {
   139  				return err
   140  			}
   141  
   142  			// Store the updated MeshConfig back into the config map.
   143  			newYAML, err := injectConfigToYaml(mc)
   144  			if err != nil {
   145  				return err
   146  			}
   147  			cfgMap.Data["config"] = newYAML
   148  
   149  			// Write the config map back to the cluster.
   150  			if err := ic.updateConfigMap(c, cfgMap); err != nil {
   151  				return err
   152  			}
   153  			scopes.Framework.Debugf("patched %s injection configmap:\n%s", c.Name(), cfgMap.Data["config"])
   154  			return nil
   155  		})
   156  	}
   157  
   158  	// Restore the original value of the MeshConfig when the context completes.
   159  	t.CleanupStrategy(cleanupStrategy, func() {
   160  		// Invalidate the member mesh config again, since we're rolling back the changes.
   161  		ic.mu.Lock()
   162  		ic.injectConfig = nil
   163  		ic.mu.Unlock()
   164  
   165  		errG := multierror.Group{}
   166  		mu.RLock()
   167  		defer mu.RUnlock()
   168  		for cn, mcYAML := range origCfg {
   169  			cn, mcYAML := cn, mcYAML
   170  			c := ic.ctx.AllClusters().GetByName(cn)
   171  			errG.Go(func() error {
   172  				cfgMap, err := ic.getConfigMap(c, ic.configMapName())
   173  				if err != nil {
   174  					return err
   175  				}
   176  
   177  				cfgMap.Data["config"] = mcYAML
   178  				if err := ic.updateConfigMap(c, cfgMap); err != nil {
   179  					return err
   180  				}
   181  				scopes.Framework.Debugf("cleanup patched %s injection configmap:\n%s", c.Name(), cfgMap.Data["config"])
   182  				return nil
   183  			})
   184  		}
   185  		if err := errG.Wait().ErrorOrNil(); err != nil {
   186  			scopes.Framework.Errorf("failed cleaning up cluster-local config: %v", err)
   187  		}
   188  	})
   189  	return errG.Wait().ErrorOrNil()
   190  }
   191  
   192  func (ic *injectConfig) ValuesConfig() (*inject.ValuesConfig, error) {
   193  	ic.mu.Lock()
   194  	myic := ic.values
   195  	ic.mu.Unlock()
   196  
   197  	if myic == nil {
   198  		c := ic.ctx.AllClusters().Configs()[0]
   199  
   200  		cfgMap, err := ic.getConfigMap(c, ic.configMapName())
   201  		if err != nil {
   202  			return nil, err
   203  		}
   204  
   205  		// Get the MeshConfig yaml from the config map.
   206  		icYAML, err := getInjectConfigYaml(cfgMap, "values")
   207  		if err != nil {
   208  			return nil, err
   209  		}
   210  
   211  		// Parse the YAML.
   212  		s, err := inject.NewValuesConfig(icYAML)
   213  		if err != nil {
   214  			return nil, err
   215  		}
   216  		myic = &s
   217  
   218  		// Save the updated mesh config.
   219  		ic.mu.Lock()
   220  		ic.values = myic
   221  		ic.mu.Unlock()
   222  	}
   223  
   224  	return myic, nil
   225  }
   226  
   227  func (ic *injectConfig) configMapName() string {
   228  	cmName := "istio-sidecar-injector"
   229  	if rev := ic.revisions.Default(); rev != "default" && rev != "" {
   230  		cmName += "-" + rev
   231  	}
   232  	return cmName
   233  }
   234  
   235  func (mc *meshConfig) MeshConfig() (*meshconfig.MeshConfig, error) {
   236  	mc.mu.Lock()
   237  	mymc := mc.meshConfig
   238  	mc.mu.Unlock()
   239  
   240  	if mymc == nil {
   241  		c := mc.ctx.AllClusters().Configs()[0]
   242  
   243  		cfgMapName, err := mc.configMapName()
   244  		if err != nil {
   245  			return nil, err
   246  		}
   247  
   248  		cfgMap, err := mc.getConfigMap(c, cfgMapName)
   249  		if err != nil {
   250  			return nil, err
   251  		}
   252  
   253  		// Get the MeshConfig yaml from the config map.
   254  		mcYAML, err := getMeshConfigData(c, cfgMap)
   255  		if err != nil {
   256  			return nil, err
   257  		}
   258  
   259  		// Parse the YAML.
   260  		mymc, err = yamlToMeshConfig(mcYAML)
   261  		if err != nil {
   262  			return nil, err
   263  		}
   264  
   265  		// Save the updated mesh config.
   266  		mc.mu.Lock()
   267  		mc.meshConfig = mymc
   268  		mc.mu.Unlock()
   269  	}
   270  
   271  	return mymc, nil
   272  }
   273  
   274  func (mc *meshConfig) MeshConfigOrFail(t test.Failer) *meshconfig.MeshConfig {
   275  	t.Helper()
   276  	out, err := mc.MeshConfig()
   277  	if err != nil {
   278  		t.Fatal(err)
   279  	}
   280  	return out
   281  }
   282  
   283  func (mc *meshConfig) UpdateMeshConfig(t resource.Context, update func(*meshconfig.MeshConfig) error, cleanupStrategy cleanup.Strategy) error {
   284  	// Invalidate the member variable. The next time it's requested, it will get a fresh value.
   285  	mc.mu.Lock()
   286  	mc.meshConfig = nil
   287  	mc.mu.Unlock()
   288  
   289  	errG := multierror.Group{}
   290  	origCfg := map[string]string{}
   291  	mu := sync.RWMutex{}
   292  
   293  	for _, c := range mc.ctx.AllClusters().Kube() {
   294  		c := c
   295  		errG.Go(func() error {
   296  			cfgMapName, err := mc.configMapName()
   297  			if err != nil {
   298  				return err
   299  			}
   300  
   301  			cfgMap, err := mc.getConfigMap(c, cfgMapName)
   302  			if err != nil {
   303  				// Remote clusters typically don't have mesh config, allow it to skip
   304  				if c.IsRemote() && kerrors.IsNotFound(err) {
   305  					scopes.Framework.Infof("skipped %s meshconfig patch, as it is a remote", c.Name())
   306  					return nil
   307  				}
   308  				return err
   309  			}
   310  
   311  			// Get the MeshConfig yaml from the config map.
   312  			mcYAML, err := getMeshConfigData(c, cfgMap)
   313  			if err != nil {
   314  				return err
   315  			}
   316  			// Store the original YAML so we can restore later.
   317  			mu.Lock()
   318  			origCfg[c.Name()] = mcYAML
   319  			mu.Unlock()
   320  
   321  			// Parse the YAML.
   322  			ymc, err := yamlToMeshConfig(mcYAML)
   323  			if err != nil {
   324  				return err
   325  			}
   326  
   327  			// Apply the change.
   328  			if err := update(ymc); err != nil {
   329  				return err
   330  			}
   331  
   332  			// Store the updated MeshConfig back into the config map.
   333  			newYAML, err := meshConfigToYAML(ymc)
   334  			if err != nil {
   335  				return err
   336  			}
   337  			setMeshConfigData(cfgMap, newYAML)
   338  
   339  			// Write the config map back to the cluster.
   340  			if err := mc.updateConfigMap(c, cfgMap); err != nil {
   341  				return err
   342  			}
   343  			scopes.Framework.Infof("patched %s meshconfig:\n%s", c.Name(), cfgMap.Data["mesh"])
   344  			return nil
   345  		})
   346  	}
   347  
   348  	// Restore the original value of the MeshConfig when the context completes.
   349  	t.CleanupStrategy(cleanupStrategy, func() {
   350  		// Invalidate the member mesh config again, since we're rolling back the changes.
   351  		mc.mu.Lock()
   352  		mc.meshConfig = nil
   353  		mc.mu.Unlock()
   354  
   355  		errG := multierror.Group{}
   356  		mu.RLock()
   357  		defer mu.RUnlock()
   358  		for cn, mcYAML := range origCfg {
   359  			cn, mcYAML := cn, mcYAML
   360  			c := mc.ctx.AllClusters().GetByName(cn)
   361  			errG.Go(func() error {
   362  				cfgMapName, err := mc.configMapName()
   363  				if err != nil {
   364  					return err
   365  				}
   366  				cfgMap, err := mc.getConfigMap(c, cfgMapName)
   367  				if err != nil {
   368  					return err
   369  				}
   370  				setMeshConfigData(cfgMap, mcYAML)
   371  				if err := mc.updateConfigMap(c, cfgMap); err != nil {
   372  					return err
   373  				}
   374  				scopes.Framework.Infof("cleanup patched %s meshconfig:\n%s", c.Name(), cfgMap.Data["mesh"])
   375  				return nil
   376  			})
   377  		}
   378  		if err := errG.Wait().ErrorOrNil(); err != nil {
   379  			scopes.Framework.Errorf("failed cleaning up cluster-local config: %v", err)
   380  		}
   381  	})
   382  	return errG.Wait().ErrorOrNil()
   383  }
   384  
   385  func (mc *meshConfig) UpdateMeshConfigOrFail(ctx resource.Context, t test.Failer, update func(*meshconfig.MeshConfig) error, cleanupStrategy cleanup.Strategy) {
   386  	t.Helper()
   387  	if err := mc.UpdateMeshConfig(ctx, update, cleanupStrategy); err != nil {
   388  		t.Fatal(err)
   389  	}
   390  }
   391  
   392  func (mc *meshConfig) PatchMeshConfig(t resource.Context, patch string) error {
   393  	return mc.UpdateMeshConfig(t, func(mc *meshconfig.MeshConfig) error {
   394  		return protomarshal.ApplyYAML(patch, mc)
   395  	}, cleanup.Always)
   396  }
   397  
   398  func (mc *meshConfig) PatchMeshConfigOrFail(ctx resource.Context, t test.Failer, patch string) {
   399  	t.Helper()
   400  	if err := mc.PatchMeshConfig(ctx, patch); err != nil {
   401  		t.Fatal(err)
   402  	}
   403  }
   404  
   405  func (mc *meshConfig) configMapName() (string, error) {
   406  	i, err := Get(mc.ctx)
   407  	if err != nil {
   408  		return "", err
   409  	}
   410  
   411  	sharedCfgMapName := i.Settings().SharedMeshConfigName
   412  	if sharedCfgMapName != "" {
   413  		return sharedCfgMapName, nil
   414  	}
   415  
   416  	cmName := "istio"
   417  	if rev := mc.revisions.Default(); rev != "default" && rev != "" {
   418  		cmName += "-" + rev
   419  	}
   420  	return cmName, nil
   421  }
   422  
   423  func (cm *configMap) getConfigMap(c cluster.Cluster, name string) (*corev1.ConfigMap, error) {
   424  	return c.Kube().CoreV1().ConfigMaps(cm.namespace).Get(context.TODO(), name, metav1.GetOptions{})
   425  }
   426  
   427  func (cm *configMap) updateConfigMap(c cluster.Cluster, cfgMap *corev1.ConfigMap) error {
   428  	_, err := c.Kube().CoreV1().ConfigMaps(cm.namespace).Update(context.TODO(), cfgMap, metav1.UpdateOptions{})
   429  	if err != nil {
   430  		return err
   431  	}
   432  	if c.IsExternalControlPlane() {
   433  		// Normal control plane uses ConfigMap informers to load mesh config. This is ~instant.
   434  		// The external config uses a file mounted ConfigMap. This is super slow, but we can trigger it explicitly:
   435  		// https://github.com/kubernetes/kubernetes/issues/30189
   436  		pl, err := c.Kube().CoreV1().Pods(cm.namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: "app=istiod"})
   437  		if err != nil {
   438  			return err
   439  		}
   440  		for _, pod := range pl.Items {
   441  			patchBytes := fmt.Sprintf(`{ "metadata": {"annotations": { "test.istio.io/mesh-config-hash": "%s" } } }`, hash(cfgMap.Data["mesh"]))
   442  			_, err := c.Kube().CoreV1().Pods(cm.namespace).Patch(context.TODO(), pod.Name,
   443  				types.MergePatchType, []byte(patchBytes), metav1.PatchOptions{FieldManager: "istio-ci"})
   444  			if err != nil {
   445  				return fmt.Errorf("patch %v: %v", patchBytes, err)
   446  			}
   447  		}
   448  	}
   449  	return nil
   450  }
   451  
   452  func hash(s string) string {
   453  	// nolint: gosec
   454  	// Test only code
   455  	h := md5.New()
   456  	_, _ = io.WriteString(h, s)
   457  	return hex.EncodeToString(h.Sum(nil))
   458  }
   459  
   460  func getMeshConfigData(c cluster.Cluster, cm *corev1.ConfigMap) (string, error) {
   461  	// Get the MeshConfig yaml from the config map.
   462  	mcYAML, ok := cm.Data["mesh"]
   463  	if !ok {
   464  		return "", fmt.Errorf("mesh config was missing in istio config map for %s", c.Name())
   465  	}
   466  	return mcYAML, nil
   467  }
   468  
   469  func setMeshConfigData(cm *corev1.ConfigMap, mcYAML string) {
   470  	cm.Data["mesh"] = mcYAML
   471  }
   472  
   473  func yamlToMeshConfig(mcYAML string) (*meshconfig.MeshConfig, error) {
   474  	// Parse the YAML.
   475  	mc := &meshconfig.MeshConfig{}
   476  	if err := protomarshal.ApplyYAML(mcYAML, mc); err != nil {
   477  		return nil, err
   478  	}
   479  	return mc, nil
   480  }
   481  
   482  func meshConfigToYAML(mc *meshconfig.MeshConfig) (string, error) {
   483  	return protomarshal.ToYAML(mc)
   484  }
   485  
   486  func getInjectConfigYaml(cm *corev1.ConfigMap, configKey string) (string, error) {
   487  	if cm == nil {
   488  		return "", fmt.Errorf("no ConfigMap found")
   489  	}
   490  
   491  	configYaml, exists := cm.Data[configKey]
   492  	if !exists {
   493  		return "", fmt.Errorf("missing ConfigMap config key %q", configKey)
   494  	}
   495  	return configYaml, nil
   496  }
   497  
   498  func injectConfigToYaml(config *inject.Config) (string, error) {
   499  	bres, err := yaml.Marshal(config)
   500  	return string(bres), err
   501  }
   502  
   503  func yamlToInjectConfig(configYaml string) (*inject.Config, error) {
   504  	// Parse the YAML.
   505  	c, err := inject.UnmarshalConfig([]byte(configYaml))
   506  	if err != nil {
   507  		return nil, err
   508  	}
   509  	return &c, nil
   510  }