istio.io/istio@v0.0.0-20240520182934-d79c90f27776/istioctl/pkg/wait/wait.go (about)

     1  // Copyright Istio Authors
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package wait
    16  
    17  import (
    18  	"context"
    19  	"encoding/json"
    20  	"errors"
    21  	"fmt"
    22  	"strconv"
    23  	"strings"
    24  	"time"
    25  
    26  	"github.com/spf13/cobra"
    27  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    28  
    29  	"istio.io/istio/istioctl/pkg/cli"
    30  	"istio.io/istio/istioctl/pkg/clioptions"
    31  	"istio.io/istio/istioctl/pkg/util/handlers"
    32  	"istio.io/istio/pilot/pkg/xds"
    33  	"istio.io/istio/pkg/config"
    34  	"istio.io/istio/pkg/config/schema/collections"
    35  	"istio.io/istio/pkg/config/schema/resource"
    36  	"istio.io/istio/pkg/slices"
    37  )
    38  
    39  var (
    40  	forFlag      string
    41  	nameflag     string
    42  	proxyFlag    string
    43  	threshold    float32
    44  	timeout      time.Duration
    45  	generation   string
    46  	verbose      bool
    47  	targetSchema resource.Schema
    48  )
    49  
    50  const pollInterval = time.Second
    51  
    52  // Cmd represents the wait command
    53  func Cmd(cliCtx cli.Context) *cobra.Command {
    54  	namespace := cliCtx.Namespace()
    55  	var opts clioptions.ControlPlaneOptions
    56  	cmd := &cobra.Command{
    57  		Use:   "wait [flags] <type> <name>[.<namespace>]",
    58  		Short: "Wait for an Istio resource",
    59  		Long:  `Waits for the specified condition to be true of an Istio resource.`,
    60  		Example: `  # Wait until the bookinfo virtual service has been distributed to all proxies in the mesh
    61    istioctl experimental wait --for=distribution virtualservice bookinfo.default
    62  
    63    # Wait until the bookinfo virtual service has been distributed to a specific proxy
    64    istioctl experimental wait --for=distribution virtualservice bookinfo.default --proxy workload-instance.namespace
    65  
    66    # Wait until 99% of the proxies receive the distribution, timing out after 5 minutes
    67    istioctl experimental wait --for=distribution --threshold=.99 --timeout=300s virtualservice bookinfo.default
    68  `,
    69  		RunE: func(cmd *cobra.Command, args []string) error {
    70  			if forFlag == "delete" {
    71  				return errors.New("wait for delete is not yet implemented")
    72  			} else if forFlag != "distribution" {
    73  				return fmt.Errorf("--for must be 'delete' or 'distribution', got: %s", forFlag)
    74  			}
    75  			if proxyFlag != "" && threshold != 1 {
    76  				printVerbosef(cmd, "both the proxy and threshold options were provided; the threshold option is being ignored.")
    77  				threshold = 1
    78  			}
    79  			var w *watcher
    80  			ctx, cancel := context.WithTimeout(context.Background(), timeout)
    81  			defer cancel()
    82  			if generation == "" {
    83  				w = getAndWatchResource(ctx, cliCtx) // setup version getter from kubernetes
    84  			} else {
    85  				w = withContext(ctx)
    86  				w.Go(func(result chan string) error {
    87  					result <- generation
    88  					return nil
    89  				})
    90  			}
    91  			// wait for all deployed versions to be contained in generations
    92  			t := time.NewTicker(pollInterval)
    93  			printVerbosef(cmd, "getting first version from chan")
    94  			firstVersion, err := w.BlockingRead()
    95  			if err != nil {
    96  				return fmt.Errorf("unable to retrieve Kubernetes resource %s: %v", "", err)
    97  			}
    98  			generations := []string{firstVersion}
    99  			targetResource := config.Key(
   100  				targetSchema.Group(), targetSchema.Version(), targetSchema.Kind(),
   101  				nameflag, namespace)
   102  			for {
   103  				// run the check here as soon as we start
   104  				// because tickers won't run immediately
   105  				present, notpresent, sdcnum, err := poll(cliCtx, cmd, generations, targetResource, proxyFlag, opts)
   106  				printVerbosef(cmd, "Received poll result: %d/%d", present, present+notpresent)
   107  				if err != nil {
   108  					return err
   109  				} else if float32(present)/float32(present+notpresent) >= threshold {
   110  					_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Resource %s present on %d out of %d configurations across %d sidecars\n",
   111  						targetResource, present, present+notpresent, sdcnum)
   112  					return nil
   113  				}
   114  				select {
   115  				case newVersion := <-w.resultsChan:
   116  					printVerbosef(cmd, "received new target version: %s", newVersion)
   117  					generations = append(generations, newVersion)
   118  				case <-t.C:
   119  					printVerbosef(cmd, "tick")
   120  					continue
   121  				case err = <-w.errorChan:
   122  					t.Stop()
   123  					return fmt.Errorf("unable to retrieve Kubernetes resource2 %s: %v", "", err)
   124  				case <-ctx.Done():
   125  					printVerbosef(cmd, "timeout")
   126  					// I think this means the timeout has happened:
   127  					t.Stop()
   128  					errTmpl := "timeout expired before resource %s became effective on %s"
   129  					var errMsg string
   130  					if proxyFlag != "" {
   131  						errMsg = fmt.Sprintf(errTmpl, targetResource, proxyFlag)
   132  					} else {
   133  						errMsg = fmt.Sprintf(errTmpl, targetResource, "all sidecars")
   134  					}
   135  					return fmt.Errorf(errMsg)
   136  				}
   137  			}
   138  		},
   139  		Args: func(cmd *cobra.Command, args []string) error {
   140  			if err := cobra.ExactArgs(2)(cmd, args); err != nil {
   141  				return err
   142  			}
   143  			nameflag, namespace = handlers.InferPodInfo(args[1], cliCtx.NamespaceOrDefault(namespace))
   144  			return validateType(args[0])
   145  		},
   146  	}
   147  	cmd.PersistentFlags().StringVar(&forFlag, "for", "distribution",
   148  		"Wait condition, must be 'distribution' or 'delete'")
   149  	cmd.PersistentFlags().StringVar(&proxyFlag, "proxy", "",
   150  		"Name of a specific proxy to wait for the condition to be satisfied")
   151  	cmd.PersistentFlags().DurationVar(&timeout, "timeout", time.Second*30,
   152  		"The duration to wait before failing")
   153  	cmd.PersistentFlags().Float32Var(&threshold, "threshold", 1,
   154  		"The ratio of distribution required for success")
   155  	cmd.PersistentFlags().StringVar(&generation, "generation", "",
   156  		"Wait for a specific generation of config to become current, rather than using whatever is latest in "+
   157  			"Kubernetes")
   158  	cmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "enables verbose output")
   159  	_ = cmd.PersistentFlags().MarkHidden("verbose")
   160  	opts.AttachControlPlaneFlags(cmd)
   161  	return cmd
   162  }
   163  
   164  func printVerbosef(cmd *cobra.Command, template string, args ...any) {
   165  	if verbose {
   166  		_, _ = fmt.Fprintf(cmd.OutOrStdout(), template+"\n", args...)
   167  	}
   168  }
   169  
   170  func validateType(kind string) error {
   171  	originalKind := kind
   172  
   173  	// Remove any dashes.
   174  	kind = strings.ReplaceAll(kind, "-", "")
   175  
   176  	for _, s := range collections.Pilot.All() {
   177  		if strings.EqualFold(kind, s.Kind()) {
   178  			targetSchema = s
   179  			return nil
   180  		}
   181  	}
   182  	return fmt.Errorf("type %s is not recognized", originalKind)
   183  }
   184  
   185  func countVersions(versionCount map[string]int, configVersion string) {
   186  	if count, ok := versionCount[configVersion]; ok {
   187  		versionCount[configVersion] = count + 1
   188  	} else {
   189  		versionCount[configVersion] = 1
   190  	}
   191  }
   192  
   193  const distributionTrackingDisabledErrorString = "pilot version tracking is disabled " +
   194  	"(To enable this feature, please set PILOT_ENABLE_CONFIG_DISTRIBUTION_TRACKING=true)"
   195  
   196  func poll(ctx cli.Context,
   197  	cmd *cobra.Command,
   198  	acceptedVersions []string,
   199  	targetResource string,
   200  	proxyID string,
   201  	opts clioptions.ControlPlaneOptions,
   202  ) (present, notpresent, sdcnum int, err error) {
   203  	kubeClient, err := ctx.CLIClientWithRevision(opts.Revision)
   204  	if err != nil {
   205  		return 0, 0, 0, err
   206  	}
   207  	path := fmt.Sprintf("debug/config_distribution?resource=%s", targetResource)
   208  	pilotResponses, err := kubeClient.AllDiscoveryDo(context.TODO(), ctx.IstioNamespace(), path)
   209  	if err != nil {
   210  		return 0, 0, 0, fmt.Errorf("unable to query pilot for distribution "+
   211  			"(are you using pilot version >= 1.4 with config distribution tracking on): %s", err)
   212  	}
   213  	sdcnum = 0
   214  	versionCount := make(map[string]int)
   215  	for _, response := range pilotResponses {
   216  		var configVersions []xds.SyncedVersions
   217  		err = json.Unmarshal(response, &configVersions)
   218  		if err != nil {
   219  			respStr := string(response)
   220  			if strings.Contains(respStr, xds.DistributionTrackingDisabledMessage) {
   221  				return 0, 0, 0, fmt.Errorf("%s", distributionTrackingDisabledErrorString)
   222  			}
   223  			return 0, 0, 0, err
   224  		}
   225  		printVerbosef(cmd, "sync status: %+v", configVersions)
   226  		sdcnum += len(configVersions)
   227  		for _, configVersion := range configVersions {
   228  			if proxyID != "" && configVersion.ProxyID != proxyID {
   229  				continue
   230  			}
   231  			countVersions(versionCount, configVersion.ClusterVersion)
   232  			countVersions(versionCount, configVersion.RouteVersion)
   233  			countVersions(versionCount, configVersion.ListenerVersion)
   234  			countVersions(versionCount, configVersion.EndpointVersion)
   235  		}
   236  	}
   237  
   238  	for version, count := range versionCount {
   239  		if slices.Contains(acceptedVersions, version) {
   240  			present += count
   241  		} else {
   242  			notpresent += count
   243  		}
   244  	}
   245  	return present, notpresent, sdcnum, nil
   246  }
   247  
   248  // getAndWatchResource ensures that Generations always contains
   249  // the current generation of the targetResource, adding new versions
   250  // as they are created.
   251  func getAndWatchResource(ictx context.Context, cliCtx cli.Context) *watcher {
   252  	g := withContext(ictx)
   253  	// copy nameflag to avoid race
   254  	nf := nameflag
   255  	g.Go(func(result chan string) error {
   256  		// retrieve latest generation from Kubernetes
   257  		kc, err := cliCtx.CLIClient()
   258  		if err != nil {
   259  			return err
   260  		}
   261  		dclient := kc.Dynamic()
   262  		r := dclient.Resource(targetSchema.GroupVersionResource()).Namespace(cliCtx.Namespace())
   263  		watch, err := r.Watch(context.TODO(), metav1.ListOptions{FieldSelector: "metadata.name=" + nf})
   264  		if err != nil {
   265  			return err
   266  		}
   267  		for w := range watch.ResultChan() {
   268  			o, ok := w.Object.(metav1.Object)
   269  			if !ok {
   270  				continue
   271  			}
   272  			if o.GetName() == nf {
   273  				result <- strconv.FormatInt(o.GetGeneration(), 10)
   274  			}
   275  			select {
   276  			case <-ictx.Done():
   277  				return ictx.Err()
   278  			default:
   279  				continue
   280  			}
   281  		}
   282  
   283  		return nil
   284  	})
   285  	return g
   286  }
   287  
   288  type watcher struct {
   289  	resultsChan chan string
   290  	errorChan   chan error
   291  	ctx         context.Context
   292  }
   293  
   294  func withContext(ctx context.Context) *watcher {
   295  	return &watcher{
   296  		resultsChan: make(chan string, 1),
   297  		errorChan:   make(chan error, 1),
   298  		ctx:         ctx,
   299  	}
   300  }
   301  
   302  func (w *watcher) Go(f func(chan string) error) {
   303  	go func() {
   304  		if err := f(w.resultsChan); err != nil {
   305  			w.errorChan <- err
   306  		}
   307  	}()
   308  }
   309  
   310  func (w *watcher) BlockingRead() (string, error) {
   311  	select {
   312  	case err := <-w.errorChan:
   313  		return "", err
   314  	case res := <-w.resultsChan:
   315  		return res, nil
   316  	case <-w.ctx.Done():
   317  		return "", w.ctx.Err()
   318  	}
   319  }