istio.io/istio@v0.0.0-20240520182934-d79c90f27776/istioctl/pkg/wait/wait.go (about) 1 // Copyright Istio Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package wait 16 17 import ( 18 "context" 19 "encoding/json" 20 "errors" 21 "fmt" 22 "strconv" 23 "strings" 24 "time" 25 26 "github.com/spf13/cobra" 27 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 29 "istio.io/istio/istioctl/pkg/cli" 30 "istio.io/istio/istioctl/pkg/clioptions" 31 "istio.io/istio/istioctl/pkg/util/handlers" 32 "istio.io/istio/pilot/pkg/xds" 33 "istio.io/istio/pkg/config" 34 "istio.io/istio/pkg/config/schema/collections" 35 "istio.io/istio/pkg/config/schema/resource" 36 "istio.io/istio/pkg/slices" 37 ) 38 39 var ( 40 forFlag string 41 nameflag string 42 proxyFlag string 43 threshold float32 44 timeout time.Duration 45 generation string 46 verbose bool 47 targetSchema resource.Schema 48 ) 49 50 const pollInterval = time.Second 51 52 // Cmd represents the wait command 53 func Cmd(cliCtx cli.Context) *cobra.Command { 54 namespace := cliCtx.Namespace() 55 var opts clioptions.ControlPlaneOptions 56 cmd := &cobra.Command{ 57 Use: "wait [flags] <type> <name>[.<namespace>]", 58 Short: "Wait for an Istio resource", 59 Long: `Waits for the specified condition to be true of an Istio resource.`, 60 Example: ` # Wait until the bookinfo virtual service has been distributed to all proxies in the mesh 61 istioctl experimental wait --for=distribution virtualservice bookinfo.default 62 63 # Wait until the bookinfo virtual service has been distributed to a specific proxy 64 istioctl experimental wait --for=distribution virtualservice bookinfo.default --proxy workload-instance.namespace 65 66 # Wait until 99% of the proxies receive the distribution, timing out after 5 minutes 67 istioctl experimental wait --for=distribution --threshold=.99 --timeout=300s virtualservice bookinfo.default 68 `, 69 RunE: func(cmd *cobra.Command, args []string) error { 70 if forFlag == "delete" { 71 return errors.New("wait for delete is not yet implemented") 72 } else if forFlag != "distribution" { 73 return fmt.Errorf("--for must be 'delete' or 'distribution', got: %s", forFlag) 74 } 75 if proxyFlag != "" && threshold != 1 { 76 printVerbosef(cmd, "both the proxy and threshold options were provided; the threshold option is being ignored.") 77 threshold = 1 78 } 79 var w *watcher 80 ctx, cancel := context.WithTimeout(context.Background(), timeout) 81 defer cancel() 82 if generation == "" { 83 w = getAndWatchResource(ctx, cliCtx) // setup version getter from kubernetes 84 } else { 85 w = withContext(ctx) 86 w.Go(func(result chan string) error { 87 result <- generation 88 return nil 89 }) 90 } 91 // wait for all deployed versions to be contained in generations 92 t := time.NewTicker(pollInterval) 93 printVerbosef(cmd, "getting first version from chan") 94 firstVersion, err := w.BlockingRead() 95 if err != nil { 96 return fmt.Errorf("unable to retrieve Kubernetes resource %s: %v", "", err) 97 } 98 generations := []string{firstVersion} 99 targetResource := config.Key( 100 targetSchema.Group(), targetSchema.Version(), targetSchema.Kind(), 101 nameflag, namespace) 102 for { 103 // run the check here as soon as we start 104 // because tickers won't run immediately 105 present, notpresent, sdcnum, err := poll(cliCtx, cmd, generations, targetResource, proxyFlag, opts) 106 printVerbosef(cmd, "Received poll result: %d/%d", present, present+notpresent) 107 if err != nil { 108 return err 109 } else if float32(present)/float32(present+notpresent) >= threshold { 110 _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Resource %s present on %d out of %d configurations across %d sidecars\n", 111 targetResource, present, present+notpresent, sdcnum) 112 return nil 113 } 114 select { 115 case newVersion := <-w.resultsChan: 116 printVerbosef(cmd, "received new target version: %s", newVersion) 117 generations = append(generations, newVersion) 118 case <-t.C: 119 printVerbosef(cmd, "tick") 120 continue 121 case err = <-w.errorChan: 122 t.Stop() 123 return fmt.Errorf("unable to retrieve Kubernetes resource2 %s: %v", "", err) 124 case <-ctx.Done(): 125 printVerbosef(cmd, "timeout") 126 // I think this means the timeout has happened: 127 t.Stop() 128 errTmpl := "timeout expired before resource %s became effective on %s" 129 var errMsg string 130 if proxyFlag != "" { 131 errMsg = fmt.Sprintf(errTmpl, targetResource, proxyFlag) 132 } else { 133 errMsg = fmt.Sprintf(errTmpl, targetResource, "all sidecars") 134 } 135 return fmt.Errorf(errMsg) 136 } 137 } 138 }, 139 Args: func(cmd *cobra.Command, args []string) error { 140 if err := cobra.ExactArgs(2)(cmd, args); err != nil { 141 return err 142 } 143 nameflag, namespace = handlers.InferPodInfo(args[1], cliCtx.NamespaceOrDefault(namespace)) 144 return validateType(args[0]) 145 }, 146 } 147 cmd.PersistentFlags().StringVar(&forFlag, "for", "distribution", 148 "Wait condition, must be 'distribution' or 'delete'") 149 cmd.PersistentFlags().StringVar(&proxyFlag, "proxy", "", 150 "Name of a specific proxy to wait for the condition to be satisfied") 151 cmd.PersistentFlags().DurationVar(&timeout, "timeout", time.Second*30, 152 "The duration to wait before failing") 153 cmd.PersistentFlags().Float32Var(&threshold, "threshold", 1, 154 "The ratio of distribution required for success") 155 cmd.PersistentFlags().StringVar(&generation, "generation", "", 156 "Wait for a specific generation of config to become current, rather than using whatever is latest in "+ 157 "Kubernetes") 158 cmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "enables verbose output") 159 _ = cmd.PersistentFlags().MarkHidden("verbose") 160 opts.AttachControlPlaneFlags(cmd) 161 return cmd 162 } 163 164 func printVerbosef(cmd *cobra.Command, template string, args ...any) { 165 if verbose { 166 _, _ = fmt.Fprintf(cmd.OutOrStdout(), template+"\n", args...) 167 } 168 } 169 170 func validateType(kind string) error { 171 originalKind := kind 172 173 // Remove any dashes. 174 kind = strings.ReplaceAll(kind, "-", "") 175 176 for _, s := range collections.Pilot.All() { 177 if strings.EqualFold(kind, s.Kind()) { 178 targetSchema = s 179 return nil 180 } 181 } 182 return fmt.Errorf("type %s is not recognized", originalKind) 183 } 184 185 func countVersions(versionCount map[string]int, configVersion string) { 186 if count, ok := versionCount[configVersion]; ok { 187 versionCount[configVersion] = count + 1 188 } else { 189 versionCount[configVersion] = 1 190 } 191 } 192 193 const distributionTrackingDisabledErrorString = "pilot version tracking is disabled " + 194 "(To enable this feature, please set PILOT_ENABLE_CONFIG_DISTRIBUTION_TRACKING=true)" 195 196 func poll(ctx cli.Context, 197 cmd *cobra.Command, 198 acceptedVersions []string, 199 targetResource string, 200 proxyID string, 201 opts clioptions.ControlPlaneOptions, 202 ) (present, notpresent, sdcnum int, err error) { 203 kubeClient, err := ctx.CLIClientWithRevision(opts.Revision) 204 if err != nil { 205 return 0, 0, 0, err 206 } 207 path := fmt.Sprintf("debug/config_distribution?resource=%s", targetResource) 208 pilotResponses, err := kubeClient.AllDiscoveryDo(context.TODO(), ctx.IstioNamespace(), path) 209 if err != nil { 210 return 0, 0, 0, fmt.Errorf("unable to query pilot for distribution "+ 211 "(are you using pilot version >= 1.4 with config distribution tracking on): %s", err) 212 } 213 sdcnum = 0 214 versionCount := make(map[string]int) 215 for _, response := range pilotResponses { 216 var configVersions []xds.SyncedVersions 217 err = json.Unmarshal(response, &configVersions) 218 if err != nil { 219 respStr := string(response) 220 if strings.Contains(respStr, xds.DistributionTrackingDisabledMessage) { 221 return 0, 0, 0, fmt.Errorf("%s", distributionTrackingDisabledErrorString) 222 } 223 return 0, 0, 0, err 224 } 225 printVerbosef(cmd, "sync status: %+v", configVersions) 226 sdcnum += len(configVersions) 227 for _, configVersion := range configVersions { 228 if proxyID != "" && configVersion.ProxyID != proxyID { 229 continue 230 } 231 countVersions(versionCount, configVersion.ClusterVersion) 232 countVersions(versionCount, configVersion.RouteVersion) 233 countVersions(versionCount, configVersion.ListenerVersion) 234 countVersions(versionCount, configVersion.EndpointVersion) 235 } 236 } 237 238 for version, count := range versionCount { 239 if slices.Contains(acceptedVersions, version) { 240 present += count 241 } else { 242 notpresent += count 243 } 244 } 245 return present, notpresent, sdcnum, nil 246 } 247 248 // getAndWatchResource ensures that Generations always contains 249 // the current generation of the targetResource, adding new versions 250 // as they are created. 251 func getAndWatchResource(ictx context.Context, cliCtx cli.Context) *watcher { 252 g := withContext(ictx) 253 // copy nameflag to avoid race 254 nf := nameflag 255 g.Go(func(result chan string) error { 256 // retrieve latest generation from Kubernetes 257 kc, err := cliCtx.CLIClient() 258 if err != nil { 259 return err 260 } 261 dclient := kc.Dynamic() 262 r := dclient.Resource(targetSchema.GroupVersionResource()).Namespace(cliCtx.Namespace()) 263 watch, err := r.Watch(context.TODO(), metav1.ListOptions{FieldSelector: "metadata.name=" + nf}) 264 if err != nil { 265 return err 266 } 267 for w := range watch.ResultChan() { 268 o, ok := w.Object.(metav1.Object) 269 if !ok { 270 continue 271 } 272 if o.GetName() == nf { 273 result <- strconv.FormatInt(o.GetGeneration(), 10) 274 } 275 select { 276 case <-ictx.Done(): 277 return ictx.Err() 278 default: 279 continue 280 } 281 } 282 283 return nil 284 }) 285 return g 286 } 287 288 type watcher struct { 289 resultsChan chan string 290 errorChan chan error 291 ctx context.Context 292 } 293 294 func withContext(ctx context.Context) *watcher { 295 return &watcher{ 296 resultsChan: make(chan string, 1), 297 errorChan: make(chan error, 1), 298 ctx: ctx, 299 } 300 } 301 302 func (w *watcher) Go(f func(chan string) error) { 303 go func() { 304 if err := f(w.resultsChan); err != nil { 305 w.errorChan <- err 306 } 307 }() 308 } 309 310 func (w *watcher) BlockingRead() (string, error) { 311 select { 312 case err := <-w.errorChan: 313 return "", err 314 case res := <-w.resultsChan: 315 return res, nil 316 case <-w.ctx.Done(): 317 return "", w.ctx.Err() 318 } 319 }