k8s.io/kubernetes@v1.29.3/test/integration/apiserver/watchcache_test.go (about) 1 /* 2 Copyright 2020 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package apiserver 18 19 import ( 20 "context" 21 "fmt" 22 "sync" 23 "testing" 24 "time" 25 26 v1 "k8s.io/api/core/v1" 27 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 "k8s.io/apimachinery/pkg/util/wait" 29 clientset "k8s.io/client-go/kubernetes" 30 "k8s.io/kubernetes/cmd/kube-apiserver/app/options" 31 "k8s.io/kubernetes/pkg/controlplane" 32 "k8s.io/kubernetes/pkg/controlplane/reconcilers" 33 "k8s.io/kubernetes/test/integration/framework" 34 "k8s.io/kubernetes/test/utils/ktesting" 35 ) 36 37 // setup create kube-apiserver backed up by two separate etcds, 38 // with one of them containing events and the other all other objects. 39 func multiEtcdSetup(ctx context.Context, t *testing.T) (clientset.Interface, framework.TearDownFunc) { 40 etcdArgs := []string{"--experimental-watch-progress-notify-interval", "1s"} 41 etcd0URL, stopEtcd0, err := framework.RunCustomEtcd("etcd_watchcache0", etcdArgs, nil) 42 if err != nil { 43 t.Fatalf("Couldn't start etcd: %v", err) 44 } 45 46 etcd1URL, stopEtcd1, err := framework.RunCustomEtcd("etcd_watchcache1", etcdArgs, nil) 47 if err != nil { 48 t.Fatalf("Couldn't start etcd: %v", err) 49 } 50 51 etcdOptions := framework.DefaultEtcdOptions() 52 // Overwrite etcd setup to our custom etcd instances. 53 etcdOptions.StorageConfig.Transport.ServerList = []string{etcd0URL} 54 etcdOptions.EtcdServersOverrides = []string{fmt.Sprintf("/events#%s", etcd1URL)} 55 etcdOptions.EnableWatchCache = true 56 57 clientSet, _, tearDownFn := framework.StartTestServer(ctx, t, framework.TestServerSetup{ 58 ModifyServerRunOptions: func(opts *options.ServerRunOptions) { 59 // Ensure we're using the same etcd across apiserver restarts. 60 opts.Etcd = etcdOptions 61 }, 62 ModifyServerConfig: func(config *controlplane.Config) { 63 // Switch off endpoints reconciler to avoid unnecessary operations. 64 config.ExtraConfig.EndpointReconcilerType = reconcilers.NoneEndpointReconcilerType 65 }, 66 }) 67 68 closeFn := func() { 69 tearDownFn() 70 stopEtcd1() 71 stopEtcd0() 72 } 73 74 // Wait for apiserver to be stabilized. 75 // Everything but default service creation is checked in StartTestServer above by 76 // waiting for post start hooks, so we just wait for default service to exist. 77 // TODO(wojtek-t): Figure out less fragile way. 78 if err := wait.Poll(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { 79 _, err := clientSet.CoreV1().Services("default").Get(ctx, "kubernetes", metav1.GetOptions{}) 80 return err == nil, nil 81 }); err != nil { 82 t.Fatalf("Failed to wait for kubernetes service: %v:", err) 83 } 84 return clientSet, closeFn 85 } 86 87 func TestWatchCacheUpdatedByEtcd(t *testing.T) { 88 _, ctx := ktesting.NewTestContext(t) 89 ctx, cancel := context.WithCancel(ctx) 90 defer cancel() 91 92 c, closeFn := multiEtcdSetup(ctx, t) 93 defer closeFn() 94 95 makeConfigMap := func(name string) *v1.ConfigMap { 96 return &v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: name}} 97 } 98 makeSecret := func(name string) *v1.Secret { 99 return &v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name}} 100 } 101 makeEvent := func(name string) *v1.Event { 102 return &v1.Event{ObjectMeta: metav1.ObjectMeta{Name: name}} 103 } 104 105 cm, err := c.CoreV1().ConfigMaps("default").Create(ctx, makeConfigMap("name"), metav1.CreateOptions{}) 106 if err != nil { 107 t.Errorf("Couldn't create configmap: %v", err) 108 } 109 ev, err := c.CoreV1().Events("default").Create(ctx, makeEvent("name"), metav1.CreateOptions{}) 110 if err != nil { 111 t.Errorf("Couldn't create event: %v", err) 112 } 113 114 listOptions := metav1.ListOptions{ 115 ResourceVersion: "0", 116 ResourceVersionMatch: metav1.ResourceVersionMatchNotOlderThan, 117 } 118 119 // Wait until listing from cache returns resource version of corresponding 120 // resources (being the last updates). 121 t.Logf("Waiting for configmaps watchcache synced to %s", cm.ResourceVersion) 122 if err := wait.Poll(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { 123 res, err := c.CoreV1().ConfigMaps("default").List(ctx, listOptions) 124 if err != nil { 125 return false, nil 126 } 127 return res.ResourceVersion == cm.ResourceVersion, nil 128 }); err != nil { 129 t.Errorf("Failed to wait for configmaps watchcache synced: %v", err) 130 } 131 t.Logf("Waiting for events watchcache synced to %s", ev.ResourceVersion) 132 if err := wait.Poll(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { 133 res, err := c.CoreV1().Events("default").List(ctx, listOptions) 134 if err != nil { 135 return false, nil 136 } 137 return res.ResourceVersion == ev.ResourceVersion, nil 138 }); err != nil { 139 t.Errorf("Failed to wait for events watchcache synced: %v", err) 140 } 141 142 // Create a secret, that is stored in the same etcd as configmap, but 143 // different than events. 144 se, err := c.CoreV1().Secrets("default").Create(ctx, makeSecret("name"), metav1.CreateOptions{}) 145 if err != nil { 146 t.Errorf("Couldn't create secret: %v", err) 147 } 148 149 t.Logf("Waiting for configmaps watchcache synced to %s", se.ResourceVersion) 150 if err := wait.Poll(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { 151 res, err := c.CoreV1().ConfigMaps("default").List(ctx, listOptions) 152 if err != nil { 153 return false, nil 154 } 155 return res.ResourceVersion == se.ResourceVersion, nil 156 }); err != nil { 157 t.Errorf("Failed to wait for configmaps watchcache synced: %v", err) 158 } 159 t.Logf("Waiting for events watchcache NOT synced to %s", se.ResourceVersion) 160 if err := wait.Poll(100*time.Millisecond, 5*time.Second, func() (bool, error) { 161 res, err := c.CoreV1().Events("default").List(ctx, listOptions) 162 if err != nil { 163 return false, nil 164 } 165 return res.ResourceVersion == se.ResourceVersion, nil 166 }); err == nil || err != wait.ErrWaitTimeout { 167 t.Errorf("Events watchcache unexpected synced: %v", err) 168 } 169 } 170 171 func BenchmarkListFromWatchCache(b *testing.B) { 172 _, ctx := ktesting.NewTestContext(b) 173 ctx, cancel := context.WithCancel(ctx) 174 defer cancel() 175 176 c, _, tearDownFn := framework.StartTestServer(ctx, b, framework.TestServerSetup{ 177 ModifyServerConfig: func(config *controlplane.Config) { 178 // Switch off endpoints reconciler to avoid unnecessary operations. 179 config.ExtraConfig.EndpointReconcilerType = reconcilers.NoneEndpointReconcilerType 180 }, 181 }) 182 defer tearDownFn() 183 184 namespaces, secretsPerNamespace := 100, 1000 185 wg := sync.WaitGroup{} 186 187 errCh := make(chan error, namespaces) 188 for i := 0; i < namespaces; i++ { 189 wg.Add(1) 190 index := i 191 go func() { 192 defer wg.Done() 193 194 ns := &v1.Namespace{ 195 ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("namespace-%d", index)}, 196 } 197 ns, err := c.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) 198 if err != nil { 199 errCh <- err 200 return 201 } 202 203 for j := 0; j < secretsPerNamespace; j++ { 204 secret := &v1.Secret{ 205 ObjectMeta: metav1.ObjectMeta{ 206 Name: fmt.Sprintf("secret-%d", j), 207 }, 208 } 209 _, err := c.CoreV1().Secrets(ns.Name).Create(ctx, secret, metav1.CreateOptions{}) 210 if err != nil { 211 errCh <- err 212 return 213 } 214 } 215 }() 216 } 217 218 wg.Wait() 219 close(errCh) 220 for err := range errCh { 221 b.Error(err) 222 } 223 224 b.ResetTimer() 225 226 opts := metav1.ListOptions{ 227 ResourceVersion: "0", 228 } 229 for i := 0; i < b.N; i++ { 230 secrets, err := c.CoreV1().Secrets("").List(ctx, opts) 231 if err != nil { 232 b.Errorf("failed to list secrets: %v", err) 233 } 234 b.Logf("Number of secrets: %d", len(secrets.Items)) 235 } 236 }