k8s.io/kubernetes@v1.29.3/test/e2e/apimachinery/chunking.go (about) 1 /* 2 Copyright 2017 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package apimachinery 18 19 import ( 20 "context" 21 "fmt" 22 "reflect" 23 "time" 24 25 "github.com/onsi/ginkgo/v2" 26 "github.com/onsi/gomega" 27 28 v1 "k8s.io/api/core/v1" 29 apierrors "k8s.io/apimachinery/pkg/api/errors" 30 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 31 "k8s.io/apimachinery/pkg/util/wait" 32 "k8s.io/apiserver/pkg/storage/storagebackend" 33 "k8s.io/client-go/util/workqueue" 34 "k8s.io/kubernetes/test/e2e/framework" 35 admissionapi "k8s.io/pod-security-admission/api" 36 ) 37 38 const numberOfTotalResources = 400 39 40 var _ = SIGDescribe("Servers with support for API chunking", func() { 41 f := framework.NewDefaultFramework("chunking") 42 f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged 43 44 ginkgo.BeforeEach(func(ctx context.Context) { 45 ns := f.Namespace.Name 46 c := f.ClientSet 47 client := c.CoreV1().PodTemplates(ns) 48 ginkgo.By("creating a large number of resources") 49 workqueue.ParallelizeUntil(ctx, 20, numberOfTotalResources, func(i int) { 50 for tries := 3; tries >= 0; tries-- { 51 _, err := client.Create(ctx, &v1.PodTemplate{ 52 ObjectMeta: metav1.ObjectMeta{ 53 Name: fmt.Sprintf("template-%04d", i), 54 }, 55 Template: v1.PodTemplateSpec{ 56 Spec: v1.PodSpec{ 57 Containers: []v1.Container{ 58 {Name: "test", Image: "test2"}, 59 }, 60 }, 61 }, 62 }, metav1.CreateOptions{}) 63 if err == nil { 64 return 65 } 66 framework.Logf("Got an error creating template %d: %v", i, err) 67 } 68 framework.Failf("Unable to create template %d, exiting", i) 69 }) 70 }) 71 72 /* 73 Release: v1.29 74 Testname: API Chunking, server should return chunks of results for list calls 75 Description: Create a large number of PodTemplates. Attempt to retrieve the first chunk with limit set; 76 the server MUST return the chunk of the size not exceeding the limit with RemainingItems set in the response. 77 Attempt to retrieve the remaining items by providing the received continuation token and limit; 78 the server MUST return the remaining items in chunks of the size not exceeding the limit, with appropriately 79 set RemainingItems field in the response and with the ResourceVersion returned in the first response. 80 Attempt to list all objects at once without setting the limit; the server MUST return all items in a single 81 response. 82 */ 83 framework.ConformanceIt("should return chunks of results for list calls", func(ctx context.Context) { 84 ns := f.Namespace.Name 85 c := f.ClientSet 86 client := c.CoreV1().PodTemplates(ns) 87 ginkgo.By("retrieving those results in paged fashion several times") 88 for i := 0; i < 3; i++ { 89 opts := metav1.ListOptions{} 90 found := 0 91 var lastRV string 92 for { 93 // With numberOfTotalResources=400, we want to ensure that both 94 // number of items per page and number of pages are non-trivial. 95 opts.Limit = 17 96 list, err := client.List(ctx, opts) 97 framework.ExpectNoError(err, "failed to list pod templates in namespace: %s, given limit: %d", ns, opts.Limit) 98 framework.Logf("Retrieved %d/%d results with rv %s and continue %s", len(list.Items), opts.Limit, list.ResourceVersion, list.Continue) 99 gomega.Expect(len(list.Items)).To(gomega.BeNumerically("<=", opts.Limit)) 100 101 if len(lastRV) == 0 { 102 lastRV = list.ResourceVersion 103 } 104 gomega.Expect(list.ResourceVersion).To(gomega.Equal(lastRV)) 105 if list.GetContinue() == "" { 106 gomega.Expect(list.GetRemainingItemCount()).To(gomega.BeNil()) 107 } else { 108 gomega.Expect(list.GetRemainingItemCount()).ToNot(gomega.BeNil()) 109 gomega.Expect(int(*list.GetRemainingItemCount()) + len(list.Items) + found).To(gomega.BeNumerically("==", numberOfTotalResources)) 110 } 111 for _, item := range list.Items { 112 gomega.Expect(item.Name).To(gomega.Equal(fmt.Sprintf("template-%04d", found))) 113 found++ 114 } 115 if len(list.Continue) == 0 { 116 break 117 } 118 opts.Continue = list.Continue 119 } 120 gomega.Expect(found).To(gomega.BeNumerically("==", numberOfTotalResources)) 121 } 122 123 ginkgo.By("retrieving those results all at once") 124 opts := metav1.ListOptions{Limit: numberOfTotalResources + 1} 125 list, err := client.List(ctx, opts) 126 framework.ExpectNoError(err, "failed to list pod templates in namespace: %s, given limit: %d", ns, opts.Limit) 127 gomega.Expect(list.Items).To(gomega.HaveLen(numberOfTotalResources)) 128 }) 129 130 /* 131 Release: v1.29 132 Testname: API Chunking, server should support continue listing from the last key even if the original version has been compacted away 133 Description: Create a large number of PodTemplates. Attempt to retrieve the first chunk with limit set; 134 the server MUST return the chunk of the size not exceeding the limit with RemainingItems set in the response. 135 Attempt to retrieve the second page until the continuation token expires; the server MUST return a 136 continuation token for inconsistent list continuation. 137 Attempt to retrieve the second page with the received inconsistent list continuation token; the server 138 MUST return the number of items not exceeding the limit, a new continuation token and appropriately set 139 RemainingItems field in the response. 140 Attempt to retrieve the remaining pages by passing the received continuation token; the server 141 MUST return the remaining items in chunks of the size not exceeding the limit, with appropriately 142 set RemainingItems field in the response and with the ResourceVersion returned as part of the inconsistent list. 143 */ 144 framework.ConformanceIt("should support continue listing from the last key if the original version has been compacted away, though the list is inconsistent", f.WithSlow(), func(ctx context.Context) { 145 ns := f.Namespace.Name 146 c := f.ClientSet 147 client := c.CoreV1().PodTemplates(ns) 148 149 ginkgo.By("retrieving the first page") 150 oneTenth := int64(numberOfTotalResources / 10) 151 opts := metav1.ListOptions{} 152 opts.Limit = oneTenth 153 list, err := client.List(ctx, opts) 154 framework.ExpectNoError(err, "failed to list pod templates in namespace: %s, given limit: %d", ns, opts.Limit) 155 firstToken := list.Continue 156 firstRV := list.ResourceVersion 157 if list.GetContinue() == "" { 158 gomega.Expect(list.GetRemainingItemCount()).To(gomega.BeNil()) 159 } else { 160 gomega.Expect(list.GetRemainingItemCount()).ToNot(gomega.BeNil()) 161 gomega.Expect(int(*list.GetRemainingItemCount()) + len(list.Items)).To(gomega.BeNumerically("==", numberOfTotalResources)) 162 } 163 framework.Logf("Retrieved %d/%d results with rv %s and continue %s", len(list.Items), opts.Limit, list.ResourceVersion, firstToken) 164 165 ginkgo.By("retrieving the second page until the token expires") 166 opts.Continue = firstToken 167 var inconsistentToken string 168 wait.Poll(20*time.Second, 2*storagebackend.DefaultCompactInterval, func() (bool, error) { 169 _, err := client.List(ctx, opts) 170 if err == nil { 171 framework.Logf("Token %s has not expired yet", firstToken) 172 return false, nil 173 } 174 if err != nil && !apierrors.IsResourceExpired(err) { 175 return false, err 176 } 177 framework.Logf("got error %s", err) 178 status, ok := err.(apierrors.APIStatus) 179 if !ok { 180 return false, fmt.Errorf("expect error to implement the APIStatus interface, got %v", reflect.TypeOf(err)) 181 } 182 inconsistentToken = status.Status().ListMeta.Continue 183 if len(inconsistentToken) == 0 { 184 return false, fmt.Errorf("expect non empty continue token") 185 } 186 framework.Logf("Retrieved inconsistent continue %s", inconsistentToken) 187 return true, nil 188 }) 189 190 ginkgo.By("retrieving the second page again with the token received with the error message") 191 opts.Continue = inconsistentToken 192 list, err = client.List(ctx, opts) 193 framework.ExpectNoError(err, "failed to list pod templates in namespace: %s, given inconsistent continue token %s and limit: %d", ns, opts.Continue, opts.Limit) 194 framework.ExpectNotEqual(list.ResourceVersion, firstRV) 195 gomega.Expect(len(list.Items)).To(gomega.BeNumerically("==", opts.Limit)) 196 found := int(oneTenth) 197 198 if list.GetContinue() == "" { 199 gomega.Expect(list.GetRemainingItemCount()).To(gomega.BeNil()) 200 } else { 201 gomega.Expect(list.GetRemainingItemCount()).ToNot(gomega.BeNil()) 202 gomega.Expect(int(*list.GetRemainingItemCount()) + len(list.Items) + found).To(gomega.BeNumerically("==", numberOfTotalResources)) 203 } 204 for _, item := range list.Items { 205 gomega.Expect(item.Name).To(gomega.Equal(fmt.Sprintf("template-%04d", found))) 206 found++ 207 } 208 209 ginkgo.By("retrieving all remaining pages") 210 opts.Continue = list.Continue 211 lastRV := list.ResourceVersion 212 for { 213 list, err := client.List(ctx, opts) 214 framework.ExpectNoError(err, "failed to list pod templates in namespace: %s, given limit: %d", ns, opts.Limit) 215 if list.GetContinue() == "" { 216 gomega.Expect(list.GetRemainingItemCount()).To(gomega.BeNil()) 217 } else { 218 gomega.Expect(list.GetRemainingItemCount()).ToNot(gomega.BeNil()) 219 gomega.Expect(int(*list.GetRemainingItemCount()) + len(list.Items) + found).To(gomega.BeNumerically("==", numberOfTotalResources)) 220 } 221 framework.Logf("Retrieved %d/%d results with rv %s and continue %s", len(list.Items), opts.Limit, list.ResourceVersion, list.Continue) 222 gomega.Expect(len(list.Items)).To(gomega.BeNumerically("<=", opts.Limit)) 223 gomega.Expect(list.ResourceVersion).To(gomega.Equal(lastRV)) 224 for _, item := range list.Items { 225 gomega.Expect(item.Name).To(gomega.Equal(fmt.Sprintf("template-%04d", found))) 226 found++ 227 } 228 if len(list.Continue) == 0 { 229 break 230 } 231 opts.Continue = list.Continue 232 } 233 gomega.Expect(found).To(gomega.BeNumerically("==", numberOfTotalResources)) 234 }) 235 })