k8s.io/kubernetes@v1.29.3/test/e2e_node/apparmor_test.go (about) 1 /* 2 Copyright 2016 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package e2enode 18 19 import ( 20 "bytes" 21 "context" 22 "fmt" 23 "os" 24 "os/exec" 25 "regexp" 26 "strconv" 27 "strings" 28 29 v1 "k8s.io/api/core/v1" 30 apierrors "k8s.io/apimachinery/pkg/api/errors" 31 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 32 "k8s.io/apimachinery/pkg/fields" 33 "k8s.io/apimachinery/pkg/runtime" 34 "k8s.io/apimachinery/pkg/runtime/schema" 35 "k8s.io/apimachinery/pkg/util/dump" 36 "k8s.io/apimachinery/pkg/watch" 37 "k8s.io/client-go/tools/cache" 38 watchtools "k8s.io/client-go/tools/watch" 39 "k8s.io/klog/v2" 40 "k8s.io/kubernetes/pkg/kubelet/kuberuntime" 41 "k8s.io/kubernetes/test/e2e/feature" 42 "k8s.io/kubernetes/test/e2e/framework" 43 e2epod "k8s.io/kubernetes/test/e2e/framework/pod" 44 "k8s.io/kubernetes/test/e2e/nodefeature" 45 admissionapi "k8s.io/pod-security-admission/api" 46 47 "github.com/onsi/ginkgo/v2" 48 "github.com/onsi/gomega" 49 "github.com/opencontainers/runc/libcontainer/apparmor" 50 ) 51 52 var _ = SIGDescribe("AppArmor", feature.AppArmor, nodefeature.AppArmor, func() { 53 if isAppArmorEnabled() { 54 ginkgo.BeforeEach(func() { 55 ginkgo.By("Loading AppArmor profiles for testing") 56 framework.ExpectNoError(loadTestProfiles(), "Could not load AppArmor test profiles") 57 }) 58 ginkgo.Context("when running with AppArmor", func() { 59 f := framework.NewDefaultFramework("apparmor-test") 60 f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged 61 62 ginkgo.It("should reject an unloaded profile", func(ctx context.Context) { 63 status := runAppArmorTest(ctx, f, false, v1.AppArmorBetaProfileNamePrefix+"non-existent-profile") 64 gomega.Expect(status.ContainerStatuses[0].State.Waiting.Message).To(gomega.ContainSubstring("apparmor")) 65 }) 66 ginkgo.It("should enforce a profile blocking writes", func(ctx context.Context) { 67 status := runAppArmorTest(ctx, f, true, v1.AppArmorBetaProfileNamePrefix+apparmorProfilePrefix+"deny-write") 68 if len(status.ContainerStatuses) == 0 { 69 framework.Failf("Unexpected pod status: %s", dump.Pretty(status)) 70 return 71 } 72 state := status.ContainerStatuses[0].State.Terminated 73 gomega.Expect(state).ToNot(gomega.BeNil(), "ContainerState: %+v", status.ContainerStatuses[0].State) 74 gomega.Expect(state.ExitCode).To(gomega.Not(gomega.BeZero()), "ContainerStateTerminated: %+v", state) 75 76 }) 77 ginkgo.It("should enforce a permissive profile", func(ctx context.Context) { 78 status := runAppArmorTest(ctx, f, true, v1.AppArmorBetaProfileNamePrefix+apparmorProfilePrefix+"audit-write") 79 if len(status.ContainerStatuses) == 0 { 80 framework.Failf("Unexpected pod status: %s", dump.Pretty(status)) 81 return 82 } 83 state := status.ContainerStatuses[0].State.Terminated 84 gomega.Expect(state).ToNot(gomega.BeNil(), "ContainerState: %+v", status.ContainerStatuses[0].State) 85 gomega.Expect(state.ExitCode).To(gomega.BeZero(), "ContainerStateTerminated: %+v", state) 86 }) 87 }) 88 } else { 89 ginkgo.Context("when running without AppArmor", func() { 90 f := framework.NewDefaultFramework("apparmor-test") 91 f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged 92 93 ginkgo.It("should reject a pod with an AppArmor profile", func(ctx context.Context) { 94 status := runAppArmorTest(ctx, f, false, v1.AppArmorBetaProfileRuntimeDefault) 95 expectSoftRejection(status) 96 }) 97 }) 98 } 99 }) 100 101 const apparmorProfilePrefix = "e2e-node-apparmor-test-" 102 const testProfiles = ` 103 #include <tunables/global> 104 105 profile e2e-node-apparmor-test-deny-write flags=(attach_disconnected) { 106 #include <abstractions/base> 107 108 file, 109 110 # Deny all file writes. 111 deny /** w, 112 } 113 114 profile e2e-node-apparmor-test-audit-write flags=(attach_disconnected) { 115 #include <abstractions/base> 116 117 file, 118 119 # Only audit file writes. 120 audit /** w, 121 } 122 ` 123 124 func loadTestProfiles() error { 125 f, err := os.CreateTemp("/tmp", "apparmor") 126 if err != nil { 127 return fmt.Errorf("failed to open temp file: %w", err) 128 } 129 defer os.Remove(f.Name()) 130 defer f.Close() 131 132 if _, err := f.WriteString(testProfiles); err != nil { 133 return fmt.Errorf("failed to write profiles to file: %w", err) 134 } 135 136 cmd := exec.Command("apparmor_parser", "-r", "-W", f.Name()) 137 stderr := &bytes.Buffer{} 138 cmd.Stderr = stderr 139 out, err := cmd.Output() 140 // apparmor_parser does not always return an error code, so consider any stderr output an error. 141 if err != nil || stderr.Len() > 0 { 142 if stderr.Len() > 0 { 143 klog.Warning(stderr.String()) 144 } 145 if len(out) > 0 { 146 klog.Infof("apparmor_parser: %s", out) 147 } 148 return fmt.Errorf("failed to load profiles: %w", err) 149 } 150 klog.V(2).Infof("Loaded profiles: %v", out) 151 return nil 152 } 153 154 func runAppArmorTest(ctx context.Context, f *framework.Framework, shouldRun bool, profile string) v1.PodStatus { 155 pod := createPodWithAppArmor(ctx, f, profile) 156 if shouldRun { 157 // The pod needs to start before it stops, so wait for the longer start timeout. 158 framework.ExpectNoError(e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(ctx, 159 f.ClientSet, pod.Name, f.Namespace.Name, framework.PodStartTimeout)) 160 } else { 161 // Pod should remain in the pending state. Wait for the Reason to be set to "AppArmor". 162 fieldSelector := fields.OneTermEqualSelector("metadata.name", pod.Name).String() 163 w := &cache.ListWatch{ 164 ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { 165 options.FieldSelector = fieldSelector 166 return e2epod.NewPodClient(f).List(ctx, options) 167 }, 168 WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { 169 options.FieldSelector = fieldSelector 170 return e2epod.NewPodClient(f).Watch(ctx, options) 171 }, 172 } 173 preconditionFunc := func(store cache.Store) (bool, error) { 174 _, exists, err := store.Get(&metav1.ObjectMeta{Namespace: pod.Namespace, Name: pod.Name}) 175 if err != nil { 176 return true, err 177 } 178 if !exists { 179 // We need to make sure we see the object in the cache before we start waiting for events 180 // or we would be waiting for the timeout if such object didn't exist. 181 return true, apierrors.NewNotFound(v1.Resource("pods"), pod.Name) 182 } 183 184 return false, nil 185 } 186 ctx, cancel := watchtools.ContextWithOptionalTimeout(ctx, framework.PodStartTimeout) 187 defer cancel() 188 _, err := watchtools.UntilWithSync(ctx, w, &v1.Pod{}, preconditionFunc, func(e watch.Event) (bool, error) { 189 switch e.Type { 190 case watch.Deleted: 191 return false, apierrors.NewNotFound(schema.GroupResource{Resource: "pods"}, pod.Name) 192 } 193 switch t := e.Object.(type) { 194 case *v1.Pod: 195 if t.Status.Reason == "AppArmor" { 196 return true, nil 197 } 198 // Loading a profile not available on disk should return a container creation error 199 if len(t.Status.ContainerStatuses) > 0 && t.Status.ContainerStatuses[0].State.Waiting.Reason == kuberuntime.ErrCreateContainer.Error() { 200 return true, nil 201 } 202 } 203 return false, nil 204 }) 205 framework.ExpectNoError(err) 206 } 207 p, err := e2epod.NewPodClient(f).Get(ctx, pod.Name, metav1.GetOptions{}) 208 framework.ExpectNoError(err) 209 return p.Status 210 } 211 212 func createPodWithAppArmor(ctx context.Context, f *framework.Framework, profile string) *v1.Pod { 213 pod := &v1.Pod{ 214 ObjectMeta: metav1.ObjectMeta{ 215 Name: fmt.Sprintf("test-apparmor-%s", strings.Replace(profile, "/", "-", -1)), 216 Annotations: map[string]string{ 217 v1.AppArmorBetaContainerAnnotationKeyPrefix + "test": profile, 218 }, 219 }, 220 Spec: v1.PodSpec{ 221 Containers: []v1.Container{{ 222 Name: "test", 223 Image: busyboxImage, 224 Command: []string{"touch", "foo"}, 225 }}, 226 RestartPolicy: v1.RestartPolicyNever, 227 }, 228 } 229 return e2epod.NewPodClient(f).Create(ctx, pod) 230 } 231 232 func expectSoftRejection(status v1.PodStatus) { 233 args := []interface{}{"PodStatus: %+v", status} 234 gomega.Expect(status.Phase).To(gomega.Equal(v1.PodPending), args...) 235 gomega.Expect(status.Reason).To(gomega.Equal("AppArmor"), args...) 236 gomega.Expect(status.Message).To(gomega.ContainSubstring("AppArmor"), args...) 237 gomega.Expect(status.ContainerStatuses[0].State.Waiting.Reason).To(gomega.Equal("Blocked"), args...) 238 } 239 240 func isAppArmorEnabled() bool { 241 // TODO(tallclair): Pass this through the image setup rather than hardcoding. 242 if strings.Contains(framework.TestContext.NodeName, "-gci-dev-") { 243 gciVersionRe := regexp.MustCompile("-gci-dev-([0-9]+)-") 244 matches := gciVersionRe.FindStringSubmatch(framework.TestContext.NodeName) 245 if len(matches) == 2 { 246 version, err := strconv.Atoi(matches[1]) 247 if err != nil { 248 klog.Errorf("Error parsing GCI version from NodeName %q: %v", framework.TestContext.NodeName, err) 249 return false 250 } 251 return version >= 54 252 } 253 return false 254 } 255 if strings.Contains(framework.TestContext.NodeName, "-ubuntu-") { 256 return true 257 } 258 return apparmor.IsEnabled() 259 }