github.com/containers/libpod@v1.9.4-0.20220419124438-4284fd425507/test/e2e/common_test.go (about) 1 package integration 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "io/ioutil" 7 "math/rand" 8 "os" 9 "os/exec" 10 "path/filepath" 11 "sort" 12 "strconv" 13 "strings" 14 "testing" 15 "time" 16 17 "github.com/containers/libpod/libpod" 18 "github.com/containers/libpod/libpod/define" 19 "github.com/containers/libpod/pkg/inspect" 20 "github.com/containers/libpod/pkg/rootless" 21 . "github.com/containers/libpod/test/utils" 22 "github.com/containers/storage" 23 "github.com/containers/storage/pkg/reexec" 24 "github.com/containers/storage/pkg/stringid" 25 . "github.com/onsi/ginkgo" 26 . "github.com/onsi/gomega" 27 "github.com/onsi/gomega/gexec" 28 "github.com/pkg/errors" 29 ) 30 31 var ( 32 PODMAN_BINARY string 33 CONMON_BINARY string 34 CNI_CONFIG_DIR string 35 RUNC_BINARY string 36 INTEGRATION_ROOT string 37 CGROUP_MANAGER = "systemd" 38 ARTIFACT_DIR = "/tmp/.artifacts" 39 RESTORE_IMAGES = []string{ALPINE, BB} 40 defaultWaitTimeout = 90 41 ) 42 43 // PodmanTestIntegration struct for command line options 44 type PodmanTestIntegration struct { 45 PodmanTest 46 ConmonBinary string 47 CrioRoot string 48 CNIConfigDir string 49 OCIRuntime string 50 RunRoot string 51 StorageOptions string 52 SignaturePolicyPath string 53 CgroupManager string 54 Host HostOS 55 Timings []string 56 TmpDir string 57 } 58 59 var LockTmpDir string 60 61 // PodmanSessionIntegration sturct for command line session 62 type PodmanSessionIntegration struct { 63 *PodmanSession 64 } 65 66 type testResult struct { 67 name string 68 length float64 69 } 70 71 type testResultsSorted []testResult 72 73 func (a testResultsSorted) Len() int { return len(a) } 74 func (a testResultsSorted) Swap(i, j int) { a[i], a[j] = a[j], a[i] } 75 76 type testResultsSortedLength struct{ testResultsSorted } 77 78 func (a testResultsSorted) Less(i, j int) bool { return a[i].length < a[j].length } 79 80 var testResults []testResult 81 82 func TestMain(m *testing.M) { 83 if reexec.Init() { 84 return 85 } 86 os.Exit(m.Run()) 87 } 88 89 // TestLibpod ginkgo master function 90 func TestLibpod(t *testing.T) { 91 if os.Getenv("NOCACHE") == "1" { 92 CACHE_IMAGES = []string{} 93 RESTORE_IMAGES = []string{} 94 } 95 RegisterFailHandler(Fail) 96 RunSpecs(t, "Libpod Suite") 97 } 98 99 var _ = SynchronizedBeforeSuite(func() []byte { 100 // Cache images 101 cwd, _ := os.Getwd() 102 INTEGRATION_ROOT = filepath.Join(cwd, "../../") 103 podman := PodmanTestCreate("/tmp") 104 podman.ArtifactPath = ARTIFACT_DIR 105 if _, err := os.Stat(ARTIFACT_DIR); os.IsNotExist(err) { 106 if err = os.Mkdir(ARTIFACT_DIR, 0777); err != nil { 107 fmt.Printf("%q\n", err) 108 os.Exit(1) 109 } 110 } 111 112 // make cache dir 113 if err := os.MkdirAll(ImageCacheDir, 0777); err != nil { 114 fmt.Printf("%q\n", err) 115 os.Exit(1) 116 } 117 118 for _, image := range CACHE_IMAGES { 119 podman.createArtifact(image) 120 } 121 122 // If running localized tests, the cache dir is created and populated. if the 123 // tests are remote, this is a no-op 124 populateCache(podman) 125 126 host := GetHostDistributionInfo() 127 if host.Distribution == "rhel" && strings.HasPrefix(host.Version, "7") { 128 f, err := os.OpenFile("/proc/sys/user/max_user_namespaces", os.O_WRONLY, 0644) 129 if err != nil { 130 fmt.Println("Unable to enable userspace on RHEL 7") 131 os.Exit(1) 132 } 133 _, err = f.WriteString("15000") 134 if err != nil { 135 fmt.Println("Unable to enable userspace on RHEL 7") 136 os.Exit(1) 137 } 138 f.Close() 139 } 140 path, err := ioutil.TempDir("", "libpodlock") 141 if err != nil { 142 fmt.Println(err) 143 os.Exit(1) 144 } 145 return []byte(path) 146 }, func(data []byte) { 147 LockTmpDir = string(data) 148 }) 149 150 func (p *PodmanTestIntegration) Setup() { 151 cwd, _ := os.Getwd() 152 INTEGRATION_ROOT = filepath.Join(cwd, "../../") 153 p.ArtifactPath = ARTIFACT_DIR 154 } 155 156 var _ = SynchronizedAfterSuite(func() {}, 157 func() { 158 sort.Sort(testResultsSortedLength{testResults}) 159 fmt.Println("integration timing results") 160 for _, result := range testResults { 161 fmt.Printf("%s\t\t%f\n", result.name, result.length) 162 } 163 164 // previous crio-run 165 tempdir, err := CreateTempDirInTempDir() 166 if err != nil { 167 os.Exit(1) 168 } 169 podmanTest := PodmanTestCreate(tempdir) 170 171 if err := os.RemoveAll(podmanTest.CrioRoot); err != nil { 172 fmt.Printf("%q\n", err) 173 } 174 175 // for localized tests, this removes the image cache dir and for remote tests 176 // this is a no-op 177 removeCache() 178 }) 179 180 // PodmanTestCreate creates a PodmanTestIntegration instance for the tests 181 func PodmanTestCreateUtil(tempDir string, remote bool) *PodmanTestIntegration { 182 var ( 183 podmanRemoteBinary string 184 ) 185 186 host := GetHostDistributionInfo() 187 cwd, _ := os.Getwd() 188 189 podmanBinary := filepath.Join(cwd, "../../bin/podman") 190 if os.Getenv("PODMAN_BINARY") != "" { 191 podmanBinary = os.Getenv("PODMAN_BINARY") 192 } 193 194 if remote { 195 podmanRemoteBinary = filepath.Join(cwd, "../../bin/podman-remote") 196 if os.Getenv("PODMAN_REMOTE_BINARY") != "" { 197 podmanRemoteBinary = os.Getenv("PODMAN_REMOTE_BINARY") 198 } 199 } 200 conmonBinary := filepath.Join("/usr/libexec/podman/conmon") 201 altConmonBinary := "/usr/bin/conmon" 202 if _, err := os.Stat(conmonBinary); os.IsNotExist(err) { 203 conmonBinary = altConmonBinary 204 } 205 if os.Getenv("CONMON_BINARY") != "" { 206 conmonBinary = os.Getenv("CONMON_BINARY") 207 } 208 storageOptions := STORAGE_OPTIONS 209 if os.Getenv("STORAGE_OPTIONS") != "" { 210 storageOptions = os.Getenv("STORAGE_OPTIONS") 211 } 212 213 cgroupManager := CGROUP_MANAGER 214 if rootless.IsRootless() { 215 cgroupManager = "cgroupfs" 216 } 217 if os.Getenv("CGROUP_MANAGER") != "" { 218 cgroupManager = os.Getenv("CGROUP_MANAGER") 219 } 220 221 ociRuntime := os.Getenv("OCI_RUNTIME") 222 if ociRuntime == "" { 223 var err error 224 ociRuntime, err = exec.LookPath("crun") 225 // If we cannot find the crun binary, setting to something static as we have no way 226 // to return an error. The tests will fail and point out that the runc binary could 227 // not be found nicely. 228 if err != nil { 229 ociRuntime = "/usr/bin/runc" 230 } 231 } 232 os.Setenv("DISABLE_HC_SYSTEMD", "true") 233 CNIConfigDir := "/etc/cni/net.d" 234 235 storageFs := STORAGE_FS 236 if rootless.IsRootless() { 237 storageFs = ROOTLESS_STORAGE_FS 238 } 239 p := &PodmanTestIntegration{ 240 PodmanTest: PodmanTest{ 241 PodmanBinary: podmanBinary, 242 ArtifactPath: ARTIFACT_DIR, 243 TempDir: tempDir, 244 RemoteTest: remote, 245 ImageCacheFS: storageFs, 246 ImageCacheDir: ImageCacheDir, 247 }, 248 ConmonBinary: conmonBinary, 249 CrioRoot: filepath.Join(tempDir, "crio"), 250 TmpDir: tempDir, 251 CNIConfigDir: CNIConfigDir, 252 OCIRuntime: ociRuntime, 253 RunRoot: filepath.Join(tempDir, "crio-run"), 254 StorageOptions: storageOptions, 255 SignaturePolicyPath: filepath.Join(INTEGRATION_ROOT, "test/policy.json"), 256 CgroupManager: cgroupManager, 257 Host: host, 258 } 259 if remote { 260 p.PodmanTest.RemotePodmanBinary = podmanRemoteBinary 261 uuid := stringid.GenerateNonCryptoID() 262 if !rootless.IsRootless() { 263 p.VarlinkEndpoint = fmt.Sprintf("unix:/run/podman/io.podman-%s", uuid) 264 } else { 265 runtimeDir := os.Getenv("XDG_RUNTIME_DIR") 266 socket := fmt.Sprintf("io.podman-%s", uuid) 267 fqpath := filepath.Join(runtimeDir, socket) 268 p.VarlinkEndpoint = fmt.Sprintf("unix:%s", fqpath) 269 } 270 } 271 272 // Setup registries.conf ENV variable 273 p.setDefaultRegistriesConfigEnv() 274 // Rewrite the PodmanAsUser function 275 p.PodmanMakeOptions = p.makeOptions 276 return p 277 } 278 279 // RestoreAllArtifacts unpacks all cached images 280 func (p *PodmanTestIntegration) RestoreAllArtifacts() error { 281 if os.Getenv("NO_TEST_CACHE") != "" { 282 return nil 283 } 284 for _, image := range RESTORE_IMAGES { 285 if err := p.RestoreArtifact(image); err != nil { 286 return err 287 } 288 } 289 return nil 290 } 291 292 // createArtifact creates a cached image in the artifact dir 293 func (p *PodmanTestIntegration) createArtifact(image string) { 294 if os.Getenv("NO_TEST_CACHE") != "" { 295 return 296 } 297 dest := strings.Split(image, "/") 298 destName := fmt.Sprintf("/tmp/%s.tar", strings.Replace(strings.Join(strings.Split(dest[len(dest)-1], "/"), ""), ":", "-", -1)) 299 fmt.Printf("Caching %s at %s...", image, destName) 300 if _, err := os.Stat(destName); os.IsNotExist(err) { 301 pull := p.PodmanNoCache([]string{"pull", image}) 302 pull.Wait(90) 303 Expect(pull.ExitCode()).To(Equal(0)) 304 305 save := p.PodmanNoCache([]string{"save", "-o", destName, image}) 306 save.Wait(90) 307 Expect(save.ExitCode()).To(Equal(0)) 308 fmt.Printf("\n") 309 } else { 310 fmt.Printf(" already exists.\n") 311 } 312 } 313 314 // InspectImageJSON takes the session output of an inspect 315 // image and returns json 316 func (s *PodmanSessionIntegration) InspectImageJSON() []inspect.ImageData { 317 var i []inspect.ImageData 318 err := json.Unmarshal(s.Out.Contents(), &i) 319 Expect(err).To(BeNil()) 320 return i 321 } 322 323 // InspectContainer returns a container's inspect data in JSON format 324 func (p *PodmanTestIntegration) InspectContainer(name string) []define.InspectContainerData { 325 cmd := []string{"inspect", name} 326 session := p.Podman(cmd) 327 session.WaitWithDefaultTimeout() 328 Expect(session.ExitCode()).To(Equal(0)) 329 return session.InspectContainerToJSON() 330 } 331 332 func processTestResult(f GinkgoTestDescription) { 333 tr := testResult{length: f.Duration.Seconds(), name: f.TestText} 334 testResults = append(testResults, tr) 335 } 336 337 func GetPortLock(port string) storage.Locker { 338 lockFile := filepath.Join(LockTmpDir, port) 339 lock, err := storage.GetLockfile(lockFile) 340 if err != nil { 341 fmt.Println(err) 342 os.Exit(1) 343 } 344 lock.Lock() 345 return lock 346 } 347 348 // GetRandomIPAddress returns a random IP address to avoid IP 349 // collisions during parallel tests 350 func GetRandomIPAddress() string { 351 // To avoid IP collisions of initialize random seed for random IP addresses 352 rand.Seed(time.Now().UnixNano()) 353 // Add GinkgoParallelNode() on top of the IP address 354 // in case of the same random seed 355 ip3 := strconv.Itoa(rand.Intn(230) + GinkgoParallelNode()) 356 ip4 := strconv.Itoa(rand.Intn(230) + GinkgoParallelNode()) 357 return "10.88." + ip3 + "." + ip4 358 } 359 360 // RunTopContainer runs a simple container in the background that 361 // runs top. If the name passed != "", it will have a name 362 func (p *PodmanTestIntegration) RunTopContainer(name string) *PodmanSessionIntegration { 363 var podmanArgs = []string{"run"} 364 if name != "" { 365 podmanArgs = append(podmanArgs, "--name", name) 366 } 367 podmanArgs = append(podmanArgs, "-d", ALPINE, "top") 368 return p.Podman(podmanArgs) 369 } 370 371 // RunLsContainer runs a simple container in the background that 372 // simply runs ls. If the name passed != "", it will have a name 373 func (p *PodmanTestIntegration) RunLsContainer(name string) (*PodmanSessionIntegration, int, string) { 374 var podmanArgs = []string{"run"} 375 if name != "" { 376 podmanArgs = append(podmanArgs, "--name", name) 377 } 378 podmanArgs = append(podmanArgs, "-d", ALPINE, "ls") 379 session := p.Podman(podmanArgs) 380 session.WaitWithDefaultTimeout() 381 return session, session.ExitCode(), session.OutputToString() 382 } 383 384 // RunNginxWithHealthCheck runs the alpine nginx container with an optional name and adds a healthcheck into it 385 func (p *PodmanTestIntegration) RunNginxWithHealthCheck(name string) (*PodmanSessionIntegration, string) { 386 var podmanArgs = []string{"run"} 387 if name != "" { 388 podmanArgs = append(podmanArgs, "--name", name) 389 } 390 podmanArgs = append(podmanArgs, "-dt", "-P", "--health-cmd", "curl http://localhost/", nginx) 391 session := p.Podman(podmanArgs) 392 session.WaitWithDefaultTimeout() 393 return session, session.OutputToString() 394 } 395 396 func (p *PodmanTestIntegration) RunLsContainerInPod(name, pod string) (*PodmanSessionIntegration, int, string) { 397 var podmanArgs = []string{"run", "--pod", pod} 398 if name != "" { 399 podmanArgs = append(podmanArgs, "--name", name) 400 } 401 podmanArgs = append(podmanArgs, "-d", ALPINE, "ls") 402 session := p.Podman(podmanArgs) 403 session.WaitWithDefaultTimeout() 404 return session, session.ExitCode(), session.OutputToString() 405 } 406 407 // BuildImage uses podman build and buildah to build an image 408 // called imageName based on a string dockerfile 409 func (p *PodmanTestIntegration) BuildImage(dockerfile, imageName string, layers string) { 410 dockerfilePath := filepath.Join(p.TempDir, "Dockerfile") 411 err := ioutil.WriteFile(dockerfilePath, []byte(dockerfile), 0755) 412 Expect(err).To(BeNil()) 413 session := p.PodmanNoCache([]string{"build", "--layers=" + layers, "-t", imageName, "--file", dockerfilePath, p.TempDir}) 414 session.Wait(120) 415 Expect(session.ExitCode()).To(Equal(0)) 416 } 417 418 // PodmanPID execs podman and returns its PID 419 func (p *PodmanTestIntegration) PodmanPID(args []string) (*PodmanSessionIntegration, int) { 420 podmanOptions := p.MakeOptions(args, false, false) 421 fmt.Printf("Running: %s %s\n", p.PodmanBinary, strings.Join(podmanOptions, " ")) 422 command := exec.Command(p.PodmanBinary, podmanOptions...) 423 session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) 424 if err != nil { 425 Fail(fmt.Sprintf("unable to run podman command: %s", strings.Join(podmanOptions, " "))) 426 } 427 podmanSession := &PodmanSession{session} 428 return &PodmanSessionIntegration{podmanSession}, command.Process.Pid 429 } 430 431 // Cleanup cleans up the temporary store 432 func (p *PodmanTestIntegration) Cleanup() { 433 // Remove all containers 434 stopall := p.Podman([]string{"stop", "-a", "--time", "0"}) 435 stopall.Wait(90) 436 437 podstop := p.Podman([]string{"pod", "stop", "-a", "-t", "0"}) 438 podstop.WaitWithDefaultTimeout() 439 podrm := p.Podman([]string{"pod", "rm", "-fa"}) 440 podrm.WaitWithDefaultTimeout() 441 442 session := p.Podman([]string{"rm", "-fa"}) 443 session.Wait(90) 444 445 p.StopVarlink() 446 // Nuke tempdir 447 if err := os.RemoveAll(p.TempDir); err != nil { 448 fmt.Printf("%q\n", err) 449 } 450 451 // Clean up the registries configuration file ENV variable set in Create 452 resetRegistriesConfigEnv() 453 } 454 455 // CleanupPod cleans up the temporary store 456 func (p *PodmanTestIntegration) CleanupPod() { 457 // Remove all containers 458 session := p.Podman([]string{"pod", "rm", "-fa"}) 459 session.Wait(90) 460 // Nuke tempdir 461 if err := os.RemoveAll(p.TempDir); err != nil { 462 fmt.Printf("%q\n", err) 463 } 464 } 465 466 // CleanupVolume cleans up the temporary store 467 func (p *PodmanTestIntegration) CleanupVolume() { 468 // Remove all containers 469 session := p.Podman([]string{"volume", "rm", "-fa"}) 470 session.Wait(90) 471 // Nuke tempdir 472 if err := os.RemoveAll(p.TempDir); err != nil { 473 fmt.Printf("%q\n", err) 474 } 475 } 476 477 // PullImages pulls multiple images 478 func (p *PodmanTestIntegration) PullImages(images []string) error { 479 for _, i := range images { 480 p.PullImage(i) 481 } 482 return nil 483 } 484 485 // PullImage pulls a single image 486 // TODO should the timeout be configurable? 487 func (p *PodmanTestIntegration) PullImage(image string) error { 488 session := p.PodmanNoCache([]string{"pull", image}) 489 session.Wait(60) 490 Expect(session.ExitCode()).To(Equal(0)) 491 return nil 492 } 493 494 // InspectContainerToJSON takes the session output of an inspect 495 // container and returns json 496 func (s *PodmanSessionIntegration) InspectContainerToJSON() []define.InspectContainerData { 497 var i []define.InspectContainerData 498 err := json.Unmarshal(s.Out.Contents(), &i) 499 Expect(err).To(BeNil()) 500 return i 501 } 502 503 // InspectPodToJSON takes the sessions output from a pod inspect and returns json 504 func (s *PodmanSessionIntegration) InspectPodToJSON() libpod.PodInspect { 505 var i libpod.PodInspect 506 err := json.Unmarshal(s.Out.Contents(), &i) 507 Expect(err).To(BeNil()) 508 return i 509 } 510 511 // CreatePod creates a pod with no infra container 512 // it optionally takes a pod name 513 func (p *PodmanTestIntegration) CreatePod(name string) (*PodmanSessionIntegration, int, string) { 514 var podmanArgs = []string{"pod", "create", "--infra=false", "--share", ""} 515 if name != "" { 516 podmanArgs = append(podmanArgs, "--name", name) 517 } 518 session := p.Podman(podmanArgs) 519 session.WaitWithDefaultTimeout() 520 return session, session.ExitCode(), session.OutputToString() 521 } 522 523 // CreatePod creates a pod with no infra container and some labels. 524 // it optionally takes a pod name 525 func (p *PodmanTestIntegration) CreatePodWithLabels(name string, labels map[string]string) (*PodmanSessionIntegration, int, string) { 526 var podmanArgs = []string{"pod", "create", "--infra=false", "--share", ""} 527 if name != "" { 528 podmanArgs = append(podmanArgs, "--name", name) 529 } 530 for labelKey, labelValue := range labels { 531 podmanArgs = append(podmanArgs, "--label", fmt.Sprintf("%s=%s", labelKey, labelValue)) 532 } 533 session := p.Podman(podmanArgs) 534 session.WaitWithDefaultTimeout() 535 return session, session.ExitCode(), session.OutputToString() 536 } 537 538 func (p *PodmanTestIntegration) RunTopContainerInPod(name, pod string) *PodmanSessionIntegration { 539 var podmanArgs = []string{"run", "--pod", pod} 540 if name != "" { 541 podmanArgs = append(podmanArgs, "--name", name) 542 } 543 podmanArgs = append(podmanArgs, "-d", ALPINE, "top") 544 return p.Podman(podmanArgs) 545 } 546 547 func (p *PodmanTestIntegration) ImageExistsInMainStore(idOrName string) bool { 548 results := p.PodmanNoCache([]string{"image", "exists", idOrName}) 549 results.WaitWithDefaultTimeout() 550 return Expect(results.ExitCode()).To(Equal(0)) 551 } 552 553 func (p *PodmanTestIntegration) RunHealthCheck(cid string) error { 554 for i := 0; i < 10; i++ { 555 hc := p.Podman([]string{"healthcheck", "run", cid}) 556 hc.WaitWithDefaultTimeout() 557 if hc.ExitCode() == 0 { 558 return nil 559 } 560 // Restart container if it's not running 561 ps := p.Podman([]string{"ps", "--no-trunc", "--quiet", "--filter", fmt.Sprintf("id=%s", cid)}) 562 ps.WaitWithDefaultTimeout() 563 if ps.ExitCode() == 0 { 564 if !strings.Contains(ps.OutputToString(), cid) { 565 fmt.Printf("Container %s is not running, restarting", cid) 566 restart := p.Podman([]string{"restart", cid}) 567 restart.WaitWithDefaultTimeout() 568 if restart.ExitCode() != 0 { 569 return errors.Errorf("unable to restart %s", cid) 570 } 571 } 572 } 573 fmt.Printf("Waiting for %s to pass healthcheck\n", cid) 574 time.Sleep(1 * time.Second) 575 } 576 return errors.Errorf("unable to detect %s as running", cid) 577 } 578 579 func (p *PodmanTestIntegration) CreateSeccompJson(in []byte) (string, error) { 580 jsonFile := filepath.Join(p.TempDir, "seccomp.json") 581 err := WriteJsonFile(in, jsonFile) 582 if err != nil { 583 return "", err 584 } 585 return jsonFile, nil 586 }