github.com/hanks177/podman/v4@v4.1.3-0.20220613032544-16d90015bc83/test/e2e/common_test.go (about) 1 package integration 2 3 import ( 4 "bytes" 5 "fmt" 6 "io/ioutil" 7 "math/rand" 8 "net" 9 "net/url" 10 "os" 11 "os/exec" 12 "path/filepath" 13 "sort" 14 "strconv" 15 "strings" 16 "sync" 17 "testing" 18 "time" 19 20 "github.com/containers/common/pkg/cgroups" 21 "github.com/hanks177/podman/v4/libpod/define" 22 "github.com/hanks177/podman/v4/pkg/inspect" 23 "github.com/hanks177/podman/v4/pkg/rootless" 24 "github.com/hanks177/podman/v4/pkg/util" 25 . "github.com/hanks177/podman/v4/test/utils" 26 "github.com/containers/storage" 27 "github.com/containers/storage/pkg/reexec" 28 "github.com/containers/storage/pkg/stringid" 29 jsoniter "github.com/json-iterator/go" 30 . "github.com/onsi/ginkgo" 31 . "github.com/onsi/gomega" 32 . "github.com/onsi/gomega/gexec" 33 "github.com/pkg/errors" 34 "github.com/sirupsen/logrus" 35 ) 36 37 var ( 38 //lint:ignore ST1003 39 PODMAN_BINARY string //nolint:revive,stylecheck 40 INTEGRATION_ROOT string //nolint:revive,stylecheck 41 CGROUP_MANAGER = "systemd" //nolint:revive,stylecheck 42 RESTORE_IMAGES = []string{ALPINE, BB, nginx} //nolint:revive,stylecheck 43 defaultWaitTimeout = 90 44 CGROUPSV2, _ = cgroups.IsCgroup2UnifiedMode() //nolint:revive,stylecheck 45 ) 46 47 // PodmanTestIntegration struct for command line options 48 type PodmanTestIntegration struct { 49 PodmanTest 50 ConmonBinary string 51 Root string 52 NetworkConfigDir string 53 OCIRuntime string 54 RunRoot string 55 StorageOptions string 56 SignaturePolicyPath string 57 CgroupManager string 58 Host HostOS 59 Timings []string 60 TmpDir string 61 RemoteStartErr error 62 } 63 64 var LockTmpDir string 65 66 // PodmanSessionIntegration struct for command line session 67 type PodmanSessionIntegration struct { 68 *PodmanSession 69 } 70 71 type testResult struct { 72 name string 73 length float64 74 } 75 76 type testResultsSorted []testResult 77 78 func (a testResultsSorted) Len() int { return len(a) } 79 func (a testResultsSorted) Swap(i, j int) { a[i], a[j] = a[j], a[i] } 80 81 type testResultsSortedLength struct{ testResultsSorted } 82 83 func (a testResultsSorted) Less(i, j int) bool { return a[i].length < a[j].length } 84 85 var testResults []testResult 86 var testResultsMutex sync.Mutex 87 88 func TestMain(m *testing.M) { 89 if reexec.Init() { 90 return 91 } 92 os.Exit(m.Run()) 93 } 94 95 // TestLibpod ginkgo master function 96 func TestLibpod(t *testing.T) { 97 if os.Getenv("NOCACHE") == "1" { 98 CACHE_IMAGES = []string{} 99 RESTORE_IMAGES = []string{} 100 } 101 RegisterFailHandler(Fail) 102 RunSpecs(t, "Libpod Suite") 103 } 104 105 var _ = SynchronizedBeforeSuite(func() []byte { 106 // make cache dir 107 if err := os.MkdirAll(ImageCacheDir, 0777); err != nil { 108 fmt.Printf("%q\n", err) 109 os.Exit(1) 110 } 111 112 // Cache images 113 cwd, _ := os.Getwd() 114 INTEGRATION_ROOT = filepath.Join(cwd, "../../") 115 podman := PodmanTestSetup("/tmp") 116 117 // Pull cirros but don't put it into the cache 118 pullImages := []string{cirros, fedoraToolbox, volumeTest} 119 pullImages = append(pullImages, CACHE_IMAGES...) 120 for _, image := range pullImages { 121 podman.createArtifact(image) 122 } 123 124 if err := os.MkdirAll(filepath.Join(ImageCacheDir, podman.ImageCacheFS+"-images"), 0777); err != nil { 125 fmt.Printf("%q\n", err) 126 os.Exit(1) 127 } 128 podman.Root = ImageCacheDir 129 // If running localized tests, the cache dir is created and populated. if the 130 // tests are remote, this is a no-op 131 populateCache(podman) 132 133 host := GetHostDistributionInfo() 134 if host.Distribution == "rhel" && strings.HasPrefix(host.Version, "7") { 135 f, err := os.OpenFile("/proc/sys/user/max_user_namespaces", os.O_WRONLY, 0644) 136 if err != nil { 137 fmt.Println("Unable to enable userspace on RHEL 7") 138 os.Exit(1) 139 } 140 _, err = f.WriteString("15000") 141 if err != nil { 142 fmt.Println("Unable to enable userspace on RHEL 7") 143 os.Exit(1) 144 } 145 f.Close() 146 } 147 path, err := ioutil.TempDir("", "libpodlock") 148 if err != nil { 149 fmt.Println(err) 150 os.Exit(1) 151 } 152 153 // If running remote, we need to stop the associated podman system service 154 if podman.RemoteTest { 155 podman.StopRemoteService() 156 } 157 158 return []byte(path) 159 }, func(data []byte) { 160 cwd, _ := os.Getwd() 161 INTEGRATION_ROOT = filepath.Join(cwd, "../../") 162 LockTmpDir = string(data) 163 }) 164 165 func (p *PodmanTestIntegration) Setup() { 166 cwd, _ := os.Getwd() 167 INTEGRATION_ROOT = filepath.Join(cwd, "../../") 168 } 169 170 var _ = SynchronizedAfterSuite(func() {}, 171 func() { 172 sort.Sort(testResultsSortedLength{testResults}) 173 fmt.Println("integration timing results") 174 for _, result := range testResults { 175 fmt.Printf("%s\t\t%f\n", result.name, result.length) 176 } 177 178 // previous runroot 179 tempdir, err := CreateTempDirInTempDir() 180 if err != nil { 181 os.Exit(1) 182 } 183 podmanTest := PodmanTestCreate(tempdir) 184 185 if err := os.RemoveAll(podmanTest.Root); err != nil { 186 fmt.Printf("%q\n", err) 187 } 188 189 // If running remote, we need to stop the associated podman system service 190 if podmanTest.RemoteTest { 191 podmanTest.StopRemoteService() 192 } 193 // for localized tests, this removes the image cache dir and for remote tests 194 // this is a no-op 195 removeCache() 196 }) 197 198 // PodmanTestCreate creates a PodmanTestIntegration instance for the tests 199 func PodmanTestCreateUtil(tempDir string, remote bool) *PodmanTestIntegration { 200 var podmanRemoteBinary string 201 202 host := GetHostDistributionInfo() 203 cwd, _ := os.Getwd() 204 205 root := filepath.Join(tempDir, "root") 206 podmanBinary := filepath.Join(cwd, "../../bin/podman") 207 if os.Getenv("PODMAN_BINARY") != "" { 208 podmanBinary = os.Getenv("PODMAN_BINARY") 209 } 210 211 podmanRemoteBinary = filepath.Join(cwd, "../../bin/podman-remote") 212 if os.Getenv("PODMAN_REMOTE_BINARY") != "" { 213 podmanRemoteBinary = os.Getenv("PODMAN_REMOTE_BINARY") 214 } 215 216 conmonBinary := "/usr/libexec/podman/conmon" 217 altConmonBinary := "/usr/bin/conmon" 218 if _, err := os.Stat(conmonBinary); os.IsNotExist(err) { 219 conmonBinary = altConmonBinary 220 } 221 if os.Getenv("CONMON_BINARY") != "" { 222 conmonBinary = os.Getenv("CONMON_BINARY") 223 } 224 storageOptions := STORAGE_OPTIONS 225 if os.Getenv("STORAGE_OPTIONS") != "" { 226 storageOptions = os.Getenv("STORAGE_OPTIONS") 227 } 228 229 cgroupManager := CGROUP_MANAGER 230 if rootless.IsRootless() { 231 cgroupManager = "cgroupfs" 232 } 233 if os.Getenv("CGROUP_MANAGER") != "" { 234 cgroupManager = os.Getenv("CGROUP_MANAGER") 235 } 236 237 ociRuntime := os.Getenv("OCI_RUNTIME") 238 if ociRuntime == "" { 239 ociRuntime = "crun" 240 } 241 os.Setenv("DISABLE_HC_SYSTEMD", "true") 242 243 networkBackend := CNI 244 networkConfigDir := "/etc/cni/net.d" 245 if rootless.IsRootless() { 246 networkConfigDir = filepath.Join(os.Getenv("HOME"), ".config/cni/net.d") 247 } 248 249 if strings.ToLower(os.Getenv("NETWORK_BACKEND")) == "netavark" { 250 networkBackend = Netavark 251 networkConfigDir = "/etc/containers/networks" 252 if rootless.IsRootless() { 253 networkConfigDir = filepath.Join(root, "etc", "networks") 254 } 255 } 256 257 if err := os.MkdirAll(root, 0755); err != nil { 258 panic(err) 259 } 260 261 if err := os.MkdirAll(networkConfigDir, 0755); err != nil { 262 panic(err) 263 } 264 265 storageFs := STORAGE_FS 266 if rootless.IsRootless() { 267 storageFs = ROOTLESS_STORAGE_FS 268 } 269 if os.Getenv("STORAGE_FS") != "" { 270 storageFs = os.Getenv("STORAGE_FS") 271 storageOptions = "--storage-driver " + storageFs 272 } 273 p := &PodmanTestIntegration{ 274 PodmanTest: PodmanTest{ 275 PodmanBinary: podmanBinary, 276 RemotePodmanBinary: podmanRemoteBinary, 277 TempDir: tempDir, 278 RemoteTest: remote, 279 ImageCacheFS: storageFs, 280 ImageCacheDir: ImageCacheDir, 281 NetworkBackend: networkBackend, 282 }, 283 ConmonBinary: conmonBinary, 284 Root: root, 285 TmpDir: tempDir, 286 NetworkConfigDir: networkConfigDir, 287 OCIRuntime: ociRuntime, 288 RunRoot: filepath.Join(tempDir, "runroot"), 289 StorageOptions: storageOptions, 290 SignaturePolicyPath: filepath.Join(INTEGRATION_ROOT, "test/policy.json"), 291 CgroupManager: cgroupManager, 292 Host: host, 293 } 294 295 if remote { 296 var pathPrefix string 297 if !rootless.IsRootless() { 298 pathPrefix = "/run/podman/podman" 299 } else { 300 runtimeDir := os.Getenv("XDG_RUNTIME_DIR") 301 pathPrefix = filepath.Join(runtimeDir, "podman") 302 } 303 // We want to avoid collisions in socket paths, but using the 304 // socket directly for a collision check doesn’t work; bind(2) on AF_UNIX 305 // creates the file, and we need to pass a unique path now before the bind(2) 306 // happens. So, use a podman-%s.sock-lock empty file as a marker. 307 tries := 0 308 for { 309 uuid := stringid.GenerateNonCryptoID() 310 lockPath := fmt.Sprintf("%s-%s.sock-lock", pathPrefix, uuid) 311 lockFile, err := os.OpenFile(lockPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0700) 312 if err == nil { 313 lockFile.Close() 314 p.RemoteSocketLock = lockPath 315 p.RemoteSocket = fmt.Sprintf("unix:%s-%s.sock", pathPrefix, uuid) 316 break 317 } 318 tries++ 319 if tries >= 1000 { 320 panic("Too many RemoteSocket collisions") 321 } 322 } 323 } 324 325 // Setup registries.conf ENV variable 326 p.setDefaultRegistriesConfigEnv() 327 // Rewrite the PodmanAsUser function 328 p.PodmanMakeOptions = p.makeOptions 329 return p 330 } 331 332 func (p PodmanTestIntegration) AddImageToRWStore(image string) { 333 if err := p.RestoreArtifact(image); err != nil { 334 logrus.Errorf("Unable to restore %s to RW store", image) 335 } 336 } 337 338 func imageTarPath(image string) string { 339 cacheDir := os.Getenv("PODMAN_TEST_IMAGE_CACHE_DIR") 340 if cacheDir == "" { 341 cacheDir = os.Getenv("TMPDIR") 342 if cacheDir == "" { 343 cacheDir = "/tmp" 344 } 345 } 346 347 // e.g., registry.com/fubar:latest -> registry.com-fubar-latest.tar 348 imageCacheName := strings.ReplaceAll(strings.ReplaceAll(image, ":", "-"), "/", "-") + ".tar" 349 350 return filepath.Join(cacheDir, imageCacheName) 351 } 352 353 // createArtifact creates a cached image tarball in a local directory 354 func (p *PodmanTestIntegration) createArtifact(image string) { 355 if os.Getenv("NO_TEST_CACHE") != "" { 356 return 357 } 358 destName := imageTarPath(image) 359 if _, err := os.Stat(destName); os.IsNotExist(err) { 360 fmt.Printf("Caching %s at %s...\n", image, destName) 361 pull := p.PodmanNoCache([]string{"pull", image}) 362 pull.Wait(440) 363 Expect(pull).Should(Exit(0)) 364 365 save := p.PodmanNoCache([]string{"save", "-o", destName, image}) 366 save.Wait(90) 367 Expect(save).Should(Exit(0)) 368 fmt.Printf("\n") 369 } else { 370 fmt.Printf("[image already cached: %s]\n", destName) 371 } 372 } 373 374 // InspectImageJSON takes the session output of an inspect 375 // image and returns json 376 func (s *PodmanSessionIntegration) InspectImageJSON() []inspect.ImageData { 377 var i []inspect.ImageData 378 err := jsoniter.Unmarshal(s.Out.Contents(), &i) 379 Expect(err).To(BeNil()) 380 return i 381 } 382 383 // InspectContainer returns a container's inspect data in JSON format 384 func (p *PodmanTestIntegration) InspectContainer(name string) []define.InspectContainerData { 385 cmd := []string{"inspect", name} 386 session := p.Podman(cmd) 387 session.WaitWithDefaultTimeout() 388 Expect(session).Should(Exit(0)) 389 return session.InspectContainerToJSON() 390 } 391 392 func processTestResult(f GinkgoTestDescription) { 393 tr := testResult{length: f.Duration.Seconds(), name: f.TestText} 394 testResultsMutex.Lock() 395 testResults = append(testResults, tr) 396 testResultsMutex.Unlock() 397 } 398 399 func GetPortLock(port string) storage.Locker { 400 lockFile := filepath.Join(LockTmpDir, port) 401 lock, err := storage.GetLockfile(lockFile) 402 if err != nil { 403 fmt.Println(err) 404 os.Exit(1) 405 } 406 lock.Lock() 407 return lock 408 } 409 410 // GetRandomIPAddress returns a random IP address to avoid IP 411 // collisions during parallel tests 412 func GetRandomIPAddress() string { 413 // To avoid IP collisions of initialize random seed for random IP addresses 414 rand.Seed(time.Now().UnixNano()) 415 // Add GinkgoParallelNode() on top of the IP address 416 // in case of the same random seed 417 ip3 := strconv.Itoa(rand.Intn(230) + GinkgoParallelNode()) 418 ip4 := strconv.Itoa(rand.Intn(230) + GinkgoParallelNode()) 419 return "10.88." + ip3 + "." + ip4 420 } 421 422 // RunTopContainer runs a simple container in the background that 423 // runs top. If the name passed != "", it will have a name 424 func (p *PodmanTestIntegration) RunTopContainer(name string) *PodmanSessionIntegration { 425 return p.RunTopContainerWithArgs(name, nil) 426 } 427 428 // RunTopContainerWithArgs runs a simple container in the background that 429 // runs top. If the name passed != "", it will have a name, command args can also be passed in 430 func (p *PodmanTestIntegration) RunTopContainerWithArgs(name string, args []string) *PodmanSessionIntegration { 431 var podmanArgs = []string{"run"} 432 if name != "" { 433 podmanArgs = append(podmanArgs, "--name", name) 434 } 435 podmanArgs = append(podmanArgs, args...) 436 podmanArgs = append(podmanArgs, "-d", ALPINE, "top") 437 return p.Podman(podmanArgs) 438 } 439 440 // RunLsContainer runs a simple container in the background that 441 // simply runs ls. If the name passed != "", it will have a name 442 func (p *PodmanTestIntegration) RunLsContainer(name string) (*PodmanSessionIntegration, int, string) { 443 var podmanArgs = []string{"run"} 444 if name != "" { 445 podmanArgs = append(podmanArgs, "--name", name) 446 } 447 podmanArgs = append(podmanArgs, "-d", ALPINE, "ls") 448 session := p.Podman(podmanArgs) 449 session.WaitWithDefaultTimeout() 450 if session.ExitCode() != 0 { 451 return session, session.ExitCode(), session.OutputToString() 452 } 453 cid := session.OutputToString() 454 455 wsession := p.Podman([]string{"wait", cid}) 456 wsession.WaitWithDefaultTimeout() 457 return session, wsession.ExitCode(), cid 458 } 459 460 // RunNginxWithHealthCheck runs the alpine nginx container with an optional name and adds a healthcheck into it 461 func (p *PodmanTestIntegration) RunNginxWithHealthCheck(name string) (*PodmanSessionIntegration, string) { 462 var podmanArgs = []string{"run"} 463 if name != "" { 464 podmanArgs = append(podmanArgs, "--name", name) 465 } 466 // curl without -f exits 0 even if http code >= 400! 467 podmanArgs = append(podmanArgs, "-dt", "-P", "--health-cmd", "curl -f http://localhost/", nginx) 468 session := p.Podman(podmanArgs) 469 session.WaitWithDefaultTimeout() 470 return session, session.OutputToString() 471 } 472 473 func (p *PodmanTestIntegration) RunLsContainerInPod(name, pod string) (*PodmanSessionIntegration, int, string) { 474 var podmanArgs = []string{"run", "--pod", pod} 475 if name != "" { 476 podmanArgs = append(podmanArgs, "--name", name) 477 } 478 podmanArgs = append(podmanArgs, "-d", ALPINE, "ls") 479 session := p.Podman(podmanArgs) 480 session.WaitWithDefaultTimeout() 481 if session.ExitCode() != 0 { 482 return session, session.ExitCode(), session.OutputToString() 483 } 484 cid := session.OutputToString() 485 486 wsession := p.Podman([]string{"wait", cid}) 487 wsession.WaitWithDefaultTimeout() 488 return session, wsession.ExitCode(), cid 489 } 490 491 // BuildImage uses podman build and buildah to build an image 492 // called imageName based on a string dockerfile 493 func (p *PodmanTestIntegration) BuildImage(dockerfile, imageName string, layers string) string { 494 return p.buildImage(dockerfile, imageName, layers, "") 495 } 496 497 // BuildImageWithLabel uses podman build and buildah to build an image 498 // called imageName based on a string dockerfile, adds desired label to paramset 499 func (p *PodmanTestIntegration) BuildImageWithLabel(dockerfile, imageName string, layers string, label string) string { 500 return p.buildImage(dockerfile, imageName, layers, label) 501 } 502 503 // PodmanPID execs podman and returns its PID 504 func (p *PodmanTestIntegration) PodmanPID(args []string) (*PodmanSessionIntegration, int) { 505 podmanOptions := p.MakeOptions(args, false, false) 506 fmt.Printf("Running: %s %s\n", p.PodmanBinary, strings.Join(podmanOptions, " ")) 507 508 command := exec.Command(p.PodmanBinary, podmanOptions...) 509 session, err := Start(command, GinkgoWriter, GinkgoWriter) 510 if err != nil { 511 Fail("unable to run podman command: " + strings.Join(podmanOptions, " ")) 512 } 513 podmanSession := &PodmanSession{Session: session} 514 return &PodmanSessionIntegration{podmanSession}, command.Process.Pid 515 } 516 517 // Cleanup cleans up the temporary store 518 func (p *PodmanTestIntegration) Cleanup() { 519 // Remove all pods... 520 podrm := p.Podman([]string{"pod", "rm", "-fa", "-t", "0"}) 521 podrm.WaitWithDefaultTimeout() 522 523 // ...and containers 524 rmall := p.Podman([]string{"rm", "-fa", "-t", "0"}) 525 rmall.WaitWithDefaultTimeout() 526 527 p.StopRemoteService() 528 // Nuke tempdir 529 if err := os.RemoveAll(p.TempDir); err != nil { 530 fmt.Printf("%q\n", err) 531 } 532 533 // Clean up the registries configuration file ENV variable set in Create 534 resetRegistriesConfigEnv() 535 } 536 537 // CleanupVolume cleans up the temporary store 538 func (p *PodmanTestIntegration) CleanupVolume() { 539 // Remove all containers 540 session := p.Podman([]string{"volume", "rm", "-fa"}) 541 session.Wait(90) 542 543 p.Cleanup() 544 } 545 546 // CleanupSecret cleans up the temporary store 547 func (p *PodmanTestIntegration) CleanupSecrets() { 548 // Remove all containers 549 session := p.Podman([]string{"secret", "rm", "-a"}) 550 session.Wait(90) 551 552 // Stop remove service on secret cleanup 553 p.StopRemoteService() 554 555 // Nuke tempdir 556 if err := os.RemoveAll(p.TempDir); err != nil { 557 fmt.Printf("%q\n", err) 558 } 559 } 560 561 // InspectContainerToJSON takes the session output of an inspect 562 // container and returns json 563 func (s *PodmanSessionIntegration) InspectContainerToJSON() []define.InspectContainerData { 564 var i []define.InspectContainerData 565 err := jsoniter.Unmarshal(s.Out.Contents(), &i) 566 Expect(err).To(BeNil()) 567 return i 568 } 569 570 // InspectPodToJSON takes the sessions output from a pod inspect and returns json 571 func (s *PodmanSessionIntegration) InspectPodToJSON() define.InspectPodData { 572 var i define.InspectPodData 573 err := jsoniter.Unmarshal(s.Out.Contents(), &i) 574 Expect(err).To(BeNil()) 575 return i 576 } 577 578 // InspectPodToJSON takes the sessions output from an inspect and returns json 579 func (s *PodmanSessionIntegration) InspectPodArrToJSON() []define.InspectPodData { 580 var i []define.InspectPodData 581 err := jsoniter.Unmarshal(s.Out.Contents(), &i) 582 Expect(err).To(BeNil()) 583 return i 584 } 585 586 // CreatePod creates a pod with no infra container 587 // it optionally takes a pod name 588 func (p *PodmanTestIntegration) CreatePod(options map[string][]string) (*PodmanSessionIntegration, int, string) { 589 var args = []string{"pod", "create", "--infra=false", "--share", ""} 590 for k, values := range options { 591 for _, v := range values { 592 args = append(args, k+"="+v) 593 } 594 } 595 596 session := p.Podman(args) 597 session.WaitWithDefaultTimeout() 598 return session, session.ExitCode(), session.OutputToString() 599 } 600 601 func (p *PodmanTestIntegration) RunTopContainerInPod(name, pod string) *PodmanSessionIntegration { 602 return p.RunTopContainerWithArgs(name, []string{"--pod", pod}) 603 } 604 605 func (p *PodmanTestIntegration) RunHealthCheck(cid string) error { 606 for i := 0; i < 10; i++ { 607 hc := p.Podman([]string{"healthcheck", "run", cid}) 608 hc.WaitWithDefaultTimeout() 609 if hc.ExitCode() == 0 { 610 return nil 611 } 612 // Restart container if it's not running 613 ps := p.Podman([]string{"ps", "--no-trunc", "--quiet", "--filter", fmt.Sprintf("id=%s", cid)}) 614 ps.WaitWithDefaultTimeout() 615 if ps.ExitCode() == 0 { 616 if !strings.Contains(ps.OutputToString(), cid) { 617 fmt.Printf("Container %s is not running, restarting", cid) 618 restart := p.Podman([]string{"restart", cid}) 619 restart.WaitWithDefaultTimeout() 620 if restart.ExitCode() != 0 { 621 return errors.Errorf("unable to restart %s", cid) 622 } 623 } 624 } 625 fmt.Printf("Waiting for %s to pass healthcheck\n", cid) 626 time.Sleep(1 * time.Second) 627 } 628 return errors.Errorf("unable to detect %s as running", cid) 629 } 630 631 func (p *PodmanTestIntegration) CreateSeccompJSON(in []byte) (string, error) { 632 jsonFile := filepath.Join(p.TempDir, "seccomp.json") 633 err := WriteJSONFile(in, jsonFile) 634 if err != nil { 635 return "", err 636 } 637 return jsonFile, nil 638 } 639 640 func checkReason(reason string) { 641 if len(reason) < 5 { 642 panic("Test must specify a reason to skip") 643 } 644 } 645 646 func SkipIfRootlessCgroupsV1(reason string) { 647 checkReason(reason) 648 if os.Geteuid() != 0 && !CGROUPSV2 { 649 Skip("[rootless]: " + reason) 650 } 651 } 652 653 func SkipIfRootless(reason string) { 654 checkReason(reason) 655 if os.Geteuid() != 0 { 656 Skip("[rootless]: " + reason) 657 } 658 } 659 660 func SkipIfNotRootless(reason string) { 661 checkReason(reason) 662 if os.Geteuid() == 0 { 663 Skip("[notRootless]: " + reason) 664 } 665 } 666 667 func SkipIfSystemdNotRunning(reason string) { 668 checkReason(reason) 669 670 cmd := exec.Command("systemctl", "list-units") 671 err := cmd.Run() 672 if err != nil { 673 if _, ok := err.(*exec.Error); ok { 674 Skip("[notSystemd]: not running " + reason) 675 } 676 Expect(err).ToNot(HaveOccurred()) 677 } 678 } 679 680 func SkipIfNotSystemd(manager, reason string) { 681 checkReason(reason) 682 if manager != "systemd" { 683 Skip("[notSystemd]: " + reason) 684 } 685 } 686 687 func SkipIfNotFedora() { 688 info := GetHostDistributionInfo() 689 if info.Distribution != "fedora" { 690 Skip("Test can only run on Fedora") 691 } 692 } 693 694 func isRootless() bool { 695 return os.Geteuid() != 0 696 } 697 698 func isCgroupsV1() bool { 699 return !CGROUPSV2 700 } 701 702 func SkipIfCgroupV1(reason string) { 703 checkReason(reason) 704 if isCgroupsV1() { 705 Skip(reason) 706 } 707 } 708 709 func SkipIfCgroupV2(reason string) { 710 checkReason(reason) 711 if CGROUPSV2 { 712 Skip(reason) 713 } 714 } 715 716 func isContainerized() bool { 717 // This is set to "podman" by podman automatically 718 return os.Getenv("container") != "" 719 } 720 721 func SkipIfContainerized(reason string) { 722 checkReason(reason) 723 if isContainerized() { 724 Skip(reason) 725 } 726 } 727 728 func SkipIfRemote(reason string) { 729 checkReason(reason) 730 if !IsRemote() { 731 return 732 } 733 Skip("[remote]: " + reason) 734 } 735 736 func SkipIfNotRemote(reason string) { 737 checkReason(reason) 738 if IsRemote() { 739 return 740 } 741 Skip("[local]: " + reason) 742 } 743 744 // SkipIfInContainer skips a test if the test is run inside a container 745 func SkipIfInContainer(reason string) { 746 checkReason(reason) 747 if os.Getenv("TEST_ENVIRON") == "container" { 748 Skip("[container]: " + reason) 749 } 750 } 751 752 // SkipIfNotActive skips a test if the given systemd unit is not active 753 func SkipIfNotActive(unit string, reason string) { 754 checkReason(reason) 755 756 var buffer bytes.Buffer 757 cmd := exec.Command("systemctl", "is-active", unit) 758 cmd.Stdout = &buffer 759 err := cmd.Start() 760 Expect(err).ToNot(HaveOccurred()) 761 762 err = cmd.Wait() 763 Expect(err).ToNot(HaveOccurred()) 764 765 Expect(err).ToNot(HaveOccurred()) 766 if strings.TrimSpace(buffer.String()) != "active" { 767 Skip(fmt.Sprintf("[systemd]: unit %s is not active: %s", unit, reason)) 768 } 769 } 770 771 func SkipIfCNI(p *PodmanTestIntegration) { 772 if p.NetworkBackend == CNI { 773 Skip("this test is not compatible with the CNI network backend") 774 } 775 } 776 777 func SkipIfNetavark(p *PodmanTestIntegration) { 778 if p.NetworkBackend == Netavark { 779 Skip("This test is not compatible with the netavark network backend") 780 } 781 } 782 783 // PodmanAsUser is the exec call to podman on the filesystem with the specified uid/gid and environment 784 func (p *PodmanTestIntegration) PodmanAsUser(args []string, uid, gid uint32, cwd string, env []string) *PodmanSessionIntegration { 785 podmanSession := p.PodmanAsUserBase(args, uid, gid, cwd, env, false, false, nil, nil) 786 return &PodmanSessionIntegration{podmanSession} 787 } 788 789 // RestartRemoteService stop and start API Server, usually to change config 790 func (p *PodmanTestIntegration) RestartRemoteService() { 791 p.StopRemoteService() 792 p.StartRemoteService() 793 } 794 795 // RestoreArtifactToCache populates the imagecache from tarballs that were cached earlier 796 func (p *PodmanTestIntegration) RestoreArtifactToCache(image string) error { 797 tarball := imageTarPath(image) 798 if _, err := os.Stat(tarball); err == nil { 799 fmt.Printf("Restoring %s...\n", image) 800 p.Root = p.ImageCacheDir 801 restore := p.PodmanNoEvents([]string{"load", "-q", "-i", tarball}) 802 restore.WaitWithDefaultTimeout() 803 } 804 return nil 805 } 806 807 func populateCache(podman *PodmanTestIntegration) { 808 for _, image := range CACHE_IMAGES { 809 err := podman.RestoreArtifactToCache(image) 810 Expect(err).To(BeNil()) 811 } 812 // logformatter uses this to recognize the first test 813 fmt.Printf("-----------------------------\n") 814 } 815 816 func removeCache() { 817 // Remove cache dirs 818 if err := os.RemoveAll(ImageCacheDir); err != nil { 819 fmt.Printf("%q\n", err) 820 } 821 } 822 823 // PodmanNoCache calls the podman command with no configured imagecache 824 func (p *PodmanTestIntegration) PodmanNoCache(args []string) *PodmanSessionIntegration { 825 podmanSession := p.PodmanBase(args, false, true) 826 return &PodmanSessionIntegration{podmanSession} 827 } 828 829 func PodmanTestSetup(tempDir string) *PodmanTestIntegration { 830 return PodmanTestCreateUtil(tempDir, false) 831 } 832 833 // PodmanNoEvents calls the Podman command without an imagecache and without an 834 // events backend. It is used mostly for caching and uncaching images. 835 func (p *PodmanTestIntegration) PodmanNoEvents(args []string) *PodmanSessionIntegration { 836 podmanSession := p.PodmanBase(args, true, true) 837 return &PodmanSessionIntegration{podmanSession} 838 } 839 840 // MakeOptions assembles all the podman main options 841 func (p *PodmanTestIntegration) makeOptions(args []string, noEvents, noCache bool) []string { 842 if p.RemoteTest { 843 if !util.StringInSlice("--remote", args) { 844 return append([]string{"--remote", "--url", p.RemoteSocket}, args...) 845 } 846 return args 847 } 848 849 var debug string 850 if _, ok := os.LookupEnv("E2E_DEBUG"); ok { 851 debug = "--log-level=debug --syslog=true " 852 } 853 854 eventsType := "file" 855 if noEvents { 856 eventsType = "none" 857 } 858 859 podmanOptions := strings.Split(fmt.Sprintf("%s--root %s --runroot %s --runtime %s --conmon %s --network-config-dir %s --cgroup-manager %s --tmpdir %s --events-backend %s", 860 debug, p.Root, p.RunRoot, p.OCIRuntime, p.ConmonBinary, p.NetworkConfigDir, p.CgroupManager, p.TmpDir, eventsType), " ") 861 862 if !p.RemoteTest { 863 podmanOptions = append(podmanOptions, "--network-backend", p.NetworkBackend.ToString()) 864 } 865 866 podmanOptions = append(podmanOptions, strings.Split(p.StorageOptions, " ")...) 867 if !noCache { 868 cacheOptions := []string{"--storage-opt", 869 fmt.Sprintf("%s.imagestore=%s", p.PodmanTest.ImageCacheFS, p.PodmanTest.ImageCacheDir)} 870 podmanOptions = append(cacheOptions, podmanOptions...) 871 } 872 podmanOptions = append(podmanOptions, args...) 873 return podmanOptions 874 } 875 876 func writeConf(conf []byte, confPath string) { 877 if _, err := os.Stat(filepath.Dir(confPath)); os.IsNotExist(err) { 878 if err := os.MkdirAll(filepath.Dir(confPath), 0o777); err != nil { 879 fmt.Println(err) 880 } 881 } 882 if err := ioutil.WriteFile(confPath, conf, 0o777); err != nil { 883 fmt.Println(err) 884 } 885 } 886 887 func removeConf(confPath string) { 888 if err := os.Remove(confPath); err != nil { 889 fmt.Println(err) 890 } 891 } 892 893 // generateNetworkConfig generates a CNI or Netavark config with a random name 894 // it returns the network name and the filepath 895 func generateNetworkConfig(p *PodmanTestIntegration) (string, string) { 896 var ( 897 path string 898 conf string 899 ) 900 // generate a random name to prevent conflicts with other tests 901 name := "net" + stringid.GenerateNonCryptoID() 902 if p.NetworkBackend != Netavark { 903 path = filepath.Join(p.NetworkConfigDir, fmt.Sprintf("%s.conflist", name)) 904 conf = fmt.Sprintf(`{ 905 "cniVersion": "0.3.0", 906 "name": "%s", 907 "plugins": [ 908 { 909 "type": "bridge", 910 "bridge": "cni1", 911 "isGateway": true, 912 "ipMasq": true, 913 "ipam": { 914 "type": "host-local", 915 "subnet": "10.99.0.0/16", 916 "routes": [ 917 { "dst": "0.0.0.0/0" } 918 ] 919 } 920 }, 921 { 922 "type": "portmap", 923 "capabilities": { 924 "portMappings": true 925 } 926 } 927 ] 928 }`, name) 929 } else { 930 path = filepath.Join(p.NetworkConfigDir, fmt.Sprintf("%s.json", name)) 931 conf = fmt.Sprintf(` 932 { 933 "name": "%s", 934 "id": "e1ef2749024b88f5663ca693a9118e036d6bfc48bcfe460faf45e9614a513e5c", 935 "driver": "bridge", 936 "network_interface": "netavark1", 937 "created": "2022-01-05T14:15:10.975493521-06:00", 938 "subnets": [ 939 { 940 "subnet": "10.100.0.0/16", 941 "gateway": "10.100.0.1" 942 } 943 ], 944 "ipv6_enabled": false, 945 "internal": false, 946 "dns_enabled": true, 947 "ipam_options": { 948 "driver": "host-local" 949 } 950 } 951 `, name) 952 } 953 writeConf([]byte(conf), path) 954 return name, path 955 } 956 957 func (p *PodmanTestIntegration) removeNetwork(name string) { 958 session := p.Podman([]string{"network", "rm", "-f", name}) 959 session.WaitWithDefaultTimeout() 960 Expect(session.ExitCode()).To(BeNumerically("<=", 1), "Exit code must be 0 or 1") 961 } 962 963 func (s *PodmanSessionIntegration) jq(jqCommand string) (string, error) { 964 var out bytes.Buffer 965 cmd := exec.Command("jq", jqCommand) 966 cmd.Stdin = strings.NewReader(s.OutputToString()) 967 cmd.Stdout = &out 968 err := cmd.Run() 969 return strings.TrimRight(out.String(), "\n"), err 970 } 971 972 func (p *PodmanTestIntegration) buildImage(dockerfile, imageName string, layers string, label string) string { 973 dockerfilePath := filepath.Join(p.TempDir, "Dockerfile") 974 err := ioutil.WriteFile(dockerfilePath, []byte(dockerfile), 0755) 975 Expect(err).To(BeNil()) 976 cmd := []string{"build", "--pull-never", "--layers=" + layers, "--file", dockerfilePath} 977 if label != "" { 978 cmd = append(cmd, "--label="+label) 979 } 980 if len(imageName) > 0 { 981 cmd = append(cmd, []string{"-t", imageName}...) 982 } 983 cmd = append(cmd, p.TempDir) 984 session := p.Podman(cmd) 985 session.Wait(240) 986 Expect(session).Should(Exit(0), fmt.Sprintf("BuildImage session output: %q", session.OutputToString())) 987 output := session.OutputToStringArray() 988 return output[len(output)-1] 989 } 990 991 func writeYaml(content string, fileName string) error { 992 f, err := os.Create(fileName) 993 if err != nil { 994 return err 995 } 996 defer f.Close() 997 998 _, err = f.WriteString(content) 999 if err != nil { 1000 return err 1001 } 1002 1003 return nil 1004 } 1005 1006 // GetPort finds an unused port on the system 1007 func GetPort() int { 1008 a, err := net.ResolveTCPAddr("tcp", "localhost:0") 1009 if err != nil { 1010 Fail(fmt.Sprintf("unable to get free port: %v", err)) 1011 } 1012 1013 l, err := net.ListenTCP("tcp", a) 1014 if err != nil { 1015 Fail(fmt.Sprintf("unable to get free port: %v", err)) 1016 } 1017 defer l.Close() 1018 return l.Addr().(*net.TCPAddr).Port 1019 } 1020 1021 func ncz(port int) bool { 1022 timeout := 500 * time.Millisecond 1023 for i := 0; i < 5; i++ { 1024 ncCmd := []string{"-z", "localhost", fmt.Sprintf("%d", port)} 1025 fmt.Printf("Running: nc %s\n", strings.Join(ncCmd, " ")) 1026 check := SystemExec("nc", ncCmd) 1027 if check.ExitCode() == 0 { 1028 return true 1029 } 1030 time.Sleep(timeout) 1031 timeout++ 1032 } 1033 return false 1034 } 1035 1036 func createNetworkName(name string) string { 1037 return name + stringid.GenerateNonCryptoID()[:10] 1038 } 1039 1040 var IPRegex = `(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}` 1041 1042 // digShort execs into the given container and does a dig lookup with a timeout 1043 // backoff. If it gets a response, it ensures that the output is in the correct 1044 // format and iterates a string array for match 1045 func digShort(container, lookupName string, matchNames []string, p *PodmanTestIntegration) { 1046 digInterval := time.Millisecond * 250 1047 for i := 0; i < 6; i++ { 1048 time.Sleep(digInterval * time.Duration(i)) 1049 dig := p.Podman([]string{"exec", container, "dig", "+short", lookupName}) 1050 dig.WaitWithDefaultTimeout() 1051 if dig.ExitCode() == 0 { 1052 output := dig.OutputToString() 1053 Expect(output).To(MatchRegexp(IPRegex)) 1054 for _, name := range matchNames { 1055 Expect(output).To(Equal(name)) 1056 } 1057 // success 1058 return 1059 } 1060 } 1061 Fail("dns is not responding") 1062 } 1063 1064 // WaitForFile to be created in defaultWaitTimeout seconds, returns false if file not created 1065 func WaitForFile(path string) (err error) { 1066 until := time.Now().Add(time.Duration(defaultWaitTimeout) * time.Second) 1067 for i := 1; time.Now().Before(until); i++ { 1068 _, err = os.Stat(path) 1069 switch { 1070 case err == nil: 1071 return nil 1072 case errors.Is(err, os.ErrNotExist): 1073 time.Sleep(time.Duration(i) * time.Second) 1074 default: 1075 return err 1076 } 1077 } 1078 return err 1079 } 1080 1081 // WaitForService blocks, waiting for some service listening on given host:port 1082 func WaitForService(address url.URL) { 1083 // Wait for podman to be ready 1084 var conn net.Conn 1085 var err error 1086 for i := 1; i <= 5; i++ { 1087 conn, err = net.Dial("tcp", address.Host) 1088 if err != nil { 1089 // Podman not available yet... 1090 time.Sleep(time.Duration(i) * time.Second) 1091 } 1092 } 1093 Expect(err).ShouldNot(HaveOccurred()) 1094 conn.Close() 1095 }