gitee.com/leisunstar/runtime@v0.0.0-20200521203717-5cef3e7b53f9/pkg/katautils/config.go (about) 1 // Copyright (c) 2018 Intel Corporation 2 // Copyright (c) 2018 HyperHQ Inc. 3 // 4 // SPDX-License-Identifier: Apache-2.0 5 // 6 7 package katautils 8 9 import ( 10 "errors" 11 "fmt" 12 "io/ioutil" 13 goruntime "runtime" 14 "strings" 15 16 "github.com/BurntSushi/toml" 17 govmmQemu "github.com/intel/govmm/qemu" 18 vc "github.com/kata-containers/runtime/virtcontainers" 19 "github.com/kata-containers/runtime/virtcontainers/device/config" 20 exp "github.com/kata-containers/runtime/virtcontainers/experimental" 21 "github.com/kata-containers/runtime/virtcontainers/pkg/oci" 22 "github.com/kata-containers/runtime/virtcontainers/utils" 23 "github.com/sirupsen/logrus" 24 ) 25 26 const ( 27 defaultHypervisor = vc.QemuHypervisor 28 defaultAgent = vc.KataContainersAgent 29 ) 30 31 var ( 32 defaultProxy = vc.KataProxyType 33 defaultShim = vc.KataShimType 34 35 // if true, enable opentracing support. 36 tracing = false 37 ) 38 39 // The TOML configuration file contains a number of sections (or 40 // tables). The names of these tables are in dotted ("nested table") 41 // form: 42 // 43 // [<component>.<type>] 44 // 45 // The components are hypervisor, proxy, shim and agent. For example, 46 // 47 // [proxy.kata] 48 // 49 // The currently supported types are listed below: 50 const ( 51 // supported hypervisor component types 52 firecrackerHypervisorTableType = "firecracker" 53 clhHypervisorTableType = "clh" 54 qemuHypervisorTableType = "qemu" 55 acrnHypervisorTableType = "acrn" 56 57 // supported proxy component types 58 kataProxyTableType = "kata" 59 60 // supported shim component types 61 kataShimTableType = "kata" 62 63 // supported agent component types 64 kataAgentTableType = "kata" 65 66 // the maximum amount of PCI bridges that can be cold plugged in a VM 67 maxPCIBridges uint32 = 5 68 ) 69 70 type tomlConfig struct { 71 Hypervisor map[string]hypervisor 72 Proxy map[string]proxy 73 Shim map[string]shim 74 Agent map[string]agent 75 Runtime runtime 76 Factory factory 77 Netmon netmon 78 } 79 80 type factory struct { 81 Template bool `toml:"enable_template"` 82 TemplatePath string `toml:"template_path"` 83 VMCacheNumber uint `toml:"vm_cache_number"` 84 VMCacheEndpoint string `toml:"vm_cache_endpoint"` 85 } 86 87 type hypervisor struct { 88 Path string `toml:"path"` 89 JailerPath string `toml:"jailer_path"` 90 Kernel string `toml:"kernel"` 91 CtlPath string `toml:"ctlpath"` 92 Initrd string `toml:"initrd"` 93 Image string `toml:"image"` 94 Firmware string `toml:"firmware"` 95 MachineAccelerators string `toml:"machine_accelerators"` 96 CPUFeatures string `toml:"cpu_features"` 97 KernelParams string `toml:"kernel_params"` 98 MachineType string `toml:"machine_type"` 99 BlockDeviceDriver string `toml:"block_device_driver"` 100 EntropySource string `toml:"entropy_source"` 101 SharedFS string `toml:"shared_fs"` 102 VirtioFSDaemon string `toml:"virtio_fs_daemon"` 103 VirtioFSCache string `toml:"virtio_fs_cache"` 104 VirtioFSExtraArgs []string `toml:"virtio_fs_extra_args"` 105 VirtioFSCacheSize uint32 `toml:"virtio_fs_cache_size"` 106 BlockDeviceCacheSet bool `toml:"block_device_cache_set"` 107 BlockDeviceCacheDirect bool `toml:"block_device_cache_direct"` 108 BlockDeviceCacheNoflush bool `toml:"block_device_cache_noflush"` 109 EnableVhostUserStore bool `toml:"enable_vhost_user_store"` 110 VhostUserStorePath string `toml:"vhost_user_store_path"` 111 NumVCPUs int32 `toml:"default_vcpus"` 112 DefaultMaxVCPUs uint32 `toml:"default_maxvcpus"` 113 MemorySize uint32 `toml:"default_memory"` 114 MemSlots uint32 `toml:"memory_slots"` 115 MemOffset uint32 `toml:"memory_offset"` 116 DefaultBridges uint32 `toml:"default_bridges"` 117 Msize9p uint32 `toml:"msize_9p"` 118 PCIeRootPort uint32 `toml:"pcie_root_port"` 119 DisableBlockDeviceUse bool `toml:"disable_block_device_use"` 120 MemPrealloc bool `toml:"enable_mem_prealloc"` 121 HugePages bool `toml:"enable_hugepages"` 122 VirtioMem bool `toml:"enable_virtio_mem"` 123 FileBackedMemRootDir string `toml:"file_mem_backend"` 124 Swap bool `toml:"enable_swap"` 125 Debug bool `toml:"enable_debug"` 126 DisableNestingChecks bool `toml:"disable_nesting_checks"` 127 EnableIOThreads bool `toml:"enable_iothreads"` 128 UseVSock bool `toml:"use_vsock"` 129 DisableImageNvdimm bool `toml:"disable_image_nvdimm"` 130 HotplugVFIOOnRootBus bool `toml:"hotplug_vfio_on_root_bus"` 131 DisableVhostNet bool `toml:"disable_vhost_net"` 132 GuestHookPath string `toml:"guest_hook_path"` 133 } 134 135 type proxy struct { 136 Path string `toml:"path"` 137 Debug bool `toml:"enable_debug"` 138 } 139 140 type runtime struct { 141 Debug bool `toml:"enable_debug"` 142 Tracing bool `toml:"enable_tracing"` 143 DisableNewNetNs bool `toml:"disable_new_netns"` 144 DisableGuestSeccomp bool `toml:"disable_guest_seccomp"` 145 SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"` 146 Experimental []string `toml:"experimental"` 147 InterNetworkModel string `toml:"internetworking_model"` 148 } 149 150 type shim struct { 151 Path string `toml:"path"` 152 Debug bool `toml:"enable_debug"` 153 Tracing bool `toml:"enable_tracing"` 154 } 155 156 type agent struct { 157 Debug bool `toml:"enable_debug"` 158 Tracing bool `toml:"enable_tracing"` 159 TraceMode string `toml:"trace_mode"` 160 TraceType string `toml:"trace_type"` 161 KernelModules []string `toml:"kernel_modules"` 162 } 163 164 type netmon struct { 165 Path string `toml:"path"` 166 Debug bool `toml:"enable_debug"` 167 Enable bool `toml:"enable_netmon"` 168 } 169 170 func (h hypervisor) path() (string, error) { 171 p := h.Path 172 173 if h.Path == "" { 174 p = defaultHypervisorPath 175 } 176 177 return ResolvePath(p) 178 } 179 180 func (h hypervisor) ctlpath() (string, error) { 181 p := h.CtlPath 182 183 if h.CtlPath == "" { 184 p = defaultHypervisorCtlPath 185 } 186 187 return ResolvePath(p) 188 } 189 190 func (h hypervisor) jailerPath() (string, error) { 191 p := h.JailerPath 192 193 if h.JailerPath == "" { 194 return "", nil 195 } 196 197 return ResolvePath(p) 198 } 199 200 func (h hypervisor) kernel() (string, error) { 201 p := h.Kernel 202 203 if p == "" { 204 p = defaultKernelPath 205 } 206 207 return ResolvePath(p) 208 } 209 210 func (h hypervisor) initrd() (string, error) { 211 p := h.Initrd 212 213 if p == "" { 214 return "", errors.New("initrd is not set") 215 } 216 217 return ResolvePath(p) 218 } 219 220 func (h hypervisor) image() (string, error) { 221 p := h.Image 222 223 if p == "" { 224 return "", errors.New("image is not set") 225 } 226 227 return ResolvePath(p) 228 } 229 230 func (h hypervisor) firmware() (string, error) { 231 p := h.Firmware 232 233 if p == "" { 234 if defaultFirmwarePath == "" { 235 return "", nil 236 } 237 p = defaultFirmwarePath 238 } 239 240 return ResolvePath(p) 241 } 242 243 func (h hypervisor) machineAccelerators() string { 244 var machineAccelerators string 245 for _, accelerator := range strings.Split(h.MachineAccelerators, ",") { 246 if accelerator != "" { 247 machineAccelerators += strings.TrimSpace(accelerator) + "," 248 } 249 } 250 251 machineAccelerators = strings.Trim(machineAccelerators, ",") 252 253 return machineAccelerators 254 } 255 256 func (h hypervisor) cpuFeatures() string { 257 var cpuFeatures string 258 for _, feature := range strings.Split(h.CPUFeatures, ",") { 259 if feature != "" { 260 cpuFeatures += strings.TrimSpace(feature) + "," 261 } 262 } 263 264 cpuFeatures = strings.Trim(cpuFeatures, ",") 265 266 return cpuFeatures 267 } 268 269 func (h hypervisor) kernelParams() string { 270 if h.KernelParams == "" { 271 return defaultKernelParams 272 } 273 274 return h.KernelParams 275 } 276 277 func (h hypervisor) machineType() string { 278 if h.MachineType == "" { 279 return defaultMachineType 280 } 281 282 return h.MachineType 283 } 284 285 func (h hypervisor) GetEntropySource() string { 286 if h.EntropySource == "" { 287 return defaultEntropySource 288 } 289 290 return h.EntropySource 291 } 292 293 func (h hypervisor) defaultVCPUs() uint32 { 294 numCPUs := goruntime.NumCPU() 295 296 if h.NumVCPUs < 0 || h.NumVCPUs > int32(numCPUs) { 297 return uint32(numCPUs) 298 } 299 if h.NumVCPUs == 0 { // or unspecified 300 return defaultVCPUCount 301 } 302 303 return uint32(h.NumVCPUs) 304 } 305 306 func (h hypervisor) defaultMaxVCPUs() uint32 { 307 numcpus := uint32(goruntime.NumCPU()) 308 maxvcpus := vc.MaxQemuVCPUs() 309 reqVCPUs := h.DefaultMaxVCPUs 310 311 //don't exceed the number of physical CPUs. If a default is not provided, use the 312 // numbers of physical CPUs 313 if reqVCPUs >= numcpus || reqVCPUs == 0 { 314 reqVCPUs = numcpus 315 } 316 317 // Don't exceed the maximum number of vCPUs supported by hypervisor 318 if reqVCPUs > maxvcpus { 319 return maxvcpus 320 } 321 322 return reqVCPUs 323 } 324 325 func (h hypervisor) defaultMemSz() uint32 { 326 if h.MemorySize < vc.MinHypervisorMemory { 327 return defaultMemSize // MiB 328 } 329 330 return h.MemorySize 331 } 332 333 func (h hypervisor) defaultMemSlots() uint32 { 334 slots := h.MemSlots 335 if slots == 0 { 336 slots = defaultMemSlots 337 } 338 339 return slots 340 } 341 342 func (h hypervisor) defaultMemOffset() uint32 { 343 offset := h.MemOffset 344 if offset == 0 { 345 offset = defaultMemOffset 346 } 347 348 return offset 349 } 350 351 func (h hypervisor) defaultBridges() uint32 { 352 if h.DefaultBridges == 0 { 353 return defaultBridgesCount 354 } 355 356 if h.DefaultBridges > maxPCIBridges { 357 return maxPCIBridges 358 } 359 360 return h.DefaultBridges 361 } 362 363 func (h hypervisor) defaultVirtioFSCache() string { 364 if h.VirtioFSCache == "" { 365 return defaultVirtioFSCacheMode 366 } 367 368 return h.VirtioFSCache 369 } 370 371 func (h hypervisor) blockDeviceDriver() (string, error) { 372 supportedBlockDrivers := []string{config.VirtioSCSI, config.VirtioBlock, config.VirtioMmio, config.Nvdimm, config.VirtioBlockCCW} 373 374 if h.BlockDeviceDriver == "" { 375 return defaultBlockDeviceDriver, nil 376 } 377 378 for _, b := range supportedBlockDrivers { 379 if b == h.BlockDeviceDriver { 380 return h.BlockDeviceDriver, nil 381 } 382 } 383 384 return "", fmt.Errorf("Invalid hypervisor block storage driver %v specified (supported drivers: %v)", h.BlockDeviceDriver, supportedBlockDrivers) 385 } 386 387 func (h hypervisor) sharedFS() (string, error) { 388 supportedSharedFS := []string{config.Virtio9P, config.VirtioFS} 389 390 if h.SharedFS == "" { 391 return config.Virtio9P, nil 392 } 393 394 for _, fs := range supportedSharedFS { 395 if fs == h.SharedFS { 396 return h.SharedFS, nil 397 } 398 } 399 400 return "", fmt.Errorf("Invalid hypervisor shared file system %v specified (supported file systems: %v)", h.SharedFS, supportedSharedFS) 401 } 402 403 func (h hypervisor) msize9p() uint32 { 404 if h.Msize9p == 0 { 405 return defaultMsize9p 406 } 407 408 return h.Msize9p 409 } 410 411 func (h hypervisor) useVSock() bool { 412 return h.UseVSock 413 } 414 415 func (h hypervisor) guestHookPath() string { 416 if h.GuestHookPath == "" { 417 return defaultGuestHookPath 418 } 419 return h.GuestHookPath 420 } 421 422 func (h hypervisor) vhostUserStorePath() string { 423 if h.VhostUserStorePath == "" { 424 return defaultVhostUserStorePath 425 } 426 return h.VhostUserStorePath 427 } 428 429 func (h hypervisor) getInitrdAndImage() (initrd string, image string, err error) { 430 initrd, errInitrd := h.initrd() 431 432 image, errImage := h.image() 433 434 if image != "" && initrd != "" { 435 return "", "", errors.New("having both an image and an initrd defined in the configuration file is not supported") 436 } 437 438 if errInitrd != nil && errImage != nil { 439 return "", "", fmt.Errorf("Either initrd or image must be set to a valid path (initrd: %v) (image: %v)", errInitrd, errImage) 440 } 441 442 return 443 } 444 445 func (p proxy) path() (string, error) { 446 path := p.Path 447 if path == "" { 448 path = defaultProxyPath 449 } 450 451 return ResolvePath(path) 452 } 453 454 func (p proxy) debug() bool { 455 return p.Debug 456 } 457 458 func (s shim) path() (string, error) { 459 p := s.Path 460 461 if p == "" { 462 p = defaultShimPath 463 } 464 465 return ResolvePath(p) 466 } 467 468 func (s shim) debug() bool { 469 return s.Debug 470 } 471 472 func (s shim) trace() bool { 473 return s.Tracing 474 } 475 476 func (a agent) debug() bool { 477 return a.Debug 478 } 479 480 func (a agent) trace() bool { 481 return a.Tracing 482 } 483 484 func (a agent) traceMode() string { 485 return a.TraceMode 486 } 487 488 func (a agent) traceType() string { 489 return a.TraceType 490 } 491 492 func (a agent) kernelModules() []string { 493 return a.KernelModules 494 } 495 496 func (n netmon) enable() bool { 497 return n.Enable 498 } 499 500 func (n netmon) path() string { 501 if n.Path == "" { 502 return defaultNetmonPath 503 } 504 505 return n.Path 506 } 507 508 func (n netmon) debug() bool { 509 return n.Debug 510 } 511 512 func newFirecrackerHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { 513 hypervisor, err := h.path() 514 if err != nil { 515 return vc.HypervisorConfig{}, err 516 } 517 518 jailer, err := h.jailerPath() 519 if err != nil { 520 return vc.HypervisorConfig{}, err 521 } 522 523 kernel, err := h.kernel() 524 if err != nil { 525 return vc.HypervisorConfig{}, err 526 } 527 528 initrd, image, err := h.getInitrdAndImage() 529 if err != nil { 530 return vc.HypervisorConfig{}, err 531 } 532 533 firmware, err := h.firmware() 534 if err != nil { 535 return vc.HypervisorConfig{}, err 536 } 537 538 kernelParams := h.kernelParams() 539 540 blockDriver, err := h.blockDeviceDriver() 541 if err != nil { 542 return vc.HypervisorConfig{}, err 543 } 544 545 if !utils.SupportsVsocks() { 546 return vc.HypervisorConfig{}, errors.New("No vsock support, firecracker cannot be used") 547 } 548 549 return vc.HypervisorConfig{ 550 HypervisorPath: hypervisor, 551 JailerPath: jailer, 552 KernelPath: kernel, 553 InitrdPath: initrd, 554 ImagePath: image, 555 FirmwarePath: firmware, 556 KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), 557 NumVCPUs: h.defaultVCPUs(), 558 DefaultMaxVCPUs: h.defaultMaxVCPUs(), 559 MemorySize: h.defaultMemSz(), 560 MemSlots: h.defaultMemSlots(), 561 EntropySource: h.GetEntropySource(), 562 DefaultBridges: h.defaultBridges(), 563 DisableBlockDeviceUse: h.DisableBlockDeviceUse, 564 HugePages: h.HugePages, 565 Mlock: !h.Swap, 566 Debug: h.Debug, 567 DisableNestingChecks: h.DisableNestingChecks, 568 BlockDeviceDriver: blockDriver, 569 EnableIOThreads: h.EnableIOThreads, 570 DisableVhostNet: true, // vhost-net backend is not supported in Firecracker 571 UseVSock: true, 572 GuestHookPath: h.guestHookPath(), 573 }, nil 574 } 575 576 func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { 577 hypervisor, err := h.path() 578 if err != nil { 579 return vc.HypervisorConfig{}, err 580 } 581 582 kernel, err := h.kernel() 583 if err != nil { 584 return vc.HypervisorConfig{}, err 585 } 586 587 initrd, image, err := h.getInitrdAndImage() 588 if err != nil { 589 return vc.HypervisorConfig{}, err 590 } 591 592 if image != "" && initrd != "" { 593 return vc.HypervisorConfig{}, 594 errors.New("having both an image and an initrd defined in the configuration file is not supported") 595 } 596 597 if image == "" && initrd == "" { 598 return vc.HypervisorConfig{}, 599 errors.New("either image or initrd must be defined in the configuration file") 600 } 601 602 firmware, err := h.firmware() 603 if err != nil { 604 return vc.HypervisorConfig{}, err 605 } 606 607 machineAccelerators := h.machineAccelerators() 608 cpuFeatures := h.cpuFeatures() 609 kernelParams := h.kernelParams() 610 machineType := h.machineType() 611 612 // The "microvm" machine type doesn't support NVDIMM so override the 613 // config setting to explicitly disable it (i.e. don't require the 614 // user to add 'disable_image_nvdimm = true' in the .toml file). 615 if machineType == govmmQemu.MachineTypeMicrovm && !h.DisableImageNvdimm { 616 h.DisableImageNvdimm = true 617 kataUtilsLogger.Info("Setting 'disable_image_nvdimm = true' as microvm does not support NVDIMM") 618 } 619 620 blockDriver, err := h.blockDeviceDriver() 621 if err != nil { 622 return vc.HypervisorConfig{}, err 623 } 624 625 sharedFS, err := h.sharedFS() 626 if err != nil { 627 return vc.HypervisorConfig{}, err 628 } 629 630 if sharedFS == config.VirtioFS && h.VirtioFSDaemon == "" { 631 return vc.HypervisorConfig{}, 632 errors.New("cannot enable virtio-fs without daemon path in configuration file") 633 } 634 635 useVSock := false 636 if h.useVSock() { 637 if utils.SupportsVsocks() { 638 kataUtilsLogger.Info("vsock supported") 639 useVSock = true 640 } else { 641 kataUtilsLogger.Warn("No vsock support, falling back to legacy serial port") 642 } 643 } 644 645 return vc.HypervisorConfig{ 646 HypervisorPath: hypervisor, 647 KernelPath: kernel, 648 InitrdPath: initrd, 649 ImagePath: image, 650 FirmwarePath: firmware, 651 MachineAccelerators: machineAccelerators, 652 CPUFeatures: cpuFeatures, 653 KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), 654 HypervisorMachineType: machineType, 655 NumVCPUs: h.defaultVCPUs(), 656 DefaultMaxVCPUs: h.defaultMaxVCPUs(), 657 MemorySize: h.defaultMemSz(), 658 MemSlots: h.defaultMemSlots(), 659 MemOffset: h.defaultMemOffset(), 660 VirtioMem: h.VirtioMem, 661 EntropySource: h.GetEntropySource(), 662 DefaultBridges: h.defaultBridges(), 663 DisableBlockDeviceUse: h.DisableBlockDeviceUse, 664 SharedFS: sharedFS, 665 VirtioFSDaemon: h.VirtioFSDaemon, 666 VirtioFSCacheSize: h.VirtioFSCacheSize, 667 VirtioFSCache: h.defaultVirtioFSCache(), 668 VirtioFSExtraArgs: h.VirtioFSExtraArgs, 669 MemPrealloc: h.MemPrealloc, 670 HugePages: h.HugePages, 671 FileBackedMemRootDir: h.FileBackedMemRootDir, 672 Mlock: !h.Swap, 673 Debug: h.Debug, 674 DisableNestingChecks: h.DisableNestingChecks, 675 BlockDeviceDriver: blockDriver, 676 BlockDeviceCacheSet: h.BlockDeviceCacheSet, 677 BlockDeviceCacheDirect: h.BlockDeviceCacheDirect, 678 BlockDeviceCacheNoflush: h.BlockDeviceCacheNoflush, 679 EnableIOThreads: h.EnableIOThreads, 680 Msize9p: h.msize9p(), 681 UseVSock: useVSock, 682 DisableImageNvdimm: h.DisableImageNvdimm, 683 HotplugVFIOOnRootBus: h.HotplugVFIOOnRootBus, 684 PCIeRootPort: h.PCIeRootPort, 685 DisableVhostNet: h.DisableVhostNet, 686 EnableVhostUserStore: h.EnableVhostUserStore, 687 VhostUserStorePath: h.vhostUserStorePath(), 688 GuestHookPath: h.guestHookPath(), 689 }, nil 690 } 691 692 func newAcrnHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { 693 hypervisor, err := h.path() 694 if err != nil { 695 return vc.HypervisorConfig{}, err 696 } 697 698 hypervisorctl, err := h.ctlpath() 699 if err != nil { 700 return vc.HypervisorConfig{}, err 701 } 702 703 kernel, err := h.kernel() 704 if err != nil { 705 return vc.HypervisorConfig{}, err 706 } 707 708 image, err := h.image() 709 if err != nil { 710 return vc.HypervisorConfig{}, err 711 } 712 713 if image == "" { 714 return vc.HypervisorConfig{}, 715 errors.New("image must be defined in the configuration file") 716 } 717 718 firmware, err := h.firmware() 719 if err != nil { 720 return vc.HypervisorConfig{}, err 721 } 722 723 kernelParams := h.kernelParams() 724 725 blockDriver, err := h.blockDeviceDriver() 726 if err != nil { 727 return vc.HypervisorConfig{}, err 728 } 729 730 return vc.HypervisorConfig{ 731 HypervisorPath: hypervisor, 732 KernelPath: kernel, 733 ImagePath: image, 734 HypervisorCtlPath: hypervisorctl, 735 FirmwarePath: firmware, 736 KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), 737 NumVCPUs: h.defaultVCPUs(), 738 DefaultMaxVCPUs: h.defaultMaxVCPUs(), 739 MemorySize: h.defaultMemSz(), 740 MemSlots: h.defaultMemSlots(), 741 EntropySource: h.GetEntropySource(), 742 DefaultBridges: h.defaultBridges(), 743 HugePages: h.HugePages, 744 Mlock: !h.Swap, 745 Debug: h.Debug, 746 DisableNestingChecks: h.DisableNestingChecks, 747 BlockDeviceDriver: blockDriver, 748 DisableVhostNet: h.DisableVhostNet, 749 GuestHookPath: h.guestHookPath(), 750 }, nil 751 } 752 753 func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { 754 hypervisor, err := h.path() 755 if err != nil { 756 return vc.HypervisorConfig{}, err 757 } 758 759 kernel, err := h.kernel() 760 if err != nil { 761 return vc.HypervisorConfig{}, err 762 } 763 764 initrd, image, err := h.getInitrdAndImage() 765 if err != nil { 766 return vc.HypervisorConfig{}, err 767 } 768 769 if initrd != "" { 770 return vc.HypervisorConfig{}, 771 errors.New("having an initrd defined in the configuration file is not supported") 772 } 773 774 if image == "" { 775 return vc.HypervisorConfig{}, 776 errors.New("image must be defined in the configuration file") 777 } 778 779 firmware, err := h.firmware() 780 if err != nil { 781 return vc.HypervisorConfig{}, err 782 } 783 784 machineAccelerators := h.machineAccelerators() 785 kernelParams := h.kernelParams() 786 machineType := h.machineType() 787 788 blockDriver, err := h.blockDeviceDriver() 789 if err != nil { 790 return vc.HypervisorConfig{}, err 791 } 792 793 sharedFS := config.VirtioFS 794 795 if h.VirtioFSDaemon == "" { 796 return vc.HypervisorConfig{}, 797 errors.New("virtio-fs daemon path is missing in configuration file") 798 } 799 800 return vc.HypervisorConfig{ 801 HypervisorPath: hypervisor, 802 KernelPath: kernel, 803 InitrdPath: initrd, 804 ImagePath: image, 805 FirmwarePath: firmware, 806 MachineAccelerators: machineAccelerators, 807 KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), 808 HypervisorMachineType: machineType, 809 NumVCPUs: h.defaultVCPUs(), 810 DefaultMaxVCPUs: h.defaultMaxVCPUs(), 811 MemorySize: h.defaultMemSz(), 812 MemSlots: h.defaultMemSlots(), 813 MemOffset: h.defaultMemOffset(), 814 VirtioMem: h.VirtioMem, 815 EntropySource: h.GetEntropySource(), 816 DefaultBridges: h.defaultBridges(), 817 DisableBlockDeviceUse: h.DisableBlockDeviceUse, 818 SharedFS: sharedFS, 819 VirtioFSDaemon: h.VirtioFSDaemon, 820 VirtioFSCacheSize: h.VirtioFSCacheSize, 821 VirtioFSCache: h.VirtioFSCache, 822 MemPrealloc: h.MemPrealloc, 823 HugePages: h.HugePages, 824 FileBackedMemRootDir: h.FileBackedMemRootDir, 825 Mlock: !h.Swap, 826 Debug: h.Debug, 827 DisableNestingChecks: h.DisableNestingChecks, 828 BlockDeviceDriver: blockDriver, 829 BlockDeviceCacheSet: h.BlockDeviceCacheSet, 830 BlockDeviceCacheDirect: h.BlockDeviceCacheDirect, 831 BlockDeviceCacheNoflush: h.BlockDeviceCacheNoflush, 832 EnableIOThreads: h.EnableIOThreads, 833 Msize9p: h.msize9p(), 834 HotplugVFIOOnRootBus: h.HotplugVFIOOnRootBus, 835 PCIeRootPort: h.PCIeRootPort, 836 DisableVhostNet: true, 837 UseVSock: true, 838 }, nil 839 } 840 841 func newFactoryConfig(f factory) (oci.FactoryConfig, error) { 842 if f.TemplatePath == "" { 843 f.TemplatePath = defaultTemplatePath 844 } 845 if f.VMCacheEndpoint == "" { 846 f.VMCacheEndpoint = defaultVMCacheEndpoint 847 } 848 return oci.FactoryConfig{ 849 Template: f.Template, 850 TemplatePath: f.TemplatePath, 851 VMCacheNumber: f.VMCacheNumber, 852 VMCacheEndpoint: f.VMCacheEndpoint, 853 }, nil 854 } 855 856 func newShimConfig(s shim) (vc.ShimConfig, error) { 857 path, err := s.path() 858 if err != nil { 859 return vc.ShimConfig{}, err 860 } 861 862 return vc.ShimConfig{ 863 Path: path, 864 Debug: s.debug(), 865 Trace: s.trace(), 866 }, nil 867 } 868 869 func updateRuntimeConfigHypervisor(configPath string, tomlConf tomlConfig, config *oci.RuntimeConfig) error { 870 for k, hypervisor := range tomlConf.Hypervisor { 871 var err error 872 var hConfig vc.HypervisorConfig 873 874 switch k { 875 case firecrackerHypervisorTableType: 876 config.HypervisorType = vc.FirecrackerHypervisor 877 hConfig, err = newFirecrackerHypervisorConfig(hypervisor) 878 case qemuHypervisorTableType: 879 config.HypervisorType = vc.QemuHypervisor 880 hConfig, err = newQemuHypervisorConfig(hypervisor) 881 case acrnHypervisorTableType: 882 config.HypervisorType = vc.AcrnHypervisor 883 hConfig, err = newAcrnHypervisorConfig(hypervisor) 884 case clhHypervisorTableType: 885 config.HypervisorType = vc.ClhHypervisor 886 hConfig, err = newClhHypervisorConfig(hypervisor) 887 } 888 889 if err != nil { 890 return fmt.Errorf("%v: %v", configPath, err) 891 } 892 893 config.HypervisorConfig = hConfig 894 } 895 896 return nil 897 } 898 899 func updateRuntimeConfigProxy(configPath string, tomlConf tomlConfig, config *oci.RuntimeConfig, builtIn bool) error { 900 if builtIn { 901 config.ProxyType = vc.KataBuiltInProxyType 902 config.ProxyConfig = vc.ProxyConfig{ 903 Debug: config.Debug, 904 } 905 return nil 906 } 907 908 for k, proxy := range tomlConf.Proxy { 909 switch k { 910 case kataProxyTableType: 911 config.ProxyType = vc.KataProxyType 912 default: 913 return fmt.Errorf("%s proxy type not supported", k) 914 } 915 916 path, err := proxy.path() 917 if err != nil { 918 return err 919 } 920 921 config.ProxyConfig = vc.ProxyConfig{ 922 Path: path, 923 Debug: proxy.debug(), 924 } 925 } 926 927 return nil 928 } 929 930 func updateRuntimeConfigAgent(configPath string, tomlConf tomlConfig, config *oci.RuntimeConfig, builtIn bool) error { 931 if builtIn { 932 var agentConfig vc.KataAgentConfig 933 934 // If the agent config section isn't a Kata one, just default 935 // to everything being disabled. 936 agentConfig, _ = config.AgentConfig.(vc.KataAgentConfig) 937 938 config.AgentType = vc.KataContainersAgent 939 config.AgentConfig = vc.KataAgentConfig{ 940 LongLiveConn: true, 941 UseVSock: config.HypervisorConfig.UseVSock, 942 Debug: agentConfig.Debug, 943 KernelModules: agentConfig.KernelModules, 944 } 945 946 return nil 947 } 948 949 for k, agent := range tomlConf.Agent { 950 switch k { 951 case kataAgentTableType: 952 config.AgentType = vc.KataContainersAgent 953 config.AgentConfig = vc.KataAgentConfig{ 954 UseVSock: config.HypervisorConfig.UseVSock, 955 Debug: agent.debug(), 956 Trace: agent.trace(), 957 TraceMode: agent.traceMode(), 958 TraceType: agent.traceType(), 959 KernelModules: agent.kernelModules(), 960 } 961 default: 962 return fmt.Errorf("%s agent type is not supported", k) 963 } 964 } 965 966 return nil 967 } 968 969 func updateRuntimeConfigShim(configPath string, tomlConf tomlConfig, config *oci.RuntimeConfig, builtIn bool) error { 970 if builtIn { 971 config.ShimType = vc.KataBuiltInShimType 972 config.ShimConfig = vc.ShimConfig{} 973 return nil 974 } 975 976 for k, shim := range tomlConf.Shim { 977 switch k { 978 case kataShimTableType: 979 config.ShimType = vc.KataShimType 980 default: 981 return fmt.Errorf("%s shim is not supported", k) 982 } 983 984 shConfig, err := newShimConfig(shim) 985 if err != nil { 986 return fmt.Errorf("%v: %v", configPath, err) 987 } 988 989 config.ShimConfig = shConfig 990 } 991 992 return nil 993 } 994 995 // SetKernelParams adds the user-specified kernel parameters (from the 996 // configuration file) to the defaults so that the former take priority. 997 func SetKernelParams(runtimeConfig *oci.RuntimeConfig) error { 998 defaultKernelParams := GetKernelParamsFunc(needSystemd(runtimeConfig.HypervisorConfig), runtimeConfig.Trace) 999 1000 if runtimeConfig.HypervisorConfig.Debug { 1001 strParams := vc.SerializeParams(defaultKernelParams, "=") 1002 formatted := strings.Join(strParams, " ") 1003 kataUtilsLogger.WithField("default-kernel-parameters", formatted).Debug() 1004 } 1005 1006 // retrieve the parameters specified in the config file 1007 userKernelParams := runtimeConfig.HypervisorConfig.KernelParams 1008 1009 // reset 1010 runtimeConfig.HypervisorConfig.KernelParams = []vc.Param{} 1011 1012 // first, add default values 1013 for _, p := range defaultKernelParams { 1014 if err := runtimeConfig.AddKernelParam(p); err != nil { 1015 return err 1016 } 1017 } 1018 1019 // set the scsi scan mode to none for virtio-scsi 1020 if runtimeConfig.HypervisorConfig.BlockDeviceDriver == config.VirtioSCSI { 1021 p := vc.Param{ 1022 Key: "scsi_mod.scan", 1023 Value: "none", 1024 } 1025 if err := runtimeConfig.AddKernelParam(p); err != nil { 1026 return err 1027 } 1028 } 1029 1030 // next, check for agent specific kernel params 1031 if agentConfig, ok := runtimeConfig.AgentConfig.(vc.KataAgentConfig); ok { 1032 err := vc.KataAgentSetDefaultTraceConfigOptions(&agentConfig) 1033 if err != nil { 1034 return err 1035 } 1036 1037 params := vc.KataAgentKernelParams(agentConfig) 1038 1039 for _, p := range params { 1040 if err := runtimeConfig.AddKernelParam(p); err != nil { 1041 return err 1042 } 1043 } 1044 } 1045 1046 // now re-add the user-specified values so that they take priority. 1047 for _, p := range userKernelParams { 1048 if err := runtimeConfig.AddKernelParam(p); err != nil { 1049 return err 1050 } 1051 } 1052 1053 return nil 1054 } 1055 1056 func updateRuntimeConfig(configPath string, tomlConf tomlConfig, config *oci.RuntimeConfig, builtIn bool) error { 1057 if err := updateRuntimeConfigHypervisor(configPath, tomlConf, config); err != nil { 1058 return err 1059 } 1060 1061 if err := updateRuntimeConfigProxy(configPath, tomlConf, config, builtIn); err != nil { 1062 return err 1063 } 1064 1065 if err := updateRuntimeConfigAgent(configPath, tomlConf, config, builtIn); err != nil { 1066 return err 1067 } 1068 1069 if err := updateRuntimeConfigShim(configPath, tomlConf, config, builtIn); err != nil { 1070 return err 1071 } 1072 1073 fConfig, err := newFactoryConfig(tomlConf.Factory) 1074 if err != nil { 1075 return fmt.Errorf("%v: %v", configPath, err) 1076 } 1077 config.FactoryConfig = fConfig 1078 1079 config.NetmonConfig = vc.NetmonConfig{ 1080 Path: tomlConf.Netmon.path(), 1081 Debug: tomlConf.Netmon.debug(), 1082 Enable: tomlConf.Netmon.enable(), 1083 } 1084 1085 err = SetKernelParams(config) 1086 if err != nil { 1087 return err 1088 } 1089 1090 return nil 1091 } 1092 1093 func GetDefaultHypervisorConfig() vc.HypervisorConfig { 1094 return vc.HypervisorConfig{ 1095 HypervisorPath: defaultHypervisorPath, 1096 JailerPath: defaultJailerPath, 1097 KernelPath: defaultKernelPath, 1098 ImagePath: defaultImagePath, 1099 InitrdPath: defaultInitrdPath, 1100 FirmwarePath: defaultFirmwarePath, 1101 MachineAccelerators: defaultMachineAccelerators, 1102 CPUFeatures: defaultCPUFeatures, 1103 HypervisorMachineType: defaultMachineType, 1104 NumVCPUs: defaultVCPUCount, 1105 DefaultMaxVCPUs: defaultMaxVCPUCount, 1106 MemorySize: defaultMemSize, 1107 MemOffset: defaultMemOffset, 1108 VirtioMem: defaultVirtioMem, 1109 DisableBlockDeviceUse: defaultDisableBlockDeviceUse, 1110 DefaultBridges: defaultBridgesCount, 1111 MemPrealloc: defaultEnableMemPrealloc, 1112 HugePages: defaultEnableHugePages, 1113 FileBackedMemRootDir: defaultFileBackedMemRootDir, 1114 Mlock: !defaultEnableSwap, 1115 Debug: defaultEnableDebug, 1116 DisableNestingChecks: defaultDisableNestingChecks, 1117 BlockDeviceDriver: defaultBlockDeviceDriver, 1118 BlockDeviceCacheSet: defaultBlockDeviceCacheSet, 1119 BlockDeviceCacheDirect: defaultBlockDeviceCacheDirect, 1120 BlockDeviceCacheNoflush: defaultBlockDeviceCacheNoflush, 1121 EnableIOThreads: defaultEnableIOThreads, 1122 Msize9p: defaultMsize9p, 1123 HotplugVFIOOnRootBus: defaultHotplugVFIOOnRootBus, 1124 PCIeRootPort: defaultPCIeRootPort, 1125 GuestHookPath: defaultGuestHookPath, 1126 VhostUserStorePath: defaultVhostUserStorePath, 1127 VirtioFSCache: defaultVirtioFSCacheMode, 1128 DisableImageNvdimm: defaultDisableImageNvdimm, 1129 } 1130 } 1131 1132 func initConfig() (config oci.RuntimeConfig, err error) { 1133 var defaultAgentConfig interface{} 1134 1135 err = config.InterNetworkModel.SetModel(defaultInterNetworkingModel) 1136 if err != nil { 1137 return oci.RuntimeConfig{}, err 1138 } 1139 1140 defaultAgentConfig = vc.KataAgentConfig{} 1141 1142 config = oci.RuntimeConfig{ 1143 HypervisorType: defaultHypervisor, 1144 HypervisorConfig: GetDefaultHypervisorConfig(), 1145 AgentType: defaultAgent, 1146 AgentConfig: defaultAgentConfig, 1147 ProxyType: defaultProxy, 1148 ShimType: defaultShim, 1149 } 1150 1151 return config, nil 1152 } 1153 1154 // LoadConfiguration loads the configuration file and converts it into a 1155 // runtime configuration. 1156 // 1157 // If ignoreLogging is true, the system logger will not be initialised nor 1158 // will this function make any log calls. 1159 // 1160 // All paths are resolved fully meaning if this function does not return an 1161 // error, all paths are valid at the time of the call. 1162 func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolvedConfigPath string, config oci.RuntimeConfig, err error) { 1163 1164 config, err = initConfig() 1165 if err != nil { 1166 return "", oci.RuntimeConfig{}, err 1167 } 1168 1169 tomlConf, resolved, err := decodeConfig(configPath) 1170 if err != nil { 1171 return "", oci.RuntimeConfig{}, err 1172 } 1173 1174 config.Debug = tomlConf.Runtime.Debug 1175 if !tomlConf.Runtime.Debug { 1176 // If debug is not required, switch back to the original 1177 // default log priority, otherwise continue in debug mode. 1178 kataUtilsLogger.Logger.Level = originalLoggerLevel 1179 } 1180 1181 config.Trace = tomlConf.Runtime.Tracing 1182 tracing = config.Trace 1183 1184 if tomlConf.Runtime.InterNetworkModel != "" { 1185 err = config.InterNetworkModel.SetModel(tomlConf.Runtime.InterNetworkModel) 1186 if err != nil { 1187 return "", config, err 1188 } 1189 } 1190 1191 if !ignoreLogging { 1192 err := handleSystemLog("", "") 1193 if err != nil { 1194 return "", config, err 1195 } 1196 1197 kataUtilsLogger.WithFields( 1198 logrus.Fields{ 1199 "format": "TOML", 1200 "file": resolved, 1201 }).Info("loaded configuration") 1202 } 1203 1204 if err := updateRuntimeConfig(resolved, tomlConf, &config, builtIn); err != nil { 1205 return "", config, err 1206 } 1207 1208 config.DisableGuestSeccomp = tomlConf.Runtime.DisableGuestSeccomp 1209 1210 // use no proxy if HypervisorConfig.UseVSock is true 1211 if config.HypervisorConfig.UseVSock { 1212 kataUtilsLogger.Info("VSOCK supported, configure to not use proxy") 1213 config.ProxyType = vc.NoProxyType 1214 config.ProxyConfig = vc.ProxyConfig{Debug: config.Debug} 1215 } 1216 1217 config.SandboxCgroupOnly = tomlConf.Runtime.SandboxCgroupOnly 1218 config.DisableNewNetNs = tomlConf.Runtime.DisableNewNetNs 1219 for _, f := range tomlConf.Runtime.Experimental { 1220 feature := exp.Get(f) 1221 if feature == nil { 1222 return "", config, fmt.Errorf("Unsupported experimental feature %q", f) 1223 } 1224 config.Experimental = append(config.Experimental, *feature) 1225 } 1226 1227 if err := checkConfig(config); err != nil { 1228 return "", config, err 1229 } 1230 1231 return resolved, config, nil 1232 } 1233 1234 func decodeConfig(configPath string) (tomlConfig, string, error) { 1235 var ( 1236 resolved string 1237 tomlConf tomlConfig 1238 err error 1239 ) 1240 1241 if configPath == "" { 1242 resolved, err = getDefaultConfigFile() 1243 } else { 1244 resolved, err = ResolvePath(configPath) 1245 } 1246 1247 if err != nil { 1248 return tomlConf, "", fmt.Errorf("Cannot find usable config file (%v)", err) 1249 } 1250 1251 configData, err := ioutil.ReadFile(resolved) 1252 if err != nil { 1253 return tomlConf, resolved, err 1254 } 1255 1256 _, err = toml.Decode(string(configData), &tomlConf) 1257 if err != nil { 1258 return tomlConf, resolved, err 1259 } 1260 1261 return tomlConf, resolved, nil 1262 } 1263 1264 // checkConfig checks the validity of the specified config. 1265 func checkConfig(config oci.RuntimeConfig) error { 1266 if err := checkNetNsConfig(config); err != nil { 1267 return err 1268 } 1269 1270 if err := checkHypervisorConfig(config.HypervisorConfig); err != nil { 1271 return err 1272 } 1273 1274 if err := checkFactoryConfig(config); err != nil { 1275 return err 1276 } 1277 1278 return nil 1279 } 1280 1281 // checkNetNsConfig performs sanity checks on disable_new_netns config. 1282 // Because it is an expert option and conflicts with some other common configs. 1283 func checkNetNsConfig(config oci.RuntimeConfig) error { 1284 if config.DisableNewNetNs { 1285 if config.NetmonConfig.Enable { 1286 return fmt.Errorf("config disable_new_netns conflicts with enable_netmon") 1287 } 1288 if config.InterNetworkModel != vc.NetXConnectNoneModel { 1289 return fmt.Errorf("config disable_new_netns only works with 'none' internetworking_model") 1290 } 1291 } else if shim, ok := config.ShimConfig.(vc.ShimConfig); ok && shim.Trace { 1292 // Normally, the shim runs in a separate network namespace. 1293 // But when tracing, the shim process needs to be able to talk 1294 // to the Jaeger agent running in the host network namespace. 1295 return errors.New("Shim tracing requires disable_new_netns for Jaeger agent communication") 1296 } 1297 1298 return nil 1299 } 1300 1301 // checkFactoryConfig ensures the VM factory configuration is valid. 1302 func checkFactoryConfig(config oci.RuntimeConfig) error { 1303 if config.FactoryConfig.Template { 1304 if config.HypervisorConfig.InitrdPath == "" { 1305 return errors.New("Factory option enable_template requires an initrd image") 1306 } 1307 } 1308 1309 if config.FactoryConfig.VMCacheNumber > 0 { 1310 if config.HypervisorType != vc.QemuHypervisor { 1311 return errors.New("VM cache just support qemu") 1312 } 1313 if config.AgentType != vc.KataContainersAgent { 1314 return errors.New("VM cache just support kata agent") 1315 } 1316 } 1317 1318 return nil 1319 } 1320 1321 // checkHypervisorConfig performs basic "sanity checks" on the hypervisor 1322 // config. 1323 func checkHypervisorConfig(config vc.HypervisorConfig) error { 1324 type image struct { 1325 path string 1326 initrd bool 1327 } 1328 1329 images := []image{ 1330 { 1331 path: config.ImagePath, 1332 initrd: false, 1333 }, 1334 { 1335 path: config.InitrdPath, 1336 initrd: true, 1337 }, 1338 } 1339 1340 memSizeMB := int64(config.MemorySize) 1341 1342 if memSizeMB == 0 { 1343 return errors.New("VM memory cannot be zero") 1344 } 1345 1346 mb := int64(1024 * 1024) 1347 1348 for _, image := range images { 1349 if image.path == "" { 1350 continue 1351 } 1352 1353 imageSizeBytes, err := fileSize(image.path) 1354 if err != nil { 1355 return err 1356 } 1357 1358 if imageSizeBytes == 0 { 1359 return fmt.Errorf("image %q is empty", image.path) 1360 } 1361 1362 if imageSizeBytes > mb { 1363 imageSizeMB := imageSizeBytes / mb 1364 1365 msg := fmt.Sprintf("VM memory (%dMB) smaller than image %q size (%dMB)", 1366 memSizeMB, image.path, imageSizeMB) 1367 if imageSizeMB >= memSizeMB { 1368 if image.initrd { 1369 // Initrd's need to be fully read into memory 1370 return errors.New(msg) 1371 } 1372 1373 // Images do not need to be fully read 1374 // into memory, but it would be highly 1375 // unusual to have an image larger 1376 // than the amount of memory assigned 1377 // to the VM. 1378 kataUtilsLogger.Warn(msg) 1379 } 1380 } 1381 } 1382 1383 return nil 1384 } 1385 1386 // GetDefaultConfigFilePaths returns a list of paths that will be 1387 // considered as configuration files in priority order. 1388 func GetDefaultConfigFilePaths() []string { 1389 return []string{ 1390 // normally below "/etc" 1391 defaultSysConfRuntimeConfiguration, 1392 1393 // normally below "/usr/share" 1394 defaultRuntimeConfiguration, 1395 } 1396 } 1397 1398 // getDefaultConfigFile looks in multiple default locations for a 1399 // configuration file and returns the resolved path for the first file 1400 // found, or an error if no config files can be found. 1401 func getDefaultConfigFile() (string, error) { 1402 var errs []string 1403 1404 for _, file := range GetDefaultConfigFilePaths() { 1405 resolved, err := ResolvePath(file) 1406 if err == nil { 1407 return resolved, nil 1408 } 1409 s := fmt.Sprintf("config file %q unresolvable: %v", file, err) 1410 errs = append(errs, s) 1411 } 1412 1413 return "", errors.New(strings.Join(errs, ", ")) 1414 } 1415 1416 // SetConfigOptions will override some of the defaults settings. 1417 func SetConfigOptions(n, runtimeConfig, sysRuntimeConfig string) { 1418 if n != "" { 1419 name = n 1420 } 1421 1422 if runtimeConfig != "" { 1423 defaultRuntimeConfiguration = runtimeConfig 1424 } 1425 1426 if sysRuntimeConfig != "" { 1427 defaultSysConfRuntimeConfiguration = sysRuntimeConfig 1428 } 1429 }