github.com/mirantis/virtlet@v1.5.2-0.20191204181327-1659b8a48e9b/cmd/vmwrapper/vmwrapper.go (about) 1 /* 2 Copyright 2016-2017 Mirantis 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package main 18 19 import ( 20 "encoding/json" 21 "flag" 22 "fmt" 23 "os" 24 "syscall" 25 26 "github.com/golang/glog" 27 28 "github.com/Mirantis/virtlet/pkg/config" 29 "github.com/Mirantis/virtlet/pkg/network" 30 "github.com/Mirantis/virtlet/pkg/nsfix" 31 "github.com/Mirantis/virtlet/pkg/tapmanager" 32 "github.com/Mirantis/virtlet/pkg/utils" 33 "github.com/Mirantis/virtlet/pkg/utils/cgroups" 34 ) 35 36 const ( 37 fdSocketPath = "/var/lib/virtlet/tapfdserver.sock" 38 defaultEmulator = "/usr/bin/qemu-system-x86_64" // FIXME 39 vmsProcFile = "/var/lib/virtlet/vms.procfile" 40 ) 41 42 type reexecArg struct { 43 Args []string 44 } 45 46 func handleReexec(arg interface{}) (interface{}, error) { 47 args := arg.(*reexecArg).Args 48 if err := syscall.Exec(args[0], args, os.Environ()); err != nil { 49 return nil, fmt.Errorf("Can't exec emulator: %v", err) 50 } 51 return nil, nil // unreachable 52 } 53 54 func main() { 55 nsfix.RegisterReexec("vmwrapper", handleReexec, reexecArg{}) 56 nsfix.HandleReexec() 57 58 // configure glog (apparently no better way to do it ...) 59 flag.CommandLine.Parse([]string{"-v=3", "-logtostderr=true"}) 60 61 runInAnotherContainer := os.Getuid() != 0 62 63 var pid int 64 var err error 65 if runInAnotherContainer { 66 glog.V(0).Infof("Obtaining PID of the VM container process...") 67 pid, err = utils.WaitForProcess(vmsProcFile) 68 if err != nil { 69 glog.Errorf("Can't obtain PID of the VM container process") 70 os.Exit(1) 71 } 72 } 73 74 // FIXME: move the pid of qemu instance out of kubelet-managed 75 // for cgroups that aren't managed by libvirt. 76 // If we don't do this, the VM pod will be killed by kubelet when Virtlet pod 77 // is removed dnd cgroup-per-qos is enabled in kubelet settings. 78 cm := cgroups.NewManager(os.Getpid(), nil) 79 for _, ctl := range []string{"hugetlb", "systemd", "pids"} { 80 if _, err := cm.GetProcessController(ctl); err == nil { 81 err = cm.MoveProcess(ctl, "/") 82 if err != nil { 83 glog.Warningf("failed to move pid into cgroup %q path /: %v", ctl, err) 84 } 85 } 86 } 87 88 emulator := os.Getenv(config.EmulatorEnvVarName) 89 emulatorArgs := os.Args[1:] 90 var netArgs []string 91 if emulator == "" { 92 // this happens during 'qemu -help' invocation by libvirt 93 // (capability check) 94 emulator = defaultEmulator 95 } else { 96 netFdKey := os.Getenv(config.NetKeyEnvVarName) 97 nextToUseHostdevNo := 0 98 99 if netFdKey != "" { 100 c := tapmanager.NewFDClient(fdSocketPath) 101 fds, marshaledData, err := c.GetFDs(netFdKey) 102 if err != nil { 103 glog.Errorf("Failed to obtain tap fds for key %q: %v", netFdKey, err) 104 os.Exit(1) 105 } 106 107 var descriptions []tapmanager.InterfaceDescription 108 if err := json.Unmarshal(marshaledData, &descriptions); err != nil { 109 glog.Errorf("Failed to unmarshal network interface info: %v", err) 110 os.Exit(1) 111 } 112 113 for i, desc := range descriptions { 114 switch desc.Type { 115 case network.InterfaceTypeTap: 116 netArgs = append(netArgs, 117 "-netdev", 118 fmt.Sprintf("tap,id=tap%d,fd=%d", desc.FdIndex, fds[desc.FdIndex]), 119 "-device", 120 fmt.Sprintf("virtio-net-pci,netdev=tap%d,id=net%d,mac=%s", desc.FdIndex, i, desc.HardwareAddr), 121 ) 122 case network.InterfaceTypeVF: 123 netArgs = append(netArgs, 124 "-device", 125 fmt.Sprintf("vfio-pci,host=%s,id=hostdev%d", 126 desc.PCIAddress[5:], 127 nextToUseHostdevNo, 128 ), 129 ) 130 nextToUseHostdevNo += 1 131 default: 132 // Impossible situation when tapmanager is built from other sources than vmwrapper 133 glog.Errorf("Received unknown interface type: %d", int(desc.Type)) 134 os.Exit(1) 135 } 136 } 137 } 138 } 139 140 args := append([]string{emulator}, emulatorArgs...) 141 args = append(args, netArgs...) 142 env := os.Environ() 143 if runInAnotherContainer { 144 nsFixCall := nsfix.NewCall("vmwrapper"). 145 TargetPid(pid). 146 Arg(&reexecArg{args}). 147 RemountSys() 148 // Currently we don't drop privs when SR-IOV support is enabled 149 // because of an unresolved emulator permission problem. 150 if os.Getenv("VMWRAPPER_KEEP_PRIVS") == "" { 151 nsFixCall.DropPrivs() 152 } 153 if err := nsFixCall.SwitchToNamespaces(); err != nil { 154 glog.Fatalf("Error reexecuting vmwrapper: %v", err) 155 } 156 } else { 157 if err := setupCPUSets(cm); err != nil { 158 glog.Errorf("Can't set cpusets for emulator: %v", err) 159 os.Exit(1) 160 } 161 // this log hides errors returned by libvirt virError 162 // because of libvirt's output parsing approach 163 // glog.V(0).Infof("Executing emulator: %s", strings.Join(args, " ")) 164 if err := syscall.Exec(args[0], args, env); err != nil { 165 glog.Errorf("Can't exec emulator: %v", err) 166 os.Exit(1) 167 } 168 } 169 } 170 171 func setupCPUSets(cm cgroups.Manager) error { 172 cpusets := os.Getenv(config.CpusetsEnvVarName) 173 if cpusets == "" { 174 return nil 175 } 176 177 controller, err := cm.GetProcessController("cpuset") 178 if err != nil { 179 return err 180 } 181 182 return controller.Set("cpus", cpusets) 183 }