github.com/kata-containers/runtime@v0.0.0-20210505125100-04f29832a923/netmon/netmon.go (about) 1 // Copyright (c) 2018 Intel Corporation 2 // 3 // SPDX-License-Identifier: Apache-2.0 4 // 5 6 package main 7 8 import ( 9 "encoding/json" 10 "errors" 11 "flag" 12 "fmt" 13 "io/ioutil" 14 "log/syslog" 15 "net" 16 "os" 17 "os/exec" 18 "os/signal" 19 "path/filepath" 20 "strings" 21 "syscall" 22 "time" 23 24 "github.com/kata-containers/runtime/pkg/signals" 25 vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" 26 "github.com/sirupsen/logrus" 27 lSyslog "github.com/sirupsen/logrus/hooks/syslog" 28 "github.com/vishvananda/netlink" 29 "golang.org/x/sys/unix" 30 ) 31 32 const ( 33 netmonName = "kata-netmon" 34 35 kataCmd = "kata-network" 36 kataCLIAddIfaceCmd = "add-iface" 37 kataCLIDelIfaceCmd = "del-iface" 38 kataCLIUpdtRoutesCmd = "update-routes" 39 40 kataSuffix = "kata" 41 42 // sharedFile is the name of the file that will be used to share 43 // the data between this process and the kata-runtime process 44 // responsible for updating the network. 45 sharedFile = "shared.json" 46 storageFilePerm = os.FileMode(0640) 47 storageDirPerm = os.FileMode(0750) 48 ) 49 50 var ( 51 // version is the netmon version. This variable is populated at build time. 52 version = "unknown" 53 54 // For simplicity the code will only focus on IPv4 addresses for now. 55 netlinkFamily = netlink.FAMILY_ALL 56 57 storageParentPath = "/var/run/kata-containers/netmon/sbs" 58 ) 59 60 type netmonParams struct { 61 sandboxID string 62 runtimePath string 63 debug bool 64 logLevel string 65 } 66 67 type netmon struct { 68 netmonParams 69 70 storagePath string 71 sharedFile string 72 73 netIfaces map[int]vcTypes.Interface 74 75 linkUpdateCh chan netlink.LinkUpdate 76 linkDoneCh chan struct{} 77 78 rtUpdateCh chan netlink.RouteUpdate 79 rtDoneCh chan struct{} 80 81 netHandler *netlink.Handle 82 } 83 84 var netmonLog = logrus.New() 85 86 func printVersion() { 87 fmt.Printf("%s version %s\n", netmonName, version) 88 } 89 90 const componentDescription = `is a network monitoring process that is intended to be started in the 91 appropriate network namespace so that it can listen to any event related to 92 link and routes. Whenever a new interface or route is created/updated, it is 93 responsible for calling into the kata-runtime CLI to ask for the actual 94 creation/update of the given interface or route. 95 ` 96 97 func printComponentDescription() { 98 fmt.Printf("\n%s %s\n", netmonName, componentDescription) 99 } 100 101 func parseOptions() netmonParams { 102 var version, help bool 103 104 params := netmonParams{} 105 106 flag.BoolVar(&help, "h", false, "describe component usage") 107 flag.BoolVar(&help, "help", false, "") 108 flag.BoolVar(¶ms.debug, "d", false, "enable debug mode") 109 flag.BoolVar(&version, "v", false, "display program version and exit") 110 flag.BoolVar(&version, "version", false, "") 111 flag.StringVar(¶ms.sandboxID, "s", "", "sandbox id (required)") 112 flag.StringVar(¶ms.runtimePath, "r", "", "runtime path (required)") 113 flag.StringVar(¶ms.logLevel, "log", "warn", 114 "log messages above specified level: debug, warn, error, fatal or panic") 115 116 flag.Parse() 117 118 if help { 119 printComponentDescription() 120 flag.PrintDefaults() 121 os.Exit(0) 122 } 123 124 if version { 125 printVersion() 126 os.Exit(0) 127 } 128 129 if params.sandboxID == "" { 130 fmt.Fprintf(os.Stderr, "Error: sandbox id is empty, one must be provided\n") 131 flag.PrintDefaults() 132 os.Exit(1) 133 } 134 135 if params.runtimePath == "" { 136 fmt.Fprintf(os.Stderr, "Error: runtime path is empty, one must be provided\n") 137 flag.PrintDefaults() 138 os.Exit(1) 139 } 140 141 return params 142 } 143 144 func newNetmon(params netmonParams) (*netmon, error) { 145 handler, err := netlink.NewHandle(netlinkFamily) 146 if err != nil { 147 return nil, err 148 } 149 150 n := &netmon{ 151 netmonParams: params, 152 storagePath: filepath.Join(storageParentPath, params.sandboxID), 153 sharedFile: filepath.Join(storageParentPath, params.sandboxID, sharedFile), 154 netIfaces: make(map[int]vcTypes.Interface), 155 linkUpdateCh: make(chan netlink.LinkUpdate), 156 linkDoneCh: make(chan struct{}), 157 rtUpdateCh: make(chan netlink.RouteUpdate), 158 rtDoneCh: make(chan struct{}), 159 netHandler: handler, 160 } 161 162 if err := os.MkdirAll(n.storagePath, storageDirPerm); err != nil { 163 return nil, err 164 } 165 166 return n, nil 167 } 168 169 func (n *netmon) cleanup() { 170 os.RemoveAll(n.storagePath) 171 n.netHandler.Delete() 172 close(n.linkDoneCh) 173 close(n.rtDoneCh) 174 } 175 176 // setupSignalHandler sets up signal handling, starting a go routine to deal 177 // with signals as they arrive. 178 func (n *netmon) setupSignalHandler() { 179 signals.SetLogger(n.logger()) 180 181 sigCh := make(chan os.Signal, 8) 182 183 for _, sig := range signals.HandledSignals() { 184 signal.Notify(sigCh, sig) 185 } 186 187 go func() { 188 for { 189 sig := <-sigCh 190 191 nativeSignal, ok := sig.(syscall.Signal) 192 if !ok { 193 err := errors.New("unknown signal") 194 netmonLog.WithError(err).WithField("signal", sig.String()).Error() 195 continue 196 } 197 198 if signals.FatalSignal(nativeSignal) { 199 netmonLog.WithField("signal", sig).Error("received fatal signal") 200 signals.Die(nil) 201 } else if n.debug && signals.NonFatalSignal(nativeSignal) { 202 netmonLog.WithField("signal", sig).Debug("handling signal") 203 signals.Backtrace() 204 } 205 } 206 }() 207 } 208 209 func (n *netmon) logger() *logrus.Entry { 210 fields := logrus.Fields{ 211 "name": netmonName, 212 "pid": os.Getpid(), 213 "source": "netmon", 214 } 215 216 if n.sandboxID != "" { 217 fields["sandbox"] = n.sandboxID 218 } 219 220 return netmonLog.WithFields(fields) 221 } 222 223 func (n *netmon) setupLogger() error { 224 level, err := logrus.ParseLevel(n.logLevel) 225 if err != nil { 226 return err 227 } 228 229 netmonLog.SetLevel(level) 230 231 netmonLog.Formatter = &logrus.TextFormatter{TimestampFormat: time.RFC3339Nano} 232 233 hook, err := lSyslog.NewSyslogHook("", "", syslog.LOG_INFO|syslog.LOG_USER, netmonName) 234 if err != nil { 235 return err 236 } 237 238 netmonLog.AddHook(hook) 239 240 announceFields := logrus.Fields{ 241 "runtime-path": n.runtimePath, 242 "debug": n.debug, 243 "log-level": n.logLevel, 244 } 245 246 n.logger().WithFields(announceFields).Info("announce") 247 248 return nil 249 } 250 251 func (n *netmon) listenNetlinkEvents() error { 252 if err := netlink.LinkSubscribe(n.linkUpdateCh, n.linkDoneCh); err != nil { 253 return err 254 } 255 256 return netlink.RouteSubscribe(n.rtUpdateCh, n.rtDoneCh) 257 } 258 259 // convertInterface converts a link and its IP addresses as defined by netlink 260 // package, into the Interface structure format expected by kata-runtime to 261 // describe an interface and its associated IP addresses. 262 func convertInterface(linkAttrs *netlink.LinkAttrs, linkType string, addrs []netlink.Addr) vcTypes.Interface { 263 if linkAttrs == nil { 264 netmonLog.Warn("Link attributes are nil") 265 return vcTypes.Interface{} 266 } 267 268 var ipAddrs []*vcTypes.IPAddress 269 270 for _, addr := range addrs { 271 if addr.IPNet == nil { 272 continue 273 } 274 275 netMask, _ := addr.Mask.Size() 276 277 ipAddr := &vcTypes.IPAddress{ 278 Address: addr.IP.String(), 279 Mask: fmt.Sprintf("%d", netMask), 280 } 281 282 if addr.IP.To4() != nil { 283 ipAddr.Family = netlink.FAMILY_V4 284 } else { 285 ipAddr.Family = netlink.FAMILY_V6 286 } 287 288 ipAddrs = append(ipAddrs, ipAddr) 289 } 290 291 iface := vcTypes.Interface{ 292 Device: linkAttrs.Name, 293 Name: linkAttrs.Name, 294 IPAddresses: ipAddrs, 295 Mtu: uint64(linkAttrs.MTU), 296 HwAddr: linkAttrs.HardwareAddr.String(), 297 LinkType: linkType, 298 } 299 300 netmonLog.WithField("interface", iface).Debug("Interface converted") 301 302 return iface 303 } 304 305 // convertRoutes converts a list of routes as defined by netlink package, 306 // into a list of Route structure format expected by kata-runtime to 307 // describe a set of routes. 308 func convertRoutes(netRoutes []netlink.Route) []vcTypes.Route { 309 var routes []vcTypes.Route 310 311 for _, netRoute := range netRoutes { 312 dst := "" 313 314 if netRoute.Protocol == unix.RTPROT_KERNEL { 315 continue 316 } 317 318 if netRoute.Dst != nil { 319 dst = netRoute.Dst.String() 320 if netRoute.Dst.IP.To4() != nil || netRoute.Dst.IP.To16() != nil { 321 dst = netRoute.Dst.String() 322 } else { 323 netmonLog.WithField("destination", netRoute.Dst.IP.String()).Warn("Unexpected network address format") 324 } 325 } 326 327 src := "" 328 if netRoute.Src != nil { 329 if netRoute.Src.To4() != nil || netRoute.Src.To16() != nil { 330 src = netRoute.Src.String() 331 } else { 332 netmonLog.WithField("source", netRoute.Src.String()).Warn("Unexpected network address format") 333 } 334 } 335 336 gw := "" 337 if netRoute.Gw != nil { 338 if netRoute.Gw.To4() != nil || netRoute.Gw.To16() != nil { 339 gw = netRoute.Gw.String() 340 } else { 341 netmonLog.WithField("gateway", netRoute.Gw.String()).Warn("Unexpected network address format") 342 } 343 } 344 345 dev := "" 346 iface, err := net.InterfaceByIndex(netRoute.LinkIndex) 347 if err == nil { 348 dev = iface.Name 349 } 350 351 route := vcTypes.Route{ 352 Dest: dst, 353 Gateway: gw, 354 Device: dev, 355 Source: src, 356 Scope: uint32(netRoute.Scope), 357 } 358 359 routes = append(routes, route) 360 } 361 362 netmonLog.WithField("routes", routes).Debug("Routes converted") 363 364 return routes 365 } 366 367 // scanNetwork lists all the interfaces it can find inside the current 368 // network namespace, and store them in-memory to keep track of them. 369 func (n *netmon) scanNetwork() error { 370 links, err := n.netHandler.LinkList() 371 if err != nil { 372 return err 373 } 374 375 for _, link := range links { 376 addrs, err := n.netHandler.AddrList(link, netlinkFamily) 377 if err != nil { 378 return err 379 } 380 381 linkAttrs := link.Attrs() 382 if linkAttrs == nil { 383 continue 384 } 385 386 iface := convertInterface(linkAttrs, link.Type(), addrs) 387 n.netIfaces[linkAttrs.Index] = iface 388 } 389 390 n.logger().Debug("Network scanned") 391 392 return nil 393 } 394 395 func (n *netmon) storeDataToSend(data interface{}) error { 396 // Marshal the data structure into a JSON bytes array. 397 jsonArray, err := json.Marshal(data) 398 if err != nil { 399 return err 400 } 401 402 // Store the JSON bytes array at the specified path. 403 return ioutil.WriteFile(n.sharedFile, jsonArray, storageFilePerm) 404 } 405 406 func (n *netmon) execKataCmd(subCmd string) error { 407 execCmd := exec.Command(n.runtimePath, kataCmd, subCmd, n.sandboxID, n.sharedFile) 408 409 n.logger().WithField("command", execCmd).Debug("Running runtime command") 410 411 // Make use of Run() to ensure the kata-runtime process has correctly 412 // terminated before to go further. 413 if err := execCmd.Run(); err != nil { 414 return err 415 } 416 417 // Remove the shared file after the command returned. At this point 418 // we know the content of the file is not going to be used anymore, 419 // and the file path can be reused for further commands. 420 return os.Remove(n.sharedFile) 421 } 422 423 func (n *netmon) addInterfaceCLI(iface vcTypes.Interface) error { 424 if err := n.storeDataToSend(iface); err != nil { 425 return err 426 } 427 428 return n.execKataCmd(kataCLIAddIfaceCmd) 429 } 430 431 func (n *netmon) delInterfaceCLI(iface vcTypes.Interface) error { 432 if err := n.storeDataToSend(iface); err != nil { 433 return err 434 } 435 436 return n.execKataCmd(kataCLIDelIfaceCmd) 437 } 438 439 func (n *netmon) updateRoutesCLI(routes []vcTypes.Route) error { 440 if err := n.storeDataToSend(routes); err != nil { 441 return err 442 } 443 444 return n.execKataCmd(kataCLIUpdtRoutesCmd) 445 } 446 447 func (n *netmon) updateRoutes() error { 448 // Get all the routes. 449 netlinkRoutes, err := n.netHandler.RouteList(nil, netlinkFamily) 450 if err != nil { 451 return err 452 } 453 454 // Translate them into Route structures. 455 routes := convertRoutes(netlinkRoutes) 456 457 // Update the routes through the Kata CLI. 458 return n.updateRoutesCLI(routes) 459 } 460 461 func (n *netmon) handleRTMNewAddr(ev netlink.LinkUpdate) error { 462 n.logger().Debug("Interface update not supported") 463 return nil 464 } 465 466 func (n *netmon) handleRTMDelAddr(ev netlink.LinkUpdate) error { 467 n.logger().Debug("Interface update not supported") 468 return nil 469 } 470 471 func (n *netmon) handleRTMNewLink(ev netlink.LinkUpdate) error { 472 // NEWLINK might be a lot of different things. We're interested in 473 // adding the interface (both to our list and by calling into the 474 // Kata CLI API) only if this has the flags UP and RUNNING, meaning 475 // we don't expect any further change on the interface, and that we 476 // are ready to add it. 477 478 linkAttrs := ev.Link.Attrs() 479 if linkAttrs == nil { 480 n.logger().Warn("The link attributes are nil") 481 return nil 482 } 483 484 // First, ignore if the interface name contains "kata". This way we 485 // are preventing from adding interfaces created by Kata Containers. 486 if strings.HasSuffix(linkAttrs.Name, kataSuffix) { 487 n.logger().Debugf("Ignore the interface %s because found %q", 488 linkAttrs.Name, kataSuffix) 489 return nil 490 } 491 492 // Check if the interface exist in the internal list. 493 if _, exist := n.netIfaces[int(ev.Index)]; exist { 494 n.logger().Debugf("Ignoring interface %s because already exist", 495 linkAttrs.Name) 496 return nil 497 } 498 499 // Now, check if the interface has been enabled to UP and RUNNING. 500 if (ev.Flags&unix.IFF_UP) != unix.IFF_UP || 501 (ev.Flags&unix.IFF_RUNNING) != unix.IFF_RUNNING { 502 n.logger().Debugf("Ignore the interface %s because not UP and RUNNING", 503 linkAttrs.Name) 504 return nil 505 } 506 507 // Get the list of IP addresses associated with this interface. 508 addrs, err := n.netHandler.AddrList(ev.Link, netlinkFamily) 509 if err != nil { 510 return err 511 } 512 513 // Convert the interfaces in the appropriate structure format. 514 iface := convertInterface(linkAttrs, ev.Link.Type(), addrs) 515 516 // Add the interface through the Kata CLI. 517 if err := n.addInterfaceCLI(iface); err != nil { 518 return err 519 } 520 521 // Add the interface to the internal list. 522 n.netIfaces[linkAttrs.Index] = iface 523 524 // Complete by updating the routes. 525 return n.updateRoutes() 526 } 527 528 func (n *netmon) handleRTMDelLink(ev netlink.LinkUpdate) error { 529 // It can only delete if identical interface is found in the internal 530 // list of interfaces. Otherwise, the deletion will be ignored. 531 linkAttrs := ev.Link.Attrs() 532 if linkAttrs == nil { 533 n.logger().Warn("Link attributes are nil") 534 return nil 535 } 536 537 // First, ignore if the interface name contains "kata". This way we 538 // are preventing from deleting interfaces created by Kata Containers. 539 if strings.Contains(linkAttrs.Name, kataSuffix) { 540 n.logger().Debugf("Ignore the interface %s because found %q", 541 linkAttrs.Name, kataSuffix) 542 return nil 543 } 544 545 // Check if the interface exist in the internal list. 546 iface, exist := n.netIfaces[int(ev.Index)] 547 if !exist { 548 n.logger().Debugf("Ignoring interface %s because not found", 549 linkAttrs.Name) 550 return nil 551 } 552 553 if err := n.delInterfaceCLI(iface); err != nil { 554 return err 555 } 556 557 // Delete the interface from the internal list. 558 delete(n.netIfaces, linkAttrs.Index) 559 560 // Complete by updating the routes. 561 return n.updateRoutes() 562 } 563 564 func (n *netmon) handleRTMNewRoute(ev netlink.RouteUpdate) error { 565 // Add the route through updateRoutes(), only if the route refer to an 566 // interface that already exists in the internal list of interfaces. 567 if _, exist := n.netIfaces[ev.Route.LinkIndex]; !exist { 568 n.logger().Debugf("Ignoring route %+v since interface %d not found", 569 ev.Route, ev.Route.LinkIndex) 570 return nil 571 } 572 573 return n.updateRoutes() 574 } 575 576 func (n *netmon) handleRTMDelRoute(ev netlink.RouteUpdate) error { 577 // Remove the route through updateRoutes(), only if the route refer to 578 // an interface that already exists in the internal list of interfaces. 579 return n.updateRoutes() 580 } 581 582 func (n *netmon) handleLinkEvent(ev netlink.LinkUpdate) error { 583 n.logger().Debug("handleLinkEvent: netlink event received") 584 585 switch ev.Header.Type { 586 case unix.NLMSG_DONE: 587 n.logger().Debug("NLMSG_DONE") 588 return nil 589 case unix.NLMSG_ERROR: 590 n.logger().Error("NLMSG_ERROR") 591 return fmt.Errorf("Error while listening on netlink socket") 592 case unix.RTM_NEWADDR: 593 n.logger().Debug("RTM_NEWADDR") 594 return n.handleRTMNewAddr(ev) 595 case unix.RTM_DELADDR: 596 n.logger().Debug("RTM_DELADDR") 597 return n.handleRTMDelAddr(ev) 598 case unix.RTM_NEWLINK: 599 n.logger().Debug("RTM_NEWLINK") 600 return n.handleRTMNewLink(ev) 601 case unix.RTM_DELLINK: 602 n.logger().Debug("RTM_DELLINK") 603 return n.handleRTMDelLink(ev) 604 default: 605 n.logger().Warnf("Unknown msg type %v", ev.Header.Type) 606 } 607 608 return nil 609 } 610 611 func (n *netmon) handleRouteEvent(ev netlink.RouteUpdate) error { 612 n.logger().Debug("handleRouteEvent: netlink event received") 613 614 switch ev.Type { 615 case unix.RTM_NEWROUTE: 616 n.logger().Debug("RTM_NEWROUTE") 617 return n.handleRTMNewRoute(ev) 618 case unix.RTM_DELROUTE: 619 n.logger().Debug("RTM_DELROUTE") 620 return n.handleRTMDelRoute(ev) 621 default: 622 n.logger().Warnf("Unknown msg type %v", ev.Type) 623 } 624 625 return nil 626 } 627 628 func (n *netmon) handleEvents() (err error) { 629 for { 630 select { 631 case ev := <-n.linkUpdateCh: 632 if err = n.handleLinkEvent(ev); err != nil { 633 return err 634 } 635 case ev := <-n.rtUpdateCh: 636 if err = n.handleRouteEvent(ev); err != nil { 637 return err 638 } 639 } 640 } 641 } 642 643 func main() { 644 // Parse parameters. 645 params := parseOptions() 646 647 // Create netmon handler. 648 n, err := newNetmon(params) 649 if err != nil { 650 netmonLog.WithError(err).Fatal("newNetmon()") 651 os.Exit(1) 652 } 653 defer n.cleanup() 654 655 // Init logger. 656 if err := n.setupLogger(); err != nil { 657 netmonLog.WithError(err).Fatal("setupLogger()") 658 os.Exit(1) 659 } 660 661 // Setup signal handlers 662 n.setupSignalHandler() 663 664 // Scan the current interfaces. 665 if err := n.scanNetwork(); err != nil { 666 n.logger().WithError(err).Fatal("scanNetwork()") 667 os.Exit(1) 668 } 669 670 // Subscribe to the link listener. 671 if err := n.listenNetlinkEvents(); err != nil { 672 n.logger().WithError(err).Fatal("listenNetlinkEvents()") 673 os.Exit(1) 674 } 675 676 // Go into the main loop. 677 if err := n.handleEvents(); err != nil { 678 n.logger().WithError(err).Fatal("handleEvents()") 679 os.Exit(1) 680 } 681 }