github.com/jiasir/docker@v1.3.3-0.20170609024000-252e610103e7/daemon/daemon_windows.go (about) 1 package daemon 2 3 import ( 4 "fmt" 5 "os" 6 "path/filepath" 7 "strings" 8 "syscall" 9 10 "github.com/Microsoft/hcsshim" 11 "github.com/Sirupsen/logrus" 12 "github.com/docker/docker/api/types" 13 containertypes "github.com/docker/docker/api/types/container" 14 "github.com/docker/docker/container" 15 "github.com/docker/docker/daemon/config" 16 "github.com/docker/docker/image" 17 "github.com/docker/docker/pkg/idtools" 18 "github.com/docker/docker/pkg/parsers" 19 "github.com/docker/docker/pkg/platform" 20 "github.com/docker/docker/pkg/sysinfo" 21 "github.com/docker/docker/pkg/system" 22 "github.com/docker/docker/runconfig" 23 "github.com/docker/libnetwork" 24 nwconfig "github.com/docker/libnetwork/config" 25 "github.com/docker/libnetwork/datastore" 26 winlibnetwork "github.com/docker/libnetwork/drivers/windows" 27 "github.com/docker/libnetwork/netlabel" 28 "github.com/docker/libnetwork/options" 29 blkiodev "github.com/opencontainers/runc/libcontainer/configs" 30 "golang.org/x/sys/windows" 31 ) 32 33 const ( 34 defaultNetworkSpace = "172.16.0.0/12" 35 platformSupported = true 36 windowsMinCPUShares = 1 37 windowsMaxCPUShares = 10000 38 windowsMinCPUPercent = 1 39 windowsMaxCPUPercent = 100 40 windowsMinCPUCount = 1 41 42 errInvalidState = syscall.Errno(0x139F) 43 ) 44 45 // Windows has no concept of an execution state directory. So use config.Root here. 46 func getPluginExecRoot(root string) string { 47 return filepath.Join(root, "plugins") 48 } 49 50 func getBlkioWeightDevices(config *containertypes.HostConfig) ([]blkiodev.WeightDevice, error) { 51 return nil, nil 52 } 53 54 func (daemon *Daemon) parseSecurityOpt(container *container.Container, hostConfig *containertypes.HostConfig) error { 55 return parseSecurityOpt(container, hostConfig) 56 } 57 58 func parseSecurityOpt(container *container.Container, config *containertypes.HostConfig) error { 59 return nil 60 } 61 62 func getBlkioReadIOpsDevices(config *containertypes.HostConfig) ([]blkiodev.ThrottleDevice, error) { 63 return nil, nil 64 } 65 66 func getBlkioWriteIOpsDevices(config *containertypes.HostConfig) ([]blkiodev.ThrottleDevice, error) { 67 return nil, nil 68 } 69 70 func getBlkioReadBpsDevices(config *containertypes.HostConfig) ([]blkiodev.ThrottleDevice, error) { 71 return nil, nil 72 } 73 74 func getBlkioWriteBpsDevices(config *containertypes.HostConfig) ([]blkiodev.ThrottleDevice, error) { 75 return nil, nil 76 } 77 78 func (daemon *Daemon) getLayerInit() func(string) error { 79 return nil 80 } 81 82 func checkKernel() error { 83 return nil 84 } 85 86 func (daemon *Daemon) getCgroupDriver() string { 87 return "" 88 } 89 90 // adaptContainerSettings is called during container creation to modify any 91 // settings necessary in the HostConfig structure. 92 func (daemon *Daemon) adaptContainerSettings(hostConfig *containertypes.HostConfig, adjustCPUShares bool) error { 93 if hostConfig == nil { 94 return nil 95 } 96 97 return nil 98 } 99 100 func verifyContainerResources(resources *containertypes.Resources, isHyperv bool) ([]string, error) { 101 warnings := []string{} 102 103 if !isHyperv { 104 // The processor resource controls are mutually exclusive on 105 // Windows Server Containers, the order of precedence is 106 // CPUCount first, then CPUShares, and CPUPercent last. 107 if resources.CPUCount > 0 { 108 if resources.CPUShares > 0 { 109 warnings = append(warnings, "Conflicting options: CPU count takes priority over CPU shares on Windows Server Containers. CPU shares discarded") 110 logrus.Warn("Conflicting options: CPU count takes priority over CPU shares on Windows Server Containers. CPU shares discarded") 111 resources.CPUShares = 0 112 } 113 if resources.CPUPercent > 0 { 114 warnings = append(warnings, "Conflicting options: CPU count takes priority over CPU percent on Windows Server Containers. CPU percent discarded") 115 logrus.Warn("Conflicting options: CPU count takes priority over CPU percent on Windows Server Containers. CPU percent discarded") 116 resources.CPUPercent = 0 117 } 118 } else if resources.CPUShares > 0 { 119 if resources.CPUPercent > 0 { 120 warnings = append(warnings, "Conflicting options: CPU shares takes priority over CPU percent on Windows Server Containers. CPU percent discarded") 121 logrus.Warn("Conflicting options: CPU shares takes priority over CPU percent on Windows Server Containers. CPU percent discarded") 122 resources.CPUPercent = 0 123 } 124 } 125 } 126 127 if resources.CPUShares < 0 || resources.CPUShares > windowsMaxCPUShares { 128 return warnings, fmt.Errorf("range of CPUShares is from %d to %d", windowsMinCPUShares, windowsMaxCPUShares) 129 } 130 if resources.CPUPercent < 0 || resources.CPUPercent > windowsMaxCPUPercent { 131 return warnings, fmt.Errorf("range of CPUPercent is from %d to %d", windowsMinCPUPercent, windowsMaxCPUPercent) 132 } 133 if resources.CPUCount < 0 { 134 return warnings, fmt.Errorf("invalid CPUCount: CPUCount cannot be negative") 135 } 136 137 if resources.NanoCPUs > 0 && resources.CPUPercent > 0 { 138 return warnings, fmt.Errorf("conflicting options: Nano CPUs and CPU Percent cannot both be set") 139 } 140 if resources.NanoCPUs > 0 && resources.CPUShares > 0 { 141 return warnings, fmt.Errorf("conflicting options: Nano CPUs and CPU Shares cannot both be set") 142 } 143 // The precision we could get is 0.01, because on Windows we have to convert to CPUPercent. 144 // We don't set the lower limit here and it is up to the underlying platform (e.g., Windows) to return an error. 145 if resources.NanoCPUs < 0 || resources.NanoCPUs > int64(sysinfo.NumCPU())*1e9 { 146 return warnings, fmt.Errorf("range of CPUs is from 0.01 to %d.00, as there are only %d CPUs available", sysinfo.NumCPU(), sysinfo.NumCPU()) 147 } 148 149 osv := system.GetOSVersion() 150 if resources.NanoCPUs > 0 && isHyperv && osv.Build < 16175 { 151 leftoverNanoCPUs := resources.NanoCPUs % 1e9 152 if leftoverNanoCPUs != 0 && resources.NanoCPUs > 1e9 { 153 resources.NanoCPUs = ((resources.NanoCPUs + 1e9/2) / 1e9) * 1e9 154 warningString := fmt.Sprintf("Your current OS version does not support Hyper-V containers with NanoCPUs greater than 1000000000 but not divisible by 1000000000. NanoCPUs rounded to %d", resources.NanoCPUs) 155 warnings = append(warnings, warningString) 156 logrus.Warn(warningString) 157 } 158 } 159 160 if len(resources.BlkioDeviceReadBps) > 0 { 161 return warnings, fmt.Errorf("invalid option: Windows does not support BlkioDeviceReadBps") 162 } 163 if len(resources.BlkioDeviceReadIOps) > 0 { 164 return warnings, fmt.Errorf("invalid option: Windows does not support BlkioDeviceReadIOps") 165 } 166 if len(resources.BlkioDeviceWriteBps) > 0 { 167 return warnings, fmt.Errorf("invalid option: Windows does not support BlkioDeviceWriteBps") 168 } 169 if len(resources.BlkioDeviceWriteIOps) > 0 { 170 return warnings, fmt.Errorf("invalid option: Windows does not support BlkioDeviceWriteIOps") 171 } 172 if resources.BlkioWeight > 0 { 173 return warnings, fmt.Errorf("invalid option: Windows does not support BlkioWeight") 174 } 175 if len(resources.BlkioWeightDevice) > 0 { 176 return warnings, fmt.Errorf("invalid option: Windows does not support BlkioWeightDevice") 177 } 178 if resources.CgroupParent != "" { 179 return warnings, fmt.Errorf("invalid option: Windows does not support CgroupParent") 180 } 181 if resources.CPUPeriod != 0 { 182 return warnings, fmt.Errorf("invalid option: Windows does not support CPUPeriod") 183 } 184 if resources.CpusetCpus != "" { 185 return warnings, fmt.Errorf("invalid option: Windows does not support CpusetCpus") 186 } 187 if resources.CpusetMems != "" { 188 return warnings, fmt.Errorf("invalid option: Windows does not support CpusetMems") 189 } 190 if resources.KernelMemory != 0 { 191 return warnings, fmt.Errorf("invalid option: Windows does not support KernelMemory") 192 } 193 if resources.MemoryReservation != 0 { 194 return warnings, fmt.Errorf("invalid option: Windows does not support MemoryReservation") 195 } 196 if resources.MemorySwap != 0 { 197 return warnings, fmt.Errorf("invalid option: Windows does not support MemorySwap") 198 } 199 if resources.MemorySwappiness != nil && *resources.MemorySwappiness != -1 { 200 return warnings, fmt.Errorf("invalid option: Windows does not support MemorySwappiness") 201 } 202 if resources.OomKillDisable != nil && *resources.OomKillDisable { 203 return warnings, fmt.Errorf("invalid option: Windows does not support OomKillDisable") 204 } 205 if resources.PidsLimit != 0 { 206 return warnings, fmt.Errorf("invalid option: Windows does not support PidsLimit") 207 } 208 if len(resources.Ulimits) != 0 { 209 return warnings, fmt.Errorf("invalid option: Windows does not support Ulimits") 210 } 211 return warnings, nil 212 } 213 214 // verifyPlatformContainerSettings performs platform-specific validation of the 215 // hostconfig and config structures. 216 func verifyPlatformContainerSettings(daemon *Daemon, hostConfig *containertypes.HostConfig, config *containertypes.Config, update bool) ([]string, error) { 217 warnings := []string{} 218 219 hyperv := daemon.runAsHyperVContainer(hostConfig) 220 if !hyperv && system.IsWindowsClient() { 221 // @engine maintainers. This block should not be removed. It partially enforces licensing 222 // restrictions on Windows. Ping @jhowardmsft if there are concerns or PRs to change this. 223 return warnings, fmt.Errorf("Windows client operating systems only support Hyper-V containers") 224 } 225 226 w, err := verifyContainerResources(&hostConfig.Resources, hyperv) 227 warnings = append(warnings, w...) 228 return warnings, err 229 } 230 231 // reloadPlatform updates configuration with platform specific options 232 // and updates the passed attributes 233 func (daemon *Daemon) reloadPlatform(config *config.Config, attributes map[string]string) { 234 } 235 236 // verifyDaemonSettings performs validation of daemon config struct 237 func verifyDaemonSettings(config *config.Config) error { 238 return nil 239 } 240 241 // checkSystem validates platform-specific requirements 242 func checkSystem() error { 243 // Validate the OS version. Note that docker.exe must be manifested for this 244 // call to return the correct version. 245 osv := system.GetOSVersion() 246 if osv.MajorVersion < 10 { 247 return fmt.Errorf("This version of Windows does not support the docker daemon") 248 } 249 if osv.Build < 14393 { 250 return fmt.Errorf("The docker daemon requires build 14393 or later of Windows Server 2016 or Windows 10") 251 } 252 253 vmcompute := windows.NewLazySystemDLL("vmcompute.dll") 254 if vmcompute.Load() != nil { 255 return fmt.Errorf("Failed to load vmcompute.dll. Ensure that the Containers role is installed.") 256 } 257 258 return nil 259 } 260 261 // configureKernelSecuritySupport configures and validate security support for the kernel 262 func configureKernelSecuritySupport(config *config.Config, driverName string) error { 263 return nil 264 } 265 266 // configureMaxThreads sets the Go runtime max threads threshold 267 func configureMaxThreads(config *config.Config) error { 268 return nil 269 } 270 271 func (daemon *Daemon) initNetworkController(config *config.Config, activeSandboxes map[string]interface{}) (libnetwork.NetworkController, error) { 272 netOptions, err := daemon.networkOptions(config, nil, nil) 273 if err != nil { 274 return nil, err 275 } 276 controller, err := libnetwork.New(netOptions...) 277 if err != nil { 278 return nil, fmt.Errorf("error obtaining controller instance: %v", err) 279 } 280 281 hnsresponse, err := hcsshim.HNSListNetworkRequest("GET", "", "") 282 if err != nil { 283 return nil, err 284 } 285 286 // Remove networks not present in HNS 287 for _, v := range controller.Networks() { 288 options := v.Info().DriverOptions() 289 hnsid := options[winlibnetwork.HNSID] 290 found := false 291 292 for _, v := range hnsresponse { 293 if v.Id == hnsid { 294 found = true 295 break 296 } 297 } 298 299 if !found { 300 // global networks should not be deleted by local HNS 301 if v.Info().Scope() != datastore.GlobalScope { 302 err = v.Delete() 303 if err != nil { 304 logrus.Errorf("Error occurred when removing network %v", err) 305 } 306 } 307 } 308 } 309 310 _, err = controller.NewNetwork("null", "none", "", libnetwork.NetworkOptionPersist(false)) 311 if err != nil { 312 return nil, err 313 } 314 315 defaultNetworkExists := false 316 317 if network, err := controller.NetworkByName(runconfig.DefaultDaemonNetworkMode().NetworkName()); err == nil { 318 options := network.Info().DriverOptions() 319 for _, v := range hnsresponse { 320 if options[winlibnetwork.HNSID] == v.Id { 321 defaultNetworkExists = true 322 break 323 } 324 } 325 } 326 327 // discover and add HNS networks to windows 328 // network that exist are removed and added again 329 for _, v := range hnsresponse { 330 var n libnetwork.Network 331 s := func(current libnetwork.Network) bool { 332 options := current.Info().DriverOptions() 333 if options[winlibnetwork.HNSID] == v.Id { 334 n = current 335 return true 336 } 337 return false 338 } 339 340 controller.WalkNetworks(s) 341 if n != nil { 342 // global networks should not be deleted by local HNS 343 if n.Info().Scope() == datastore.GlobalScope { 344 continue 345 } 346 v.Name = n.Name() 347 // This will not cause network delete from HNS as the network 348 // is not yet populated in the libnetwork windows driver 349 n.Delete() 350 } 351 352 netOption := map[string]string{ 353 winlibnetwork.NetworkName: v.Name, 354 winlibnetwork.HNSID: v.Id, 355 } 356 357 v4Conf := []*libnetwork.IpamConf{} 358 for _, subnet := range v.Subnets { 359 ipamV4Conf := libnetwork.IpamConf{} 360 ipamV4Conf.PreferredPool = subnet.AddressPrefix 361 ipamV4Conf.Gateway = subnet.GatewayAddress 362 v4Conf = append(v4Conf, &ipamV4Conf) 363 } 364 365 name := v.Name 366 367 // If there is no nat network create one from the first NAT network 368 // encountered if it doesn't already exist 369 if !defaultNetworkExists && 370 runconfig.DefaultDaemonNetworkMode() == containertypes.NetworkMode(strings.ToLower(v.Type)) && 371 n == nil { 372 name = runconfig.DefaultDaemonNetworkMode().NetworkName() 373 defaultNetworkExists = true 374 } 375 376 v6Conf := []*libnetwork.IpamConf{} 377 _, err := controller.NewNetwork(strings.ToLower(v.Type), name, "", 378 libnetwork.NetworkOptionGeneric(options.Generic{ 379 netlabel.GenericData: netOption, 380 }), 381 libnetwork.NetworkOptionIpam("default", "", v4Conf, v6Conf, nil), 382 ) 383 384 if err != nil { 385 logrus.Errorf("Error occurred when creating network %v", err) 386 } 387 } 388 389 if !config.DisableBridge { 390 // Initialize default driver "bridge" 391 if err := initBridgeDriver(controller, config); err != nil { 392 return nil, err 393 } 394 } 395 396 return controller, nil 397 } 398 399 func initBridgeDriver(controller libnetwork.NetworkController, config *config.Config) error { 400 if _, err := controller.NetworkByName(runconfig.DefaultDaemonNetworkMode().NetworkName()); err == nil { 401 return nil 402 } 403 404 netOption := map[string]string{ 405 winlibnetwork.NetworkName: runconfig.DefaultDaemonNetworkMode().NetworkName(), 406 } 407 408 var ipamOption libnetwork.NetworkOption 409 var subnetPrefix string 410 411 if config.BridgeConfig.FixedCIDR != "" { 412 subnetPrefix = config.BridgeConfig.FixedCIDR 413 } else { 414 // TP5 doesn't support properly detecting subnet 415 osv := system.GetOSVersion() 416 if osv.Build < 14360 { 417 subnetPrefix = defaultNetworkSpace 418 } 419 } 420 421 if subnetPrefix != "" { 422 ipamV4Conf := libnetwork.IpamConf{} 423 ipamV4Conf.PreferredPool = subnetPrefix 424 v4Conf := []*libnetwork.IpamConf{&ipamV4Conf} 425 v6Conf := []*libnetwork.IpamConf{} 426 ipamOption = libnetwork.NetworkOptionIpam("default", "", v4Conf, v6Conf, nil) 427 } 428 429 _, err := controller.NewNetwork(string(runconfig.DefaultDaemonNetworkMode()), runconfig.DefaultDaemonNetworkMode().NetworkName(), "", 430 libnetwork.NetworkOptionGeneric(options.Generic{ 431 netlabel.GenericData: netOption, 432 }), 433 ipamOption, 434 ) 435 436 if err != nil { 437 return fmt.Errorf("Error creating default network: %v", err) 438 } 439 440 return nil 441 } 442 443 // registerLinks sets up links between containers and writes the 444 // configuration out for persistence. As of Windows TP4, links are not supported. 445 func (daemon *Daemon) registerLinks(container *container.Container, hostConfig *containertypes.HostConfig) error { 446 return nil 447 } 448 449 func (daemon *Daemon) cleanupMountsByID(in string) error { 450 return nil 451 } 452 453 func (daemon *Daemon) cleanupMounts() error { 454 return nil 455 } 456 457 func setupRemappedRoot(config *config.Config) (*idtools.IDMappings, error) { 458 return &idtools.IDMappings{}, nil 459 } 460 461 func setupDaemonRoot(config *config.Config, rootDir string, rootIDs idtools.IDPair) error { 462 config.Root = rootDir 463 // Create the root directory if it doesn't exists 464 if err := system.MkdirAllWithACL(config.Root, 0); err != nil && !os.IsExist(err) { 465 return err 466 } 467 return nil 468 } 469 470 // runasHyperVContainer returns true if we are going to run as a Hyper-V container 471 func (daemon *Daemon) runAsHyperVContainer(hostConfig *containertypes.HostConfig) bool { 472 if hostConfig.Isolation.IsDefault() { 473 // Container is set to use the default, so take the default from the daemon configuration 474 return daemon.defaultIsolation.IsHyperV() 475 } 476 477 // Container is requesting an isolation mode. Honour it. 478 return hostConfig.Isolation.IsHyperV() 479 480 } 481 482 // conditionalMountOnStart is a platform specific helper function during the 483 // container start to call mount. 484 func (daemon *Daemon) conditionalMountOnStart(container *container.Container) error { 485 // We do not mount if a Hyper-V container 486 if !daemon.runAsHyperVContainer(container.HostConfig) { 487 return daemon.Mount(container) 488 } 489 return nil 490 } 491 492 // conditionalUnmountOnCleanup is a platform specific helper function called 493 // during the cleanup of a container to unmount. 494 func (daemon *Daemon) conditionalUnmountOnCleanup(container *container.Container) error { 495 // We do not unmount if a Hyper-V container 496 if !daemon.runAsHyperVContainer(container.HostConfig) { 497 return daemon.Unmount(container) 498 } 499 return nil 500 } 501 502 func driverOptions(config *config.Config) []nwconfig.Option { 503 return []nwconfig.Option{} 504 } 505 506 func (daemon *Daemon) stats(c *container.Container) (*types.StatsJSON, error) { 507 if !c.IsRunning() { 508 return nil, errNotRunning{c.ID} 509 } 510 511 // Obtain the stats from HCS via libcontainerd 512 stats, err := daemon.containerd.Stats(c.ID) 513 if err != nil { 514 return nil, err 515 } 516 517 // Start with an empty structure 518 s := &types.StatsJSON{} 519 520 // Populate the CPU/processor statistics 521 s.CPUStats = types.CPUStats{ 522 CPUUsage: types.CPUUsage{ 523 TotalUsage: stats.Processor.TotalRuntime100ns, 524 UsageInKernelmode: stats.Processor.RuntimeKernel100ns, 525 UsageInUsermode: stats.Processor.RuntimeKernel100ns, 526 }, 527 } 528 529 // Populate the memory statistics 530 s.MemoryStats = types.MemoryStats{ 531 Commit: stats.Memory.UsageCommitBytes, 532 CommitPeak: stats.Memory.UsageCommitPeakBytes, 533 PrivateWorkingSet: stats.Memory.UsagePrivateWorkingSetBytes, 534 } 535 536 // Populate the storage statistics 537 s.StorageStats = types.StorageStats{ 538 ReadCountNormalized: stats.Storage.ReadCountNormalized, 539 ReadSizeBytes: stats.Storage.ReadSizeBytes, 540 WriteCountNormalized: stats.Storage.WriteCountNormalized, 541 WriteSizeBytes: stats.Storage.WriteSizeBytes, 542 } 543 544 // Populate the network statistics 545 s.Networks = make(map[string]types.NetworkStats) 546 547 for _, nstats := range stats.Network { 548 s.Networks[nstats.EndpointId] = types.NetworkStats{ 549 RxBytes: nstats.BytesReceived, 550 RxPackets: nstats.PacketsReceived, 551 RxDropped: nstats.DroppedPacketsIncoming, 552 TxBytes: nstats.BytesSent, 553 TxPackets: nstats.PacketsSent, 554 TxDropped: nstats.DroppedPacketsOutgoing, 555 } 556 } 557 558 // Set the timestamp 559 s.Stats.Read = stats.Timestamp 560 s.Stats.NumProcs = platform.NumProcs() 561 562 return s, nil 563 } 564 565 // setDefaultIsolation determine the default isolation mode for the 566 // daemon to run in. This is only applicable on Windows 567 func (daemon *Daemon) setDefaultIsolation() error { 568 daemon.defaultIsolation = containertypes.Isolation("process") 569 // On client SKUs, default to Hyper-V 570 if system.IsWindowsClient() { 571 daemon.defaultIsolation = containertypes.Isolation("hyperv") 572 } 573 for _, option := range daemon.configStore.ExecOptions { 574 key, val, err := parsers.ParseKeyValueOpt(option) 575 if err != nil { 576 return err 577 } 578 key = strings.ToLower(key) 579 switch key { 580 581 case "isolation": 582 if !containertypes.Isolation(val).IsValid() { 583 return fmt.Errorf("Invalid exec-opt value for 'isolation':'%s'", val) 584 } 585 if containertypes.Isolation(val).IsHyperV() { 586 daemon.defaultIsolation = containertypes.Isolation("hyperv") 587 } 588 if containertypes.Isolation(val).IsProcess() { 589 if system.IsWindowsClient() { 590 // @engine maintainers. This block should not be removed. It partially enforces licensing 591 // restrictions on Windows. Ping @jhowardmsft if there are concerns or PRs to change this. 592 return fmt.Errorf("Windows client operating systems only support Hyper-V containers") 593 } 594 daemon.defaultIsolation = containertypes.Isolation("process") 595 } 596 default: 597 return fmt.Errorf("Unrecognised exec-opt '%s'\n", key) 598 } 599 } 600 601 logrus.Infof("Windows default isolation mode: %s", daemon.defaultIsolation) 602 return nil 603 } 604 605 func rootFSToAPIType(rootfs *image.RootFS) types.RootFS { 606 var layers []string 607 for _, l := range rootfs.DiffIDs { 608 layers = append(layers, l.String()) 609 } 610 return types.RootFS{ 611 Type: rootfs.Type, 612 Layers: layers, 613 } 614 } 615 616 func setupDaemonProcess(config *config.Config) error { 617 return nil 618 } 619 620 // verifyVolumesInfo is a no-op on windows. 621 // This is called during daemon initialization to migrate volumes from pre-1.7. 622 // volumes were not supported on windows pre-1.7 623 func (daemon *Daemon) verifyVolumesInfo(container *container.Container) error { 624 return nil 625 } 626 627 func (daemon *Daemon) setupSeccompProfile() error { 628 return nil 629 }