github.com/robryk/drone@v0.2.1-0.20140602202253-40fe4305815d/pkg/build/build.go (about) 1 package build 2 3 import ( 4 "fmt" 5 "io" 6 "io/ioutil" 7 "os" 8 "os/exec" 9 "path/filepath" 10 "strings" 11 "time" 12 13 "github.com/drone/drone/pkg/build/buildfile" 14 "github.com/drone/drone/pkg/build/docker" 15 "github.com/drone/drone/pkg/build/dockerfile" 16 "github.com/drone/drone/pkg/build/log" 17 "github.com/drone/drone/pkg/build/proxy" 18 "github.com/drone/drone/pkg/build/repo" 19 "github.com/drone/drone/pkg/build/script" 20 ) 21 22 // BuildState stores information about a build 23 // process including the Exit status and various 24 // Runtime statistics (coming soon). 25 type BuildState struct { 26 Started int64 27 Finished int64 28 ExitCode int 29 30 // we may eventually include detailed resource 31 // usage statistics, including including CPU time, 32 // Max RAM, Max Swap, Disk space, and more. 33 } 34 35 func New(dockerClient *docker.Client) *Builder { 36 return &Builder{ 37 dockerClient: dockerClient, 38 } 39 } 40 41 // Builder represents a build process being prepared 42 // to run. 43 type Builder struct { 44 // Image specifies the Docker Image that will be 45 // used to virtualize the Build process. 46 Build *script.Build 47 48 // Source specifies the Repository path of the code 49 // that we are testing. 50 // 51 // The source repository may be a local repository 52 // on the current filesystem, or a remote repository 53 // on GitHub, Bitbucket, etc. 54 Repo *repo.Repo 55 56 // Key is an identify file, such as an RSA private key, that 57 // will be copied into the environments ~/.ssh/id_rsa file. 58 Key []byte 59 60 // Timeout is the maximum amount of to will wait for a process 61 // to exit. The default is no timeout. 62 Timeout time.Duration 63 64 // Privileged indicates the build should be executed in privileged 65 // mode. The default is false. 66 Privileged bool 67 68 // Stdout specifies the builds's standard output. 69 // 70 // If stdout is nil, Run connects the corresponding file descriptor 71 // to the null device (os.DevNull). 72 Stdout io.Writer 73 74 // BuildState contains information about an exited build, 75 // available after a call to Run. 76 BuildState *BuildState 77 78 // Docker image that was created for 79 // this build. 80 image *docker.Image 81 82 // Docker container was that created 83 // for this build. 84 container *docker.Run 85 86 // Docker containers created for the 87 // specified services and linked to 88 // this build. 89 services []*docker.Container 90 91 dockerClient *docker.Client 92 } 93 94 func (b *Builder) Run() error { 95 // teardown will remove the Image and stop and 96 // remove the service containers after the 97 // build is done running. 98 defer b.teardown() 99 100 // setup will create the Image and supporting 101 // service containers. 102 if err := b.setup(); err != nil { 103 return err 104 } 105 106 // make sure build state is not nil 107 b.BuildState = &BuildState{} 108 b.BuildState.ExitCode = 0 109 b.BuildState.Started = time.Now().UTC().Unix() 110 111 c := make(chan error, 1) 112 go func() { 113 c <- b.run() 114 }() 115 116 // wait for either a) the job to complete or b) the job to timeout 117 select { 118 case err := <-c: 119 return err 120 case <-time.After(b.Timeout): 121 log.Errf("time limit exceeded for build %s", b.Build.Name) 122 b.BuildState.ExitCode = 124 123 b.BuildState.Finished = time.Now().UTC().Unix() 124 return nil 125 } 126 } 127 128 func (b *Builder) setup() error { 129 130 // temp directory to store all files required 131 // to generate the Docker image. 132 dir, err := ioutil.TempDir("", "drone-") 133 if err != nil { 134 return err 135 } 136 137 // clean up after our mess. 138 defer os.RemoveAll(dir) 139 140 // make sure the image isn't empty. this would be bad 141 if len(b.Build.Image) == 0 { 142 log.Err("Fatal Error, No Docker Image specified") 143 return fmt.Errorf("Error: missing Docker image") 144 } 145 146 // if we're using an alias for the build name we 147 // should substitute it now 148 if alias, ok := builders[b.Build.Image]; ok { 149 b.Build.Image = alias.Tag 150 } 151 152 // if this is a local repository we should symlink 153 // to the source code in our temp directory 154 if b.Repo.IsLocal() { 155 // this is where we used to use symlinks. We should 156 // talk to the docker team about this, since copying 157 // the entire repository is slow :( 158 // 159 // see https://github.com/dotcloud/docker/pull/3567 160 161 //src := filepath.Join(dir, "src") 162 //err = os.Symlink(b.Repo.Path, src) 163 //if err != nil { 164 // return err 165 //} 166 167 src := filepath.Join(dir, "src") 168 cmd := exec.Command("cp", "-a", b.Repo.Path, src) 169 if err := cmd.Run(); err != nil { 170 return err 171 } 172 } 173 174 // start all services required for the build 175 // that will get linked to the container. 176 for _, service := range b.Build.Services { 177 178 // Parse the name of the Docker image 179 // And then construct a fully qualified image name 180 owner, name, tag := parseImageName(service) 181 cname := fmt.Sprintf("%s/%s:%s", owner, name, tag) 182 183 // Get the image info 184 img, err := b.dockerClient.Images.Inspect(cname) 185 if err != nil { 186 // Get the image if it doesn't exist 187 if err := b.dockerClient.Images.Pull(cname); err != nil { 188 return fmt.Errorf("Error: Unable to pull image %s", cname) 189 } 190 191 img, err = b.dockerClient.Images.Inspect(cname) 192 if err != nil { 193 return fmt.Errorf("Error: Invalid or unknown image %s", cname) 194 } 195 } 196 197 // debugging 198 log.Infof("starting service container %s", cname) 199 200 // Run the contianer 201 run, err := b.dockerClient.Containers.RunDaemonPorts(cname, img.Config.ExposedPorts) 202 if err != nil { 203 return err 204 } 205 206 // Get the container info 207 info, err := b.dockerClient.Containers.Inspect(run.ID) 208 if err != nil { 209 // on error kill the container since it hasn't yet been 210 // added to the array and would therefore not get 211 // removed in the defer statement. 212 b.dockerClient.Containers.Stop(run.ID, 10) 213 b.dockerClient.Containers.Remove(run.ID) 214 return err 215 } 216 217 // Add the running service to the list 218 b.services = append(b.services, info) 219 } 220 221 if err := b.writeIdentifyFile(dir); err != nil { 222 return err 223 } 224 225 if err := b.writeBuildScript(dir); err != nil { 226 return err 227 } 228 229 if err := b.writeProxyScript(dir); err != nil { 230 return err 231 } 232 233 if err := b.writeDockerfile(dir); err != nil { 234 return err 235 } 236 237 // debugging 238 log.Info("creating build image") 239 240 // check for build container (ie bradrydzewski/go:1.2) 241 // and download if it doesn't already exist 242 if _, err := b.dockerClient.Images.Inspect(b.Build.Image); err == docker.ErrNotFound { 243 // download the image if it doesn't exist 244 if err := b.dockerClient.Images.Pull(b.Build.Image); err != nil { 245 return err 246 } 247 } 248 249 // create the Docker image 250 id := createUID() 251 if err := b.dockerClient.Images.Build(id, dir); err != nil { 252 return err 253 } 254 255 // debugging 256 log.Infof("copying repository to %s", b.Repo.Dir) 257 258 // get the image details 259 b.image, err = b.dockerClient.Images.Inspect(id) 260 if err != nil { 261 // if we have problems with the image make sure 262 // we remove it before we exit 263 b.dockerClient.Images.Remove(id) 264 return err 265 } 266 267 return nil 268 } 269 270 // teardown is a helper function that we can use to 271 // stop and remove the build container, its supporting image, 272 // and the supporting service containers. 273 func (b *Builder) teardown() error { 274 275 // stop and destroy the container 276 if b.container != nil { 277 278 // debugging 279 log.Info("removing build container") 280 281 // stop the container, ignore error message 282 b.dockerClient.Containers.Stop(b.container.ID, 15) 283 284 // remove the container, ignore error message 285 if err := b.dockerClient.Containers.Remove(b.container.ID); err != nil { 286 log.Errf("failed to delete build container %s", b.container.ID) 287 } 288 } 289 290 // stop and destroy the container services 291 for i, container := range b.services { 292 // debugging 293 log.Infof("removing service container %s", b.Build.Services[i]) 294 295 // stop the service container, ignore the error 296 b.dockerClient.Containers.Stop(container.ID, 15) 297 298 // remove the service container, ignore the error 299 if err := b.dockerClient.Containers.Remove(container.ID); err != nil { 300 log.Errf("failed to delete service container %s", container.ID) 301 } 302 } 303 304 // destroy the underlying image 305 if b.image != nil { 306 // debugging 307 log.Info("removing build image") 308 309 if _, err := b.dockerClient.Images.Remove(b.image.ID); err != nil { 310 log.Errf("failed to completely delete build image %s. %s", b.image.ID, err.Error()) 311 } 312 } 313 314 return nil 315 } 316 317 func (b *Builder) run() error { 318 // create and run the container 319 conf := docker.Config{ 320 Image: b.image.ID, 321 AttachStdin: false, 322 AttachStdout: true, 323 AttachStderr: true, 324 } 325 326 // configure if Docker should run in privileged mode 327 host := docker.HostConfig{ 328 Privileged: (b.Privileged && len(b.Repo.PR) == 0), 329 } 330 331 // debugging 332 log.Noticef("starting build %s", b.Build.Name) 333 334 // link service containers 335 for i, service := range b.services { 336 // convert name of the image to a slug 337 _, name, _ := parseImageName(b.Build.Services[i]) 338 339 // link the service container to our 340 // build container. 341 host.Links = append(host.Links, service.Name[1:]+":"+name) 342 } 343 344 // where are temp files going to go? 345 tmpPath := "/tmp/drone" 346 if len(os.Getenv("DRONE_TMP")) > 0 { 347 tmpPath = os.Getenv("DRONE_TMP") 348 } 349 350 log.Infof("temp directory is %s", tmpPath) 351 352 if err := os.MkdirAll(tmpPath, 0777); err != nil { 353 return fmt.Errorf("Failed to create temp directory at %s: %s", tmpPath, err) 354 } 355 356 // link cached volumes 357 conf.Volumes = make(map[string]struct{}) 358 for _, volume := range b.Build.Cache { 359 name := filepath.Clean(b.Repo.Name) 360 branch := filepath.Clean(b.Repo.Branch) 361 volume := filepath.Clean(volume) 362 363 // with Docker, volumes must be an absolute path. If an absolute 364 // path is not provided, then assume it is for the repository 365 // working directory. 366 if strings.HasPrefix(volume, "/") == false { 367 volume = filepath.Join(b.Repo.Dir, volume) 368 } 369 370 // local cache path on the host machine 371 // this path is going to be really long 372 hostpath := filepath.Join(tmpPath, name, branch, volume) 373 374 // check if the volume is created 375 if _, err := os.Stat(hostpath); err != nil { 376 // if does not exist then create 377 os.MkdirAll(hostpath, 0777) 378 } 379 380 host.Binds = append(host.Binds, hostpath+":"+volume) 381 conf.Volumes[volume] = struct{}{} 382 383 // debugging 384 log.Infof("mounting volume %s:%s", hostpath, volume) 385 } 386 387 // create the container from the image 388 run, err := b.dockerClient.Containers.Create(&conf) 389 if err != nil { 390 return err 391 } 392 393 // cache instance of docker.Run 394 b.container = run 395 396 // attach to the container 397 go func() { 398 b.dockerClient.Containers.Attach(run.ID, &writer{b.Stdout}) 399 }() 400 401 // start the container 402 if err := b.dockerClient.Containers.Start(run.ID, &host); err != nil { 403 b.BuildState.ExitCode = 1 404 b.BuildState.Finished = time.Now().UTC().Unix() 405 return err 406 } 407 408 // wait for the container to stop 409 wait, err := b.dockerClient.Containers.Wait(run.ID) 410 if err != nil { 411 b.BuildState.ExitCode = 1 412 b.BuildState.Finished = time.Now().UTC().Unix() 413 return err 414 } 415 416 // set completion time 417 b.BuildState.Finished = time.Now().UTC().Unix() 418 419 // get the exit code if possible 420 b.BuildState.ExitCode = wait.StatusCode 421 422 return nil 423 } 424 425 // writeDockerfile is a helper function that generates a 426 // Dockerfile and writes to the builds temporary directory 427 // so that it can be used to create the Image. 428 func (b *Builder) writeDockerfile(dir string) error { 429 var dockerfile = dockerfile.New(b.Build.Image) 430 dockerfile.WriteWorkdir(b.Repo.Dir) 431 dockerfile.WriteAdd("drone", "/usr/local/bin/") 432 433 // upload source code if repository is stored 434 // on the host machine 435 if b.Repo.IsRemote() == false { 436 dockerfile.WriteAdd("src", filepath.Join(b.Repo.Dir)) 437 } 438 439 switch { 440 case strings.HasPrefix(b.Build.Image, "bradrydzewski/"), 441 strings.HasPrefix(b.Build.Image, "drone/"): 442 // the default user for all official Drone imnage 443 // is the "ubuntu" user, since all build images 444 // inherit from the ubuntu cloud ISO 445 dockerfile.WriteUser("ubuntu") 446 dockerfile.WriteEnv("HOME", "/home/ubuntu") 447 dockerfile.WriteEnv("LANG", "en_US.UTF-8") 448 dockerfile.WriteEnv("LANGUAGE", "en_US:en") 449 dockerfile.WriteEnv("LOGNAME", "ubuntu") 450 dockerfile.WriteEnv("TERM", "xterm") 451 dockerfile.WriteEnv("SHELL", "/bin/bash") 452 dockerfile.WriteAdd("id_rsa", "/home/ubuntu/.ssh/id_rsa") 453 dockerfile.WriteRun("sudo chown -R ubuntu:ubuntu /home/ubuntu/.ssh") 454 dockerfile.WriteRun("sudo chown -R ubuntu:ubuntu /var/cache/drone") 455 dockerfile.WriteRun("sudo chown -R ubuntu:ubuntu /usr/local/bin/drone") 456 dockerfile.WriteRun("sudo chmod 600 /home/ubuntu/.ssh/id_rsa") 457 default: 458 // all other images are assumed to use 459 // the root user. 460 dockerfile.WriteUser("root") 461 dockerfile.WriteEnv("HOME", "/root") 462 dockerfile.WriteEnv("LANG", "en_US.UTF-8") 463 dockerfile.WriteEnv("LANGUAGE", "en_US:en") 464 dockerfile.WriteEnv("LOGNAME", "root") 465 dockerfile.WriteEnv("TERM", "xterm") 466 dockerfile.WriteEnv("SHELL", "/bin/bash") 467 dockerfile.WriteEnv("GOPATH", "/var/cache/drone") 468 dockerfile.WriteAdd("id_rsa", "/root/.ssh/id_rsa") 469 dockerfile.WriteRun("chmod 600 /root/.ssh/id_rsa") 470 dockerfile.WriteRun("echo 'StrictHostKeyChecking no' > /root/.ssh/config") 471 } 472 473 dockerfile.WriteAdd("proxy.sh", "/etc/drone.d/") 474 dockerfile.WriteEntrypoint("/bin/bash -e /usr/local/bin/drone") 475 476 // write the Dockerfile to the temporary directory 477 return ioutil.WriteFile(filepath.Join(dir, "Dockerfile"), dockerfile.Bytes(), 0700) 478 } 479 480 // writeBuildScript is a helper function that 481 // will generate the build script file in the builder's 482 // temp directory to be added to the Image. 483 func (b *Builder) writeBuildScript(dir string) error { 484 f := buildfile.New() 485 486 // add environment variables about the build 487 f.WriteEnv("CI", "true") 488 f.WriteEnv("DRONE", "true") 489 f.WriteEnv("DRONE_BRANCH", b.Repo.Branch) 490 f.WriteEnv("DRONE_COMMIT", b.Repo.Commit) 491 f.WriteEnv("DRONE_PR", b.Repo.PR) 492 f.WriteEnv("DRONE_BUILD_DIR", b.Repo.Dir) 493 494 // add environment variables for code coverage 495 // systems, like coveralls. 496 f.WriteEnv("CI_NAME", "DRONE") 497 f.WriteEnv("CI_BUILD_NUMBER", b.Repo.Commit) 498 f.WriteEnv("CI_BUILD_URL", "") 499 f.WriteEnv("CI_BRANCH", b.Repo.Branch) 500 f.WriteEnv("CI_PULL_REQUEST", b.Repo.PR) 501 502 // add /etc/hosts entries 503 for _, mapping := range b.Build.Hosts { 504 f.WriteHost(mapping) 505 } 506 507 // if the repository is remote then we should 508 // add the commands to the build script to 509 // clone the repository 510 if b.Repo.IsRemote() { 511 for _, cmd := range b.Repo.Commands() { 512 f.WriteCmd(cmd) 513 } 514 } 515 516 // if the commit is for merging a pull request 517 // we should only execute the build commands, 518 // and omit the deploy and publish commands. 519 if len(b.Repo.PR) == 0 { 520 b.Build.Write(f, b.Repo) 521 } else { 522 // only write the build commands 523 b.Build.WriteBuild(f) 524 } 525 526 scriptfilePath := filepath.Join(dir, "drone") 527 return ioutil.WriteFile(scriptfilePath, f.Bytes(), 0700) 528 } 529 530 // writeProxyScript is a helper function that 531 // will generate the proxy.sh file in the builder's 532 // temp directory to be added to the Image. 533 func (b *Builder) writeProxyScript(dir string) error { 534 var proxyfile = proxy.Proxy{} 535 536 // loop through services so that we can 537 // map ip address to localhost 538 for _, container := range b.services { 539 // create an entry for each port 540 for port := range container.NetworkSettings.Ports { 541 proxyfile.Set(port.Port(), container.NetworkSettings.IPAddress) 542 } 543 } 544 545 // write the proxyfile to the temp directory 546 proxyfilePath := filepath.Join(dir, "proxy.sh") 547 return ioutil.WriteFile(proxyfilePath, proxyfile.Bytes(), 0755) 548 } 549 550 // writeIdentifyFile is a helper function that 551 // will generate the id_rsa file in the builder's 552 // temp directory to be added to the Image. 553 func (b *Builder) writeIdentifyFile(dir string) error { 554 keyfilePath := filepath.Join(dir, "id_rsa") 555 return ioutil.WriteFile(keyfilePath, b.Key, 0700) 556 }