github.com/kvattikuti/drone@v0.2.1-0.20140603034306-d400229a327a/pkg/build/build.go (about) 1 package build 2 3 import ( 4 "fmt" 5 "io" 6 "io/ioutil" 7 "os" 8 "os/exec" 9 "path/filepath" 10 "strings" 11 "time" 12 13 "github.com/drone/drone/pkg/build/buildfile" 14 "github.com/drone/drone/pkg/build/docker" 15 "github.com/drone/drone/pkg/build/dockerfile" 16 "github.com/drone/drone/pkg/build/log" 17 "github.com/drone/drone/pkg/build/proxy" 18 "github.com/drone/drone/pkg/build/repo" 19 "github.com/drone/drone/pkg/build/script" 20 ) 21 22 // BuildState stores information about a build 23 // process including the Exit status and various 24 // Runtime statistics (coming soon). 25 type BuildState struct { 26 Started int64 27 Finished int64 28 ExitCode int 29 30 // we may eventually include detailed resource 31 // usage statistics, including including CPU time, 32 // Max RAM, Max Swap, Disk space, and more. 33 } 34 35 func New(dockerClient *docker.Client) *Builder { 36 return &Builder{ 37 dockerClient: dockerClient, 38 } 39 } 40 41 // Builder represents a build process being prepared 42 // to run. 43 type Builder struct { 44 // Image specifies the Docker Image that will be 45 // used to virtualize the Build process. 46 Build *script.Build 47 48 // Source specifies the Repository path of the code 49 // that we are testing. 50 // 51 // The source repository may be a local repository 52 // on the current filesystem, or a remote repository 53 // on GitHub, Bitbucket, etc. 54 Repo *repo.Repo 55 56 // Key is an identify file, such as an RSA private key, that 57 // will be copied into the environments ~/.ssh/id_rsa file. 58 Key []byte 59 60 // Timeout is the maximum amount of to will wait for a process 61 // to exit. The default is no timeout. 62 Timeout time.Duration 63 64 // Privileged indicates the build should be executed in privileged 65 // mode. The default is false. 66 Privileged bool 67 68 // Stdout specifies the builds's standard output. 69 // 70 // If stdout is nil, Run connects the corresponding file descriptor 71 // to the null device (os.DevNull). 72 Stdout io.Writer 73 74 // BuildState contains information about an exited build, 75 // available after a call to Run. 76 BuildState *BuildState 77 78 // Docker image that was created for 79 // this build. 80 image *docker.Image 81 82 // Docker container was that created 83 // for this build. 84 container *docker.Run 85 86 // Docker containers created for the 87 // specified services and linked to 88 // this build. 89 services []*docker.Container 90 91 dockerClient *docker.Client 92 } 93 94 func (b *Builder) Run() error { 95 // teardown will remove the Image and stop and 96 // remove the service containers after the 97 // build is done running. 98 //defer b.teardown() 99 100 println("before build setup") 101 // setup will create the Image and supporting 102 // service containers. 103 if err := b.setup(); err != nil { 104 return err 105 } 106 107 println("before make") 108 // make sure build state is not nil 109 b.BuildState = &BuildState{} 110 b.BuildState.ExitCode = 0 111 b.BuildState.Started = time.Now().UTC().Unix() 112 113 c := make(chan error, 1) 114 println("before run") 115 println("build timeout is ", b.Timeout) 116 go func() { 117 c <- b.run() 118 }() 119 120 // wait for either a) the job to complete or b) the job to timeout 121 select { 122 case err := <-c: 123 return err 124 case <-time.After(b.Timeout): 125 log.Errf("time limit exceeded for build %s", b.Build.Name) 126 b.BuildState.ExitCode = 124 127 b.BuildState.Finished = time.Now().UTC().Unix() 128 return nil 129 } 130 } 131 132 func (b *Builder) setup() error { 133 134 // temp directory to store all files required 135 // to generate the Docker image. 136 dir, err := ioutil.TempDir("", "drone-") 137 if err != nil { 138 return err 139 } 140 println("temp dir is ", dir) 141 // clean up after our mess. 142 defer os.RemoveAll(dir) 143 println("after removeall files from temp dir") 144 println("is b.Build nil? ", b.Build) 145 146 // make sure the image isn't empty. this would be bad 147 if len(b.Build.Image) == 0 { 148 log.Err("Fatal Error, No Docker Image specified") 149 return fmt.Errorf("Error: missing Docker image") 150 } 151 println("after checking image is not empty") 152 153 // if we're using an alias for the build name we 154 // should substitute it now 155 if alias, ok := builders[b.Build.Image]; ok { 156 b.Build.Image = alias.Tag 157 } 158 159 // if this is a local repository we should symlink 160 // to the source code in our temp directory 161 if b.Repo.IsLocal() { 162 // this is where we used to use symlinks. We should 163 // talk to the docker team about this, since copying 164 // the entire repository is slow :( 165 // 166 // see https://github.com/dotcloud/docker/pull/3567 167 168 //src := filepath.Join(dir, "src") 169 //err = os.Symlink(b.Repo.Path, src) 170 //if err != nil { 171 // return err 172 //} 173 174 src := filepath.Join(dir, "src") 175 cmd := exec.Command("cp", "-a", b.Repo.Path, src) 176 if err := cmd.Run(); err != nil { 177 return err 178 } 179 } 180 println("after checking if repo is local") 181 println("list of services: ", b.Build.Services) 182 183 // start all services required for the build 184 // that will get linked to the container. 185 for _, service := range b.Build.Services { 186 187 // Parse the name of the Docker image 188 // And then construct a fully qualified image name 189 owner, name, tag := parseImageName(service) 190 cname := fmt.Sprintf("%s/%s:%s", owner, name, tag) 191 192 println("cname of service is ", cname) 193 // Get the image info 194 img, err := b.dockerClient.Images.Inspect(cname) 195 if err != nil { 196 // Get the image if it doesn't exist 197 if err := b.dockerClient.Images.Pull(cname); err != nil { 198 return fmt.Errorf("Error: Unable to pull image %s", cname) 199 } 200 201 img, err = b.dockerClient.Images.Inspect(cname) 202 if err != nil { 203 return fmt.Errorf("Error: Invalid or unknown image %s", cname) 204 } 205 } 206 207 // debugging 208 log.Infof("starting service container %s", cname) 209 210 // Run the contianer 211 run, err := b.dockerClient.Containers.RunDaemonPorts(cname, img.Config.ExposedPorts) 212 if err != nil { 213 return err 214 } 215 216 // Get the container info 217 info, err := b.dockerClient.Containers.Inspect(run.ID) 218 if err != nil { 219 // on error kill the container since it hasn't yet been 220 // added to the array and would therefore not get 221 // removed in the defer statement. 222 b.dockerClient.Containers.Stop(run.ID, 10) 223 b.dockerClient.Containers.Remove(run.ID) 224 return err 225 } 226 227 // Add the running service to the list 228 b.services = append(b.services, info) 229 } 230 231 if err := b.writeIdentifyFile(dir); err != nil { 232 return err 233 } 234 235 if err := b.writeBuildScript(dir); err != nil { 236 return err 237 } 238 239 if err := b.writeProxyScript(dir); err != nil { 240 return err 241 } 242 243 if err := b.writeDockerfile(dir); err != nil { 244 return err 245 } 246 247 // debugging 248 log.Info("creating build image") 249 250 // check for build container (ie bradrydzewski/go:1.2) 251 // and download if it doesn't already exist 252 if _, err := b.dockerClient.Images.Inspect(b.Build.Image); err == docker.ErrNotFound { 253 // download the image if it doesn't exist 254 if err := b.dockerClient.Images.Pull(b.Build.Image); err != nil { 255 return err 256 } 257 } 258 259 // create the Docker image 260 id := createUID() 261 if err := b.dockerClient.Images.Build(id, dir); err != nil { 262 return err 263 } 264 265 // debugging 266 log.Infof("copying repository to %s", b.Repo.Dir) 267 268 // get the image details 269 b.image, err = b.dockerClient.Images.Inspect(id) 270 if err != nil { 271 // if we have problems with the image make sure 272 // we remove it before we exit 273 b.dockerClient.Images.Remove(id) 274 return err 275 } 276 277 return nil 278 } 279 280 // teardown is a helper function that we can use to 281 // stop and remove the build container, its supporting image, 282 // and the supporting service containers. 283 func (b *Builder) teardown() error { 284 285 // stop and destroy the container 286 if b.container != nil { 287 288 // debugging 289 log.Info("removing build container") 290 291 // stop the container, ignore error message 292 b.dockerClient.Containers.Stop(b.container.ID, 15) 293 294 // remove the container, ignore error message 295 if err := b.dockerClient.Containers.Remove(b.container.ID); err != nil { 296 log.Errf("failed to delete build container %s", b.container.ID) 297 } 298 } 299 300 // stop and destroy the container services 301 for i, container := range b.services { 302 // debugging 303 log.Infof("removing service container %s", b.Build.Services[i]) 304 305 // stop the service container, ignore the error 306 b.dockerClient.Containers.Stop(container.ID, 15) 307 308 // remove the service container, ignore the error 309 if err := b.dockerClient.Containers.Remove(container.ID); err != nil { 310 log.Errf("failed to delete service container %s", container.ID) 311 } 312 } 313 314 // destroy the underlying image 315 if b.image != nil { 316 // debugging 317 log.Info("removing build image") 318 319 if _, err := b.dockerClient.Images.Remove(b.image.ID); err != nil { 320 log.Errf("failed to completely delete build image %s. %s", b.image.ID, err.Error()) 321 } 322 } 323 324 return nil 325 } 326 327 func (b *Builder) run() error { 328 // create and run the container 329 conf := docker.Config{ 330 Image: b.image.ID, 331 AttachStdin: false, 332 AttachStdout: true, 333 AttachStderr: true, 334 } 335 336 // configure if Docker should run in privileged mode 337 host := docker.HostConfig{ 338 Privileged: (b.Privileged && len(b.Repo.PR) == 0), 339 } 340 341 // debugging 342 log.Noticef("starting build %s", b.Build.Name) 343 344 // link service containers 345 for i, service := range b.services { 346 // convert name of the image to a slug 347 _, name, _ := parseImageName(b.Build.Services[i]) 348 349 // link the service container to our 350 // build container. 351 host.Links = append(host.Links, service.Name[1:]+":"+name) 352 } 353 354 // where are temp files going to go? 355 tmpPath := "/tmp/drone" 356 if len(os.Getenv("DRONE_TMP")) > 0 { 357 tmpPath = os.Getenv("DRONE_TMP") 358 } 359 360 log.Infof("temp directory is %s", tmpPath) 361 362 if err := os.MkdirAll(tmpPath, 0777); err != nil { 363 return fmt.Errorf("Failed to create temp directory at %s: %s", tmpPath, err) 364 } 365 println("done creating temporary directory") 366 367 // link cached volumes 368 conf.Volumes = make(map[string]struct{}) 369 for _, volume := range b.Build.Cache { 370 println("in the mounting volume loop") 371 name := filepath.Clean(b.Repo.Name) 372 branch := filepath.Clean(b.Repo.Branch) 373 volume := filepath.Clean(volume) 374 375 // with Docker, volumes must be an absolute path. If an absolute 376 // path is not provided, then assume it is for the repository 377 // working directory. 378 if strings.HasPrefix(volume, "/") == false { 379 volume = filepath.Join(b.Repo.Dir, volume) 380 } 381 println("volume is", volume) 382 // local cache path on the host machine 383 // this path is going to be really long 384 hostpath := filepath.Join(tmpPath, name, branch, volume) 385 println("hostpath is ", hostpath) 386 // check if the volume is created 387 if _, err := os.Stat(hostpath); err != nil { 388 // if does not exist then create 389 os.MkdirAll(hostpath, 0777) 390 } 391 println("hostpath creation done, if not exists") 392 host.Binds = append(host.Binds, hostpath+":"+volume) 393 conf.Volumes[volume] = struct{}{} 394 395 // debugging 396 log.Infof("mounting volume %s:%s", hostpath, volume) 397 } 398 399 println("mounting cached volumes done") 400 401 println("creating container") 402 // create the container from the image 403 run, err := b.dockerClient.Containers.Create(&conf) 404 if err != nil { 405 return err 406 } 407 println("creating container done") 408 409 // cache instance of docker.Run 410 b.container = run 411 412 // attach to the container 413 println("attach to container") 414 go func() { 415 b.dockerClient.Containers.Attach(run.ID, &writer{b.Stdout}) 416 }() 417 418 // start the container 419 println("start container") 420 if err := b.dockerClient.Containers.Start(run.ID, &host); err != nil { 421 b.BuildState.ExitCode = 1 422 b.BuildState.Finished = time.Now().UTC().Unix() 423 return err 424 } 425 426 // wait for the container to stop 427 println("wait for container to stop") 428 wait, err := b.dockerClient.Containers.Wait(run.ID) 429 if err != nil { 430 b.BuildState.ExitCode = 1 431 b.BuildState.Finished = time.Now().UTC().Unix() 432 println("build in error: ", err.Error()); 433 return err 434 } 435 436 // set completion time 437 b.BuildState.Finished = time.Now().UTC().Unix() 438 println("build finished cleanly"); 439 // get the exit code if possible 440 b.BuildState.ExitCode = wait.StatusCode 441 442 return nil 443 } 444 445 // writeDockerfile is a helper function that generates a 446 // Dockerfile and writes to the builds temporary directory 447 // so that it can be used to create the Image. 448 func (b *Builder) writeDockerfile(dir string) error { 449 var dockerfile = dockerfile.New(b.Build.Image) 450 dockerfile.WriteWorkdir(b.Repo.Dir) 451 dockerfile.WriteAdd("drone", "/usr/local/bin/") 452 453 // upload source code if repository is stored 454 // on the host machine 455 if b.Repo.IsRemote() == false { 456 dockerfile.WriteAdd("src", filepath.Join(b.Repo.Dir)) 457 } 458 459 switch { 460 case strings.HasPrefix(b.Build.Image, "bradrydzewski/"), 461 strings.HasPrefix(b.Build.Image, "drone/"): 462 // the default user for all official Drone imnage 463 // is the "ubuntu" user, since all build images 464 // inherit from the ubuntu cloud ISO 465 dockerfile.WriteUser("ubuntu") 466 dockerfile.WriteEnv("HOME", "/home/ubuntu") 467 dockerfile.WriteEnv("LANG", "en_US.UTF-8") 468 dockerfile.WriteEnv("LANGUAGE", "en_US:en") 469 dockerfile.WriteEnv("LOGNAME", "ubuntu") 470 dockerfile.WriteEnv("TERM", "xterm") 471 dockerfile.WriteEnv("SHELL", "/bin/bash") 472 dockerfile.WriteAdd("id_rsa", "/home/ubuntu/.ssh/id_rsa") 473 dockerfile.WriteRun("sudo chown -R ubuntu:ubuntu /home/ubuntu/.ssh") 474 dockerfile.WriteRun("sudo chown -R ubuntu:ubuntu /var/cache/drone") 475 dockerfile.WriteRun("sudo chown -R ubuntu:ubuntu /usr/local/bin/drone") 476 dockerfile.WriteRun("sudo chmod 600 /home/ubuntu/.ssh/id_rsa") 477 default: 478 // all other images are assumed to use 479 // the root user. 480 dockerfile.WriteUser("root") 481 dockerfile.WriteEnv("HOME", "/root") 482 dockerfile.WriteEnv("LANG", "en_US.UTF-8") 483 dockerfile.WriteEnv("LANGUAGE", "en_US:en") 484 dockerfile.WriteEnv("LOGNAME", "root") 485 dockerfile.WriteEnv("TERM", "xterm") 486 dockerfile.WriteEnv("SHELL", "/bin/bash") 487 dockerfile.WriteEnv("GOPATH", "/var/cache/drone") 488 dockerfile.WriteAdd("id_rsa", "/root/.ssh/id_rsa") 489 dockerfile.WriteRun("chmod 600 /root/.ssh/id_rsa") 490 dockerfile.WriteRun("echo 'StrictHostKeyChecking no' > /root/.ssh/config") 491 } 492 493 dockerfile.WriteAdd("proxy.sh", "/etc/drone.d/") 494 dockerfile.WriteEntrypoint("/bin/bash -e /usr/local/bin/drone") 495 496 // write the Dockerfile to the temporary directory 497 return ioutil.WriteFile(filepath.Join(dir, "Dockerfile"), dockerfile.Bytes(), 0700) 498 } 499 500 // writeBuildScript is a helper function that 501 // will generate the build script file in the builder's 502 // temp directory to be added to the Image. 503 func (b *Builder) writeBuildScript(dir string) error { 504 f := buildfile.New() 505 506 // add environment variables about the build 507 f.WriteEnv("CI", "true") 508 f.WriteEnv("DRONE", "true") 509 f.WriteEnv("DRONE_BRANCH", b.Repo.Branch) 510 f.WriteEnv("DRONE_COMMIT", b.Repo.Commit) 511 f.WriteEnv("DRONE_PR", b.Repo.PR) 512 f.WriteEnv("DRONE_BUILD_DIR", b.Repo.Dir) 513 514 // add environment variables for code coverage 515 // systems, like coveralls. 516 f.WriteEnv("CI_NAME", "DRONE") 517 f.WriteEnv("CI_BUILD_NUMBER", b.Repo.Commit) 518 f.WriteEnv("CI_BUILD_URL", "") 519 f.WriteEnv("CI_BRANCH", b.Repo.Branch) 520 f.WriteEnv("CI_PULL_REQUEST", b.Repo.PR) 521 522 // add /etc/hosts entries 523 for _, mapping := range b.Build.Hosts { 524 f.WriteHost(mapping) 525 } 526 527 // if the repository is remote then we should 528 // add the commands to the build script to 529 // clone the repository 530 if b.Repo.IsRemote() { 531 for _, cmd := range b.Repo.Commands() { 532 f.WriteCmd(cmd) 533 println(cmd) 534 } 535 } 536 537 // if the commit is for merging a pull request 538 // we should only execute the build commands, 539 // and omit the deploy and publish commands. 540 if len(b.Repo.PR) == 0 { 541 b.Build.Write(f, b.Repo) 542 } else { 543 // only write the build commands 544 b.Build.WriteBuild(f) 545 } 546 547 scriptfilePath := filepath.Join(dir, "drone") 548 return ioutil.WriteFile(scriptfilePath, f.Bytes(), 0700) 549 } 550 551 // writeProxyScript is a helper function that 552 // will generate the proxy.sh file in the builder's 553 // temp directory to be added to the Image. 554 func (b *Builder) writeProxyScript(dir string) error { 555 var proxyfile = proxy.Proxy{} 556 557 // loop through services so that we can 558 // map ip address to localhost 559 for _, container := range b.services { 560 // create an entry for each port 561 for port := range container.NetworkSettings.Ports { 562 proxyfile.Set(port.Port(), container.NetworkSettings.IPAddress) 563 } 564 } 565 566 // write the proxyfile to the temp directory 567 proxyfilePath := filepath.Join(dir, "proxy.sh") 568 return ioutil.WriteFile(proxyfilePath, proxyfile.Bytes(), 0755) 569 } 570 571 // writeIdentifyFile is a helper function that 572 // will generate the id_rsa file in the builder's 573 // temp directory to be added to the Image. 574 func (b *Builder) writeIdentifyFile(dir string) error { 575 keyfilePath := filepath.Join(dir, "id_rsa") 576 return ioutil.WriteFile(keyfilePath, b.Key, 0700) 577 }