github.com/juju/charm/v11@v11.2.0/charmdir.go (about) 1 // Copyright 2011, 2012, 2013 Canonical Ltd. 2 // Licensed under the LGPLv3, see LICENCE file for details. 3 4 package charm 5 6 import ( 7 "archive/zip" 8 "context" 9 "fmt" 10 "io" 11 "os" 12 "os/exec" 13 "path/filepath" 14 "strconv" 15 "strings" 16 "syscall" 17 "time" 18 19 "github.com/juju/errors" 20 ) 21 22 // defaultJujuIgnore contains jujuignore directives for excluding VCS- and 23 // build-related directories when archiving. The following set of directives 24 // will be prepended to the contents of the charm's .jujuignore file if one is 25 // provided. 26 // 27 // NOTE: writeArchive auto-generates its own revision and version files so they 28 // need to be excluded here to prevent anyone from overriding their contents by 29 // adding files with the same name to their charm repo. 30 var defaultJujuIgnore = ` 31 .git 32 .svn 33 .hg 34 .bzr 35 .tox 36 37 /build/ 38 /revision 39 /version 40 41 .jujuignore 42 ` 43 44 // CharmDir encapsulates access to data and operations 45 // on a charm directory. 46 type CharmDir struct { 47 Path string 48 *charmBase 49 } 50 51 // Trick to ensure *CharmDir implements the Charm interface. 52 var _ Charm = (*CharmDir)(nil) 53 54 // IsCharmDir report whether the path is likely to represent 55 // a charm, even it may be incomplete. 56 func IsCharmDir(path string) bool { 57 dir := &CharmDir{Path: path} 58 _, err := os.Stat(dir.join("metadata.yaml")) 59 return err == nil 60 } 61 62 // ReadCharmDir returns a CharmDir representing an expanded charm directory. 63 func ReadCharmDir(path string) (*CharmDir, error) { 64 b := &CharmDir{ 65 Path: path, 66 charmBase: &charmBase{}, 67 } 68 reader, err := os.Open(b.join("metadata.yaml")) 69 if err != nil { 70 return nil, errors.Annotatef(err, `reading "metadata.yaml" file`) 71 } 72 b.meta, err = ReadMeta(reader) 73 _ = reader.Close() 74 if err != nil { 75 return nil, errors.Annotatef(err, `parsing "metadata.yaml" file`) 76 } 77 78 // Try to read the optional manifest.yaml, it's required to determine if 79 // this charm is v1 or not. 80 reader, err = os.Open(b.join("manifest.yaml")) 81 if _, ok := err.(*os.PathError); ok { 82 b.manifest = nil 83 } else if err != nil { 84 return nil, errors.Annotatef(err, `reading "manifest.yaml" file`) 85 } else { 86 b.manifest, err = ReadManifest(reader) 87 _ = reader.Close() 88 if err != nil { 89 return nil, errors.Annotatef(err, `parsing "manifest.yaml" file`) 90 } 91 } 92 93 reader, err = os.Open(b.join("config.yaml")) 94 if _, ok := err.(*os.PathError); ok { 95 b.config = NewConfig() 96 } else if err != nil { 97 return nil, errors.Annotatef(err, `reading "config.yaml" file`) 98 } else { 99 b.config, err = ReadConfig(reader) 100 _ = reader.Close() 101 if err != nil { 102 return nil, errors.Annotatef(err, `parsing "config.yaml" file`) 103 } 104 } 105 106 reader, err = os.Open(b.join("metrics.yaml")) 107 if err == nil { 108 b.metrics, err = ReadMetrics(reader) 109 _ = reader.Close() 110 if err != nil { 111 return nil, errors.Annotatef(err, `parsing "metrics.yaml" file`) 112 } 113 } else if !os.IsNotExist(err) { 114 return nil, errors.Annotatef(err, `reading "metrics.yaml" file`) 115 } 116 117 if b.actions, err = getActions( 118 b.meta.Name, 119 func(file string) (io.ReadCloser, error) { 120 return os.Open(b.join(file)) 121 }, 122 func(err error) bool { 123 _, ok := err.(*os.PathError) 124 return ok 125 }, 126 ); err != nil { 127 return nil, err 128 } 129 130 if reader, err = os.Open(b.join("revision")); err == nil { 131 _, err = fmt.Fscan(reader, &b.revision) 132 _ = reader.Close() 133 if err != nil { 134 return nil, errors.New("invalid revision file") 135 } 136 } 137 138 reader, err = os.Open(b.join("lxd-profile.yaml")) 139 if _, ok := err.(*os.PathError); ok { 140 b.lxdProfile = NewLXDProfile() 141 } else if err != nil { 142 return nil, errors.Annotatef(err, `reading "lxd-profile.yaml" file`) 143 } else { 144 b.lxdProfile, err = ReadLXDProfile(reader) 145 _ = reader.Close() 146 if err != nil { 147 return nil, errors.Annotatef(err, `parsing "lxd-profile.yaml" file`) 148 } 149 } 150 151 reader, err = os.Open(b.join("version")) 152 if err != nil { 153 if _, ok := err.(*os.PathError); !ok { 154 return nil, errors.Annotatef(err, `reading "version" file`) 155 } 156 } else { 157 b.version, err = ReadVersion(reader) 158 _ = reader.Close() 159 if err != nil { 160 return nil, errors.Annotatef(err, `parsing "version" file`) 161 } 162 } 163 164 return b, nil 165 } 166 167 // buildIgnoreRules parses the contents of the charm's .jujuignore file and 168 // compiles a set of rules that are used to decide which files should be 169 // archived. 170 func (dir *CharmDir) buildIgnoreRules() (ignoreRuleset, error) { 171 // Start with a set of sane defaults to ensure backwards-compatibility 172 // for charms that do not use a .jujuignore file. 173 rules, err := newIgnoreRuleset(strings.NewReader(defaultJujuIgnore)) 174 if err != nil { 175 return nil, err 176 } 177 178 pathToJujuignore := dir.join(".jujuignore") 179 if _, err := os.Stat(pathToJujuignore); err == nil { 180 file, err := os.Open(dir.join(".jujuignore")) 181 if err != nil { 182 return nil, errors.Annotatef(err, `reading ".jujuignore" file`) 183 } 184 defer func() { _ = file.Close() }() 185 186 jujuignoreRules, err := newIgnoreRuleset(file) 187 if err != nil { 188 return nil, errors.Annotate(err, `parsing ".jujuignore" file`) 189 } 190 191 rules = append(rules, jujuignoreRules...) 192 } 193 194 return rules, nil 195 } 196 197 // join builds a path rooted at the charm's expanded directory 198 // path and the extra path components provided. 199 func (dir *CharmDir) join(parts ...string) string { 200 parts = append([]string{dir.Path}, parts...) 201 return filepath.Join(parts...) 202 } 203 204 // SetDiskRevision does the same as SetRevision but also changes 205 // the revision file in the charm directory. 206 func (dir *CharmDir) SetDiskRevision(revision int) error { 207 dir.SetRevision(revision) 208 file, err := os.OpenFile(dir.join("revision"), os.O_WRONLY|os.O_CREATE, 0644) 209 if err != nil { 210 return err 211 } 212 _, err = file.Write([]byte(strconv.Itoa(revision))) 213 file.Close() 214 return err 215 } 216 217 // resolveSymlinkedRoot returns the target destination of a 218 // charm root directory if the root directory is a symlink. 219 func resolveSymlinkedRoot(rootPath string) (string, error) { 220 info, err := os.Lstat(rootPath) 221 if err == nil && info.Mode()&os.ModeSymlink != 0 { 222 rootPath, err = filepath.EvalSymlinks(rootPath) 223 if err != nil { 224 return "", fmt.Errorf("cannot read path symlink at %q: %v", rootPath, err) 225 } 226 } 227 return rootPath, nil 228 } 229 230 // ArchiveTo creates a charm file from the charm expanded in dir. 231 // By convention a charm archive should have a ".charm" suffix. 232 func (dir *CharmDir) ArchiveTo(w io.Writer) error { 233 ignoreRules, err := dir.buildIgnoreRules() 234 if err != nil { 235 return err 236 } 237 // We update the version to make sure we don't lag behind 238 dir.version, _, err = dir.MaybeGenerateVersionString(logger) 239 if err != nil { 240 // We don't want to stop, even if the version cannot be generated 241 logger.Warningf("trying to generate version string: %v", err) 242 } 243 244 return writeArchive(w, dir.Path, dir.revision, dir.version, dir.Meta().Hooks(), ignoreRules) 245 } 246 247 func writeArchive(w io.Writer, path string, revision int, versionString string, hooks map[string]bool, ignoreRules ignoreRuleset) error { 248 zipw := zip.NewWriter(w) 249 defer zipw.Close() 250 251 // The root directory may be symlinked elsewhere so 252 // resolve that before creating the zip. 253 rootPath, err := resolveSymlinkedRoot(path) 254 if err != nil { 255 return err 256 } 257 zp := zipPacker{zipw, rootPath, hooks, ignoreRules} 258 if revision != -1 { 259 zp.AddFile("revision", strconv.Itoa(revision)) 260 } 261 if versionString != "" { 262 zp.AddFile("version", versionString) 263 } 264 return filepath.Walk(rootPath, zp.WalkFunc()) 265 } 266 267 type zipPacker struct { 268 *zip.Writer 269 root string 270 hooks map[string]bool 271 ignoreRules ignoreRuleset 272 } 273 274 func (zp *zipPacker) WalkFunc() filepath.WalkFunc { 275 return func(path string, fi os.FileInfo, err error) error { 276 return zp.visit(path, fi, err) 277 } 278 } 279 280 func (zp *zipPacker) AddFile(filename string, value string) error { 281 h := &zip.FileHeader{Name: filename} 282 h.SetMode(syscall.S_IFREG | 0644) 283 w, err := zp.CreateHeader(h) 284 if err == nil { 285 _, err = w.Write([]byte(value)) 286 } 287 return err 288 } 289 290 func (zp *zipPacker) visit(path string, fi os.FileInfo, err error) error { 291 if err != nil { 292 return err 293 } 294 295 relpath, err := filepath.Rel(zp.root, path) 296 if err != nil { 297 return err 298 } 299 300 // Replace any Windows path separators with "/". 301 // zip file spec 4.4.17.1 says that separators are always "/" even on Windows. 302 relpath = filepath.ToSlash(relpath) 303 304 // Check if this file or dir needs to be ignored 305 if zp.ignoreRules.Match(relpath, fi.IsDir()) { 306 if fi.IsDir() { 307 return filepath.SkipDir 308 } 309 310 return nil 311 } 312 313 method := zip.Deflate 314 if fi.IsDir() { 315 relpath += "/" 316 method = zip.Store 317 } 318 319 mode := fi.Mode() 320 if err := checkFileType(relpath, mode); err != nil { 321 return err 322 } 323 if mode&os.ModeSymlink != 0 { 324 method = zip.Store 325 } 326 h := &zip.FileHeader{ 327 Name: relpath, 328 Method: method, 329 } 330 331 perm := os.FileMode(0644) 332 if mode&os.ModeSymlink != 0 { 333 perm = 0777 334 } else if mode&0100 != 0 { 335 perm = 0755 336 } 337 if filepath.Dir(relpath) == "hooks" { 338 hookName := filepath.Base(relpath) 339 if _, ok := zp.hooks[hookName]; ok && !fi.IsDir() && mode&0100 == 0 { 340 logger.Warningf("making %q executable in charm", path) 341 perm = perm | 0100 342 } 343 } 344 h.SetMode(mode&^0777 | perm) 345 346 w, err := zp.CreateHeader(h) 347 if err != nil || fi.IsDir() { 348 return err 349 } 350 var data []byte 351 if mode&os.ModeSymlink != 0 { 352 target, err := os.Readlink(path) 353 if err != nil { 354 return err 355 } 356 if err := checkSymlinkTarget(zp.root, relpath, target); err != nil { 357 return err 358 } 359 data = []byte(target) 360 _, err = w.Write(data) 361 } else { 362 file, err := os.Open(path) 363 if err != nil { 364 return err 365 } 366 defer file.Close() 367 _, err = io.Copy(w, file) 368 } 369 return err 370 } 371 372 func checkSymlinkTarget(basedir, symlink, target string) error { 373 if filepath.IsAbs(target) { 374 return fmt.Errorf("symlink %q is absolute: %q", symlink, target) 375 } 376 p := filepath.Join(filepath.Dir(symlink), target) 377 if p == ".." || strings.HasPrefix(p, "../") { 378 return fmt.Errorf("symlink %q links out of charm: %q", symlink, target) 379 } 380 return nil 381 } 382 383 func checkFileType(path string, mode os.FileMode) error { 384 e := "file has an unknown type: %q" 385 switch mode & os.ModeType { 386 case os.ModeDir, os.ModeSymlink, 0: 387 return nil 388 case os.ModeNamedPipe: 389 e = "file is a named pipe: %q" 390 case os.ModeSocket: 391 e = "file is a socket: %q" 392 case os.ModeDevice: 393 e = "file is a device: %q" 394 } 395 return fmt.Errorf(e, path) 396 } 397 398 // Logger represents the logging methods called. 399 type Logger interface { 400 Warningf(message string, args ...interface{}) 401 Debugf(message string, args ...interface{}) 402 Errorf(message string, args ...interface{}) 403 Tracef(message string, args ...interface{}) 404 Infof(message string, args ...interface{}) 405 } 406 407 type vcsCMD struct { 408 vcsType string 409 args []string 410 usesTypeCheck func(ctx context.Context, charmPath string, CancelFunc func()) bool 411 } 412 413 func (v *vcsCMD) commonErrHandler(err error, charmPath string) error { 414 return errors.Errorf("%q version string generation failed : "+ 415 "%v\nThis means that the charm version won't show in juju status. Charm path %q", v.vcsType, err, charmPath) 416 } 417 418 // usesGit first check checks for the easy case of the current charmdir has a 419 // git folder. 420 // There can be cases when the charmdir actually uses git and is just a subdir, 421 // hence the below check 422 func usesGit(ctx context.Context, charmPath string, cancelFunc func()) bool { 423 defer cancelFunc() 424 if _, err := os.Stat(filepath.Join(charmPath, ".git")); err == nil { 425 return true 426 } 427 args := []string{"rev-parse", "--is-inside-work-tree"} 428 execCmd := exec.CommandContext(ctx, "git", args...) 429 execCmd.Dir = charmPath 430 431 _, err := execCmd.Output() 432 433 if ctx.Err() == context.DeadlineExceeded { 434 logger.Debugf("git command timed out for charm in path: %q", charmPath) 435 return false 436 } 437 438 if err == nil { 439 return true 440 } 441 return false 442 } 443 444 func usesBzr(ctx context.Context, charmPath string, cancelFunc func()) bool { 445 defer cancelFunc() 446 if _, err := os.Stat(filepath.Join(charmPath, ".bzr")); err == nil { 447 return true 448 } 449 return false 450 } 451 452 func usesHg(ctx context.Context, charmPath string, cancelFunc func()) bool { 453 defer cancelFunc() 454 if _, err := os.Stat(filepath.Join(charmPath, ".hg")); err == nil { 455 return true 456 } 457 return false 458 } 459 460 // VersionFileVersionType holds the type of the versioned file type, either 461 // git, hg, bzr or a raw version file. 462 const versionFileVersionType = "versionFile" 463 464 // MaybeGenerateVersionString generates charm version string. 465 // We want to know whether parent folders use one of these vcs, that's why we 466 // try to execute each one of them 467 // The second return value is the detected vcs type. 468 func (dir *CharmDir) MaybeGenerateVersionString(logger Logger) (string, string, error) { 469 // vcsStrategies is the strategies to use to access the version file content. 470 vcsStrategies := map[string]vcsCMD{ 471 "hg": vcsCMD{ 472 vcsType: "hg", 473 args: []string{"id", "-n"}, 474 usesTypeCheck: usesHg, 475 }, 476 "git": vcsCMD{ 477 vcsType: "git", 478 args: []string{"describe", "--dirty", "--always"}, 479 usesTypeCheck: usesGit, 480 }, 481 "bzr": vcsCMD{ 482 vcsType: "bzr", 483 args: []string{"version-info"}, 484 usesTypeCheck: usesBzr, 485 }, 486 } 487 488 // Nowadays most vcs used are git, we want to make sure that git is the first one we test 489 vcsOrder := [...]string{"git", "hg", "bzr"} 490 cmdWaitTime := 2 * time.Second 491 492 absPath := dir.Path 493 if !filepath.IsAbs(absPath) { 494 var err error 495 absPath, err = filepath.Abs(dir.Path) 496 if err != nil { 497 return "", "", errors.Annotatef(err, "failed resolving relative path %q", dir.Path) 498 } 499 } 500 501 for _, vcsType := range vcsOrder { 502 vcsCmd := vcsStrategies[vcsType] 503 ctx, cancel := context.WithTimeout(context.Background(), cmdWaitTime) 504 if vcsCmd.usesTypeCheck(ctx, dir.Path, cancel) { 505 cmd := exec.Command(vcsCmd.vcsType, vcsCmd.args...) 506 // We need to make sure that the working directory will be the one we execute the commands from. 507 cmd.Dir = dir.Path 508 // Version string value is written to stdout if successful. 509 out, err := cmd.Output() 510 if err != nil { 511 // We had an error but we still know that we use a vcs, hence we can stop here and handle it. 512 return "", vcsType, vcsCmd.commonErrHandler(err, absPath) 513 } 514 output := strings.TrimSuffix(string(out), "\n") 515 return output, vcsType, nil 516 } 517 } 518 519 // If all strategies fail we fallback to check the version below 520 if file, err := os.Open(dir.join("version")); err == nil { 521 logger.Debugf("charm is not in version control, but uses a version file, charm path %q", absPath) 522 ver, err := ReadVersion(file) 523 file.Close() 524 if err != nil { 525 return "", versionFileVersionType, err 526 } 527 return ver, versionFileVersionType, nil 528 } 529 logger.Infof("charm is not versioned, charm path %q", absPath) 530 return "", "", nil 531 }