github.com/Pankov404/juju@v0.0.0-20150703034450-be266991dceb/environs/sync/sync.go (about) 1 // Copyright 2012, 2013 Canonical Ltd. 2 // Licensed under the AGPLv3, see LICENCE file for details. 3 4 package sync 5 6 import ( 7 "bytes" 8 "fmt" 9 "io" 10 "io/ioutil" 11 "os" 12 "path/filepath" 13 14 "github.com/juju/errors" 15 "github.com/juju/loggo" 16 "github.com/juju/utils" 17 18 "github.com/juju/juju/environs/filestorage" 19 "github.com/juju/juju/environs/simplestreams" 20 "github.com/juju/juju/environs/storage" 21 envtools "github.com/juju/juju/environs/tools" 22 coretools "github.com/juju/juju/tools" 23 "github.com/juju/juju/version" 24 ) 25 26 var logger = loggo.GetLogger("juju.environs.sync") 27 28 // SyncContext describes the context for tool synchronization. 29 type SyncContext struct { 30 // TargetToolsFinder is a ToolsFinder provided to find existing 31 // tools in the target destination. 32 TargetToolsFinder ToolsFinder 33 34 // TargetToolsUploader is a ToolsUploader provided to upload 35 // tools to the target destination. 36 TargetToolsUploader ToolsUploader 37 38 // AllVersions controls the copy of all versions, not only the latest. 39 AllVersions bool 40 41 // Copy tools with major version, if MajorVersion > 0. 42 MajorVersion int 43 44 // Copy tools with minor version, if MinorVersion > 0. 45 MinorVersion int 46 47 // DryRun controls that nothing is copied. Instead it's logged 48 // what would be coppied. 49 DryRun bool 50 51 // Stream specifies the simplestreams stream to use (defaults to "Released"). 52 Stream string 53 54 // Source, if non-empty, specifies a directory in the local file system 55 // to use as a source. 56 Source string 57 } 58 59 // ToolsFinder provides an interface for finding tools of a specified version. 60 type ToolsFinder interface { 61 // FindTools returns a list of tools with the specified major version in the specified stream. 62 FindTools(major int, stream string) (coretools.List, error) 63 } 64 65 // ToolsUploader provides an interface for uploading tools and associated 66 // metadata. 67 type ToolsUploader interface { 68 // UploadTools uploads the tools with the specified version and tarball contents. 69 UploadTools(toolsDir, stream string, tools *coretools.Tools, data []byte) error 70 } 71 72 // SyncTools copies the Juju tools tarball from the official bucket 73 // or a specified source directory into the user's environment. 74 func SyncTools(syncContext *SyncContext) error { 75 sourceDataSource, err := selectSourceDatasource(syncContext) 76 if err != nil { 77 return err 78 } 79 80 logger.Infof("listing available tools") 81 if syncContext.MajorVersion == 0 && syncContext.MinorVersion == 0 { 82 syncContext.MajorVersion = version.Current.Major 83 syncContext.MinorVersion = -1 84 if !syncContext.AllVersions { 85 syncContext.MinorVersion = version.Current.Minor 86 } 87 } 88 89 toolsDir := syncContext.Stream 90 // If no stream has been specified, assume "released" for non-devel versions of Juju. 91 if syncContext.Stream == "" { 92 // We now store the tools in a directory named after their stream, but the 93 // legacy behaviour is to store all tools in a single "releases" directory. 94 toolsDir = envtools.LegacyReleaseDirectory 95 if !version.Current.IsDev() { 96 syncContext.Stream = envtools.ReleasedStream 97 } else { 98 syncContext.Stream = envtools.TestingStream 99 } 100 } 101 sourceTools, err := envtools.FindToolsForCloud( 102 []simplestreams.DataSource{sourceDataSource}, simplestreams.CloudSpec{}, 103 syncContext.Stream, syncContext.MajorVersion, syncContext.MinorVersion, coretools.Filter{}) 104 // For backwards compatibility with cloud storage, if there are no tools in the specified stream, 105 // double check the release stream. 106 // TODO - remove this when we no longer need to support cloud storage upgrades. 107 if err == envtools.ErrNoTools { 108 sourceTools, err = envtools.FindToolsForCloud( 109 []simplestreams.DataSource{sourceDataSource}, simplestreams.CloudSpec{}, 110 envtools.ReleasedStream, syncContext.MajorVersion, syncContext.MinorVersion, coretools.Filter{}) 111 } 112 if err != nil { 113 return err 114 } 115 116 logger.Infof("found %d tools", len(sourceTools)) 117 if !syncContext.AllVersions { 118 var latest version.Number 119 latest, sourceTools = sourceTools.Newest() 120 logger.Infof("found %d recent tools (version %s)", len(sourceTools), latest) 121 } 122 for _, tool := range sourceTools { 123 logger.Debugf("found source tool: %v", tool) 124 } 125 126 logger.Infof("listing target tools storage") 127 targetTools, err := syncContext.TargetToolsFinder.FindTools(syncContext.MajorVersion, syncContext.Stream) 128 switch err { 129 case nil, coretools.ErrNoMatches, envtools.ErrNoTools: 130 default: 131 return err 132 } 133 for _, tool := range targetTools { 134 logger.Debugf("found target tool: %v", tool) 135 } 136 137 missing := sourceTools.Exclude(targetTools) 138 logger.Infof("found %d tools in target; %d tools to be copied", len(targetTools), len(missing)) 139 if syncContext.DryRun { 140 for _, tools := range missing { 141 logger.Infof("copying %s from %s", tools.Version, tools.URL) 142 } 143 return nil 144 } 145 146 err = copyTools(toolsDir, syncContext.Stream, missing, syncContext.TargetToolsUploader) 147 if err != nil { 148 return err 149 } 150 logger.Infof("copied %d tools", len(missing)) 151 return nil 152 } 153 154 // selectSourceDatasource returns a storage reader based on the source setting. 155 func selectSourceDatasource(syncContext *SyncContext) (simplestreams.DataSource, error) { 156 source := syncContext.Source 157 if source == "" { 158 source = envtools.DefaultBaseURL 159 } 160 sourceURL, err := envtools.ToolsURL(source) 161 if err != nil { 162 return nil, err 163 } 164 logger.Infof("using sync tools source: %v", sourceURL) 165 return simplestreams.NewURLDataSource("sync tools source", sourceURL, utils.VerifySSLHostnames), nil 166 } 167 168 // copyTools copies a set of tools from the source to the target. 169 func copyTools(toolsDir, stream string, tools []*coretools.Tools, u ToolsUploader) error { 170 for _, tool := range tools { 171 logger.Infof("copying %s from %s", tool.Version, tool.URL) 172 if err := copyOneToolsPackage(toolsDir, stream, tool, u); err != nil { 173 return err 174 } 175 } 176 return nil 177 } 178 179 // copyOneToolsPackage copies one tool from the source to the target. 180 func copyOneToolsPackage(toolsDir, stream string, tools *coretools.Tools, u ToolsUploader) error { 181 toolsName := envtools.StorageName(tools.Version, toolsDir) 182 logger.Infof("downloading %q %v (%v)", stream, toolsName, tools.URL) 183 resp, err := utils.GetValidatingHTTPClient().Get(tools.URL) 184 if err != nil { 185 return err 186 } 187 defer resp.Body.Close() 188 // Verify SHA-256 hash. 189 var buf bytes.Buffer 190 sha256, size, err := utils.ReadSHA256(io.TeeReader(resp.Body, &buf)) 191 if err != nil { 192 return err 193 } 194 if tools.SHA256 == "" { 195 logger.Warningf("no SHA-256 hash for %v", tools.SHA256) 196 } else if sha256 != tools.SHA256 { 197 return errors.Errorf("SHA-256 hash mismatch (%v/%v)", sha256, tools.SHA256) 198 } 199 sizeInKB := (size + 512) / 1024 200 logger.Infof("uploading %v (%dkB) to environment", toolsName, sizeInKB) 201 return u.UploadTools(toolsDir, stream, tools, buf.Bytes()) 202 } 203 204 // UploadFunc is the type of Upload, which may be 205 // reassigned to control the behaviour of tools 206 // uploading. 207 type UploadFunc func(stor storage.Storage, stream string, forceVersion *version.Number, series ...string) (*coretools.Tools, error) 208 209 // Exported for testing. 210 var Upload UploadFunc = upload 211 212 // upload builds whatever version of github.com/juju/juju is in $GOPATH, 213 // uploads it to the given storage, and returns a Tools instance describing 214 // them. If forceVersion is not nil, the uploaded tools bundle will report 215 // the given version number; if any fakeSeries are supplied, additional copies 216 // of the built tools will be uploaded for use by machines of those series. 217 // Juju tools built for one series do not necessarily run on another, but this 218 // func exists only for development use cases. 219 func upload(stor storage.Storage, stream string, forceVersion *version.Number, fakeSeries ...string) (*coretools.Tools, error) { 220 builtTools, err := BuildToolsTarball(forceVersion, stream) 221 if err != nil { 222 return nil, err 223 } 224 defer os.RemoveAll(builtTools.Dir) 225 logger.Debugf("Uploading tools for %v", fakeSeries) 226 return syncBuiltTools(stor, stream, builtTools, fakeSeries...) 227 } 228 229 // cloneToolsForSeries copies the built tools tarball into a tarball for the specified 230 // stream and series and generates corresponding metadata. 231 func cloneToolsForSeries(toolsInfo *BuiltTools, stream string, series ...string) error { 232 // Copy the tools to the target storage, recording a Tools struct for each one. 233 var targetTools coretools.List 234 targetTools = append(targetTools, &coretools.Tools{ 235 Version: toolsInfo.Version, 236 Size: toolsInfo.Size, 237 SHA256: toolsInfo.Sha256Hash, 238 }) 239 putTools := func(vers version.Binary) (string, error) { 240 name := envtools.StorageName(vers, stream) 241 src := filepath.Join(toolsInfo.Dir, toolsInfo.StorageName) 242 dest := filepath.Join(toolsInfo.Dir, name) 243 destDir := filepath.Dir(dest) 244 if err := os.MkdirAll(destDir, 0755); err != nil { 245 return "", err 246 } 247 if err := utils.CopyFile(dest, src); err != nil { 248 return "", err 249 } 250 // Append to targetTools the attributes required to write out tools metadata. 251 targetTools = append(targetTools, &coretools.Tools{ 252 Version: vers, 253 Size: toolsInfo.Size, 254 SHA256: toolsInfo.Sha256Hash, 255 }) 256 return name, nil 257 } 258 logger.Debugf("generating tarballs for %v", series) 259 for _, series := range series { 260 _, err := version.SeriesVersion(series) 261 if err != nil { 262 return err 263 } 264 if series != toolsInfo.Version.Series { 265 fakeVersion := toolsInfo.Version 266 fakeVersion.Series = series 267 if _, err := putTools(fakeVersion); err != nil { 268 return err 269 } 270 } 271 } 272 // The tools have been copied to a temp location from which they will be uploaded, 273 // now write out the matching simplestreams metadata so that SyncTools can find them. 274 metadataStore, err := filestorage.NewFileStorageWriter(toolsInfo.Dir) 275 if err != nil { 276 return err 277 } 278 logger.Debugf("generating tools metadata") 279 return envtools.MergeAndWriteMetadata(metadataStore, stream, stream, targetTools, false) 280 } 281 282 // BuiltTools contains metadata for a tools tarball resulting from 283 // a call to BundleTools. 284 type BuiltTools struct { 285 Version version.Binary 286 Dir string 287 StorageName string 288 Sha256Hash string 289 Size int64 290 } 291 292 // BuildToolsTarballFunc is a function which can build a tools tarball. 293 type BuildToolsTarballFunc func(forceVersion *version.Number, stream string) (*BuiltTools, error) 294 295 // Override for testing. 296 var BuildToolsTarball BuildToolsTarballFunc = buildToolsTarball 297 298 // buildToolsTarball bundles a tools tarball and places it in a temp directory in 299 // the expected tools path. 300 func buildToolsTarball(forceVersion *version.Number, stream string) (builtTools *BuiltTools, err error) { 301 // TODO(rog) find binaries from $PATH when not using a development 302 // version of juju within a $GOPATH. 303 304 logger.Debugf("Building tools") 305 // We create the entire archive before asking the environment to 306 // start uploading so that we can be sure we have archived 307 // correctly. 308 f, err := ioutil.TempFile("", "juju-tgz") 309 if err != nil { 310 return nil, err 311 } 312 defer f.Close() 313 defer os.Remove(f.Name()) 314 toolsVersion, sha256Hash, err := envtools.BundleTools(f, forceVersion) 315 if err != nil { 316 return nil, err 317 } 318 fileInfo, err := f.Stat() 319 if err != nil { 320 return nil, fmt.Errorf("cannot stat newly made tools archive: %v", err) 321 } 322 size := fileInfo.Size() 323 logger.Infof("built tools %v (%dkB)", toolsVersion, (size+512)/1024) 324 baseToolsDir, err := ioutil.TempDir("", "juju-tools") 325 if err != nil { 326 return nil, err 327 } 328 329 // If we exit with an error, clean up the built tools directory. 330 defer func() { 331 if err != nil { 332 os.RemoveAll(baseToolsDir) 333 } 334 }() 335 336 err = os.MkdirAll(filepath.Join(baseToolsDir, storage.BaseToolsPath, stream), 0755) 337 if err != nil { 338 return nil, err 339 } 340 storageName := envtools.StorageName(toolsVersion, stream) 341 err = utils.CopyFile(filepath.Join(baseToolsDir, storageName), f.Name()) 342 if err != nil { 343 return nil, err 344 } 345 return &BuiltTools{ 346 Version: toolsVersion, 347 Dir: baseToolsDir, 348 StorageName: storageName, 349 Size: size, 350 Sha256Hash: sha256Hash, 351 }, nil 352 } 353 354 // syncBuiltTools copies to storage a tools tarball and cloned copies for each series. 355 func syncBuiltTools(stor storage.Storage, stream string, builtTools *BuiltTools, fakeSeries ...string) (*coretools.Tools, error) { 356 if err := cloneToolsForSeries(builtTools, stream, fakeSeries...); err != nil { 357 return nil, err 358 } 359 syncContext := &SyncContext{ 360 Source: builtTools.Dir, 361 TargetToolsFinder: StorageToolsFinder{stor}, 362 TargetToolsUploader: StorageToolsUploader{stor, false, false}, 363 AllVersions: true, 364 Stream: stream, 365 MajorVersion: builtTools.Version.Major, 366 MinorVersion: -1, 367 } 368 logger.Debugf("uploading tools to cloud storage") 369 err := SyncTools(syncContext) 370 if err != nil { 371 return nil, err 372 } 373 url, err := stor.URL(builtTools.StorageName) 374 if err != nil { 375 return nil, err 376 } 377 return &coretools.Tools{ 378 Version: builtTools.Version, 379 URL: url, 380 Size: builtTools.Size, 381 SHA256: builtTools.Sha256Hash, 382 }, nil 383 } 384 385 // StorageToolsFinder is an implementation of ToolsFinder 386 // that searches for tools in the specified storage. 387 type StorageToolsFinder struct { 388 Storage storage.StorageReader 389 } 390 391 func (f StorageToolsFinder) FindTools(major int, stream string) (coretools.List, error) { 392 return envtools.ReadList(f.Storage, stream, major, -1) 393 } 394 395 // StorageToolsUplader is an implementation of ToolsUploader that 396 // writes tools to the provided storage and then writes merged 397 // metadata, optionally with mirrors. 398 type StorageToolsUploader struct { 399 Storage storage.Storage 400 WriteMetadata bool 401 WriteMirrors envtools.ShouldWriteMirrors 402 } 403 404 func (u StorageToolsUploader) UploadTools(toolsDir, stream string, tools *coretools.Tools, data []byte) error { 405 toolsName := envtools.StorageName(tools.Version, toolsDir) 406 if err := u.Storage.Put(toolsName, bytes.NewReader(data), int64(len(data))); err != nil { 407 return err 408 } 409 if !u.WriteMetadata { 410 return nil 411 } 412 err := envtools.MergeAndWriteMetadata(u.Storage, toolsDir, stream, coretools.List{tools}, u.WriteMirrors) 413 if err != nil { 414 logger.Errorf("error writing tools metadata: %v", err) 415 return err 416 } 417 return nil 418 }