github.com/lukahartwig/terraform@v0.11.4-0.20180302171601-664391c254ea/config/module/storage.go (about) 1 package module 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "io/ioutil" 7 "log" 8 "os" 9 "path/filepath" 10 11 getter "github.com/hashicorp/go-getter" 12 "github.com/hashicorp/terraform/registry" 13 "github.com/hashicorp/terraform/registry/regsrc" 14 "github.com/hashicorp/terraform/svchost/auth" 15 "github.com/hashicorp/terraform/svchost/disco" 16 "github.com/mitchellh/cli" 17 ) 18 19 const manifestName = "modules.json" 20 21 // moduleManifest is the serialization structure used to record the stored 22 // module's metadata. 23 type moduleManifest struct { 24 Modules []moduleRecord 25 } 26 27 // moduleRecords represents the stored module's metadata. 28 // This is compared for equality using '==', so all fields needs to remain 29 // comparable. 30 type moduleRecord struct { 31 // Source is the module source string from the config, minus any 32 // subdirectory. 33 Source string 34 35 // Key is the locally unique identifier for this module. 36 Key string 37 38 // Version is the exact version string for the stored module. 39 Version string 40 41 // Dir is the directory name returned by the FileStorage. This is what 42 // allows us to correlate a particular module version with the location on 43 // disk. 44 Dir string 45 46 // Root is the root directory containing the module. If the module is 47 // unpacked from an archive, and not located in the root directory, this is 48 // used to direct the loader to the correct subdirectory. This is 49 // independent from any subdirectory in the original source string, which 50 // may traverse further into the module tree. 51 Root string 52 53 // url is the location of the module source 54 url string 55 56 // Registry is true if this module is sourced from a registry 57 registry bool 58 } 59 60 // Storage implements methods to manage the storage of modules. 61 // This is used by Tree.Load to query registries, authenticate requests, and 62 // store modules locally. 63 type Storage struct { 64 // StorageDir is the full path to the directory where all modules will be 65 // stored. 66 StorageDir string 67 // Services is a required *disco.Disco, which may have services and 68 // credentials pre-loaded. 69 Services *disco.Disco 70 // Creds optionally provides credentials for communicating with service 71 // providers. 72 Creds auth.CredentialsSource 73 // Ui is an optional cli.Ui for user output 74 Ui cli.Ui 75 // Mode is the GetMode that will be used for various operations. 76 Mode GetMode 77 78 registry *registry.Client 79 } 80 81 func NewStorage(dir string, services *disco.Disco, creds auth.CredentialsSource) *Storage { 82 regClient := registry.NewClient(services, creds, nil) 83 84 return &Storage{ 85 StorageDir: dir, 86 registry: regClient, 87 } 88 } 89 90 // loadManifest returns the moduleManifest file from the parent directory. 91 func (s Storage) loadManifest() (moduleManifest, error) { 92 manifest := moduleManifest{} 93 94 manifestPath := filepath.Join(s.StorageDir, manifestName) 95 data, err := ioutil.ReadFile(manifestPath) 96 if err != nil && !os.IsNotExist(err) { 97 return manifest, err 98 } 99 100 if len(data) == 0 { 101 return manifest, nil 102 } 103 104 if err := json.Unmarshal(data, &manifest); err != nil { 105 return manifest, err 106 } 107 return manifest, nil 108 } 109 110 // Store the location of the module, along with the version used and the module 111 // root directory. The storage method loads the entire file and rewrites it 112 // each time. This is only done a few times during init, so efficiency is 113 // not a concern. 114 func (s Storage) recordModule(rec moduleRecord) error { 115 manifest, err := s.loadManifest() 116 if err != nil { 117 // if there was a problem with the file, we will attempt to write a new 118 // one. Any non-data related error should surface there. 119 log.Printf("[WARN] error reading module manifest: %s", err) 120 } 121 122 // do nothing if we already have the exact module 123 for i, stored := range manifest.Modules { 124 if rec == stored { 125 return nil 126 } 127 128 // they are not equal, but if the storage path is the same we need to 129 // remove this rec to be replaced. 130 if rec.Dir == stored.Dir { 131 manifest.Modules[i] = manifest.Modules[len(manifest.Modules)-1] 132 manifest.Modules = manifest.Modules[:len(manifest.Modules)-1] 133 break 134 } 135 } 136 137 manifest.Modules = append(manifest.Modules, rec) 138 139 js, err := json.Marshal(manifest) 140 if err != nil { 141 panic(err) 142 } 143 144 manifestPath := filepath.Join(s.StorageDir, manifestName) 145 return ioutil.WriteFile(manifestPath, js, 0644) 146 } 147 148 // load the manifest from dir, and return all module versions matching the 149 // provided source. Records with no version info will be skipped, as they need 150 // to be uniquely identified by other means. 151 func (s Storage) moduleVersions(source string) ([]moduleRecord, error) { 152 manifest, err := s.loadManifest() 153 if err != nil { 154 return manifest.Modules, err 155 } 156 157 var matching []moduleRecord 158 159 for _, m := range manifest.Modules { 160 if m.Source == source && m.Version != "" { 161 log.Printf("[DEBUG] found local version %q for module %s", m.Version, m.Source) 162 matching = append(matching, m) 163 } 164 } 165 166 return matching, nil 167 } 168 169 func (s Storage) moduleDir(key string) (string, error) { 170 manifest, err := s.loadManifest() 171 if err != nil { 172 return "", err 173 } 174 175 for _, m := range manifest.Modules { 176 if m.Key == key { 177 return m.Dir, nil 178 } 179 } 180 181 return "", nil 182 } 183 184 // return only the root directory of the module stored in dir. 185 func (s Storage) getModuleRoot(dir string) (string, error) { 186 manifest, err := s.loadManifest() 187 if err != nil { 188 return "", err 189 } 190 191 for _, mod := range manifest.Modules { 192 if mod.Dir == dir { 193 return mod.Root, nil 194 } 195 } 196 return "", nil 197 } 198 199 // record only the Root directory for the module stored at dir. 200 func (s Storage) recordModuleRoot(dir, root string) error { 201 rec := moduleRecord{ 202 Dir: dir, 203 Root: root, 204 } 205 206 return s.recordModule(rec) 207 } 208 209 func (s Storage) output(msg string) { 210 if s.Ui == nil || s.Mode == GetModeNone { 211 return 212 } 213 s.Ui.Output(msg) 214 } 215 216 func (s Storage) getStorage(key string, src string) (string, bool, error) { 217 storage := &getter.FolderStorage{ 218 StorageDir: s.StorageDir, 219 } 220 221 log.Printf("[DEBUG] fetching module from %s", src) 222 223 // Get the module with the level specified if we were told to. 224 if s.Mode > GetModeNone { 225 log.Printf("[DEBUG] fetching %q with key %q", src, key) 226 if err := storage.Get(key, src, s.Mode == GetModeUpdate); err != nil { 227 return "", false, err 228 } 229 } 230 231 // Get the directory where the module is. 232 dir, found, err := storage.Dir(key) 233 log.Printf("[DEBUG] found %q in %q: %t", src, dir, found) 234 return dir, found, err 235 } 236 237 // find a stored module that's not from a registry 238 func (s Storage) findModule(key string) (string, error) { 239 if s.Mode == GetModeUpdate { 240 return "", nil 241 } 242 243 return s.moduleDir(key) 244 } 245 246 // GetModule fetches a module source into the specified directory. This is used 247 // as a convenience function by the CLI to initialize a configuration. 248 func (s Storage) GetModule(dst, src string) error { 249 // reset this in case the caller was going to re-use it 250 mode := s.Mode 251 s.Mode = GetModeUpdate 252 defer func() { 253 s.Mode = mode 254 }() 255 256 rec, err := s.findRegistryModule(src, anyVersion) 257 if err != nil { 258 return err 259 } 260 261 pwd, err := os.Getwd() 262 if err != nil { 263 return err 264 } 265 266 source := rec.url 267 if source == "" { 268 source, err = getter.Detect(src, pwd, getter.Detectors) 269 if err != nil { 270 return fmt.Errorf("module %s: %s", src, err) 271 } 272 } 273 274 if source == "" { 275 return fmt.Errorf("module %q not found", src) 276 } 277 278 return GetCopy(dst, source) 279 } 280 281 // find a registry module 282 func (s Storage) findRegistryModule(mSource, constraint string) (moduleRecord, error) { 283 rec := moduleRecord{ 284 Source: mSource, 285 } 286 // detect if we have a registry source 287 mod, err := regsrc.ParseModuleSource(mSource) 288 switch err { 289 case nil: 290 //ok 291 case regsrc.ErrInvalidModuleSource: 292 return rec, nil 293 default: 294 return rec, err 295 } 296 rec.registry = true 297 298 log.Printf("[TRACE] %q is a registry module", mod.Display()) 299 300 versions, err := s.moduleVersions(mod.String()) 301 if err != nil { 302 log.Printf("[ERROR] error looking up versions for %q: %s", mod.Display(), err) 303 return rec, err 304 } 305 306 match, err := newestRecord(versions, constraint) 307 if err != nil { 308 log.Printf("[INFO] no matching version for %q<%s>, %s", mod.Display(), constraint, err) 309 } 310 log.Printf("[DEBUG] matched %q version %s for %s", mod, match.Version, constraint) 311 312 rec.Dir = match.Dir 313 rec.Version = match.Version 314 found := rec.Dir != "" 315 316 // we need to lookup available versions 317 // Only on Get if it's not found, on unconditionally on Update 318 if (s.Mode == GetModeGet && !found) || (s.Mode == GetModeUpdate) { 319 resp, err := s.registry.Versions(mod) 320 if err != nil { 321 return rec, err 322 } 323 324 if len(resp.Modules) == 0 { 325 return rec, fmt.Errorf("module %q not found in registry", mod.Display()) 326 } 327 328 match, err := newestVersion(resp.Modules[0].Versions, constraint) 329 if err != nil { 330 return rec, err 331 } 332 333 if match == nil { 334 return rec, fmt.Errorf("no versions for %q found matching %q", mod.Display(), constraint) 335 } 336 337 rec.Version = match.Version 338 339 rec.url, err = s.registry.Location(mod, rec.Version) 340 if err != nil { 341 return rec, err 342 } 343 344 // we've already validated this by now 345 host, _ := mod.SvcHost() 346 s.output(fmt.Sprintf(" Found version %s of %s on %s", rec.Version, mod.Module(), host.ForDisplay())) 347 348 } 349 return rec, nil 350 }