github.com/trawler/terraform@v0.10.8-0.20171106022149-4b1c7a1d9b48/config/module/storage.go (about) 1 package module 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "io/ioutil" 7 "log" 8 "os" 9 "path/filepath" 10 11 getter "github.com/hashicorp/go-getter" 12 "github.com/hashicorp/terraform/registry/regsrc" 13 "github.com/hashicorp/terraform/svchost/auth" 14 "github.com/hashicorp/terraform/svchost/disco" 15 "github.com/mitchellh/cli" 16 ) 17 18 const manifestName = "modules.json" 19 20 // moduleManifest is the serialization structure used to record the stored 21 // module's metadata. 22 type moduleManifest struct { 23 Modules []moduleRecord 24 } 25 26 // moduleRecords represents the stored module's metadata. 27 // This is compared for equality using '==', so all fields needs to remain 28 // comparable. 29 type moduleRecord struct { 30 // Source is the module source string from the config, minus any 31 // subdirectory. 32 Source string 33 34 // Key is the locally unique identifier for this module. 35 Key string 36 37 // Version is the exact version string for the stored module. 38 Version string 39 40 // Dir is the directory name returned by the FileStorage. This is what 41 // allows us to correlate a particular module version with the location on 42 // disk. 43 Dir string 44 45 // Root is the root directory containing the module. If the module is 46 // unpacked from an archive, and not located in the root directory, this is 47 // used to direct the loader to the correct subdirectory. This is 48 // independent from any subdirectory in the original source string, which 49 // may traverse further into the module tree. 50 Root string 51 52 // url is the location of the module source 53 url string 54 55 // Registry is true if this module is sourced from a registry 56 registry bool 57 } 58 59 // Storage implements methods to manage the storage of modules. 60 // This is used by Tree.Load to query registries, authenticate requests, and 61 // store modules locally. 62 type Storage struct { 63 // StorageDir is the full path to the directory where all modules will be 64 // stored. 65 StorageDir string 66 // Services is a required *disco.Disco, which may have services and 67 // credentials pre-loaded. 68 Services *disco.Disco 69 // Creds optionally provides credentials for communicating with service 70 // providers. 71 Creds auth.CredentialsSource 72 // Ui is an optional cli.Ui for user output 73 Ui cli.Ui 74 // Mode is the GetMode that will be used for various operations. 75 Mode GetMode 76 } 77 78 func NewStorage(dir string, services *disco.Disco, creds auth.CredentialsSource) *Storage { 79 s := &Storage{ 80 StorageDir: dir, 81 Services: services, 82 Creds: creds, 83 } 84 85 // make sure this isn't nil 86 if s.Services == nil { 87 s.Services = disco.NewDisco() 88 } 89 return s 90 } 91 92 // loadManifest returns the moduleManifest file from the parent directory. 93 func (s Storage) loadManifest() (moduleManifest, error) { 94 manifest := moduleManifest{} 95 96 manifestPath := filepath.Join(s.StorageDir, manifestName) 97 data, err := ioutil.ReadFile(manifestPath) 98 if err != nil && !os.IsNotExist(err) { 99 return manifest, err 100 } 101 102 if len(data) == 0 { 103 return manifest, nil 104 } 105 106 if err := json.Unmarshal(data, &manifest); err != nil { 107 return manifest, err 108 } 109 return manifest, nil 110 } 111 112 // Store the location of the module, along with the version used and the module 113 // root directory. The storage method loads the entire file and rewrites it 114 // each time. This is only done a few times during init, so efficiency is 115 // not a concern. 116 func (s Storage) recordModule(rec moduleRecord) error { 117 manifest, err := s.loadManifest() 118 if err != nil { 119 // if there was a problem with the file, we will attempt to write a new 120 // one. Any non-data related error should surface there. 121 log.Printf("[WARN] error reading module manifest: %s", err) 122 } 123 124 // do nothing if we already have the exact module 125 for i, stored := range manifest.Modules { 126 if rec == stored { 127 return nil 128 } 129 130 // they are not equal, but if the storage path is the same we need to 131 // remove this rec to be replaced. 132 if rec.Dir == stored.Dir { 133 manifest.Modules[i] = manifest.Modules[len(manifest.Modules)-1] 134 manifest.Modules = manifest.Modules[:len(manifest.Modules)-1] 135 break 136 } 137 } 138 139 manifest.Modules = append(manifest.Modules, rec) 140 141 js, err := json.Marshal(manifest) 142 if err != nil { 143 panic(err) 144 } 145 146 manifestPath := filepath.Join(s.StorageDir, manifestName) 147 return ioutil.WriteFile(manifestPath, js, 0644) 148 } 149 150 // load the manifest from dir, and return all module versions matching the 151 // provided source. Records with no version info will be skipped, as they need 152 // to be uniquely identified by other means. 153 func (s Storage) moduleVersions(source string) ([]moduleRecord, error) { 154 manifest, err := s.loadManifest() 155 if err != nil { 156 return manifest.Modules, err 157 } 158 159 var matching []moduleRecord 160 161 for _, m := range manifest.Modules { 162 if m.Source == source && m.Version != "" { 163 log.Printf("[DEBUG] found local version %q for module %s", m.Version, m.Source) 164 matching = append(matching, m) 165 } 166 } 167 168 return matching, nil 169 } 170 171 func (s Storage) moduleDir(key string) (string, error) { 172 manifest, err := s.loadManifest() 173 if err != nil { 174 return "", err 175 } 176 177 for _, m := range manifest.Modules { 178 if m.Key == key { 179 return m.Dir, nil 180 } 181 } 182 183 return "", nil 184 } 185 186 // return only the root directory of the module stored in dir. 187 func (s Storage) getModuleRoot(dir string) (string, error) { 188 manifest, err := s.loadManifest() 189 if err != nil { 190 return "", err 191 } 192 193 for _, mod := range manifest.Modules { 194 if mod.Dir == dir { 195 return mod.Root, nil 196 } 197 } 198 return "", nil 199 } 200 201 // record only the Root directory for the module stored at dir. 202 func (s Storage) recordModuleRoot(dir, root string) error { 203 rec := moduleRecord{ 204 Dir: dir, 205 Root: root, 206 } 207 208 return s.recordModule(rec) 209 } 210 211 func (s Storage) output(msg string) { 212 if s.Ui == nil || s.Mode == GetModeNone { 213 return 214 } 215 s.Ui.Output(msg) 216 } 217 218 func (s Storage) getStorage(key string, src string) (string, bool, error) { 219 storage := &getter.FolderStorage{ 220 StorageDir: s.StorageDir, 221 } 222 223 log.Printf("[DEBUG] fetching module from %s", src) 224 225 // Get the module with the level specified if we were told to. 226 if s.Mode > GetModeNone { 227 log.Printf("[DEBUG] fetching %q with key %q", src, key) 228 if err := storage.Get(key, src, s.Mode == GetModeUpdate); err != nil { 229 return "", false, err 230 } 231 } 232 233 // Get the directory where the module is. 234 dir, found, err := storage.Dir(key) 235 log.Printf("[DEBUG] found %q in %q: %t", src, dir, found) 236 return dir, found, err 237 } 238 239 // find a stored module that's not from a registry 240 func (s Storage) findModule(key string) (string, error) { 241 if s.Mode == GetModeUpdate { 242 return "", nil 243 } 244 245 return s.moduleDir(key) 246 } 247 248 // GetModule fetches a module source into the specified directory. This is used 249 // as a convenience function by the CLI to initialize a configuration. 250 func (s Storage) GetModule(dst, src string) error { 251 // reset this in case the caller was going to re-use it 252 mode := s.Mode 253 s.Mode = GetModeUpdate 254 defer func() { 255 s.Mode = mode 256 }() 257 258 rec, err := s.findRegistryModule(src, anyVersion) 259 if err != nil { 260 return err 261 } 262 263 pwd, err := os.Getwd() 264 if err != nil { 265 return err 266 } 267 268 source := rec.url 269 if source == "" { 270 source, err = getter.Detect(src, pwd, getter.Detectors) 271 if err != nil { 272 return fmt.Errorf("module %s: %s", src, err) 273 } 274 } 275 276 if source == "" { 277 return fmt.Errorf("module %q not found", src) 278 } 279 280 return GetCopy(dst, source) 281 } 282 283 // find a registry module 284 func (s Storage) findRegistryModule(mSource, constraint string) (moduleRecord, error) { 285 rec := moduleRecord{ 286 Source: mSource, 287 } 288 // detect if we have a registry source 289 mod, err := regsrc.ParseModuleSource(mSource) 290 switch err { 291 case nil: 292 //ok 293 case regsrc.ErrInvalidModuleSource: 294 return rec, nil 295 default: 296 return rec, err 297 } 298 rec.registry = true 299 300 log.Printf("[TRACE] %q is a registry module", mod.Module()) 301 302 versions, err := s.moduleVersions(mod.String()) 303 if err != nil { 304 log.Printf("[ERROR] error looking up versions for %q: %s", mod.Module(), err) 305 return rec, err 306 } 307 308 match, err := newestRecord(versions, constraint) 309 if err != nil { 310 log.Printf("[INFO] no matching version for %q<%s>, %s", mod.Module(), constraint, err) 311 } 312 log.Printf("[DEBUG] matched %q version %s for %s", mod, match.Version, constraint) 313 314 rec.Dir = match.Dir 315 rec.Version = match.Version 316 found := rec.Dir != "" 317 318 // we need to lookup available versions 319 // Only on Get if it's not found, on unconditionally on Update 320 if (s.Mode == GetModeGet && !found) || (s.Mode == GetModeUpdate) { 321 resp, err := s.lookupModuleVersions(mod) 322 if err != nil { 323 return rec, err 324 } 325 326 if len(resp.Modules) == 0 { 327 return rec, fmt.Errorf("module %q not found in registry", mod.Module()) 328 } 329 330 match, err := newestVersion(resp.Modules[0].Versions, constraint) 331 if err != nil { 332 return rec, err 333 } 334 335 if match == nil { 336 return rec, fmt.Errorf("no versions for %q found matching %q", mod.Module(), constraint) 337 } 338 339 rec.Version = match.Version 340 341 rec.url, err = s.lookupModuleLocation(mod, rec.Version) 342 if err != nil { 343 return rec, err 344 } 345 346 s.output(fmt.Sprintf(" Found version %s of %s on %s", rec.Version, mod.Module(), mod.RawHost.Display())) 347 348 } 349 return rec, nil 350 }