github.com/trawler/terraform@v0.10.8-0.20171106022149-4b1c7a1d9b48/config/module/storage.go (about)

     1  package module
     2  
     3  import (
     4  	"encoding/json"
     5  	"fmt"
     6  	"io/ioutil"
     7  	"log"
     8  	"os"
     9  	"path/filepath"
    10  
    11  	getter "github.com/hashicorp/go-getter"
    12  	"github.com/hashicorp/terraform/registry/regsrc"
    13  	"github.com/hashicorp/terraform/svchost/auth"
    14  	"github.com/hashicorp/terraform/svchost/disco"
    15  	"github.com/mitchellh/cli"
    16  )
    17  
    18  const manifestName = "modules.json"
    19  
    20  // moduleManifest is the serialization structure used to record the stored
    21  // module's metadata.
    22  type moduleManifest struct {
    23  	Modules []moduleRecord
    24  }
    25  
    26  // moduleRecords represents the stored module's metadata.
    27  // This is compared for equality using '==', so all fields needs to remain
    28  // comparable.
    29  type moduleRecord struct {
    30  	// Source is the module source string from the config, minus any
    31  	// subdirectory.
    32  	Source string
    33  
    34  	// Key is the locally unique identifier for this module.
    35  	Key string
    36  
    37  	// Version is the exact version string for the stored module.
    38  	Version string
    39  
    40  	// Dir is the directory name returned by the FileStorage. This is what
    41  	// allows us to correlate a particular module version with the location on
    42  	// disk.
    43  	Dir string
    44  
    45  	// Root is the root directory containing the module. If the module is
    46  	// unpacked from an archive, and not located in the root directory, this is
    47  	// used to direct the loader to the correct subdirectory. This is
    48  	// independent from any subdirectory in the original source string, which
    49  	// may traverse further into the module tree.
    50  	Root string
    51  
    52  	// url is the location of the module source
    53  	url string
    54  
    55  	// Registry is true if this module is sourced from a registry
    56  	registry bool
    57  }
    58  
    59  // Storage implements methods to manage the storage of modules.
    60  // This is used by Tree.Load to query registries, authenticate requests, and
    61  // store modules locally.
    62  type Storage struct {
    63  	// StorageDir is the full path to the directory where all modules will be
    64  	// stored.
    65  	StorageDir string
    66  	// Services is a required *disco.Disco, which may have services and
    67  	// credentials pre-loaded.
    68  	Services *disco.Disco
    69  	// Creds optionally provides credentials for communicating with service
    70  	// providers.
    71  	Creds auth.CredentialsSource
    72  	// Ui is an optional cli.Ui for user output
    73  	Ui cli.Ui
    74  	// Mode is the GetMode that will be used for various operations.
    75  	Mode GetMode
    76  }
    77  
    78  func NewStorage(dir string, services *disco.Disco, creds auth.CredentialsSource) *Storage {
    79  	s := &Storage{
    80  		StorageDir: dir,
    81  		Services:   services,
    82  		Creds:      creds,
    83  	}
    84  
    85  	// make sure this isn't nil
    86  	if s.Services == nil {
    87  		s.Services = disco.NewDisco()
    88  	}
    89  	return s
    90  }
    91  
    92  // loadManifest returns the moduleManifest file from the parent directory.
    93  func (s Storage) loadManifest() (moduleManifest, error) {
    94  	manifest := moduleManifest{}
    95  
    96  	manifestPath := filepath.Join(s.StorageDir, manifestName)
    97  	data, err := ioutil.ReadFile(manifestPath)
    98  	if err != nil && !os.IsNotExist(err) {
    99  		return manifest, err
   100  	}
   101  
   102  	if len(data) == 0 {
   103  		return manifest, nil
   104  	}
   105  
   106  	if err := json.Unmarshal(data, &manifest); err != nil {
   107  		return manifest, err
   108  	}
   109  	return manifest, nil
   110  }
   111  
   112  // Store the location of the module, along with the version used and the module
   113  // root directory. The storage method loads the entire file and rewrites it
   114  // each time. This is only done a few times during init, so efficiency is
   115  // not a concern.
   116  func (s Storage) recordModule(rec moduleRecord) error {
   117  	manifest, err := s.loadManifest()
   118  	if err != nil {
   119  		// if there was a problem with the file, we will attempt to write a new
   120  		// one. Any non-data related error should surface there.
   121  		log.Printf("[WARN] error reading module manifest: %s", err)
   122  	}
   123  
   124  	// do nothing if we already have the exact module
   125  	for i, stored := range manifest.Modules {
   126  		if rec == stored {
   127  			return nil
   128  		}
   129  
   130  		// they are not equal, but if the storage path is the same we need to
   131  		// remove this rec to be replaced.
   132  		if rec.Dir == stored.Dir {
   133  			manifest.Modules[i] = manifest.Modules[len(manifest.Modules)-1]
   134  			manifest.Modules = manifest.Modules[:len(manifest.Modules)-1]
   135  			break
   136  		}
   137  	}
   138  
   139  	manifest.Modules = append(manifest.Modules, rec)
   140  
   141  	js, err := json.Marshal(manifest)
   142  	if err != nil {
   143  		panic(err)
   144  	}
   145  
   146  	manifestPath := filepath.Join(s.StorageDir, manifestName)
   147  	return ioutil.WriteFile(manifestPath, js, 0644)
   148  }
   149  
   150  // load the manifest from dir, and return all module versions matching the
   151  // provided source. Records with no version info will be skipped, as they need
   152  // to be uniquely identified by other means.
   153  func (s Storage) moduleVersions(source string) ([]moduleRecord, error) {
   154  	manifest, err := s.loadManifest()
   155  	if err != nil {
   156  		return manifest.Modules, err
   157  	}
   158  
   159  	var matching []moduleRecord
   160  
   161  	for _, m := range manifest.Modules {
   162  		if m.Source == source && m.Version != "" {
   163  			log.Printf("[DEBUG] found local version %q for module %s", m.Version, m.Source)
   164  			matching = append(matching, m)
   165  		}
   166  	}
   167  
   168  	return matching, nil
   169  }
   170  
   171  func (s Storage) moduleDir(key string) (string, error) {
   172  	manifest, err := s.loadManifest()
   173  	if err != nil {
   174  		return "", err
   175  	}
   176  
   177  	for _, m := range manifest.Modules {
   178  		if m.Key == key {
   179  			return m.Dir, nil
   180  		}
   181  	}
   182  
   183  	return "", nil
   184  }
   185  
   186  // return only the root directory of the module stored in dir.
   187  func (s Storage) getModuleRoot(dir string) (string, error) {
   188  	manifest, err := s.loadManifest()
   189  	if err != nil {
   190  		return "", err
   191  	}
   192  
   193  	for _, mod := range manifest.Modules {
   194  		if mod.Dir == dir {
   195  			return mod.Root, nil
   196  		}
   197  	}
   198  	return "", nil
   199  }
   200  
   201  // record only the Root directory for the module stored at dir.
   202  func (s Storage) recordModuleRoot(dir, root string) error {
   203  	rec := moduleRecord{
   204  		Dir:  dir,
   205  		Root: root,
   206  	}
   207  
   208  	return s.recordModule(rec)
   209  }
   210  
   211  func (s Storage) output(msg string) {
   212  	if s.Ui == nil || s.Mode == GetModeNone {
   213  		return
   214  	}
   215  	s.Ui.Output(msg)
   216  }
   217  
   218  func (s Storage) getStorage(key string, src string) (string, bool, error) {
   219  	storage := &getter.FolderStorage{
   220  		StorageDir: s.StorageDir,
   221  	}
   222  
   223  	log.Printf("[DEBUG] fetching module from %s", src)
   224  
   225  	// Get the module with the level specified if we were told to.
   226  	if s.Mode > GetModeNone {
   227  		log.Printf("[DEBUG] fetching %q with key %q", src, key)
   228  		if err := storage.Get(key, src, s.Mode == GetModeUpdate); err != nil {
   229  			return "", false, err
   230  		}
   231  	}
   232  
   233  	// Get the directory where the module is.
   234  	dir, found, err := storage.Dir(key)
   235  	log.Printf("[DEBUG] found %q in %q: %t", src, dir, found)
   236  	return dir, found, err
   237  }
   238  
   239  // find a stored module that's not from a registry
   240  func (s Storage) findModule(key string) (string, error) {
   241  	if s.Mode == GetModeUpdate {
   242  		return "", nil
   243  	}
   244  
   245  	return s.moduleDir(key)
   246  }
   247  
   248  // GetModule fetches a module source into the specified directory. This is used
   249  // as a convenience function by the CLI to initialize a configuration.
   250  func (s Storage) GetModule(dst, src string) error {
   251  	// reset this in case the caller was going to re-use it
   252  	mode := s.Mode
   253  	s.Mode = GetModeUpdate
   254  	defer func() {
   255  		s.Mode = mode
   256  	}()
   257  
   258  	rec, err := s.findRegistryModule(src, anyVersion)
   259  	if err != nil {
   260  		return err
   261  	}
   262  
   263  	pwd, err := os.Getwd()
   264  	if err != nil {
   265  		return err
   266  	}
   267  
   268  	source := rec.url
   269  	if source == "" {
   270  		source, err = getter.Detect(src, pwd, getter.Detectors)
   271  		if err != nil {
   272  			return fmt.Errorf("module %s: %s", src, err)
   273  		}
   274  	}
   275  
   276  	if source == "" {
   277  		return fmt.Errorf("module %q not found", src)
   278  	}
   279  
   280  	return GetCopy(dst, source)
   281  }
   282  
   283  // find a registry module
   284  func (s Storage) findRegistryModule(mSource, constraint string) (moduleRecord, error) {
   285  	rec := moduleRecord{
   286  		Source: mSource,
   287  	}
   288  	// detect if we have a registry source
   289  	mod, err := regsrc.ParseModuleSource(mSource)
   290  	switch err {
   291  	case nil:
   292  		//ok
   293  	case regsrc.ErrInvalidModuleSource:
   294  		return rec, nil
   295  	default:
   296  		return rec, err
   297  	}
   298  	rec.registry = true
   299  
   300  	log.Printf("[TRACE] %q is a registry module", mod.Module())
   301  
   302  	versions, err := s.moduleVersions(mod.String())
   303  	if err != nil {
   304  		log.Printf("[ERROR] error looking up versions for %q: %s", mod.Module(), err)
   305  		return rec, err
   306  	}
   307  
   308  	match, err := newestRecord(versions, constraint)
   309  	if err != nil {
   310  		log.Printf("[INFO] no matching version for %q<%s>, %s", mod.Module(), constraint, err)
   311  	}
   312  	log.Printf("[DEBUG] matched %q version %s for %s", mod, match.Version, constraint)
   313  
   314  	rec.Dir = match.Dir
   315  	rec.Version = match.Version
   316  	found := rec.Dir != ""
   317  
   318  	// we need to lookup available versions
   319  	// Only on Get if it's not found, on unconditionally on Update
   320  	if (s.Mode == GetModeGet && !found) || (s.Mode == GetModeUpdate) {
   321  		resp, err := s.lookupModuleVersions(mod)
   322  		if err != nil {
   323  			return rec, err
   324  		}
   325  
   326  		if len(resp.Modules) == 0 {
   327  			return rec, fmt.Errorf("module %q not found in registry", mod.Module())
   328  		}
   329  
   330  		match, err := newestVersion(resp.Modules[0].Versions, constraint)
   331  		if err != nil {
   332  			return rec, err
   333  		}
   334  
   335  		if match == nil {
   336  			return rec, fmt.Errorf("no versions for %q found matching %q", mod.Module(), constraint)
   337  		}
   338  
   339  		rec.Version = match.Version
   340  
   341  		rec.url, err = s.lookupModuleLocation(mod, rec.Version)
   342  		if err != nil {
   343  			return rec, err
   344  		}
   345  
   346  		s.output(fmt.Sprintf("  Found version %s of %s on %s", rec.Version, mod.Module(), mod.RawHost.Display()))
   347  
   348  	}
   349  	return rec, nil
   350  }