github.com/hhrutter/nomad@v0.6.0-rc2.0.20170723054333-80c4b03f0705/client/getter/getter_test.go (about)

     1  package getter
     2  
     3  import (
     4  	"fmt"
     5  	"io/ioutil"
     6  	"net/http"
     7  	"net/http/httptest"
     8  	"os"
     9  	"path/filepath"
    10  	"reflect"
    11  	"strings"
    12  	"testing"
    13  
    14  	"github.com/hashicorp/nomad/client/driver/env"
    15  	"github.com/hashicorp/nomad/nomad/mock"
    16  	"github.com/hashicorp/nomad/nomad/structs"
    17  )
    18  
    19  // fakeReplacer is a noop version of env.TaskEnv.ReplanceEnv
    20  type fakeReplacer struct{}
    21  
    22  func (fakeReplacer) ReplaceEnv(s string) string {
    23  	return s
    24  }
    25  
    26  var taskEnv = fakeReplacer{}
    27  
    28  func TestGetArtifact_FileAndChecksum(t *testing.T) {
    29  	// Create the test server hosting the file to download
    30  	ts := httptest.NewServer(http.FileServer(http.Dir(filepath.Dir("./test-fixtures/"))))
    31  	defer ts.Close()
    32  
    33  	// Create a temp directory to download into
    34  	taskDir, err := ioutil.TempDir("", "nomad-test")
    35  	if err != nil {
    36  		t.Fatalf("failed to make temp directory: %v", err)
    37  	}
    38  	defer os.RemoveAll(taskDir)
    39  
    40  	// Create the artifact
    41  	file := "test.sh"
    42  	artifact := &structs.TaskArtifact{
    43  		GetterSource: fmt.Sprintf("%s/%s", ts.URL, file),
    44  		GetterOptions: map[string]string{
    45  			"checksum": "md5:bce963762aa2dbfed13caf492a45fb72",
    46  		},
    47  	}
    48  
    49  	// Download the artifact
    50  	if err := GetArtifact(taskEnv, artifact, taskDir); err != nil {
    51  		t.Fatalf("GetArtifact failed: %v", err)
    52  	}
    53  
    54  	// Verify artifact exists
    55  	if _, err := os.Stat(filepath.Join(taskDir, file)); err != nil {
    56  		t.Fatalf("file not found: %s", err)
    57  	}
    58  }
    59  
    60  func TestGetArtifact_File_RelativeDest(t *testing.T) {
    61  	// Create the test server hosting the file to download
    62  	ts := httptest.NewServer(http.FileServer(http.Dir(filepath.Dir("./test-fixtures/"))))
    63  	defer ts.Close()
    64  
    65  	// Create a temp directory to download into
    66  	taskDir, err := ioutil.TempDir("", "nomad-test")
    67  	if err != nil {
    68  		t.Fatalf("failed to make temp directory: %v", err)
    69  	}
    70  	defer os.RemoveAll(taskDir)
    71  
    72  	// Create the artifact
    73  	file := "test.sh"
    74  	relative := "foo/"
    75  	artifact := &structs.TaskArtifact{
    76  		GetterSource: fmt.Sprintf("%s/%s", ts.URL, file),
    77  		GetterOptions: map[string]string{
    78  			"checksum": "md5:bce963762aa2dbfed13caf492a45fb72",
    79  		},
    80  		RelativeDest: relative,
    81  	}
    82  
    83  	// Download the artifact
    84  	if err := GetArtifact(taskEnv, artifact, taskDir); err != nil {
    85  		t.Fatalf("GetArtifact failed: %v", err)
    86  	}
    87  
    88  	// Verify artifact was downloaded to the correct path
    89  	if _, err := os.Stat(filepath.Join(taskDir, relative, file)); err != nil {
    90  		t.Fatalf("file not found: %s", err)
    91  	}
    92  }
    93  
    94  func TestGetGetterUrl_Interprolation(t *testing.T) {
    95  	// Create the artifact
    96  	artifact := &structs.TaskArtifact{
    97  		GetterSource: "${NOMAD_META_ARTIFACT}",
    98  	}
    99  
   100  	url := "foo.com"
   101  	alloc := mock.Alloc()
   102  	task := alloc.Job.TaskGroups[0].Tasks[0]
   103  	task.Meta = map[string]string{"artifact": url}
   104  	taskEnv := env.NewBuilder(mock.Node(), alloc, task, "global").Build()
   105  
   106  	act, err := getGetterUrl(taskEnv, artifact)
   107  	if err != nil {
   108  		t.Fatalf("getGetterUrl() failed: %v", err)
   109  	}
   110  
   111  	if act != url {
   112  		t.Fatalf("getGetterUrl() returned %q; want %q", act, url)
   113  	}
   114  }
   115  
   116  func TestGetArtifact_InvalidChecksum(t *testing.T) {
   117  	// Create the test server hosting the file to download
   118  	ts := httptest.NewServer(http.FileServer(http.Dir(filepath.Dir("./test-fixtures/"))))
   119  	defer ts.Close()
   120  
   121  	// Create a temp directory to download into
   122  	taskDir, err := ioutil.TempDir("", "nomad-test")
   123  	if err != nil {
   124  		t.Fatalf("failed to make temp directory: %v", err)
   125  	}
   126  	defer os.RemoveAll(taskDir)
   127  
   128  	// Create the artifact with an incorrect checksum
   129  	file := "test.sh"
   130  	artifact := &structs.TaskArtifact{
   131  		GetterSource: fmt.Sprintf("%s/%s", ts.URL, file),
   132  		GetterOptions: map[string]string{
   133  			"checksum": "md5:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
   134  		},
   135  	}
   136  
   137  	// Download the artifact and expect an error
   138  	if err := GetArtifact(taskEnv, artifact, taskDir); err == nil {
   139  		t.Fatalf("GetArtifact should have failed")
   140  	}
   141  }
   142  
   143  func createContents(basedir string, fileContents map[string]string, t *testing.T) {
   144  	for relPath, content := range fileContents {
   145  		folder := basedir
   146  		if strings.Index(relPath, "/") != -1 {
   147  			// Create the folder.
   148  			folder = filepath.Join(basedir, filepath.Dir(relPath))
   149  			if err := os.Mkdir(folder, 0777); err != nil {
   150  				t.Fatalf("failed to make directory: %v", err)
   151  			}
   152  		}
   153  
   154  		// Create a file in the existing folder.
   155  		file := filepath.Join(folder, filepath.Base(relPath))
   156  		if err := ioutil.WriteFile(file, []byte(content), 0777); err != nil {
   157  			t.Fatalf("failed to write data to file %v: %v", file, err)
   158  		}
   159  	}
   160  }
   161  
   162  func checkContents(basedir string, fileContents map[string]string, t *testing.T) {
   163  	for relPath, content := range fileContents {
   164  		path := filepath.Join(basedir, relPath)
   165  		actual, err := ioutil.ReadFile(path)
   166  		if err != nil {
   167  			t.Fatalf("failed to read file %q: %v", path, err)
   168  		}
   169  
   170  		if !reflect.DeepEqual(actual, []byte(content)) {
   171  			t.Fatalf("%q: expected %q; got %q", path, content, string(actual))
   172  		}
   173  	}
   174  }
   175  
   176  func TestGetArtifact_Archive(t *testing.T) {
   177  	// Create the test server hosting the file to download
   178  	ts := httptest.NewServer(http.FileServer(http.Dir(filepath.Dir("./test-fixtures/"))))
   179  	defer ts.Close()
   180  
   181  	// Create a temp directory to download into and create some of the same
   182  	// files that exist in the artifact to ensure they are overridden
   183  	taskDir, err := ioutil.TempDir("", "nomad-test")
   184  	if err != nil {
   185  		t.Fatalf("failed to make temp directory: %v", err)
   186  	}
   187  	defer os.RemoveAll(taskDir)
   188  
   189  	create := map[string]string{
   190  		"exist/my.config": "to be replaced",
   191  		"untouched":       "existing top-level",
   192  	}
   193  	createContents(taskDir, create, t)
   194  
   195  	file := "archive.tar.gz"
   196  	artifact := &structs.TaskArtifact{
   197  		GetterSource: fmt.Sprintf("%s/%s", ts.URL, file),
   198  		GetterOptions: map[string]string{
   199  			"checksum": "sha1:20bab73c72c56490856f913cf594bad9a4d730f6",
   200  		},
   201  	}
   202  
   203  	if err := GetArtifact(taskEnv, artifact, taskDir); err != nil {
   204  		t.Fatalf("GetArtifact failed: %v", err)
   205  	}
   206  
   207  	// Verify the unarchiving overrode files properly.
   208  	expected := map[string]string{
   209  		"untouched":       "existing top-level",
   210  		"exist/my.config": "hello world\n",
   211  		"new/my.config":   "hello world\n",
   212  		"test.sh":         "sleep 1\n",
   213  	}
   214  	checkContents(taskDir, expected, t)
   215  }
   216  
   217  func TestGetGetterUrl_Queries(t *testing.T) {
   218  	cases := []struct {
   219  		name     string
   220  		artifact *structs.TaskArtifact
   221  		output   string
   222  	}{
   223  		{
   224  			name: "adds query parameters",
   225  			artifact: &structs.TaskArtifact{
   226  				GetterSource: "https://foo.com?test=1",
   227  				GetterOptions: map[string]string{
   228  					"foo": "bar",
   229  					"bam": "boom",
   230  				},
   231  			},
   232  			output: "https://foo.com?bam=boom&foo=bar&test=1",
   233  		},
   234  		{
   235  			name: "git without http",
   236  			artifact: &structs.TaskArtifact{
   237  				GetterSource: "github.com/hashicorp/nomad",
   238  				GetterOptions: map[string]string{
   239  					"ref": "abcd1234",
   240  				},
   241  			},
   242  			output: "github.com/hashicorp/nomad?ref=abcd1234",
   243  		},
   244  		{
   245  			name: "git using ssh",
   246  			artifact: &structs.TaskArtifact{
   247  				GetterSource: "git@github.com:hashicorp/nomad?sshkey=1",
   248  				GetterOptions: map[string]string{
   249  					"ref": "abcd1234",
   250  				},
   251  			},
   252  			output: "git@github.com:hashicorp/nomad?ref=abcd1234&sshkey=1",
   253  		},
   254  		{
   255  			name: "s3 scheme 1",
   256  			artifact: &structs.TaskArtifact{
   257  				GetterSource: "s3::https://s3.amazonaws.com/bucket/foo",
   258  				GetterOptions: map[string]string{
   259  					"aws_access_key_id": "abcd1234",
   260  				},
   261  			},
   262  			output: "s3::https://s3.amazonaws.com/bucket/foo?aws_access_key_id=abcd1234",
   263  		},
   264  		{
   265  			name: "s3 scheme 2",
   266  			artifact: &structs.TaskArtifact{
   267  				GetterSource: "s3::https://s3-eu-west-1.amazonaws.com/bucket/foo",
   268  				GetterOptions: map[string]string{
   269  					"aws_access_key_id": "abcd1234",
   270  				},
   271  			},
   272  			output: "s3::https://s3-eu-west-1.amazonaws.com/bucket/foo?aws_access_key_id=abcd1234",
   273  		},
   274  		{
   275  			name: "s3 scheme 3",
   276  			artifact: &structs.TaskArtifact{
   277  				GetterSource: "bucket.s3.amazonaws.com/foo",
   278  				GetterOptions: map[string]string{
   279  					"aws_access_key_id": "abcd1234",
   280  				},
   281  			},
   282  			output: "bucket.s3.amazonaws.com/foo?aws_access_key_id=abcd1234",
   283  		},
   284  		{
   285  			name: "s3 scheme 4",
   286  			artifact: &structs.TaskArtifact{
   287  				GetterSource: "bucket.s3-eu-west-1.amazonaws.com/foo/bar",
   288  				GetterOptions: map[string]string{
   289  					"aws_access_key_id": "abcd1234",
   290  				},
   291  			},
   292  			output: "bucket.s3-eu-west-1.amazonaws.com/foo/bar?aws_access_key_id=abcd1234",
   293  		},
   294  		{
   295  			name: "local file",
   296  			artifact: &structs.TaskArtifact{
   297  				GetterSource: "/foo/bar",
   298  			},
   299  			output: "/foo/bar",
   300  		},
   301  	}
   302  
   303  	for _, c := range cases {
   304  		t.Run(c.name, func(t *testing.T) {
   305  			act, err := getGetterUrl(taskEnv, c.artifact)
   306  			if err != nil {
   307  				t.Fatalf("want %q; got err %v", c.output, err)
   308  			} else if act != c.output {
   309  				t.Fatalf("want %q; got %q", c.output, act)
   310  			}
   311  		})
   312  	}
   313  }