github.com/hhrutter/nomad@v0.6.0-rc2.0.20170723054333-80c4b03f0705/client/getter/getter_test.go (about) 1 package getter 2 3 import ( 4 "fmt" 5 "io/ioutil" 6 "net/http" 7 "net/http/httptest" 8 "os" 9 "path/filepath" 10 "reflect" 11 "strings" 12 "testing" 13 14 "github.com/hashicorp/nomad/client/driver/env" 15 "github.com/hashicorp/nomad/nomad/mock" 16 "github.com/hashicorp/nomad/nomad/structs" 17 ) 18 19 // fakeReplacer is a noop version of env.TaskEnv.ReplanceEnv 20 type fakeReplacer struct{} 21 22 func (fakeReplacer) ReplaceEnv(s string) string { 23 return s 24 } 25 26 var taskEnv = fakeReplacer{} 27 28 func TestGetArtifact_FileAndChecksum(t *testing.T) { 29 // Create the test server hosting the file to download 30 ts := httptest.NewServer(http.FileServer(http.Dir(filepath.Dir("./test-fixtures/")))) 31 defer ts.Close() 32 33 // Create a temp directory to download into 34 taskDir, err := ioutil.TempDir("", "nomad-test") 35 if err != nil { 36 t.Fatalf("failed to make temp directory: %v", err) 37 } 38 defer os.RemoveAll(taskDir) 39 40 // Create the artifact 41 file := "test.sh" 42 artifact := &structs.TaskArtifact{ 43 GetterSource: fmt.Sprintf("%s/%s", ts.URL, file), 44 GetterOptions: map[string]string{ 45 "checksum": "md5:bce963762aa2dbfed13caf492a45fb72", 46 }, 47 } 48 49 // Download the artifact 50 if err := GetArtifact(taskEnv, artifact, taskDir); err != nil { 51 t.Fatalf("GetArtifact failed: %v", err) 52 } 53 54 // Verify artifact exists 55 if _, err := os.Stat(filepath.Join(taskDir, file)); err != nil { 56 t.Fatalf("file not found: %s", err) 57 } 58 } 59 60 func TestGetArtifact_File_RelativeDest(t *testing.T) { 61 // Create the test server hosting the file to download 62 ts := httptest.NewServer(http.FileServer(http.Dir(filepath.Dir("./test-fixtures/")))) 63 defer ts.Close() 64 65 // Create a temp directory to download into 66 taskDir, err := ioutil.TempDir("", "nomad-test") 67 if err != nil { 68 t.Fatalf("failed to make temp directory: %v", err) 69 } 70 defer os.RemoveAll(taskDir) 71 72 // Create the artifact 73 file := "test.sh" 74 relative := "foo/" 75 artifact := &structs.TaskArtifact{ 76 GetterSource: fmt.Sprintf("%s/%s", ts.URL, file), 77 GetterOptions: map[string]string{ 78 "checksum": "md5:bce963762aa2dbfed13caf492a45fb72", 79 }, 80 RelativeDest: relative, 81 } 82 83 // Download the artifact 84 if err := GetArtifact(taskEnv, artifact, taskDir); err != nil { 85 t.Fatalf("GetArtifact failed: %v", err) 86 } 87 88 // Verify artifact was downloaded to the correct path 89 if _, err := os.Stat(filepath.Join(taskDir, relative, file)); err != nil { 90 t.Fatalf("file not found: %s", err) 91 } 92 } 93 94 func TestGetGetterUrl_Interprolation(t *testing.T) { 95 // Create the artifact 96 artifact := &structs.TaskArtifact{ 97 GetterSource: "${NOMAD_META_ARTIFACT}", 98 } 99 100 url := "foo.com" 101 alloc := mock.Alloc() 102 task := alloc.Job.TaskGroups[0].Tasks[0] 103 task.Meta = map[string]string{"artifact": url} 104 taskEnv := env.NewBuilder(mock.Node(), alloc, task, "global").Build() 105 106 act, err := getGetterUrl(taskEnv, artifact) 107 if err != nil { 108 t.Fatalf("getGetterUrl() failed: %v", err) 109 } 110 111 if act != url { 112 t.Fatalf("getGetterUrl() returned %q; want %q", act, url) 113 } 114 } 115 116 func TestGetArtifact_InvalidChecksum(t *testing.T) { 117 // Create the test server hosting the file to download 118 ts := httptest.NewServer(http.FileServer(http.Dir(filepath.Dir("./test-fixtures/")))) 119 defer ts.Close() 120 121 // Create a temp directory to download into 122 taskDir, err := ioutil.TempDir("", "nomad-test") 123 if err != nil { 124 t.Fatalf("failed to make temp directory: %v", err) 125 } 126 defer os.RemoveAll(taskDir) 127 128 // Create the artifact with an incorrect checksum 129 file := "test.sh" 130 artifact := &structs.TaskArtifact{ 131 GetterSource: fmt.Sprintf("%s/%s", ts.URL, file), 132 GetterOptions: map[string]string{ 133 "checksum": "md5:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 134 }, 135 } 136 137 // Download the artifact and expect an error 138 if err := GetArtifact(taskEnv, artifact, taskDir); err == nil { 139 t.Fatalf("GetArtifact should have failed") 140 } 141 } 142 143 func createContents(basedir string, fileContents map[string]string, t *testing.T) { 144 for relPath, content := range fileContents { 145 folder := basedir 146 if strings.Index(relPath, "/") != -1 { 147 // Create the folder. 148 folder = filepath.Join(basedir, filepath.Dir(relPath)) 149 if err := os.Mkdir(folder, 0777); err != nil { 150 t.Fatalf("failed to make directory: %v", err) 151 } 152 } 153 154 // Create a file in the existing folder. 155 file := filepath.Join(folder, filepath.Base(relPath)) 156 if err := ioutil.WriteFile(file, []byte(content), 0777); err != nil { 157 t.Fatalf("failed to write data to file %v: %v", file, err) 158 } 159 } 160 } 161 162 func checkContents(basedir string, fileContents map[string]string, t *testing.T) { 163 for relPath, content := range fileContents { 164 path := filepath.Join(basedir, relPath) 165 actual, err := ioutil.ReadFile(path) 166 if err != nil { 167 t.Fatalf("failed to read file %q: %v", path, err) 168 } 169 170 if !reflect.DeepEqual(actual, []byte(content)) { 171 t.Fatalf("%q: expected %q; got %q", path, content, string(actual)) 172 } 173 } 174 } 175 176 func TestGetArtifact_Archive(t *testing.T) { 177 // Create the test server hosting the file to download 178 ts := httptest.NewServer(http.FileServer(http.Dir(filepath.Dir("./test-fixtures/")))) 179 defer ts.Close() 180 181 // Create a temp directory to download into and create some of the same 182 // files that exist in the artifact to ensure they are overridden 183 taskDir, err := ioutil.TempDir("", "nomad-test") 184 if err != nil { 185 t.Fatalf("failed to make temp directory: %v", err) 186 } 187 defer os.RemoveAll(taskDir) 188 189 create := map[string]string{ 190 "exist/my.config": "to be replaced", 191 "untouched": "existing top-level", 192 } 193 createContents(taskDir, create, t) 194 195 file := "archive.tar.gz" 196 artifact := &structs.TaskArtifact{ 197 GetterSource: fmt.Sprintf("%s/%s", ts.URL, file), 198 GetterOptions: map[string]string{ 199 "checksum": "sha1:20bab73c72c56490856f913cf594bad9a4d730f6", 200 }, 201 } 202 203 if err := GetArtifact(taskEnv, artifact, taskDir); err != nil { 204 t.Fatalf("GetArtifact failed: %v", err) 205 } 206 207 // Verify the unarchiving overrode files properly. 208 expected := map[string]string{ 209 "untouched": "existing top-level", 210 "exist/my.config": "hello world\n", 211 "new/my.config": "hello world\n", 212 "test.sh": "sleep 1\n", 213 } 214 checkContents(taskDir, expected, t) 215 } 216 217 func TestGetGetterUrl_Queries(t *testing.T) { 218 cases := []struct { 219 name string 220 artifact *structs.TaskArtifact 221 output string 222 }{ 223 { 224 name: "adds query parameters", 225 artifact: &structs.TaskArtifact{ 226 GetterSource: "https://foo.com?test=1", 227 GetterOptions: map[string]string{ 228 "foo": "bar", 229 "bam": "boom", 230 }, 231 }, 232 output: "https://foo.com?bam=boom&foo=bar&test=1", 233 }, 234 { 235 name: "git without http", 236 artifact: &structs.TaskArtifact{ 237 GetterSource: "github.com/hashicorp/nomad", 238 GetterOptions: map[string]string{ 239 "ref": "abcd1234", 240 }, 241 }, 242 output: "github.com/hashicorp/nomad?ref=abcd1234", 243 }, 244 { 245 name: "git using ssh", 246 artifact: &structs.TaskArtifact{ 247 GetterSource: "git@github.com:hashicorp/nomad?sshkey=1", 248 GetterOptions: map[string]string{ 249 "ref": "abcd1234", 250 }, 251 }, 252 output: "git@github.com:hashicorp/nomad?ref=abcd1234&sshkey=1", 253 }, 254 { 255 name: "s3 scheme 1", 256 artifact: &structs.TaskArtifact{ 257 GetterSource: "s3::https://s3.amazonaws.com/bucket/foo", 258 GetterOptions: map[string]string{ 259 "aws_access_key_id": "abcd1234", 260 }, 261 }, 262 output: "s3::https://s3.amazonaws.com/bucket/foo?aws_access_key_id=abcd1234", 263 }, 264 { 265 name: "s3 scheme 2", 266 artifact: &structs.TaskArtifact{ 267 GetterSource: "s3::https://s3-eu-west-1.amazonaws.com/bucket/foo", 268 GetterOptions: map[string]string{ 269 "aws_access_key_id": "abcd1234", 270 }, 271 }, 272 output: "s3::https://s3-eu-west-1.amazonaws.com/bucket/foo?aws_access_key_id=abcd1234", 273 }, 274 { 275 name: "s3 scheme 3", 276 artifact: &structs.TaskArtifact{ 277 GetterSource: "bucket.s3.amazonaws.com/foo", 278 GetterOptions: map[string]string{ 279 "aws_access_key_id": "abcd1234", 280 }, 281 }, 282 output: "bucket.s3.amazonaws.com/foo?aws_access_key_id=abcd1234", 283 }, 284 { 285 name: "s3 scheme 4", 286 artifact: &structs.TaskArtifact{ 287 GetterSource: "bucket.s3-eu-west-1.amazonaws.com/foo/bar", 288 GetterOptions: map[string]string{ 289 "aws_access_key_id": "abcd1234", 290 }, 291 }, 292 output: "bucket.s3-eu-west-1.amazonaws.com/foo/bar?aws_access_key_id=abcd1234", 293 }, 294 { 295 name: "local file", 296 artifact: &structs.TaskArtifact{ 297 GetterSource: "/foo/bar", 298 }, 299 output: "/foo/bar", 300 }, 301 } 302 303 for _, c := range cases { 304 t.Run(c.name, func(t *testing.T) { 305 act, err := getGetterUrl(taskEnv, c.artifact) 306 if err != nil { 307 t.Fatalf("want %q; got err %v", c.output, err) 308 } else if act != c.output { 309 t.Fatalf("want %q; got %q", c.output, act) 310 } 311 }) 312 } 313 }