github.com/hernad/nomad@v1.6.112/e2e/jobsubmissions/jobsubapi_test.go (about) 1 // Copyright (c) HashiCorp, Inc. 2 // SPDX-License-Identifier: MPL-2.0 3 4 package jobsubmissions 5 6 import ( 7 "fmt" 8 "os" 9 "path/filepath" 10 "strings" 11 "testing" 12 13 "github.com/hernad/nomad/api" 14 "github.com/hernad/nomad/e2e/acl" 15 "github.com/hernad/nomad/e2e/e2eutil" 16 "github.com/hernad/nomad/helper/uuid" 17 "github.com/shoenig/test/must" 18 ) 19 20 func TestJobSubmissionAPI(t *testing.T) { 21 nomad := e2eutil.NomadClient(t) 22 23 e2eutil.WaitForLeader(t, nomad) 24 e2eutil.WaitForNodesReady(t, nomad, 1) 25 26 t.Run("testParseAPI", testParseAPI) 27 t.Run("testRunCLIVarFlags", testRunCLIVarFlags) 28 t.Run("testSubmissionACL", testSubmissionACL) 29 t.Run("testMaxSize", testMaxSize) 30 t.Run("testReversion", testReversion) 31 t.Run("testVarFiles", testVarFiles) 32 } 33 34 func testParseAPI(t *testing.T) { 35 nomad := e2eutil.NomadClient(t) 36 37 jobID := "job-sub-parse-" + uuid.Short() 38 jobIDs := []string{jobID} 39 t.Cleanup(e2eutil.MaybeCleanupJobsAndGC(&jobIDs)) 40 41 spec, err := os.ReadFile("input/xyz.hcl") 42 must.NoError(t, err) 43 44 job, err := nomad.Jobs().ParseHCLOpts(&api.JobsParseRequest{ 45 JobHCL: string(spec), 46 HCLv1: false, 47 Variables: "X=\"baz\" \n Y=50 \n Z=true \n", 48 Canonicalize: true, 49 }) 50 must.NoError(t, err) 51 args := job.TaskGroups[0].Tasks[0].Config["args"] 52 must.Eq(t, []any{"X baz, Y 50, Z true"}, args.([]any)) 53 } 54 55 func testRunCLIVarFlags(t *testing.T) { 56 nomad := e2eutil.NomadClient(t) 57 58 jobID := "job-sub-cli-" + uuid.Short() 59 jobIDs := []string{jobID} 60 t.Cleanup(e2eutil.MaybeCleanupJobsAndGC(&jobIDs)) 61 62 // register job via cli with var arguments 63 err := e2eutil.RegisterWithArgs(jobID, "input/xyz.hcl", "-var=X=foo", "-var=Y=42", "-var=Z=true") 64 must.NoError(t, err) 65 66 // find our alloc id 67 allocID := e2eutil.SingleAllocID(t, jobID, "default", 0) 68 69 // wait for alloc to complete 70 _ = e2eutil.WaitForAllocStopped(t, nomad, allocID) 71 72 // inspect alloc logs making sure our variables got set 73 out, err := e2eutil.AllocLogs(allocID, "", e2eutil.LogsStdOut) 74 must.NoError(t, err) 75 must.Eq(t, "X foo, Y 42, Z true\n", out) 76 77 // check the submission api 78 sub, _, err := nomad.Jobs().Submission(jobID, 0, &api.QueryOptions{ 79 Region: "global", 80 Namespace: "default", 81 }) 82 must.NoError(t, err) 83 must.Eq(t, "hcl2", sub.Format) 84 must.NotEq(t, "", sub.Source) 85 must.Eq(t, map[string]string{"X": "foo", "Y": "42", "Z": "true"}, sub.VariableFlags) 86 must.Eq(t, "", sub.Variables) 87 88 // register job again with different var arguments 89 err = e2eutil.RegisterWithArgs(jobID, "input/xyz.hcl", "-var=X=bar", "-var=Y=99", "-var=Z=false") 90 must.NoError(t, err) 91 92 // find our alloc id 93 allocID = e2eutil.SingleAllocID(t, jobID, "default", 1) 94 95 // wait for alloc to complete 96 _ = e2eutil.WaitForAllocStopped(t, nomad, allocID) 97 98 // inspect alloc logs making sure our new variables got set 99 out, err = e2eutil.AllocLogs(allocID, "", e2eutil.LogsStdOut) 100 must.NoError(t, err) 101 must.Eq(t, "X bar, Y 99, Z false\n", out) 102 103 // check the submission api for v1 104 sub, _, err = nomad.Jobs().Submission(jobID, 1, &api.QueryOptions{ 105 Region: "global", 106 Namespace: "default", 107 }) 108 must.NoError(t, err) 109 must.Eq(t, "hcl2", sub.Format) 110 must.NotEq(t, "", sub.Source) 111 must.Eq(t, map[string]string{"X": "bar", "Y": "99", "Z": "false"}, sub.VariableFlags) 112 must.Eq(t, "", sub.Variables) 113 114 // check the submission api for v0 (make sure we still have it) 115 sub, _, err = nomad.Jobs().Submission(jobID, 0, &api.QueryOptions{ 116 Region: "global", 117 Namespace: "default", 118 }) 119 must.NoError(t, err) 120 must.Eq(t, "hcl2", sub.Format) 121 must.NotEq(t, "", sub.Source) 122 must.Eq(t, map[string]string{ 123 "X": "foo", 124 "Y": "42", 125 "Z": "true", 126 }, sub.VariableFlags) 127 must.Eq(t, "", sub.Variables) 128 129 // deregister the job with purge 130 e2eutil.WaitForJobStopped(t, nomad, jobID) 131 132 // check the submission api for v0 after deregister (make sure its gone) 133 sub, _, err = nomad.Jobs().Submission(jobID, 0, &api.QueryOptions{ 134 Region: "global", 135 Namespace: "default", 136 }) 137 must.ErrorContains(t, err, "job source not found") 138 must.Nil(t, sub) 139 } 140 141 func testSubmissionACL(t *testing.T) { 142 nomad := e2eutil.NomadClient(t) 143 144 // setup an acl cleanup thing 145 aclCleanup := acl.NewCleanup() 146 defer aclCleanup.Run(t, nomad) 147 148 // create a namespace for ourselves 149 myNamespaceName := "submission-acl-" + uuid.Short() 150 namespaceClient := nomad.Namespaces() 151 _, err := namespaceClient.Register(&api.Namespace{ 152 Name: myNamespaceName, 153 }, &api.WriteOptions{ 154 Region: "global", 155 }) 156 must.NoError(t, err) 157 aclCleanup.Add(myNamespaceName, acl.NamespaceTestResourceType) 158 159 // create a namespace for a token that will be blocked 160 otherNamespaceName := "submission-other-acl-" + uuid.Short() 161 _, err = namespaceClient.Register(&api.Namespace{ 162 Name: otherNamespaceName, 163 }, &api.WriteOptions{ 164 Region: "global", 165 }) 166 must.NoError(t, err) 167 aclCleanup.Add(otherNamespaceName, acl.NamespaceTestResourceType) 168 169 // create an ACL policy to read in our namespace 170 myNamespacePolicy := api.ACLPolicy{ 171 Name: "submission-acl-" + uuid.Short(), 172 Rules: `namespace "` + myNamespaceName + `" {policy = "write"}`, 173 Description: "This namespace is for Job Submissions e2e testing", 174 } 175 _, err = nomad.ACLPolicies().Upsert(&myNamespacePolicy, nil) 176 must.NoError(t, err) 177 aclCleanup.Add(myNamespacePolicy.Name, acl.ACLPolicyTestResourceType) 178 179 // create an ACL policy to read in the other namespace 180 otherNamespacePolicy := api.ACLPolicy{ 181 Name: "submission-other-acl-" + uuid.Short(), 182 Rules: `namespace "` + otherNamespaceName + `" {policy = "read"}`, 183 Description: "This is another namespace for Job Submissions e2e testing", 184 } 185 _, err = nomad.ACLPolicies().Upsert(&otherNamespacePolicy, nil) 186 must.NoError(t, err) 187 aclCleanup.Add(otherNamespacePolicy.Name, acl.ACLPolicyTestResourceType) 188 189 // create a token that can read in our namespace 190 aclTokensClient := nomad.ACLTokens() 191 myToken, _, err := aclTokensClient.Create(&api.ACLToken{ 192 Name: "submission-my-read-token-" + uuid.Short(), 193 Type: "client", 194 Policies: []string{myNamespacePolicy.Name}, 195 }, &api.WriteOptions{ 196 Region: "global", 197 Namespace: myNamespaceName, 198 }) 199 must.NoError(t, err) 200 aclCleanup.Add(myToken.AccessorID, acl.ACLTokenTestResourceType) 201 202 // create a token that can read in the other namespace 203 otherToken, _, err := aclTokensClient.Create(&api.ACLToken{ 204 Name: "submission-other-read-token-" + uuid.Short(), 205 Type: "client", 206 Policies: []string{otherNamespacePolicy.Name}, 207 }, &api.WriteOptions{ 208 Region: "global", 209 Namespace: otherNamespaceName, 210 }) 211 must.NoError(t, err) 212 aclCleanup.Add(otherToken.AccessorID, acl.ACLTokenTestResourceType) 213 214 // prepare to submit a job 215 jobID := "job-sub-cli-" + uuid.Short() 216 jobIDs := []string{jobID} 217 t.Cleanup(e2eutil.MaybeCleanupJobsAndGC(&jobIDs)) 218 219 // register job via cli with var arguments (using management token) 220 err = e2eutil.RegisterWithArgs(jobID, "input/xyz.hcl", "-namespace", myNamespaceName, "-var=X=foo", "-var=Y=42", "-var=Z=true") 221 must.NoError(t, err) 222 223 // find our alloc id 224 allocID := e2eutil.SingleAllocID(t, jobID, myNamespaceName, 0) 225 226 // wait for alloc to complete 227 _ = e2eutil.WaitForAllocStopped(t, nomad, allocID) 228 229 // inspect alloc logs making sure our variables got set 230 out, err := e2eutil.AllocLogs(allocID, myNamespaceName, e2eutil.LogsStdOut) 231 must.NoError(t, err) 232 must.Eq(t, "X foo, Y 42, Z true\n", out) 233 234 // get submission using my token 235 sub, _, err := nomad.Jobs().Submission(jobID, 0, &api.QueryOptions{ 236 Region: "global", 237 Namespace: myNamespaceName, 238 AuthToken: myToken.SecretID, 239 }) 240 must.NoError(t, err) 241 must.Eq(t, "hcl2", sub.Format) 242 must.NotEq(t, "", sub.Source) 243 must.Eq(t, map[string]string{"X": "foo", "Y": "42", "Z": "true"}, sub.VariableFlags) 244 must.Eq(t, "", sub.Variables) 245 246 // get submission using other token (fail) 247 sub, _, err = nomad.Jobs().Submission(jobID, 0, &api.QueryOptions{ 248 Region: "global", 249 Namespace: myNamespaceName, 250 AuthToken: otherToken.SecretID, 251 }) 252 must.ErrorContains(t, err, "Permission denied") 253 must.Nil(t, sub) 254 } 255 256 func testMaxSize(t *testing.T) { 257 jobID := "job-sub-max-" + uuid.Short() 258 jobIDs := []string{jobID} 259 t.Cleanup(e2eutil.MaybeCleanupJobsAndGC(&jobIDs)) 260 261 // modify huge.hcl to exceed the default 1 megabyte limit 262 b, err := os.ReadFile("input/huge.hcl") 263 must.NoError(t, err) 264 huge := strings.Replace(string(b), "REPLACE", strings.Repeat("A", 2e6), 1) 265 tmpDir := t.TempDir() 266 hugeFile := filepath.Join(tmpDir, "huge.hcl") 267 err = os.WriteFile(hugeFile, []byte(huge), 0o644) 268 must.NoError(t, err) 269 270 // register the huge job file and expect a warning 271 output, err := e2eutil.RegisterGetOutput(jobID, hugeFile) 272 must.NoError(t, err) 273 must.StrContains(t, output, "job source size of 2.0 MB exceeds maximum of 1.0 MB and will be discarded") 274 275 // check the submission api making sure it is not there 276 nomad := e2eutil.NomadClient(t) 277 sub, _, err := nomad.Jobs().Submission(jobID, 0, &api.QueryOptions{ 278 Region: "global", 279 Namespace: "default", 280 }) 281 must.ErrorContains(t, err, "job source not found") 282 must.Nil(t, sub) 283 } 284 285 func testReversion(t *testing.T) { 286 nomad := e2eutil.NomadClient(t) 287 288 jobID := "job-sub-reversion-" + uuid.Short() 289 jobIDs := []string{jobID} 290 t.Cleanup(e2eutil.MaybeCleanupJobsAndGC(&jobIDs)) 291 292 // register job 3 times 293 for i := 0; i < 3; i++ { 294 yVar := fmt.Sprintf("-var=Y=%d", i) 295 296 // register the job 297 err := e2eutil.RegisterWithArgs(jobID, "input/xyz.hcl", "-var=X=hello", yVar, "-var=Z=false") 298 must.NoError(t, err) 299 300 // find our alloc id 301 allocID := e2eutil.SingleAllocID(t, jobID, "", i) 302 303 // wait for alloc to complete 304 _ = e2eutil.WaitForAllocStopped(t, nomad, allocID) 305 } 306 307 // revert the job back to version 1 308 309 err := e2eutil.Revert(jobID, "input/xyz.hcl", 1) 310 must.NoError(t, err) 311 312 // there should be a submission for version 3, and it should 313 // contain Y=1 as did the version 1 of the job 314 expectY := []string{"0", "1", "2", "1"} 315 for version := 0; version < 4; version++ { 316 sub, _, err := nomad.Jobs().Submission(jobID, version, &api.QueryOptions{ 317 Region: "global", 318 Namespace: "default", 319 }) 320 must.NoError(t, err) 321 must.Eq(t, expectY[version], sub.VariableFlags["Y"]) 322 } 323 } 324 325 func testVarFiles(t *testing.T) { 326 nomad := e2eutil.NomadClient(t) 327 328 jobID := "job-sub-var-files-" + uuid.Short() 329 jobIDs := []string{jobID} 330 t.Cleanup(e2eutil.MaybeCleanupJobsAndGC(&jobIDs)) 331 332 // register the xyz job using x.hcl y.hcl z.hcl var files 333 err := e2eutil.RegisterWithArgs( 334 jobID, 335 "input/xyz.hcl", 336 "-var-file=input/x.hcl", 337 "-var-file=input/y.hcl", 338 "-var-file=input/z.hcl", 339 ) 340 must.NoError(t, err) 341 342 const version = 0 343 344 // find our alloc id 345 allocID := e2eutil.SingleAllocID(t, jobID, "", version) 346 347 // wait for alloc to complete 348 _ = e2eutil.WaitForAllocStopped(t, nomad, allocID) 349 350 // get submission 351 sub, _, err := nomad.Jobs().Submission(jobID, version, &api.QueryOptions{ 352 Region: "global", 353 Namespace: "default", 354 }) 355 must.NoError(t, err) 356 must.StrContains(t, sub.Variables, `X = "my var file x value"`) 357 must.StrContains(t, sub.Variables, `Y = 700`) 358 must.StrContains(t, sub.Variables, `Z = true`) 359 }