github.com/kubeshop/testkube@v1.17.23/pkg/executor/output/parser_test.go (about) 1 package output 2 3 import ( 4 "testing" 5 6 "github.com/stretchr/testify/assert" 7 8 "github.com/kubeshop/testkube/pkg/api/v1/testkube" 9 ) 10 11 var exampleLogEntryLine = []byte(`{"type":"line","content":" GET http://localhost:8088/health "}`) 12 var exampleLogEntryEvent = []byte(`{"type":"event","content":"running postman/collection from testkube.Execution","obj":["{\n\t\"id\": \"blablablablabla\",\n\t\"name\": \"some-testing-exec\",\n\t\"scriptContent\": \"{\\n\\t\\\"info\\\": {\\n\\t\\t\\\"_postman_id\\\": \\\"97b67bfb-c1ca-4572-af46-06cab8f68998\\\",\\n\\t\\t\\\"name\\\": \\\"Local-API-Health\\\",\\n\\t\\t\\\"schema\\\": \\\"https://schema.getpostman.com/json/collection/v2.1.0/collection.json\\\"\\n\\t},\\n\\t\\\"item\\\": [\\n\\t\\t{\\n\\t\\t\\t\\\"name\\\": \\\"Health\\\",\\n\\t\\t\\t\\\"event\\\": [\\n\\t\\t\\t\\t{\\n\\t\\t\\t\\t\\t\\\"listen\\\": \\\"test\\\",\\n\\t\\t\\t\\t\\t\\\"script\\\": {\\n\\t\\t\\t\\t\\t\\t\\\"exec\\\": [\\n\\t\\t\\t\\t\\t\\t\\t\\\"pm.test(\\\\\\\"Status code is 200\\\\\\\", function () {\\\",\\n\\t\\t\\t\\t\\t\\t\\t\\\" pm.response.to.have.status(200);\\\",\\n\\t\\t\\t\\t\\t\\t\\t\\\"});\\\"\\n\\t\\t\\t\\t\\t\\t],\\n\\t\\t\\t\\t\\t\\t\\\"type\\\": \\\"text/javascript\\\"\\n\\t\\t\\t\\t\\t}\\n\\t\\t\\t\\t}\\n\\t\\t\\t],\\n\\t\\t\\t\\\"request\\\": {\\n\\t\\t\\t\\t\\\"method\\\": \\\"GET\\\",\\n\\t\\t\\t\\t\\\"header\\\": [],\\n\\t\\t\\t\\t\\\"url\\\": {\\n\\t\\t\\t\\t\\t\\\"raw\\\": \\\"http://localhost:8088/health\\\",\\n\\t\\t\\t\\t\\t\\\"protocol\\\": \\\"http\\\",\\n\\t\\t\\t\\t\\t\\\"host\\\": [\\n\\t\\t\\t\\t\\t\\t\\\"localhost\\\"\\n\\t\\t\\t\\t\\t],\\n\\t\\t\\t\\t\\t\\\"port\\\": \\\"8088\\\",\\n\\t\\t\\t\\t\\t\\\"path\\\": [\\n\\t\\t\\t\\t\\t\\t\\\"health\\\"\\n\\t\\t\\t\\t\\t]\\n\\t\\t\\t\\t}\\n\\t\\t\\t},\\n\\t\\t\\t\\\"response\\\": []\\n\\t\\t}\\n\\t],\\n\\t\\\"event\\\": []\\n}\"\n}"]}`) 13 var exampleLogEntryError = []byte(`{"type":"error","content":"Some error message"}`) 14 var exampleLogEntryResult = []byte(`{"type":"result","result":{"status":"failed","startTime":"2021-10-29T11:35:35.759Z","endTime":"2021-10-29T11:35:36.771Z","output":"newman\n\nLocal-API-Health\n\n→ Health\n GET http://localhost:8088/health [errored]\n connect ECONNREFUSED 127.0.0.1:8088\n 2. Status code is 200\n\n┌─────────────────────────┬──────────┬──────────┐\n│ │ executed │ failed │\n├─────────────────────────┼──────────┼──────────┤\n│ iterations │ 1 │ 0 │\n├─────────────────────────┼──────────┼──────────┤\n│ requests │ 1 │ 1 │\n├─────────────────────────┼──────────┼──────────┤\n│ test-scripts │ 1 │ 0 │\n├─────────────────────────┼──────────┼──────────┤\n│ prerequest-scripts │ 0 │ 0 │\n├─────────────────────────┼──────────┼──────────┤\n│ assertions │ 1 │ 1 │\n├─────────────────────────┴──────────┴──────────┤\n│ total run duration: 1012ms │\n├───────────────────────────────────────────────┤\n│ total data received: 0B (approx) │\n└───────────────────────────────────────────────┘\n\n # failure detail \n \n 1. Error \n connect ECONNREFUSED 127.0.0.1:8088 \n at request \n inside \"Health\" \n \n 2. AssertionError Status code is 200 \n expected { Object (id, _details, ...) } to have property 'code' \n at assertion:0 in test-script \n inside \"Health\" \n","outputType":"text/plain","errorMessage":"process error: exit status 1","steps":[{"name":"Health","duration":"0s","status":"failed","assertionResults":[{"name":"Status code is 200","status":"failed","errorMessage":"expected { Object (id, _details, ...) } to have property 'code'"}]}]}}`) 15 var exampleLogEntryNormalLogs = []byte(`{"level":"info","ts":1680606449.51631,"caller":"scraper/filesystem_extractor.go:37","msg":"walking path /data/output/report/sbadmin2-1.0.7/dist"}`) 16 17 func TestGetLogEntry(t *testing.T) { 18 t.Parallel() 19 20 t.Run("get log line", func(t *testing.T) { 21 t.Parallel() 22 23 out := GetLogEntry(exampleLogEntryLine) 24 assert.Equal(t, TypeLogLine, out.Type_) 25 }) 26 27 t.Run("get event", func(t *testing.T) { 28 t.Parallel() 29 30 out := GetLogEntry(exampleLogEntryEvent) 31 assert.Equal(t, TypeLogEvent, out.Type_) 32 }) 33 34 t.Run("get error", func(t *testing.T) { 35 t.Parallel() 36 37 out := GetLogEntry(exampleLogEntryError) 38 assert.Equal(t, TypeError, out.Type_) 39 }) 40 41 t.Run("get result", func(t *testing.T) { 42 t.Parallel() 43 44 out := GetLogEntry(exampleLogEntryResult) 45 assert.Equal(t, TypeResult, out.Type_) 46 }) 47 48 t.Run("get normal logs", func(t *testing.T) { 49 t.Parallel() 50 51 out := GetLogEntry(exampleLogEntryNormalLogs) 52 assert.Equal(t, TypeUnknown, out.Type_) 53 }) 54 } 55 56 func TestParseRunnerOutput(t *testing.T) { 57 t.Parallel() 58 59 t.Run("Invalid log", func(t *testing.T) { 60 t.Parallel() 61 62 invalidOutput := []byte(`{not a json}`) 63 result, err := ParseRunnerOutput(invalidOutput, true) 64 expectedOutput := "{not a json}\n" 65 expectedErrMessage := "wrong log type was found as last log: {not a json}" 66 67 assert.Equal(t, expectedOutput, result.Output) 68 assert.NoError(t, err) 69 assert.Equal(t, testkube.ExecutionStatusFailed, result.Status) 70 assert.Equal(t, expectedErrMessage, result.ErrorMessage) 71 }) 72 73 t.Run("Runner output with no timestamps", func(t *testing.T) { 74 t.Parallel() 75 76 var exampleOutput = []byte(` 77 {"type":"event","message":"running postman/collection from testkube.Execution","content":["{\n\t\"id\": \"blablablablabla\",\n\t\"name\": \"some-testing-exec\",\n\t\"scriptContent\": \"{\\n\\t\\\"info\\\": {\\n\\t\\t\\\"_postman_id\\\": \\\"97b67bfb-c1ca-4572-af46-06cab8f68998\\\",\\n\\t\\t\\\"name\\\": \\\"Local-API-Health\\\",\\n\\t\\t\\\"schema\\\": \\\"https://schema.getpostman.com/json/collection/v2.1.0/collection.json\\\"\\n\\t},\\n\\t\\\"item\\\": [\\n\\t\\t{\\n\\t\\t\\t\\\"name\\\": \\\"Health\\\",\\n\\t\\t\\t\\\"event\\\": [\\n\\t\\t\\t\\t{\\n\\t\\t\\t\\t\\t\\\"listen\\\": \\\"test\\\",\\n\\t\\t\\t\\t\\t\\\"script\\\": {\\n\\t\\t\\t\\t\\t\\t\\\"exec\\\": [\\n\\t\\t\\t\\t\\t\\t\\t\\\"pm.test(\\\\\\\"Status code is 200\\\\\\\", function () {\\\",\\n\\t\\t\\t\\t\\t\\t\\t\\\" pm.response.to.have.status(200);\\\",\\n\\t\\t\\t\\t\\t\\t\\t\\\"});\\\"\\n\\t\\t\\t\\t\\t\\t],\\n\\t\\t\\t\\t\\t\\t\\\"type\\\": \\\"text/javascript\\\"\\n\\t\\t\\t\\t\\t}\\n\\t\\t\\t\\t}\\n\\t\\t\\t],\\n\\t\\t\\t\\\"request\\\": {\\n\\t\\t\\t\\t\\\"method\\\": \\\"GET\\\",\\n\\t\\t\\t\\t\\\"header\\\": [],\\n\\t\\t\\t\\t\\\"url\\\": {\\n\\t\\t\\t\\t\\t\\\"raw\\\": \\\"http://localhost:8088/health\\\",\\n\\t\\t\\t\\t\\t\\\"protocol\\\": \\\"http\\\",\\n\\t\\t\\t\\t\\t\\\"host\\\": [\\n\\t\\t\\t\\t\\t\\t\\\"localhost\\\"\\n\\t\\t\\t\\t\\t],\\n\\t\\t\\t\\t\\t\\\"port\\\": \\\"8088\\\",\\n\\t\\t\\t\\t\\t\\\"path\\\": [\\n\\t\\t\\t\\t\\t\\t\\\"health\\\"\\n\\t\\t\\t\\t\\t]\\n\\t\\t\\t\\t}\\n\\t\\t\\t},\\n\\t\\t\\t\\\"response\\\": []\\n\\t\\t}\\n\\t],\\n\\t\\\"event\\\": []\\n}\"\n}"]} 78 {"type":"line","content":"newman\n\n"} 79 {"type":"line","content":"Local-API-Health\n"} 80 {"type":"line","content":"\n→ Health\n"} 81 {"type":"line","content":" GET http://localhost:8088/health "} 82 {"type":"line","content":"[errored]\n"} 83 {"type":"line","content":" connect ECONNREFUSED 127.0.0.1:8088\n"} 84 {"type":"line","content":" 2. Status code is 200\n"} 85 {"type":"line","content":"\n┌─────────────────────────┬──────────┬──────────┐\n│ │ executed │ failed │\n├─────────────────────────┼──────────┼──────────┤\n│ iterations │ 1 │ 0 │\n├─────────────────────────┼──────────┼──────────┤\n│ requests │ 1 │ 1 │\n├─────────────────────────┼──────────┼──────────┤\n│ test-scripts │ 1 │ 0 │\n├─────────────────────────┼──────────┼──────────┤\n│ prerequest-scripts │ 0 │ 0 │\n├─────────────────────────┼──────────┼──────────┤\n│ assertions │ 1 │ 1 │\n├─────────────────────────┴──────────┴──────────┤\n│ total run duration: 1012ms │\n├───────────────────────────────────────────────┤\n│ total data received: 0B (approx) │\n└───────────────────────────────────────────────┘\n"} 86 {"type":"line","content":"\n # failure detail \n \n 1. Error \n connect ECONNREFUSED 127.0.0.1:8088 \n at request \n inside \"Health\" \n \n 2. AssertionError Status code is 200 \n expected { Object (id, _details, ...) } to have property 'code' \n at assertion:0 in test-script \n inside \"Health\" \n"} 87 {"type":"result","result":{"status":"failed","startTime":"2021-10-29T11:35:35.759Z","endTime":"2021-10-29T11:35:36.771Z","output":"newman\n\nLocal-API-Health\n\n→ Health\n GET http://localhost:8088/health [errored]\n connect ECONNREFUSED 127.0.0.1:8088\n 2. Status code is 200\n\n┌─────────────────────────┬──────────┬──────────┐\n│ │ executed │ failed │\n├─────────────────────────┼──────────┼──────────┤\n│ iterations │ 1 │ 0 │\n├─────────────────────────┼──────────┼──────────┤\n│ requests │ 1 │ 1 │\n├─────────────────────────┼──────────┼──────────┤\n│ test-scripts │ 1 │ 0 │\n├─────────────────────────┼──────────┼──────────┤\n│ prerequest-scripts │ 0 │ 0 │\n├─────────────────────────┼──────────┼──────────┤\n│ assertions │ 1 │ 1 │\n├─────────────────────────┴──────────┴──────────┤\n│ total run duration: 1012ms │\n├───────────────────────────────────────────────┤\n│ total data received: 0B (approx) │\n└───────────────────────────────────────────────┘\n\n # failure detail \n \n 1. Error \n connect ECONNREFUSED 127.0.0.1:8088 \n at request \n inside \"Health\" \n \n 2. AssertionError Status code is 200 \n expected { Object (id, _details, ...) } to have property 'code' \n at assertion:0 in test-script \n inside \"Health\" \n","outputType":"text/plain","errorMessage":"process error: exit status 1","steps":[{"name":"Health","duration":"0s","status":"failed","assertionResults":[{"name":"Status code is 200","status":"failed","errorMessage":"expected { Object (id, _details, ...) } to have property 'code'"}]}]}} 88 `) 89 result, err := ParseRunnerOutput(exampleOutput, true) 90 91 assert.Len(t, result.Output, 4510) 92 assert.NoError(t, err) 93 assert.Equal(t, testkube.ExecutionStatusFailed, result.Status) 94 assert.Equal(t, "process error: exit status 1", result.ErrorMessage) 95 }) 96 97 t.Run("Runner output with passed status", func(t *testing.T) { 98 t.Parallel() 99 100 var exampleOutput = []byte(` 101 {"type":"line","content":"🌍 Reading environment variables...","time":"2023-07-18T19:12:44.065916596Z"} 102 {"type":"line","content":"✅ Environment variables read successfully","time":"2023-07-18T19:12:44.066174662Z"} 103 {"type":"line","content":"RUNNER_ENDPOINT=\"testkube-minio-service-shared:9000\"","time":"2023-07-18T19:12:44.066185893Z"} 104 {"type":"line","content":"RUNNER_ACCESSKEYID=\"********\"","time":"2023-07-18T19:12:44.066190525Z"} 105 {"type":"line","content":"RUNNER_SECRETACCESSKEY=\"********\"","time":"2023-07-18T19:12:44.066198451Z"} 106 {"type":"line","content":"RUNNER_REGION=\"\"","time":"2023-07-18T19:12:44.066202295Z"} 107 {"type":"line","content":"RUNNER_TOKEN=\"\"","time":"2023-07-18T19:12:44.066206054Z"} 108 {"type":"line","content":"RUNNER_BUCKET=\"testkube-artifacts\"","time":"2023-07-18T19:12:44.066209522Z"} 109 {"type":"line","content":"RUNNER_SSL=false","time":"2023-07-18T19:12:44.066215367Z"} 110 {"type":"line","content":"RUNNER_SCRAPPERENABLED=\"true\"","time":"2023-07-18T19:12:44.066218487Z"} 111 {"type":"line","content":"RUNNER_GITUSERNAME=\"********\"","time":"2023-07-18T19:12:44.066222184Z"} 112 {"type":"line","content":"RUNNER_GITTOKEN=\"********\"","time":"2023-07-18T19:12:44.066226231Z"} 113 {"type":"line","content":"RUNNER_DATADIR=\"/data\"","time":"2023-07-18T19:12:44.066229687Z"} 114 {"type":"line","content":"RUNNER_CLUSTERID=\"clusterf83c3172f255fad68c2ab1a59be52916\"","time":"2023-07-18T19:12:44.066233268Z"} 115 {"type":"line","content":"RUNNER_CDEVENTS_TARGET=\"\"","time":"2023-07-18T19:12:44.066237063Z"} 116 {"type":"line","content":"RUNNER_DASHBOARD_URI=\"\"","time":"2023-07-18T19:12:44.066240323Z"} 117 {"type":"line","content":"RUNNER_CLOUD_MODE=\"false\"","time":"2023-07-18T19:12:44.066244115Z"} 118 {"type":"line","content":"RUNNER_CLOUD_API_TLS_INSECURE=\"false\"","time":"2023-07-18T19:12:44.066247795Z"} 119 {"type":"line","content":"RUNNER_CLOUD_API_URL=\"agent.testkube.io:443\"","time":"2023-07-18T19:12:44.066251778Z"} 120 {"type":"line","content":"RUNNER_CLOUD_API_KEY=\"\"","time":"2023-07-18T19:12:44.066255395Z"} 121 {"type":"line","content":"RUNNER_CLOUD_CONNECTION_TIMEOUT=10","time":"2023-07-18T19:12:44.066259384Z"} 122 {"type":"line","content":"🚚 Preparing test runner","time":"2023-07-18T19:12:44.066263179Z"} 123 {"type":"line","content":"✅ Uploading artifacts using MinIO Uploader","time":"2023-07-18T19:12:44.066269065Z"} 124 {"type":"event","content":"running test [64b6e3f0530875ebaf73794f]","time":"2023-07-18T19:12:44.06658648Z"} 125 {"type":"line","content":"🚚 Preparing for test run","time":"2023-07-18T19:12:44.066599252Z"} 126 {"type":"line","content":"🚀 Test run command newman run /data/repo/Core App Tests - WebPlayer.postman_collection.json -e /tmp/testkube-tmp1897306484 --reporters cli,json --reporter-json-export /tmp/testkube-tmp4151526495.json","time":"2023-07-18T19:12:44.066856036Z"} 127 {"type":"line","content":"🔬 Executing in directory : \n $ newman run /data/repo/Core App Tests - WebPlayer.postman_collection.json -e /tmp/testkube-tmp1897306484 --reporters cli,json --reporter-json-export /tmp/testkube-tmp4151526495.json","time":"2023-07-18T19:12:44.066872166Z"} 128 {"type":"line","content":"newman\n\n","time":"2023-07-18T19:12:45.654031992Z"} 129 {"type":"line","content":"Core App Tests - WebPlayer\n","time":"2023-07-18T19:12:45.654989975Z"} 130 {"type":"line","content":"\n→ na.com client=testdb sign=testct1 company=41574150-b952-413b-898b-dc5336b4bd12\n","time":"2023-07-18T19:12:45.658607412Z"} 131 {"type":"line","content":" GET https://na.com/v6-wplt/?client=testdb\u0026sign=testct1\u0026company=41574150-b952-413b-898b-dc5336b4bd12 ","time":"2023-07-18T19:12:45.685011667Z"} 132 {"type":"line","content":"[200 OK, 33.9kB, 326ms]\n","time":"2023-07-18T19:12:46.014247794Z"} 133 {"type":"line","content":" ✓ Status code is 200\n","time":"2023-07-18T19:12:46.076176552Z"} 134 {"type":"line","content":"\n┌─────────────────────────┬────────────────────┬───────────────────┐\n│ │ executed │ failed │\n├─────────────────────────┼────────────────────┼───────────────────┤\n│ iterations │ 1 │ 0 │\n├─────────────────────────┼────────────────────┼───────────────────┤\n│ requests │ 1 │ 0 │\n├─────────────────────────┼────────────────────┼───────────────────┤\n│ test-scripts │ 1 │ 0 │\n├─────────────────────────┼────────────────────┼───────────────────┤\n│ prerequest-scripts │ 0 │ 0 │\n├─────────────────────────┼────────────────────┼───────────────────┤\n│ assertions │ 1 │ 0 │\n├─────────────────────────┴────────────────────┴───────────────────┤\n│ total run duration: 429ms │\n├──────────────────────────────────────────────────────────────────┤\n│ total data received: 33.45kB (approx) │\n├──────────────────────────────────────────────────────────────────┤\n│ average response time: 326ms [min: 326ms, max: 326ms, s.d.: 0µs] │\n└──────────────────────────────────────────────────────────────────┘\n","time":"2023-07-18T19:12:46.104174132Z"} 135 {"type":"line","content":"✅ Execution succeeded","time":"2023-07-18T19:12:46.114565626Z"} 136 {"type":"line","content":"✅ Got Newman result successfully","time":"2023-07-18T19:12:46.126116248Z"} 137 {"type":"line","content":"✅ Mapped Newman result successfully","time":"2023-07-18T19:12:46.126152021Z"} 138 {"type":"result","result":{"status":"passed","output":"newman\n\nCore App Tests - WebPlayer\n\n→ core-eks-test.poppcore.co client=testdb sign=testct1 company=41574150-b952-413b-898b-dc5336b4bd12\n GET https://na.com/v6-wplt/?client=testdb\u0026sign=testct1\u0026company=41574150-b952-413b-898b-dc5336b4bd12 [200 OK, 33.9kB, 326ms]\n ✓ Status code is 200\n\n┌─────────────────────────┬────────────────────┬───────────────────┐\n│ │ executed │ failed │\n├─────────────────────────┼────────────────────┼───────────────────┤\n│ iterations │ 1 │ 0 │\n├─────────────────────────┼────────────────────┼───────────────────┤\n│ requests │ 1 │ 0 │\n├─────────────────────────┼────────────────────┼───────────────────┤\n│ test-scripts │ 1 │ 0 │\n├─────────────────────────┼────────────────────┼───────────────────┤\n│ prerequest-scripts │ 0 │ 0 │\n├─────────────────────────┼────────────────────┼───────────────────┤\n│ assertions │ 1 │ 0 │\n├─────────────────────────┴────────────────────┴───────────────────┤\n│ total run duration: 429ms │\n├──────────────────────────────────────────────────────────────────┤\n│ total data received: 33.45kB (approx) │\n├──────────────────────────────────────────────────────────────────┤\n│ average response time: 326ms [min: 326ms, max: 326ms, s.d.: 0µs] │\n└──────────────────────────────────────────────────────────────────┘\n","outputType":"text/plain","steps":[{"name":"na.com client=testdb sign=testct1 company=41574150-b952-413b-898b-dc5336b4bd12","duration":"326ms","status":"passed","assertionResults":[{"name":"Status code is 200","status":"passed"}]}]},"time":"2023-07-18T19:12:46.12615853Z"}`) 139 result, err := ParseRunnerOutput(exampleOutput, true) 140 141 assert.Len(t, result.Output, 7304) 142 assert.NoError(t, err) 143 assert.Equal(t, testkube.ExecutionStatusPassed, result.Status) 144 }) 145 146 t.Run("Wrong order in logs", func(t *testing.T) { 147 t.Parallel() 148 149 unorderedOutput := []byte(` 150 {"type":"line","content":"🚚 Preparing test runner","time":"2023-01-17T15:29:17.921466388Z"} 151 {"type":"line","content":"🌍 Reading environment variables...","time":"2023-01-17T15:29:17.921770638Z"} 152 {"type":"line","content":"✅ Environment variables read successfully","time":"2023-01-17T15:29:17.921786721Z"} 153 {"type":"line","content":"RUNNER_ENDPOINT=\"testkube-minio-service-testkube:9000\"","time":"2023-01-17T15:29:17.921788721Z"} 154 {"type":"line","content":"RUNNER_ACCESSKEYID=\"********\"","time":"2023-01-17T15:29:17.921790721Z"} 155 {"type":"line","content":"RUNNER_SECRETACCESSKEY=\"********\"","time":"2023-01-17T15:29:17.921792388Z"} 156 {"type":"line","content":"RUNNER_REGION=\"\"","time":"2023-01-17T15:29:17.921793846Z"} 157 {"type":"line","content":"RUNNER_TOKEN=\"\"","time":"2023-01-17T15:29:17.921795304Z"} 158 {"type":"line","content":"RUNNER_SSL=false","time":"2023-01-17T15:29:17.921797054Z"} 159 {"type":"line","content":"RUNNER_SCRAPPERENABLED=\"true\"","time":"2023-01-17T15:29:17.921798679Z"} 160 {"type":"line","content":"RUNNER_GITUSERNAME=\"\"","time":"2023-01-17T15:29:17.921800138Z"} 161 {"type":"line","content":"RUNNER_GITTOKEN=\"\"","time":"2023-01-17T15:29:17.921801596Z"} 162 {"type":"line","content":"RUNNER_DATADIR=\"/data\"","time":"2023-01-17T15:29:17.921803138Z"} 163 {"type":"line","content":"RUNNER_CDEVENTS_TARGET=\"\"","time":"2023-01-17T15:29:17.921801596Z"} 164 {"type":"line","content":"RUNNER_DASHBOARD_URI=\"\"","time":"2023-01-17T15:29:17.921801596Z"} 165 {"type":"line","content":"RUNNER_CLUSTERID=\"\"","time":"2023-01-17T15:29:17.921801596Z"} 166 {"type":"error","content":"❌ can't find branch or commit in params, repo:\u0026{Type_:git-file Uri:https://github.com/kubeshop/testkube.git Branch: Commit: Path:test/cypress/executor-smoke/cypress-11 Username: Token: UsernameSecret:\u003cnil\u003e TokenSecret:\u003cnil\u003e WorkingDir:}","time":"2023-01-17T15:29:17.921940304Z"} 167 {"type":"error","content":"can't find branch or commit in params, repo:\u0026{Type_:git-file Uri:https://github.com/kubeshop/testkube.git Branch: Commit: Path:test/cypress/executor-smoke/cypress-11 Username: Token: UsernameSecret:\u003cnil\u003e TokenSecret:\u003cnil\u003e WorkingDir:}","time":"2023-01-17T15:29:17.921946638Z"} 168 {"type":"event","content":"running test [63c6bec1790802b7e3e57048]","time":"2023-01-17T15:29:17.921920596Z"} 169 {"type":"line","content":"🚚 Preparing for test run","time":"2023-01-17T15:29:17.921931471Z"} 170 `) 171 expectedOutput := `🚚 Preparing test runner 172 🌍 Reading environment variables... 173 ✅ Environment variables read successfully 174 RUNNER_ENDPOINT="testkube-minio-service-testkube:9000" 175 RUNNER_ACCESSKEYID="********" 176 RUNNER_SECRETACCESSKEY="********" 177 RUNNER_REGION="" 178 RUNNER_TOKEN="" 179 RUNNER_SSL=false 180 RUNNER_SCRAPPERENABLED="true" 181 RUNNER_GITUSERNAME="" 182 RUNNER_GITTOKEN="" 183 RUNNER_DATADIR="/data" 184 RUNNER_CDEVENTS_TARGET="" 185 RUNNER_DASHBOARD_URI="" 186 RUNNER_CLUSTERID="" 187 ❌ can't find branch or commit in params, repo:&{Type_:git-file Uri:https://github.com/kubeshop/testkube.git Branch: Commit: Path:test/cypress/executor-smoke/cypress-11 Username: Token: UsernameSecret:<nil> TokenSecret:<nil> WorkingDir:} 188 can't find branch or commit in params, repo:&{Type_:git-file Uri:https://github.com/kubeshop/testkube.git Branch: Commit: Path:test/cypress/executor-smoke/cypress-11 Username: Token: UsernameSecret:<nil> TokenSecret:<nil> WorkingDir:} 189 running test [63c6bec1790802b7e3e57048] 190 🚚 Preparing for test run 191 ` 192 result, err := ParseRunnerOutput(unorderedOutput, true) 193 194 assert.Equal(t, expectedOutput, result.Output) 195 assert.NoError(t, err) 196 assert.Equal(t, testkube.ExecutionStatusFailed, result.Status) 197 assert.Equal(t, "can't find branch or commit in params, repo:&{Type_:git-file Uri:https://github.com/kubeshop/testkube.git Branch: Commit: Path:test/cypress/executor-smoke/cypress-11 Username: Token: UsernameSecret:<nil> TokenSecret:<nil> WorkingDir:}", result.ErrorMessage) 198 }) 199 200 t.Run("Output with failed result", func(t *testing.T) { 201 t.Parallel() 202 203 output := []byte(`{"type":"event","content":"running test [63c9606d7104b0fa0b7a45f1]"} 204 {"type":"event","content":"Running [ ./zap-api-scan.py [-t https://www.example.com/openapi.json -f openapi -c examples/zap-api.conf -d -D 5 -I -l INFO -n examples/context.config -S -T 60 -U anonymous -O https://www.example.com -z -config aaa=bbb -r api-test-report.html]]"} 205 {"type":"result","result":{"status":"failed","errorMessage":"could not start process: fork/exec ./zap-api-scan.py: no such file or directory"}}`) 206 expectedOutput := `running test [63c9606d7104b0fa0b7a45f1] 207 Running [ ./zap-api-scan.py [-t https://www.example.com/openapi.json -f openapi -c examples/zap-api.conf -d -D 5 -I -l INFO -n examples/context.config -S -T 60 -U anonymous -O https://www.example.com -z -config aaa=bbb -r api-test-report.html]] 208 could not start process: fork/exec ./zap-api-scan.py: no such file or directory 209 ` 210 result, err := ParseRunnerOutput(output, true) 211 212 assert.Equal(t, expectedOutput, result.Output) 213 assert.NoError(t, err) 214 assert.Equal(t, testkube.ExecutionStatusFailed, result.Status) 215 assert.Equal(t, "could not start process: fork/exec ./zap-api-scan.py: no such file or directory", result.ErrorMessage) 216 }) 217 218 t.Run("Output with error", func(t *testing.T) { 219 t.Parallel() 220 221 output := []byte(` 222 {"type":"line","content":"🚚 Preparing test runner","time":"2023-01-19T15:22:25.867379429Z"} 223 {"type":"line","content":"🌍 Reading environment variables...","time":"2023-01-19T15:22:25.867927513Z"} 224 {"type":"line","content":"✅ Environment variables read successfully","time":"2023-01-19T15:22:25.867944763Z"} 225 {"type":"line","content":"RUNNER_ENDPOINT=\"testkube-minio-service-testkube:9000\"","time":"2023-01-19T15:22:25.867946929Z"} 226 {"type":"line","content":"RUNNER_ACCESSKEYID=\"********\"","time":"2023-01-19T15:22:25.867948804Z"} 227 {"type":"line","content":"RUNNER_SECRETACCESSKEY=\"********\"","time":"2023-01-19T15:22:25.867955263Z"} 228 {"type":"line","content":"RUNNER_REGION=\"\"","time":"2023-01-19T15:22:25.867962596Z"} 229 {"type":"line","content":"RUNNER_TOKEN=\"\"","time":"2023-01-19T15:22:25.867967971Z"} 230 {"type":"line","content":"RUNNER_SSL=false","time":"2023-01-19T15:22:25.867974013Z"} 231 {"type":"line","content":"RUNNER_SCRAPPERENABLED=\"true\"","time":"2023-01-19T15:22:25.867978888Z"} 232 {"type":"line","content":"RUNNER_GITUSERNAME=\"\"","time":"2023-01-19T15:22:25.867984179Z"} 233 {"type":"line","content":"RUNNER_GITTOKEN=\"\"","time":"2023-01-19T15:22:25.867986013Z"} 234 {"type":"line","content":"RUNNER_DATADIR=\"/data\"","time":"2023-01-19T15:22:25.867987596Z"} 235 {"type":"line","content":"RUNNER_CDEVENTS_TARGET=\"\"","time":"2023-01-17T15:29:17.921801596Z"} 236 {"type":"line","content":"RUNNER_DASHBOARD_URI=\"\"","time":"2023-01-17T15:29:17.921801596Z"} 237 {"type":"line","content":"RUNNER_CLUSTERID=\"\"","time":"2023-01-17T15:29:17.921801596Z"} 238 {"type":"event","content":"running test [63c960287104b0fa0b7a45ef]","time":"2023-01-19T15:22:25.868132888Z"} 239 {"type":"line","content":"🚚 Preparing for test run","time":"2023-01-19T15:22:25.868161346Z"} 240 {"type":"line","content":"❌ can't find branch or commit in params, repo:\u0026{Type_:git-file Uri:https://github.com/kubeshop/testkube.git Branch: Commit: Path:test/cypress/executor-smoke/cypress-11 Username: Token: UsernameSecret:\u003cnil\u003e TokenSecret:\u003cnil\u003e WorkingDir:}","time":"2023-01-19T15:22:25.868183971Z"} 241 {"type":"error","content":"can't find branch or commit in params, repo:\u0026{Type_:git-file Uri:https://github.com/kubeshop/testkube.git Branch: Commit: Path:test/cypress/executor-smoke/cypress-11 Username: Token: UsernameSecret:\u003cnil\u003e TokenSecret:\u003cnil\u003e WorkingDir:}","time":"2023-01-19T15:22:25.868198429Z"} 242 `) 243 expectedOutput := `🚚 Preparing test runner 244 🌍 Reading environment variables... 245 ✅ Environment variables read successfully 246 RUNNER_ENDPOINT="testkube-minio-service-testkube:9000" 247 RUNNER_ACCESSKEYID="********" 248 RUNNER_SECRETACCESSKEY="********" 249 RUNNER_REGION="" 250 RUNNER_TOKEN="" 251 RUNNER_SSL=false 252 RUNNER_SCRAPPERENABLED="true" 253 RUNNER_GITUSERNAME="" 254 RUNNER_GITTOKEN="" 255 RUNNER_DATADIR="/data" 256 RUNNER_CDEVENTS_TARGET="" 257 RUNNER_DASHBOARD_URI="" 258 RUNNER_CLUSTERID="" 259 running test [63c960287104b0fa0b7a45ef] 260 🚚 Preparing for test run 261 ❌ can't find branch or commit in params, repo:&{Type_:git-file Uri:https://github.com/kubeshop/testkube.git Branch: Commit: Path:test/cypress/executor-smoke/cypress-11 Username: Token: UsernameSecret:<nil> TokenSecret:<nil> WorkingDir:} 262 can't find branch or commit in params, repo:&{Type_:git-file Uri:https://github.com/kubeshop/testkube.git Branch: Commit: Path:test/cypress/executor-smoke/cypress-11 Username: Token: UsernameSecret:<nil> TokenSecret:<nil> WorkingDir:} 263 ` 264 265 result, err := ParseRunnerOutput(output, true) 266 267 assert.Equal(t, expectedOutput, result.Output) 268 assert.NoError(t, err) 269 assert.Equal(t, testkube.ExecutionStatusFailed, result.Status) 270 assert.Equal(t, "can't find branch or commit in params, repo:&{Type_:git-file Uri:https://github.com/kubeshop/testkube.git Branch: Commit: Path:test/cypress/executor-smoke/cypress-11 Username: Token: UsernameSecret:<nil> TokenSecret:<nil> WorkingDir:}", result.ErrorMessage) 271 }) 272 273 t.Run("Remove internal logs", func(t *testing.T) { 274 t.Parallel() 275 276 output := []byte(` 277 {"type":"line","content":"🚚 Preparing test runner","time":"2023-01-19T15:22:25.867379429Z"} 278 {"type":"line","content":"🌍 Reading environment variables...","time":"2023-01-19T15:22:25.867927513Z"} 279 {"type":"line","content":"✅ Environment variables read successfully","time":"2023-01-19T15:22:25.867944763Z"} 280 {"type":"line","content":"RUNNER_ENDPOINT=\"testkube-minio-service-testkube:9000\"","time":"2023-01-19T15:22:25.867946929Z"} 281 {"type":"line","content":"RUNNER_ACCESSKEYID=\"********\"","time":"2023-01-19T15:22:25.867948804Z"} 282 {"type":"line","content":"RUNNER_SECRETACCESSKEY=\"********\"","time":"2023-01-19T15:22:25.867955263Z"} 283 {"type":"line","content":"RUNNER_REGION=\"\"","time":"2023-01-19T15:22:25.867962596Z"} 284 {"type":"line","content":"RUNNER_TOKEN=\"\"","time":"2023-01-19T15:22:25.867967971Z"} 285 {"type":"line","content":"RUNNER_SSL=false","time":"2023-01-19T15:22:25.867974013Z"} 286 {"type":"line","content":"RUNNER_SCRAPPERENABLED=\"true\"","time":"2023-01-19T15:22:25.867978888Z"} 287 {"type":"line","content":"RUNNER_GITUSERNAME=\"\"","time":"2023-01-19T15:22:25.867984179Z"} 288 {"type":"line","content":"RUNNER_GITTOKEN=\"\"","time":"2023-01-19T15:22:25.867986013Z"} 289 {"type":"line","content":"RUNNER_DATADIR=\"/data\"","time":"2023-01-19T15:22:25.867987596Z"} 290 {"type":"line","content":"RUNNER_CDEVENTS_TARGET=\"\"","time":"2023-01-17T15:29:17.921801596Z"} 291 {"type":"line","content":"RUNNER_DASHBOARD_URI=\"\"","time":"2023-01-17T15:29:17.921801596Z"} 292 {"type":"line","content":"RUNNER_CLUSTERID=\"\"","time":"2023-01-17T15:29:17.921801596Z"} 293 {"type":"event","content":"running test [63c960287104b0fa0b7a45ef]","time":"2023-01-19T15:22:25.868132888Z"} 294 {"type":"line","content":"🚚 Preparing for test run","time":"2023-01-19T15:22:25.868161346Z"} 295 {"type":"line","content":"can't find branch or commit in params, repo:\u0026{Type_:git-file Uri:https://github.com/kubeshop/testkube.git Branch: Commit: Path:test/cypress/executor-smoke/cypress-11 Username: Token: UsernameSecret:\u003cnil\u003e TokenSecret:\u003cnil\u003e WorkingDir:}","time":"2023-01-19T15:22:25.868183971Z"} 296 {"level":"info","ts":1678919600.1199453,"caller":"scraper/filesystem_extractor.go:25","msg":"extracting files from directories: [/data/output]"} 297 {"level":"info","ts":1678919600.143759,"caller":"scraper/filesystem_extractor.go:27","msg":"walking directory: /data/output"} 298 {"level":"info","ts":1678919600.145652,"caller":"scraper/filesystem_extractor.go:37","msg":"walking path /data/output"} 299 {"level":"info","ts":1678919600.1457458,"caller":"scraper/filesystem_extractor.go:43","msg":"skipping directory /data/output"} 300 {"level":"info","ts":1678919600.1459277,"caller":"scraper/filesystem_extractor.go:37","msg":"walking path /data/output/jmeter.log"} 301 {"level":"info","ts":1678919600.1480958,"caller":"scraper/filesystem_extractor.go:63","msg":"filesystem extractor is sending file to be processed: jmeter.log"} 302 {"type":"error","content":"can't find branch or commit in params, repo:\u0026{Type_:git-file Uri:https://github.com/kubeshop/testkube.git Branch: Commit: Path:test/cypress/executor-smoke/cypress-11 Username: Token: UsernameSecret:\u003cnil\u003e TokenSecret:\u003cnil\u003e WorkingDir:}","time":"2023-01-19T15:22:25.868198429Z"} 303 `) 304 expectedOutput := `🚚 Preparing test runner 305 🌍 Reading environment variables... 306 ✅ Environment variables read successfully 307 RUNNER_ENDPOINT="testkube-minio-service-testkube:9000" 308 RUNNER_ACCESSKEYID="********" 309 RUNNER_SECRETACCESSKEY="********" 310 RUNNER_REGION="" 311 RUNNER_TOKEN="" 312 RUNNER_SSL=false 313 RUNNER_SCRAPPERENABLED="true" 314 RUNNER_GITUSERNAME="" 315 RUNNER_GITTOKEN="" 316 RUNNER_DATADIR="/data" 317 RUNNER_CDEVENTS_TARGET="" 318 RUNNER_DASHBOARD_URI="" 319 RUNNER_CLUSTERID="" 320 running test [63c960287104b0fa0b7a45ef] 321 🚚 Preparing for test run 322 can't find branch or commit in params, repo:&{Type_:git-file Uri:https://github.com/kubeshop/testkube.git Branch: Commit: Path:test/cypress/executor-smoke/cypress-11 Username: Token: UsernameSecret:<nil> TokenSecret:<nil> WorkingDir:} 323 {"level":"info","ts":1678919600.1199453,"caller":"scraper/filesystem_extractor.go:25","msg":"extracting files from directories: [/data/output]"} 324 {"level":"info","ts":1678919600.143759,"caller":"scraper/filesystem_extractor.go:27","msg":"walking directory: /data/output"} 325 {"level":"info","ts":1678919600.145652,"caller":"scraper/filesystem_extractor.go:37","msg":"walking path /data/output"} 326 {"level":"info","ts":1678919600.1457458,"caller":"scraper/filesystem_extractor.go:43","msg":"skipping directory /data/output"} 327 {"level":"info","ts":1678919600.1459277,"caller":"scraper/filesystem_extractor.go:37","msg":"walking path /data/output/jmeter.log"} 328 {"level":"info","ts":1678919600.1480958,"caller":"scraper/filesystem_extractor.go:63","msg":"filesystem extractor is sending file to be processed: jmeter.log"} 329 can't find branch or commit in params, repo:&{Type_:git-file Uri:https://github.com/kubeshop/testkube.git Branch: Commit: Path:test/cypress/executor-smoke/cypress-11 Username: Token: UsernameSecret:<nil> TokenSecret:<nil> WorkingDir:} 330 ` 331 332 result, err := ParseRunnerOutput(output, true) 333 334 assert.Equal(t, expectedOutput, result.Output) 335 assert.NoError(t, err) 336 assert.Equal(t, testkube.ExecutionStatusFailed, result.Status) 337 assert.Equal(t, "can't find branch or commit in params, repo:&{Type_:git-file Uri:https://github.com/kubeshop/testkube.git Branch: Commit: Path:test/cypress/executor-smoke/cypress-11 Username: Token: UsernameSecret:<nil> TokenSecret:<nil> WorkingDir:}", result.ErrorMessage) 338 }) 339 340 t.Run("flaky cypress test", func(t *testing.T) { 341 t.Parallel() 342 343 output := []byte(` 344 {"type":"error","content":"can't find branch or commit in params, repo:\u0026{Type_:git-file Uri:https://github.com/kubeshop/testkube.git Branch: Commit: Path:test/cypress/executor-smoke/cypress-11 Username: Token: UsernameSecret:\u003cnil\u003e TokenSecret:\u003cnil\u003e WorkingDir:}","time":"2023-01-20T12:44:15.719459174Z"} 345 {"type":"line","content":"🚚 Preparing test runner","time":"2023-01-20T12:44:15.714299549Z"} 346 {"type":"line","content":"🌍 Reading environment variables...","time":"2023-01-20T12:44:15.718910883Z"} 347 {"type":"line","content":"✅ Environment variables read successfully","time":"2023-01-20T12:44:15.718958008Z"} 348 {"type":"line","content":"RUNNER_ENDPOINT=\"testkube-minio-service-testkube:9000\"","time":"2023-01-20T12:44:15.718962633Z"} 349 {"type":"line","content":"RUNNER_ACCESSKEYID=\"********\"","time":"2023-01-20T12:44:15.718966091Z"} 350 {"type":"line","content":"RUNNER_SECRETACCESSKEY=\"********\"","time":"2023-01-20T12:44:15.718969383Z"} 351 {"type":"line","content":"RUNNER_REGION=\"\"","time":"2023-01-20T12:44:15.718972299Z"} 352 {"type":"line","content":"RUNNER_TOKEN=\"\"","time":"2023-01-20T12:44:15.718975174Z"} 353 {"type":"line","content":"RUNNER_SSL=false","time":"2023-01-20T12:44:15.718977924Z"} 354 {"type":"line","content":"RUNNER_SCRAPPERENABLED=\"true\"","time":"2023-01-20T12:44:15.718980758Z"} 355 {"type":"line","content":"RUNNER_GITUSERNAME=\"\"","time":"2023-01-20T12:44:15.718983549Z"} 356 {"type":"line","content":"RUNNER_GITTOKEN=\"\"","time":"2023-01-20T12:44:15.718986174Z"} 357 {"type":"line","content":"RUNNER_DATADIR=\"/data\"","time":"2023-01-20T12:44:15.718989049Z"} 358 {"type":"line","content":"RUNNER_CDEVENTS_TARGET=\"\"","time":"2023-01-17T15:29:17.921801596Z"} 359 {"type":"line","content":"RUNNER_DASHBOARD_URI=\"\"","time":"2023-01-17T15:29:17.921801596Z"} 360 {"type":"line","content":"RUNNER_CLUSTERID=\"\"","time":"2023-01-17T15:29:17.921801596Z"} 361 {"type":"event","content":"running test [63ca8c8988564860327a16b5]","time":"2023-01-20T12:44:15.719276383Z"} 362 {"type":"line","content":"🚚 Preparing for test run","time":"2023-01-20T12:44:15.719285633Z"} 363 {"type":"line","content":"❌ can't find branch or commit in params, repo:\u0026{Type_:git-file Uri:https://github.com/kubeshop/testkube.git Branch: Commit: Path:test/cypress/executor-smoke/cypress-11 Username: Token: UsernameSecret:\u003cnil\u003e TokenSecret:\u003cnil\u003e WorkingDir:}","time":"2023-01-20T12:44:15.719302049Z"} 364 `) 365 expectedOutput := `can't find branch or commit in params, repo:&{Type_:git-file Uri:https://github.com/kubeshop/testkube.git Branch: Commit: Path:test/cypress/executor-smoke/cypress-11 Username: Token: UsernameSecret:<nil> TokenSecret:<nil> WorkingDir:} 366 🚚 Preparing test runner 367 🌍 Reading environment variables... 368 ✅ Environment variables read successfully 369 RUNNER_ENDPOINT="testkube-minio-service-testkube:9000" 370 RUNNER_ACCESSKEYID="********" 371 RUNNER_SECRETACCESSKEY="********" 372 RUNNER_REGION="" 373 RUNNER_TOKEN="" 374 RUNNER_SSL=false 375 RUNNER_SCRAPPERENABLED="true" 376 RUNNER_GITUSERNAME="" 377 RUNNER_GITTOKEN="" 378 RUNNER_DATADIR="/data" 379 RUNNER_CDEVENTS_TARGET="" 380 RUNNER_DASHBOARD_URI="" 381 RUNNER_CLUSTERID="" 382 running test [63ca8c8988564860327a16b5] 383 🚚 Preparing for test run 384 ❌ can't find branch or commit in params, repo:&{Type_:git-file Uri:https://github.com/kubeshop/testkube.git Branch: Commit: Path:test/cypress/executor-smoke/cypress-11 Username: Token: UsernameSecret:<nil> TokenSecret:<nil> WorkingDir:} 385 ` 386 387 result, err := ParseRunnerOutput(output, true) 388 389 assert.Equal(t, expectedOutput, result.Output) 390 assert.NoError(t, err) 391 assert.Equal(t, testkube.ExecutionStatusFailed, result.Status) 392 assert.Equal(t, "can't find branch or commit in params, repo:&{Type_:git-file Uri:https://github.com/kubeshop/testkube.git Branch: Commit: Path:test/cypress/executor-smoke/cypress-11 Username: Token: UsernameSecret:<nil> TokenSecret:<nil> WorkingDir:}", result.ErrorMessage) 393 394 }) 395 } 396 397 func TestParseContainerOutput(t *testing.T) { 398 t.Parallel() 399 400 t.Run("Runner output with no timestamps", func(t *testing.T) { 401 t.Parallel() 402 403 var exampleOutput = []byte(` 404 {"type":"event","message":"running postman/collection from testkube.Execution","content":["{\n\t\"id\": \"blablablablabla\",\n\t\"name\": \"some-testing-exec\",\n\t\"scriptContent\": \"{\\n\\t\\\"info\\\": {\\n\\t\\t\\\"_postman_id\\\": \\\"97b67bfb-c1ca-4572-af46-06cab8f68998\\\",\\n\\t\\t\\\"name\\\": \\\"Local-API-Health\\\",\\n\\t\\t\\\"schema\\\": \\\"https://schema.getpostman.com/json/collection/v2.1.0/collection.json\\\"\\n\\t},\\n\\t\\\"item\\\": [\\n\\t\\t{\\n\\t\\t\\t\\\"name\\\": \\\"Health\\\",\\n\\t\\t\\t\\\"event\\\": [\\n\\t\\t\\t\\t{\\n\\t\\t\\t\\t\\t\\\"listen\\\": \\\"test\\\",\\n\\t\\t\\t\\t\\t\\\"script\\\": {\\n\\t\\t\\t\\t\\t\\t\\\"exec\\\": [\\n\\t\\t\\t\\t\\t\\t\\t\\\"pm.test(\\\\\\\"Status code is 200\\\\\\\", function () {\\\",\\n\\t\\t\\t\\t\\t\\t\\t\\\" pm.response.to.have.status(200);\\\",\\n\\t\\t\\t\\t\\t\\t\\t\\\"});\\\"\\n\\t\\t\\t\\t\\t\\t],\\n\\t\\t\\t\\t\\t\\t\\\"type\\\": \\\"text/javascript\\\"\\n\\t\\t\\t\\t\\t}\\n\\t\\t\\t\\t}\\n\\t\\t\\t],\\n\\t\\t\\t\\\"request\\\": {\\n\\t\\t\\t\\t\\\"method\\\": \\\"GET\\\",\\n\\t\\t\\t\\t\\\"header\\\": [],\\n\\t\\t\\t\\t\\\"url\\\": {\\n\\t\\t\\t\\t\\t\\\"raw\\\": \\\"http://localhost:8088/health\\\",\\n\\t\\t\\t\\t\\t\\\"protocol\\\": \\\"http\\\",\\n\\t\\t\\t\\t\\t\\\"host\\\": [\\n\\t\\t\\t\\t\\t\\t\\\"localhost\\\"\\n\\t\\t\\t\\t\\t],\\n\\t\\t\\t\\t\\t\\\"port\\\": \\\"8088\\\",\\n\\t\\t\\t\\t\\t\\\"path\\\": [\\n\\t\\t\\t\\t\\t\\t\\\"health\\\"\\n\\t\\t\\t\\t\\t]\\n\\t\\t\\t\\t}\\n\\t\\t\\t},\\n\\t\\t\\t\\\"response\\\": []\\n\\t\\t}\\n\\t],\\n\\t\\\"event\\\": []\\n}\"\n}"]} 405 {"type":"line","content":"newman\n\n"} 406 {"type":"line","content":"Local-API-Health\n"} 407 {"type":"line","content":"\n→ Health\n"} 408 {"type":"line","content":" GET http://localhost:8088/health "} 409 {"type":"line","content":"[errored]\n"} 410 {"type":"line","content":" connect ECONNREFUSED 127.0.0.1:8088\n"} 411 {"type":"line","content":" 2. Status code is 200\n"} 412 {"type":"line","content":"\n┌─────────────────────────┬──────────┬──────────┐\n│ │ executed │ failed │\n├─────────────────────────┼──────────┼──────────┤\n│ iterations │ 1 │ 0 │\n├─────────────────────────┼──────────┼──────────┤\n│ requests │ 1 │ 1 │\n├─────────────────────────┼──────────┼──────────┤\n│ test-scripts │ 1 │ 0 │\n├─────────────────────────┼──────────┼──────────┤\n│ prerequest-scripts │ 0 │ 0 │\n├─────────────────────────┼──────────┼──────────┤\n│ assertions │ 1 │ 1 │\n├─────────────────────────┴──────────┴──────────┤\n│ total run duration: 1012ms │\n├───────────────────────────────────────────────┤\n│ total data received: 0B (approx) │\n└───────────────────────────────────────────────┘\n"} 413 {"type":"line","content":"\n # failure detail \n \n 1. Error \n connect ECONNREFUSED 127.0.0.1:8088 \n at request \n inside \"Health\" \n \n 2. AssertionError Status code is 200 \n expected { Object (id, _details, ...) } to have property 'code' \n at assertion:0 in test-script \n inside \"Health\" \n"} 414 {"type":"result","result":{"status":"failed","startTime":"2021-10-29T11:35:35.759Z","endTime":"2021-10-29T11:35:36.771Z","output":"newman\n\nLocal-API-Health\n\n→ Health\n GET http://localhost:8088/health [errored]\n connect ECONNREFUSED 127.0.0.1:8088\n 2. Status code is 200\n\n┌─────────────────────────┬──────────┬──────────┐\n│ │ executed │ failed │\n├─────────────────────────┼──────────┼──────────┤\n│ iterations │ 1 │ 0 │\n├─────────────────────────┼──────────┼──────────┤\n│ requests │ 1 │ 1 │\n├─────────────────────────┼──────────┼──────────┤\n│ test-scripts │ 1 │ 0 │\n├─────────────────────────┼──────────┼──────────┤\n│ prerequest-scripts │ 0 │ 0 │\n├─────────────────────────┼──────────┼──────────┤\n│ assertions │ 1 │ 1 │\n├─────────────────────────┴──────────┴──────────┤\n│ total run duration: 1012ms │\n├───────────────────────────────────────────────┤\n│ total data received: 0B (approx) │\n└───────────────────────────────────────────────┘\n\n # failure detail \n \n 1. Error \n connect ECONNREFUSED 127.0.0.1:8088 \n at request \n inside \"Health\" \n \n 2. AssertionError Status code is 200 \n expected { Object (id, _details, ...) } to have property 'code' \n at assertion:0 in test-script \n inside \"Health\" \n","outputType":"text/plain","errorMessage":"process error: exit status 1","steps":[{"name":"Health","duration":"0s","status":"failed","assertionResults":[{"name":"Status code is 200","status":"failed","errorMessage":"expected { Object (id, _details, ...) } to have property 'code'"}]}]}} 415 `) 416 result, output, err := ParseContainerOutput(exampleOutput) 417 418 assert.Len(t, output, 4511) 419 assert.NoError(t, err) 420 assert.Equal(t, testkube.ExecutionStatusFailed, result.Status) 421 assert.Equal(t, "process error: exit status 1", result.ErrorMessage) 422 }) 423 424 t.Run("Runner output with passed status", func(t *testing.T) { 425 t.Parallel() 426 427 var exampleOutput = []byte(` 428 {"type":"line","content":"🌍 Reading environment variables...","time":"2023-07-18T19:12:44.065916596Z"} 429 {"type":"line","content":"✅ Environment variables read successfully","time":"2023-07-18T19:12:44.066174662Z"} 430 {"type":"line","content":"RUNNER_ENDPOINT=\"testkube-minio-service-shared:9000\"","time":"2023-07-18T19:12:44.066185893Z"} 431 {"type":"line","content":"RUNNER_ACCESSKEYID=\"********\"","time":"2023-07-18T19:12:44.066190525Z"} 432 {"type":"line","content":"RUNNER_SECRETACCESSKEY=\"********\"","time":"2023-07-18T19:12:44.066198451Z"} 433 {"type":"line","content":"RUNNER_REGION=\"\"","time":"2023-07-18T19:12:44.066202295Z"} 434 {"type":"line","content":"RUNNER_TOKEN=\"\"","time":"2023-07-18T19:12:44.066206054Z"} 435 {"type":"line","content":"RUNNER_BUCKET=\"testkube-artifacts\"","time":"2023-07-18T19:12:44.066209522Z"} 436 {"type":"line","content":"RUNNER_SSL=false","time":"2023-07-18T19:12:44.066215367Z"} 437 {"type":"line","content":"RUNNER_SCRAPPERENABLED=\"true\"","time":"2023-07-18T19:12:44.066218487Z"} 438 {"type":"line","content":"RUNNER_GITUSERNAME=\"********\"","time":"2023-07-18T19:12:44.066222184Z"} 439 {"type":"line","content":"RUNNER_GITTOKEN=\"********\"","time":"2023-07-18T19:12:44.066226231Z"} 440 {"type":"line","content":"RUNNER_DATADIR=\"/data\"","time":"2023-07-18T19:12:44.066229687Z"} 441 {"type":"line","content":"RUNNER_CLUSTERID=\"clusterf83c3172f255fad68c2ab1a59be52916\"","time":"2023-07-18T19:12:44.066233268Z"} 442 {"type":"line","content":"RUNNER_CDEVENTS_TARGET=\"\"","time":"2023-07-18T19:12:44.066237063Z"} 443 {"type":"line","content":"RUNNER_DASHBOARD_URI=\"\"","time":"2023-07-18T19:12:44.066240323Z"} 444 {"type":"line","content":"RUNNER_CLOUD_MODE=\"false\"","time":"2023-07-18T19:12:44.066244115Z"} 445 {"type":"line","content":"RUNNER_CLOUD_API_TLS_INSECURE=\"false\"","time":"2023-07-18T19:12:44.066247795Z"} 446 {"type":"line","content":"RUNNER_CLOUD_API_URL=\"agent.testkube.io:443\"","time":"2023-07-18T19:12:44.066251778Z"} 447 {"type":"line","content":"RUNNER_CLOUD_API_KEY=\"\"","time":"2023-07-18T19:12:44.066255395Z"} 448 {"type":"line","content":"RUNNER_CLOUD_CONNECTION_TIMEOUT=10","time":"2023-07-18T19:12:44.066259384Z"} 449 {"type":"line","content":"🚚 Preparing test runner","time":"2023-07-18T19:12:44.066263179Z"} 450 {"type":"line","content":"✅ Uploading artifacts using MinIO Uploader","time":"2023-07-18T19:12:44.066269065Z"} 451 {"type":"event","content":"running test [64b6e3f0530875ebaf73794f]","time":"2023-07-18T19:12:44.06658648Z"} 452 {"type":"line","content":"🚚 Preparing for test run","time":"2023-07-18T19:12:44.066599252Z"} 453 {"type":"line","content":"🚀 Test run command newman run /data/repo/Core App Tests - WebPlayer.postman_collection.json -e /tmp/testkube-tmp1897306484 --reporters cli,json --reporter-json-export /tmp/testkube-tmp4151526495.json","time":"2023-07-18T19:12:44.066856036Z"} 454 {"type":"line","content":"🔬 Executing in directory : \n $ newman run /data/repo/Core App Tests - WebPlayer.postman_collection.json -e /tmp/testkube-tmp1897306484 --reporters cli,json --reporter-json-export /tmp/testkube-tmp4151526495.json","time":"2023-07-18T19:12:44.066872166Z"} 455 {"type":"line","content":"newman\n\n","time":"2023-07-18T19:12:45.654031992Z"} 456 {"type":"line","content":"Core App Tests - WebPlayer\n","time":"2023-07-18T19:12:45.654989975Z"} 457 {"type":"line","content":"\n→ na.com client=testdb sign=testct1 company=41574150-b952-413b-898b-dc5336b4bd12\n","time":"2023-07-18T19:12:45.658607412Z"} 458 {"type":"line","content":" GET https://na.com/v6-wplt/?client=testdb\u0026sign=testct1\u0026company=41574150-b952-413b-898b-dc5336b4bd12 ","time":"2023-07-18T19:12:45.685011667Z"} 459 {"type":"line","content":"[200 OK, 33.9kB, 326ms]\n","time":"2023-07-18T19:12:46.014247794Z"} 460 {"type":"line","content":" ✓ Status code is 200\n","time":"2023-07-18T19:12:46.076176552Z"} 461 {"type":"line","content":"\n┌─────────────────────────┬────────────────────┬───────────────────┐\n│ │ executed │ failed │\n├─────────────────────────┼────────────────────┼───────────────────┤\n│ iterations │ 1 │ 0 │\n├─────────────────────────┼────────────────────┼───────────────────┤\n│ requests │ 1 │ 0 │\n├─────────────────────────┼────────────────────┼───────────────────┤\n│ test-scripts │ 1 │ 0 │\n├─────────────────────────┼────────────────────┼───────────────────┤\n│ prerequest-scripts │ 0 │ 0 │\n├─────────────────────────┼────────────────────┼───────────────────┤\n│ assertions │ 1 │ 0 │\n├─────────────────────────┴────────────────────┴───────────────────┤\n│ total run duration: 429ms │\n├──────────────────────────────────────────────────────────────────┤\n│ total data received: 33.45kB (approx) │\n├──────────────────────────────────────────────────────────────────┤\n│ average response time: 326ms [min: 326ms, max: 326ms, s.d.: 0µs] │\n└──────────────────────────────────────────────────────────────────┘\n","time":"2023-07-18T19:12:46.104174132Z"} 462 {"type":"line","content":"✅ Execution succeeded","time":"2023-07-18T19:12:46.114565626Z"} 463 {"type":"line","content":"✅ Got Newman result successfully","time":"2023-07-18T19:12:46.126116248Z"} 464 {"type":"line","content":"✅ Mapped Newman result successfully","time":"2023-07-18T19:12:46.126152021Z"} 465 {"type":"result","result":{"status":"passed","output":"newman\n\nCore App Tests - WebPlayer\n\n→ core-eks-test.poppcore.co client=testdb sign=testct1 company=41574150-b952-413b-898b-dc5336b4bd12\n GET https://na.com/v6-wplt/?client=testdb\u0026sign=testct1\u0026company=41574150-b952-413b-898b-dc5336b4bd12 [200 OK, 33.9kB, 326ms]\n ✓ Status code is 200\n\n┌─────────────────────────┬────────────────────┬───────────────────┐\n│ │ executed │ failed │\n├─────────────────────────┼────────────────────┼───────────────────┤\n│ iterations │ 1 │ 0 │\n├─────────────────────────┼────────────────────┼───────────────────┤\n│ requests │ 1 │ 0 │\n├─────────────────────────┼────────────────────┼───────────────────┤\n│ test-scripts │ 1 │ 0 │\n├─────────────────────────┼────────────────────┼───────────────────┤\n│ prerequest-scripts │ 0 │ 0 │\n├─────────────────────────┼────────────────────┼───────────────────┤\n│ assertions │ 1 │ 0 │\n├─────────────────────────┴────────────────────┴───────────────────┤\n│ total run duration: 429ms │\n├──────────────────────────────────────────────────────────────────┤\n│ total data received: 33.45kB (approx) │\n├──────────────────────────────────────────────────────────────────┤\n│ average response time: 326ms [min: 326ms, max: 326ms, s.d.: 0µs] │\n└──────────────────────────────────────────────────────────────────┘\n","outputType":"text/plain","steps":[{"name":"na.com client=testdb sign=testct1 company=41574150-b952-413b-898b-dc5336b4bd12","duration":"326ms","status":"passed","assertionResults":[{"name":"Status code is 200","status":"passed"}]}]},"time":"2023-07-18T19:12:46.12615853Z"}`) 466 result, output, err := ParseContainerOutput(exampleOutput) 467 468 assert.Len(t, output, 7305) 469 assert.NoError(t, err) 470 assert.Equal(t, testkube.ExecutionStatusPassed, result.Status) 471 }) 472 473 t.Run("Output with failed result", func(t *testing.T) { 474 t.Parallel() 475 476 output := []byte(`{"type":"event","content":"running test [63c9606d7104b0fa0b7a45f1]"} 477 {"type":"event","content":"Running [ ./zap-api-scan.py [-t https://www.example.com/openapi.json -f openapi -c examples/zap-api.conf -d -D 5 -I -l INFO -n examples/context.config -S -T 60 -U anonymous -O https://www.example.com -z -config aaa=bbb -r api-test-report.html]]"} 478 {"type":"result","result":{"status":"failed","errorMessage":"could not start process: fork/exec ./zap-api-scan.py: no such file or directory"}}`) 479 expectedOutput := `running test [63c9606d7104b0fa0b7a45f1] 480 Running [ ./zap-api-scan.py [-t https://www.example.com/openapi.json -f openapi -c examples/zap-api.conf -d -D 5 -I -l INFO -n examples/context.config -S -T 60 -U anonymous -O https://www.example.com -z -config aaa=bbb -r api-test-report.html]] 481 could not start process: fork/exec ./zap-api-scan.py: no such file or directory 482 ` 483 result, data, err := ParseContainerOutput(output) 484 485 assert.Equal(t, expectedOutput, data) 486 assert.NoError(t, err) 487 assert.Equal(t, testkube.ExecutionStatusFailed, result.Status) 488 assert.Equal(t, "could not start process: fork/exec ./zap-api-scan.py: no such file or directory", result.ErrorMessage) 489 }) 490 491 t.Run("Output with error", func(t *testing.T) { 492 t.Parallel() 493 494 output := []byte(` 495 {"type":"line","content":"🚚 Preparing test runner","time":"2023-01-19T15:22:25.867379429Z"} 496 {"type":"line","content":"🌍 Reading environment variables...","time":"2023-01-19T15:22:25.867927513Z"} 497 {"type":"line","content":"✅ Environment variables read successfully","time":"2023-01-19T15:22:25.867944763Z"} 498 {"type":"line","content":"RUNNER_ENDPOINT=\"testkube-minio-service-testkube:9000\"","time":"2023-01-19T15:22:25.867946929Z"} 499 {"type":"line","content":"RUNNER_ACCESSKEYID=\"********\"","time":"2023-01-19T15:22:25.867948804Z"} 500 {"type":"line","content":"RUNNER_SECRETACCESSKEY=\"********\"","time":"2023-01-19T15:22:25.867955263Z"} 501 {"type":"line","content":"RUNNER_REGION=\"\"","time":"2023-01-19T15:22:25.867962596Z"} 502 {"type":"line","content":"RUNNER_TOKEN=\"\"","time":"2023-01-19T15:22:25.867967971Z"} 503 {"type":"line","content":"RUNNER_SSL=false","time":"2023-01-19T15:22:25.867974013Z"} 504 {"type":"line","content":"RUNNER_SCRAPPERENABLED=\"true\"","time":"2023-01-19T15:22:25.867978888Z"} 505 {"type":"line","content":"RUNNER_GITUSERNAME=\"\"","time":"2023-01-19T15:22:25.867984179Z"} 506 {"type":"line","content":"RUNNER_GITTOKEN=\"\"","time":"2023-01-19T15:22:25.867986013Z"} 507 {"type":"line","content":"RUNNER_DATADIR=\"/data\"","time":"2023-01-19T15:22:25.867987596Z"} 508 {"type":"line","content":"RUNNER_CDEVENTS_TARGET=\"\"","time":"2023-01-17T15:29:17.921801596Z"} 509 {"type":"line","content":"RUNNER_DASHBOARD_URI=\"\"","time":"2023-01-17T15:29:17.921801596Z"} 510 {"type":"line","content":"RUNNER_CLUSTERID=\"\"","time":"2023-01-17T15:29:17.921801596Z"} 511 {"type":"event","content":"running test [63c960287104b0fa0b7a45ef]","time":"2023-01-19T15:22:25.868132888Z"} 512 {"type":"line","content":"🚚 Preparing for test run","time":"2023-01-19T15:22:25.868161346Z"} 513 {"type":"line","content":"❌ can't find branch or commit in params, repo:\u0026{Type_:git-file Uri:https://github.com/kubeshop/testkube.git Branch: Commit: Path:test/cypress/executor-smoke/cypress-11 Username: Token: UsernameSecret:\u003cnil\u003e TokenSecret:\u003cnil\u003e WorkingDir:}","time":"2023-01-19T15:22:25.868183971Z"} 514 {"type":"error","content":"can't find branch or commit in params, repo:\u0026{Type_:git-file Uri:https://github.com/kubeshop/testkube.git Branch: Commit: Path:test/cypress/executor-smoke/cypress-11 Username: Token: UsernameSecret:\u003cnil\u003e TokenSecret:\u003cnil\u003e WorkingDir:}","time":"2023-01-19T15:22:25.868198429Z"} 515 `) 516 expectedOutput := ` 517 🚚 Preparing test runner 518 🌍 Reading environment variables... 519 ✅ Environment variables read successfully 520 RUNNER_ENDPOINT="testkube-minio-service-testkube:9000" 521 RUNNER_ACCESSKEYID="********" 522 RUNNER_SECRETACCESSKEY="********" 523 RUNNER_REGION="" 524 RUNNER_TOKEN="" 525 RUNNER_SSL=false 526 RUNNER_SCRAPPERENABLED="true" 527 RUNNER_GITUSERNAME="" 528 RUNNER_GITTOKEN="" 529 RUNNER_DATADIR="/data" 530 RUNNER_CDEVENTS_TARGET="" 531 RUNNER_DASHBOARD_URI="" 532 RUNNER_CLUSTERID="" 533 running test [63c960287104b0fa0b7a45ef] 534 🚚 Preparing for test run 535 ❌ can't find branch or commit in params, repo:&{Type_:git-file Uri:https://github.com/kubeshop/testkube.git Branch: Commit: Path:test/cypress/executor-smoke/cypress-11 Username: Token: UsernameSecret:<nil> TokenSecret:<nil> WorkingDir:} 536 can't find branch or commit in params, repo:&{Type_:git-file Uri:https://github.com/kubeshop/testkube.git Branch: Commit: Path:test/cypress/executor-smoke/cypress-11 Username: Token: UsernameSecret:<nil> TokenSecret:<nil> WorkingDir:} 537 ` 538 539 result, data, err := ParseContainerOutput(output) 540 541 assert.Equal(t, expectedOutput, data) 542 assert.NoError(t, err) 543 assert.Equal(t, testkube.ExecutionStatusFailed, result.Status) 544 assert.Equal(t, "can't find branch or commit in params, repo:&{Type_:git-file Uri:https://github.com/kubeshop/testkube.git Branch: Commit: Path:test/cypress/executor-smoke/cypress-11 Username: Token: UsernameSecret:<nil> TokenSecret:<nil> WorkingDir:}", result.ErrorMessage) 545 }) 546 547 t.Run("Output random json format", func(t *testing.T) { 548 t.Parallel() 549 550 output := []byte(` 551 {"level":"info","ts":1689786938.2338681,"caller":"scraper/filesystem_extractor.go:50","msg":"scraping artifacts in directory: /data/artifacts"} 552 {"level":"info","ts":1689786938.234103,"caller":"scraper/filesystem_extractor.go:96","msg":"creating artifacts tarball with 1 files"} 553 {"level":"info","ts":1689786938.234986,"caller":"scraper/minio_uploader.go:47","msg":"MinIO loader is uploading file","file":"artifacts.tar.gz","folder":"64b81a176ebd74e68253f73e","size":12181} 554 `) 555 expectedOutput := ` 556 {"level":"info","ts":1689786938.2338681,"caller":"scraper/filesystem_extractor.go:50","msg":"scraping artifacts in directory: /data/artifacts"} 557 {"level":"info","ts":1689786938.234103,"caller":"scraper/filesystem_extractor.go:96","msg":"creating artifacts tarball with 1 files"} 558 {"level":"info","ts":1689786938.234986,"caller":"scraper/minio_uploader.go:47","msg":"MinIO loader is uploading file","file":"artifacts.tar.gz","folder":"64b81a176ebd74e68253f73e","size":12181} 559 ` 560 result, data, err := ParseContainerOutput(output) 561 562 assert.NoError(t, err) 563 assert.Equal(t, expectedOutput, data) 564 assert.Nil(t, result) 565 }) 566 567 t.Run("Output random text format", func(t *testing.T) { 568 t.Parallel() 569 570 output := []byte(` 571 [STARTED] Task without title. 572 [SUCCESS] Task without title. 573 [203:0719/171524.648215:ERROR:zygote_host_impl_linux.cc(263)] Failed to adjust OOM score of renderer with pid 374: Permission denied (13) 574 libva error: vaGetDriverNameByIndex() failed with unknown libva error, driver_name = (null) 575 [374:0719/171524.697752:ERROR:gpu_memory_buffer_support_x11.cc(44)] dri3 extension not supported. 576 [374:0719/171524.697752:ERROR:gpu_memory_buffer_support_x11.cc(44)] dri3 extension not supported. 577 578 ==================================================================================================== 579 `) 580 expectedOutput := ` 581 [STARTED] Task without title. 582 [SUCCESS] Task without title. 583 [203:0719/171524.648215:ERROR:zygote_host_impl_linux.cc(263)] Failed to adjust OOM score of renderer with pid 374: Permission denied (13) 584 libva error: vaGetDriverNameByIndex() failed with unknown libva error, driver_name = (null) 585 [374:0719/171524.697752:ERROR:gpu_memory_buffer_support_x11.cc(44)] dri3 extension not supported. 586 [374:0719/171524.697752:ERROR:gpu_memory_buffer_support_x11.cc(44)] dri3 extension not supported. 587 588 ==================================================================================================== 589 ` 590 result, data, err := ParseContainerOutput(output) 591 592 assert.Equal(t, expectedOutput, data) 593 assert.NoError(t, err) 594 assert.Nil(t, result) 595 }) 596 }