github.com/onflow/flow-go@v0.33.17/utils/test_matrix/test_matrix_test.go (about) 1 package main 2 3 import ( 4 "testing" 5 6 "github.com/stretchr/testify/require" 7 ) 8 9 // Can't have a const []string so resorting to using a test helper function. 10 func getAllFlowPackages() []string { 11 return []string{ 12 flowPackagePrefix + "abc", 13 flowPackagePrefix + "abc/123", 14 flowPackagePrefix + "abc/def", 15 flowPackagePrefix + "abc/def/ghi", 16 flowPackagePrefix + "abc/def/ghi/jkl", 17 flowPackagePrefix + "abc/def/ghi/jkl/mno", 18 flowPackagePrefix + "abc/def/ghi/jkl/mno/pqr", 19 flowPackagePrefix + "abc/def/ghi/mno/abc", 20 flowPackagePrefix + "abc/def/ghi/mno/def", 21 flowPackagePrefix + "abc/def/ghi/mno/ghi", 22 flowPackagePrefix + "abc/def/jkl", 23 flowPackagePrefix + "abc/def/jkl/mno", 24 flowPackagePrefix + "abc/def/jkl/mno/pqr", 25 flowPackagePrefix + "def", 26 flowPackagePrefix + "def/abc", 27 flowPackagePrefix + "ghi", 28 flowPackagePrefix + "jkl", 29 flowPackagePrefix + "mno/abc", 30 flowPackagePrefix + "pqr", 31 flowPackagePrefix + "stu", 32 flowPackagePrefix + "vwx", 33 flowPackagePrefix + "vwx/ghi", 34 flowPackagePrefix + "yz", 35 } 36 } 37 38 // TestListTargetPackages_DefaultRunners tests that the target packages are included in the target packages and seen packages. 39 // All packages use default CI runners. 40 func TestListTargetPackages_DefaultRunners(t *testing.T) { 41 target, seenPackages := listTargetPackages([]string{"abc", "ghi"}, getAllFlowPackages()) 42 require.Equal(t, 2, len(target.packages)) 43 44 // check all TARGET 45 // these are the expected target packages that start with "abc" 46 require.Equal(t, 13, len(target.packages["abc"])) 47 require.Contains(t, target.packages["abc"], flowPackagePrefix+"abc") 48 require.Contains(t, target.packages["abc"], flowPackagePrefix+"abc/123") 49 require.Contains(t, target.packages["abc"], flowPackagePrefix+"abc/def") 50 require.Contains(t, target.packages["abc"], flowPackagePrefix+"abc/def/ghi") 51 require.Contains(t, target.packages["abc"], flowPackagePrefix+"abc/def/ghi/jkl") 52 require.Contains(t, target.packages["abc"], flowPackagePrefix+"abc/def/ghi/jkl/mno") 53 require.Contains(t, target.packages["abc"], flowPackagePrefix+"abc/def/ghi/jkl/mno/pqr") 54 require.Contains(t, target.packages["abc"], flowPackagePrefix+"abc/def/ghi/mno/abc") 55 require.Contains(t, target.packages["abc"], flowPackagePrefix+"abc/def/ghi/mno/def") 56 require.Contains(t, target.packages["abc"], flowPackagePrefix+"abc/def/ghi/mno/ghi") 57 require.Contains(t, target.packages["abc"], flowPackagePrefix+"abc/def/jkl") 58 require.Contains(t, target.packages["abc"], flowPackagePrefix+"abc/def/jkl/mno") 59 require.Contains(t, target.packages["abc"], flowPackagePrefix+"abc/def/jkl/mno/pqr") 60 61 // there should be 1 package that starts with "ghi" 62 require.Equal(t, 1, len(target.packages["ghi"])) 63 require.Contains(t, target.packages["ghi"], flowPackagePrefix+"ghi") 64 65 // check all CI RUNNERS for each target package 66 require.Equal(t, 2, len(target.runners)) 67 require.Equal(t, target.runners["abc"], ciDefaultRunner) 68 require.Equal(t, target.runners["ghi"], ciDefaultRunner) 69 70 // check all SEEN packages 71 // these are all expected packages that start with "abc" or "ghi" 72 require.Equal(t, 14, len(seenPackages)) 73 require.Contains(t, seenPackages, flowPackagePrefix+"abc") 74 require.Contains(t, seenPackages, flowPackagePrefix+"abc/123") 75 require.Contains(t, seenPackages, flowPackagePrefix+"abc/def") 76 require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/ghi") 77 require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/ghi/jkl") 78 require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/ghi/jkl/mno") 79 require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/ghi/jkl/mno/pqr") 80 require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/ghi/mno/abc") 81 require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/ghi/mno/def") 82 require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/ghi/mno/ghi") 83 require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/jkl") 84 require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/jkl/mno") 85 require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/jkl/mno/pqr") 86 87 require.Contains(t, seenPackages, flowPackagePrefix+"ghi") 88 } 89 90 // TestListTargetSubPackages_CustomRunners tests that if a subpackage is specified as a target package, then the sub package and 91 // all children of the sub package are also included in the target packages. 92 func TestListTargetSubPackages_CustomRunners(t *testing.T) { 93 target, seenPackages := listTargetPackages([]string{"abc/def:foo_runner"}, getAllFlowPackages()) 94 require.Equal(t, 1, len(target.packages)) 95 96 // check all TARGET packages 97 // there should be 2 target subpackages that starts with "abc/def" 98 require.Equal(t, 11, len(target.packages["abc/def"])) 99 require.Contains(t, target.packages["abc/def"], flowPackagePrefix+"abc/def") 100 require.Contains(t, target.packages["abc/def"], flowPackagePrefix+"abc/def/ghi") 101 require.Contains(t, target.packages["abc/def"], flowPackagePrefix+"abc/def/ghi/jkl") 102 require.Contains(t, target.packages["abc/def"], flowPackagePrefix+"abc/def/ghi/jkl/mno") 103 require.Contains(t, target.packages["abc/def"], flowPackagePrefix+"abc/def/ghi/jkl/mno/pqr") 104 require.Contains(t, target.packages["abc/def"], flowPackagePrefix+"abc/def/ghi/mno/abc") 105 require.Contains(t, target.packages["abc/def"], flowPackagePrefix+"abc/def/ghi/mno/def") 106 require.Contains(t, target.packages["abc/def"], flowPackagePrefix+"abc/def/ghi/mno/ghi") 107 require.Contains(t, target.packages["abc/def"], flowPackagePrefix+"abc/def/jkl") 108 require.Contains(t, target.packages["abc/def"], flowPackagePrefix+"abc/def/jkl/mno") 109 require.Contains(t, target.packages["abc/def"], flowPackagePrefix+"abc/def/jkl/mno/pqr") 110 111 // check all CI RUNNERS for each target package 112 require.Equal(t, 1, len(target.runners)) 113 require.Equal(t, target.runners["abc/def"], "foo_runner") 114 115 // check all SEEN packages 116 // there should be 11 seen subpackages that start with "abc/def" 117 require.Equal(t, 11, len(seenPackages)) 118 require.Contains(t, seenPackages, flowPackagePrefix+"abc/def") 119 require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/ghi") 120 require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/ghi/jkl") 121 require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/ghi/jkl/mno") 122 require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/ghi/jkl/mno/pqr") 123 require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/ghi/mno/abc") 124 require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/ghi/mno/def") 125 require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/ghi/mno/ghi") 126 require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/jkl") 127 require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/jkl/mno") 128 require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/jkl/mno/pqr") 129 } 130 131 // TestListOtherPackages tests that the remaining packages that don't match any of the target packages are included 132 func TestListOtherPackages(t *testing.T) { 133 var seenPackages = make(map[string]string) 134 seenPackages[flowPackagePrefix+"abc/def"] = flowPackagePrefix + "abc/def" 135 seenPackages[flowPackagePrefix+"abc/def/ghi"] = flowPackagePrefix + "abc/def/ghi" 136 seenPackages[flowPackagePrefix+"ghi"] = flowPackagePrefix + "ghi" 137 seenPackages[flowPackagePrefix+"mno/abc"] = flowPackagePrefix + "mno/abc" 138 seenPackages[flowPackagePrefix+"stu"] = flowPackagePrefix + "stu" 139 140 otherPackages := listOtherPackages(getAllFlowPackages(), seenPackages) 141 142 require.Equal(t, 18, len(otherPackages)) 143 144 require.Contains(t, otherPackages, flowPackagePrefix+"abc") 145 require.Contains(t, otherPackages, flowPackagePrefix+"abc/123") 146 require.Contains(t, otherPackages, flowPackagePrefix+"abc/def/ghi/jkl") 147 require.Contains(t, otherPackages, flowPackagePrefix+"abc/def/ghi/jkl/mno") 148 require.Contains(t, otherPackages, flowPackagePrefix+"abc/def/ghi/jkl/mno/pqr") 149 require.Contains(t, otherPackages, flowPackagePrefix+"abc/def/ghi/mno/abc") 150 require.Contains(t, otherPackages, flowPackagePrefix+"abc/def/ghi/mno/def") 151 require.Contains(t, otherPackages, flowPackagePrefix+"abc/def/ghi/mno/ghi") 152 require.Contains(t, otherPackages, flowPackagePrefix+"abc/def/jkl") 153 require.Contains(t, otherPackages, flowPackagePrefix+"abc/def/jkl/mno") 154 require.Contains(t, otherPackages, flowPackagePrefix+"abc/def/jkl/mno/pqr") 155 require.Contains(t, otherPackages, flowPackagePrefix+"def") 156 require.Contains(t, otherPackages, flowPackagePrefix+"def/abc") 157 require.Contains(t, otherPackages, flowPackagePrefix+"jkl") 158 require.Contains(t, otherPackages, flowPackagePrefix+"pqr") 159 require.Contains(t, otherPackages, flowPackagePrefix+"vwx") 160 require.Contains(t, otherPackages, flowPackagePrefix+"vwx/ghi") 161 require.Contains(t, otherPackages, flowPackagePrefix+"yz") 162 } 163 164 // TestGenerateTestMatrix_CustomRunners tests that the test matrix is generated correctly where the target packages include top level 165 // packages as well as sub packages. It also tests having 2 different custom CI runners, as well as default runners. 166 func TestGenerateTestMatrix_CustomRunners(t *testing.T) { 167 target, seenPackages := listTargetPackages([]string{"abc/def", "def:foo-runner", "ghi", "vwx/ghi:foo-runner2"}, getAllFlowPackages()) 168 require.Equal(t, 4, len(target.packages)) 169 require.Equal(t, 4, len(target.runners)) 170 require.Equal(t, 15, len(seenPackages)) 171 172 otherPackages := listOtherPackages(getAllFlowPackages(), seenPackages) 173 174 matrix := generateTestMatrix(target, otherPackages) 175 176 // should be 4 groups in test matrix: abc/def, def, ghi, vwx/ghi, others 177 require.Equal(t, 5, len(matrix)) 178 179 require.Contains(t, matrix, testMatrix{ 180 Name: "abc/def", 181 Packages: "github.com/onflow/flow-go/abc/def github.com/onflow/flow-go/abc/def/ghi github.com/onflow/flow-go/abc/def/ghi/jkl github.com/onflow/flow-go/abc/def/ghi/jkl/mno github.com/onflow/flow-go/abc/def/ghi/jkl/mno/pqr github.com/onflow/flow-go/abc/def/ghi/mno/abc github.com/onflow/flow-go/abc/def/ghi/mno/def github.com/onflow/flow-go/abc/def/ghi/mno/ghi github.com/onflow/flow-go/abc/def/jkl github.com/onflow/flow-go/abc/def/jkl/mno github.com/onflow/flow-go/abc/def/jkl/mno/pqr", 182 Runner: "ubuntu-latest"}, 183 ) 184 require.Contains(t, matrix, testMatrix{ 185 Name: "def", 186 Packages: "github.com/onflow/flow-go/def github.com/onflow/flow-go/def/abc", 187 Runner: "foo-runner"}, 188 ) 189 require.Contains(t, matrix, testMatrix{ 190 Name: "ghi", 191 Packages: "github.com/onflow/flow-go/ghi", 192 Runner: "ubuntu-latest"}, 193 ) 194 require.Contains(t, matrix, testMatrix{ 195 Name: "vwx/ghi", 196 Packages: "github.com/onflow/flow-go/vwx/ghi", 197 Runner: "foo-runner2"}, 198 ) 199 require.Contains(t, matrix, testMatrix{ 200 Name: "others", 201 Packages: "github.com/onflow/flow-go/abc github.com/onflow/flow-go/abc/123 github.com/onflow/flow-go/jkl github.com/onflow/flow-go/mno/abc github.com/onflow/flow-go/pqr github.com/onflow/flow-go/stu github.com/onflow/flow-go/vwx github.com/onflow/flow-go/yz", 202 Runner: "ubuntu-latest"}, 203 ) 204 } 205 206 // TestGenerateTestMatrix_SubSubPackages tests that the test matrix is generated correctly where the target packages 207 // include 2nd and 3rd level sub packages. It also tests having 2 different custom CI runners, as well as default runners. 208 func TestGenerateTestMatrix_SubSubPackages(t *testing.T) { 209 target, seenPackages := listTargetPackages([]string{"abc/def/ghi:foo-runner1", "abc/def/jkl:foo-runner2", "abc"}, getAllFlowPackages()) 210 require.Equal(t, 3, len(target.packages)) 211 require.Equal(t, 3, len(target.runners)) 212 require.Equal(t, 13, len(seenPackages)) 213 214 otherPackages := listOtherPackages(getAllFlowPackages(), seenPackages) 215 216 matrix := generateTestMatrix(target, otherPackages) 217 218 // should be 4 groups in test matrix: abc/def/ghi, abc/def/jkl, abc, others 219 require.Equal(t, 4, len(matrix)) 220 221 require.Contains(t, matrix, testMatrix{ 222 Name: "abc/def/ghi", 223 Packages: "github.com/onflow/flow-go/abc/def/ghi github.com/onflow/flow-go/abc/def/ghi/jkl github.com/onflow/flow-go/abc/def/ghi/jkl/mno github.com/onflow/flow-go/abc/def/ghi/jkl/mno/pqr github.com/onflow/flow-go/abc/def/ghi/mno/abc github.com/onflow/flow-go/abc/def/ghi/mno/def github.com/onflow/flow-go/abc/def/ghi/mno/ghi", 224 Runner: "foo-runner1"}, 225 ) 226 227 require.Contains(t, matrix, testMatrix{ 228 Name: "abc/def/jkl", 229 Packages: "github.com/onflow/flow-go/abc/def/jkl github.com/onflow/flow-go/abc/def/jkl/mno github.com/onflow/flow-go/abc/def/jkl/mno/pqr", 230 Runner: "foo-runner2"}, 231 ) 232 233 // parent package should not have any packages from its sub packages because they were already included in the sub package groups 234 require.Contains(t, matrix, testMatrix{ 235 Name: "abc", 236 Packages: "github.com/onflow/flow-go/abc github.com/onflow/flow-go/abc/123 github.com/onflow/flow-go/abc/def", 237 Runner: "ubuntu-latest"}, 238 ) 239 240 require.Contains(t, matrix, testMatrix{ 241 Name: "others", 242 Packages: "github.com/onflow/flow-go/def github.com/onflow/flow-go/def/abc github.com/onflow/flow-go/ghi github.com/onflow/flow-go/jkl github.com/onflow/flow-go/mno/abc github.com/onflow/flow-go/pqr github.com/onflow/flow-go/stu github.com/onflow/flow-go/vwx github.com/onflow/flow-go/vwx/ghi github.com/onflow/flow-go/yz", 243 Runner: "ubuntu-latest"}, 244 ) 245 }