github.com/xzntrc/go-enry/v2@v2.0.0-20230215091818-766cc1d65498/internal/code-generator/generator/generator_test.go (about) 1 package generator 2 3 import ( 4 "flag" 5 "fmt" 6 "io/ioutil" 7 "os" 8 "os/exec" 9 "path/filepath" 10 "strings" 11 "testing" 12 13 "github.com/go-enry/go-enry/v2/internal/tokenizer" 14 15 "github.com/stretchr/testify/assert" 16 "github.com/stretchr/testify/require" 17 "github.com/stretchr/testify/suite" 18 ) 19 20 var ( 21 linguistURL = "https://github.com/github/linguist.git" 22 linguistClonedEnvVar = "ENRY_TEST_REPO" 23 commit = "d7799da826e01acdb8f84694d33116dccaabe9c2" 24 samplesDir = "samples" 25 languagesFile = filepath.Join("lib", "linguist", "languages.yml") 26 27 testDir = "test_files" 28 assetsDir = filepath.Join("..", "assets") 29 30 // Extensions test 31 extensionGold = filepath.Join(testDir, "extension.gold") 32 extensionTestTmplPath = filepath.Join(assetsDir, "extension.go.tmpl") 33 extensionTestTmplName = "extension.go.tmpl" 34 35 // Heuristics test 36 heuristicsTestFile = filepath.Join("lib", "linguist", "heuristics.yml") 37 contentGold = filepath.Join(testDir, "content.gold") 38 contentTestTmplPath = filepath.Join(assetsDir, "content.go.tmpl") 39 contentTestTmplName = "content.go.tmpl" 40 41 // Vendor test 42 vendorTestFile = filepath.Join("lib", "linguist", "vendor.yml") 43 vendorGold = filepath.Join(testDir, "vendor.gold") 44 vendorTestTmplPath = filepath.Join(assetsDir, "vendor.go.tmpl") 45 vendorTestTmplName = "vendor.go.tmpl" 46 47 // Documentation test 48 documentationTestFile = filepath.Join("lib", "linguist", "documentation.yml") 49 documentationGold = filepath.Join(testDir, "documentation.gold") 50 documentationTestTmplPath = filepath.Join(assetsDir, "documentation.go.tmpl") 51 documentationTestTmplName = "documentation.go.tmpl" 52 53 // Types test 54 typeGold = filepath.Join(testDir, "type.gold") 55 typeTestTmplPath = filepath.Join(assetsDir, "type.go.tmpl") 56 typeTestTmplName = "type.go.tmpl" 57 58 // Interpreters test 59 interpreterGold = filepath.Join(testDir, "interpreter.gold") 60 interpreterTestTmplPath = filepath.Join(assetsDir, "interpreter.go.tmpl") 61 interpreterTestTmplName = "interpreter.go.tmpl" 62 63 // Filenames test 64 filenameGold = filepath.Join(testDir, "filename.gold") 65 filenameTestTmplPath = filepath.Join(assetsDir, "filename.go.tmpl") 66 filenameTestTmplName = "filename.go.tmpl" 67 68 // Aliases test 69 aliasGold = filepath.Join(testDir, "alias.gold") 70 aliasTestTmplPath = filepath.Join(assetsDir, "alias.go.tmpl") 71 aliasTestTmplName = "alias.go.tmpl" 72 73 // Frequencies test 74 frequenciesGold = filepath.Join(testDir, "frequencies.gold") 75 frequenciesTestTmplPath = filepath.Join(assetsDir, "frequencies.go.tmpl") 76 frequenciesTestTmplName = "frequencies.go.tmpl" 77 78 // commit test 79 commitGold = filepath.Join(testDir, "commit.gold") 80 commitTestTmplPath = filepath.Join(assetsDir, "commit.go.tmpl") 81 commitTestTmplName = "commit.go.tmpl" 82 83 // mime test 84 mimeTypeGold = filepath.Join(testDir, "mimeType.gold") 85 mimeTypeTestTmplPath = filepath.Join(assetsDir, "mimeType.go.tmpl") 86 mimeTypeTestTmplName = "mimeType.go.tmpl" 87 88 // colors test 89 colorsGold = filepath.Join(testDir, "colors.gold") 90 colorsTestTmplPath = filepath.Join(assetsDir, "colors.go.tmpl") 91 colorsTestTmplName = "colors.go.tmpl" 92 93 // colors test 94 groupsGold = filepath.Join(testDir, "groups.gold") 95 groupsTestTmplPath = filepath.Join(assetsDir, "groups.go.tmpl") 96 groupsTestTmplName = "groups.go.tmpl" 97 ) 98 99 type GeneratorTestSuite struct { 100 suite.Suite 101 tmpLinguistDir string 102 isCleanupNeeded bool 103 testCases []testCase 104 } 105 106 type testCase struct { 107 name string 108 fileToParse string 109 samplesDir string 110 tmplPath string 111 tmplName string 112 commit string 113 generate File 114 wantOut string 115 } 116 117 var updateGold = flag.Bool("update_gold", false, "Update golden test files") 118 119 func Test_GeneratorTestSuite(t *testing.T) { 120 suite.Run(t, new(GeneratorTestSuite)) 121 } 122 123 func (s *GeneratorTestSuite) maybeCloneLinguist() { 124 var err error 125 s.tmpLinguistDir = os.Getenv(linguistClonedEnvVar) 126 isLinguistCloned := s.tmpLinguistDir != "" 127 if !isLinguistCloned { 128 s.tmpLinguistDir, err = ioutil.TempDir("", "linguist-") 129 require.NoError(s.T(), err) 130 131 s.T().Logf("Cloning Linguist repo to '%s' as %s was not set\n", 132 s.tmpLinguistDir, linguistClonedEnvVar) 133 cmd := exec.Command("git", "clone", "--depth", "100", linguistURL, s.tmpLinguistDir) 134 err = cmd.Run() 135 require.NoError(s.T(), err) 136 s.isCleanupNeeded = true 137 } 138 139 cwd, err := os.Getwd() 140 require.NoError(s.T(), err) 141 142 err = os.Chdir(s.tmpLinguistDir) 143 require.NoError(s.T(), err) 144 145 cmd := exec.Command("git", "checkout", commit) 146 err = cmd.Run() 147 require.NoError(s.T(), err) 148 149 err = os.Chdir(cwd) 150 require.NoError(s.T(), err) 151 } 152 153 func (s *GeneratorTestSuite) SetupSuite() { 154 s.maybeCloneLinguist() 155 s.testCases = []testCase{ 156 { 157 name: "Extensions()", 158 fileToParse: filepath.Join(s.tmpLinguistDir, languagesFile), 159 samplesDir: "", 160 tmplPath: extensionTestTmplPath, 161 tmplName: extensionTestTmplName, 162 commit: commit, 163 generate: Extensions, 164 wantOut: extensionGold, 165 }, 166 { 167 name: "Heuristics()", 168 fileToParse: filepath.Join(s.tmpLinguistDir, heuristicsTestFile), 169 samplesDir: "", 170 tmplPath: contentTestTmplPath, 171 tmplName: contentTestTmplName, 172 commit: commit, 173 generate: GenHeuristics, 174 wantOut: contentGold, 175 }, 176 { 177 name: "Vendor()", 178 fileToParse: filepath.Join(s.tmpLinguistDir, vendorTestFile), 179 samplesDir: "", 180 tmplPath: vendorTestTmplPath, 181 tmplName: vendorTestTmplName, 182 commit: commit, 183 generate: Vendor, 184 wantOut: vendorGold, 185 }, 186 { 187 name: "Documentation()", 188 fileToParse: filepath.Join(s.tmpLinguistDir, documentationTestFile), 189 samplesDir: "", 190 tmplPath: documentationTestTmplPath, 191 tmplName: documentationTestTmplName, 192 commit: commit, 193 generate: Documentation, 194 wantOut: documentationGold, 195 }, 196 { 197 name: "Types()", 198 fileToParse: filepath.Join(s.tmpLinguistDir, languagesFile), 199 samplesDir: "", 200 tmplPath: typeTestTmplPath, 201 tmplName: typeTestTmplName, 202 commit: commit, 203 generate: Types, 204 wantOut: typeGold, 205 }, 206 { 207 name: "Interpreters()", 208 fileToParse: filepath.Join(s.tmpLinguistDir, languagesFile), 209 samplesDir: "", 210 tmplPath: interpreterTestTmplPath, 211 tmplName: interpreterTestTmplName, 212 commit: commit, 213 generate: Interpreters, 214 wantOut: interpreterGold, 215 }, 216 { 217 name: "Filenames()", 218 fileToParse: filepath.Join(s.tmpLinguistDir, languagesFile), 219 samplesDir: filepath.Join(s.tmpLinguistDir, samplesDir), 220 tmplPath: filenameTestTmplPath, 221 tmplName: filenameTestTmplName, 222 commit: commit, 223 generate: Filenames, 224 wantOut: filenameGold, 225 }, 226 { 227 name: "Aliases()", 228 fileToParse: filepath.Join(s.tmpLinguistDir, languagesFile), 229 samplesDir: "", 230 tmplPath: aliasTestTmplPath, 231 tmplName: aliasTestTmplName, 232 commit: commit, 233 generate: Aliases, 234 wantOut: aliasGold, 235 }, 236 { 237 name: "Frequencies()", 238 samplesDir: filepath.Join(s.tmpLinguistDir, samplesDir), 239 tmplPath: frequenciesTestTmplPath, 240 tmplName: frequenciesTestTmplName, 241 commit: commit, 242 generate: Frequencies, 243 wantOut: frequenciesGold, 244 }, 245 { 246 name: "Commit()", 247 samplesDir: "", 248 tmplPath: commitTestTmplPath, 249 tmplName: commitTestTmplName, 250 commit: commit, 251 generate: Commit, 252 wantOut: commitGold, 253 }, 254 { 255 name: "MimeType()", 256 fileToParse: filepath.Join(s.tmpLinguistDir, languagesFile), 257 samplesDir: "", 258 tmplPath: mimeTypeTestTmplPath, 259 tmplName: mimeTypeTestTmplName, 260 commit: commit, 261 generate: MimeType, 262 wantOut: mimeTypeGold, 263 }, 264 { 265 name: "Colors()", 266 fileToParse: filepath.Join(s.tmpLinguistDir, languagesFile), 267 samplesDir: "", 268 tmplPath: colorsTestTmplPath, 269 tmplName: colorsTestTmplName, 270 commit: commit, 271 generate: Colors, 272 wantOut: colorsGold, 273 }, 274 { 275 name: "Groups()", 276 fileToParse: filepath.Join(s.tmpLinguistDir, languagesFile), 277 samplesDir: "", 278 tmplPath: groupsTestTmplPath, 279 tmplName: groupsTestTmplName, 280 commit: commit, 281 generate: Groups, 282 wantOut: groupsGold, 283 }, 284 } 285 } 286 287 func (s *GeneratorTestSuite) TearDownSuite() { 288 if s.isCleanupNeeded { 289 err := os.RemoveAll(s.tmpLinguistDir) 290 assert.NoError(s.T(), err) 291 } 292 } 293 294 // TestUpdateGeneratorTestSuiteGold is a Gold results generation automation. 295 // It should only be enabled&run manually on every new Linguist version 296 // to update *.gold files. 297 func (s *GeneratorTestSuite) TestUpdateGeneratorTestSuiteGold() { 298 if !*updateGold { 299 s.T().Skip() 300 } 301 s.T().Logf("Generating new *.gold test files") 302 for _, test := range s.testCases { 303 dst := test.wantOut 304 s.T().Logf("Generating %s from %s\n", dst, test.fileToParse) 305 err := test.generate(test.fileToParse, test.samplesDir, dst, test.tmplPath, test.tmplName, test.commit) 306 assert.NoError(s.T(), err) 307 } 308 } 309 310 func (s *GeneratorTestSuite) TestGenerationFiles() { 311 for _, test := range s.testCases { 312 gold, err := ioutil.ReadFile(test.wantOut) 313 assert.NoError(s.T(), err) 314 315 outPath, err := ioutil.TempFile("", "generator-test-") 316 assert.NoError(s.T(), err) 317 defer os.Remove(outPath.Name()) 318 err = test.generate(test.fileToParse, test.samplesDir, outPath.Name(), test.tmplPath, test.tmplName, test.commit) 319 assert.NoError(s.T(), err) 320 out, err := ioutil.ReadFile(outPath.Name()) 321 assert.NoError(s.T(), err) 322 323 expected := normalizeSpaces(string(gold)) 324 actual := normalizeSpaces(string(out)) 325 // this produces large unreadable output, so we do it 'manually' instead 326 // assert.Equal(s.T(), expected, actual, "Test %s", test.name) 327 if expected != actual { 328 assert.Fail(s.T(), fmt.Sprintf("%s output is different from %q", test.name, test.wantOut)) 329 diff, err := text_diff(gold, out) 330 if err != nil { 331 s.T().Logf("Failed produce a diff between expected and actual: %s", err.Error()) 332 s.T().Logf("Expected %q", expected[:400]) 333 s.T().Logf("Actual %q", actual[:400]) 334 } 335 s.T().Logf("\n%s", diff) 336 } 337 338 } 339 } 340 341 func (s *GeneratorTestSuite) TestTokenizerOnATS() { 342 const suspiciousSample = "samples/ATS/csv_parse.hats" 343 sFile := filepath.Join(s.tmpLinguistDir, suspiciousSample) 344 content, err := ioutil.ReadFile(sFile) 345 require.NoError(s.T(), err) 346 347 tokens := tokenizer.Tokenize(content) 348 assert.Equal(s.T(), 381, len(tokens), "Number of tokens using LF as line endings") 349 } 350 351 // normalizeSpaces returns a copy of str with whitespaces normalized. 352 // We use this to compare generated source as gofmt format may change. 353 // E.g for changes between Go 1.10 and 1.11 see 354 // https://go-review.googlesource.com/c/go/+/122295/ 355 func normalizeSpaces(str string) string { 356 return strings.Join(strings.Fields(str), " ") 357 }