github.com/linchen2chris/hugo@v0.0.0-20230307053224-cec209389705/resources/transform_test.go (about) 1 // Copyright 2019 The Hugo Authors. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // http://www.apache.org/licenses/LICENSE-2.0 7 // 8 // Unless required by applicable law or agreed to in writing, software 9 // distributed under the License is distributed on an "AS IS" BASIS, 10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 package resources 15 16 import ( 17 "context" 18 "encoding/base64" 19 "fmt" 20 "io" 21 "path/filepath" 22 "strconv" 23 "strings" 24 "sync" 25 "testing" 26 27 "github.com/gohugoio/hugo/htesting" 28 29 "github.com/gohugoio/hugo/common/herrors" 30 "github.com/gohugoio/hugo/hugofs" 31 32 "github.com/gohugoio/hugo/media" 33 "github.com/gohugoio/hugo/resources/images" 34 "github.com/gohugoio/hugo/resources/internal" 35 36 "github.com/gohugoio/hugo/helpers" 37 38 "github.com/gohugoio/hugo/resources/resource" 39 "github.com/spf13/afero" 40 41 qt "github.com/frankban/quicktest" 42 ) 43 44 const gopher = `iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDsJkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAAILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36IMRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZmrSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhktrRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEjyy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5feA+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fUyLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2SnssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg==` 45 46 func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) } 47 48 func TestTransform(t *testing.T) { 49 c := qt.New(t) 50 51 createTransformer := func(spec *Spec, filename, content string) Transformer { 52 filename = filepath.FromSlash(filename) 53 fs := spec.Fs.Source 54 afero.WriteFile(fs, filename, []byte(content), 0777) 55 r, _ := spec.New(ResourceSourceDescriptor{Fs: fs, SourceFilename: filename}) 56 return r.(Transformer) 57 } 58 59 createContentReplacer := func(name, old, new string) ResourceTransformation { 60 return &testTransformation{ 61 name: name, 62 transform: func(ctx *ResourceTransformationCtx) error { 63 in := helpers.ReaderToString(ctx.From) 64 in = strings.Replace(in, old, new, 1) 65 ctx.AddOutPathIdentifier("." + name) 66 fmt.Fprint(ctx.To, in) 67 return nil 68 }, 69 } 70 } 71 72 // Verify that we publish the same file once only. 73 assertNoDuplicateWrites := func(c *qt.C, spec *Spec) { 74 c.Helper() 75 hugofs.WalkFilesystems(spec.Fs.PublishDir, func(fs afero.Fs) bool { 76 if dfs, ok := fs.(hugofs.DuplicatesReporter); ok { 77 c.Assert(dfs.ReportDuplicates(), qt.Equals, "") 78 } 79 return false 80 }) 81 } 82 83 assertShouldExist := func(c *qt.C, spec *Spec, filename string, should bool) { 84 c.Helper() 85 exists, _ := helpers.Exists(filepath.FromSlash(filename), spec.Fs.WorkingDirReadOnly) 86 c.Assert(exists, qt.Equals, should) 87 } 88 89 c.Run("All values", func(c *qt.C) { 90 c.Parallel() 91 92 spec := newTestResourceSpec(specDescriptor{c: c}) 93 94 transformation := &testTransformation{ 95 name: "test", 96 transform: func(ctx *ResourceTransformationCtx) error { 97 // Content 98 in := helpers.ReaderToString(ctx.From) 99 in = strings.Replace(in, "blue", "green", 1) 100 fmt.Fprint(ctx.To, in) 101 102 // Media type 103 ctx.OutMediaType = media.CSVType 104 105 // Change target 106 ctx.ReplaceOutPathExtension(".csv") 107 108 // Add some data to context 109 ctx.Data["mydata"] = "Hugo Rocks!" 110 111 return nil 112 }, 113 } 114 115 r := createTransformer(spec, "f1.txt", "color is blue") 116 117 tr, err := r.Transform(transformation) 118 c.Assert(err, qt.IsNil) 119 content, err := tr.(resource.ContentProvider).Content(context.Background()) 120 c.Assert(err, qt.IsNil) 121 122 c.Assert(content, qt.Equals, "color is green") 123 c.Assert(tr.MediaType(), eq, media.CSVType) 124 c.Assert(tr.RelPermalink(), qt.Equals, "/f1.csv") 125 assertShouldExist(c, spec, "public/f1.csv", true) 126 127 data := tr.Data().(map[string]any) 128 c.Assert(data["mydata"], qt.Equals, "Hugo Rocks!") 129 130 assertNoDuplicateWrites(c, spec) 131 }) 132 133 c.Run("Meta only", func(c *qt.C) { 134 c.Parallel() 135 136 spec := newTestResourceSpec(specDescriptor{c: c}) 137 138 transformation := &testTransformation{ 139 name: "test", 140 transform: func(ctx *ResourceTransformationCtx) error { 141 // Change media type only 142 ctx.OutMediaType = media.CSVType 143 ctx.ReplaceOutPathExtension(".csv") 144 145 return nil 146 }, 147 } 148 149 r := createTransformer(spec, "f1.txt", "color is blue") 150 151 tr, err := r.Transform(transformation) 152 c.Assert(err, qt.IsNil) 153 content, err := tr.(resource.ContentProvider).Content(context.Background()) 154 c.Assert(err, qt.IsNil) 155 156 c.Assert(content, qt.Equals, "color is blue") 157 c.Assert(tr.MediaType(), eq, media.CSVType) 158 159 // The transformed file should only be published if RelPermalink 160 // or Permalink is called. 161 n := htesting.Rnd.Intn(3) 162 shouldExist := true 163 switch n { 164 case 0: 165 tr.RelPermalink() 166 case 1: 167 tr.Permalink() 168 default: 169 shouldExist = false 170 } 171 172 assertShouldExist(c, spec, "public/f1.csv", shouldExist) 173 assertNoDuplicateWrites(c, spec) 174 }) 175 176 c.Run("Memory-cached transformation", func(c *qt.C) { 177 c.Parallel() 178 179 spec := newTestResourceSpec(specDescriptor{c: c}) 180 181 // Two transformations with same id, different behaviour. 182 t1 := createContentReplacer("t1", "blue", "green") 183 t2 := createContentReplacer("t1", "color", "car") 184 185 for i, transformation := range []ResourceTransformation{t1, t2} { 186 r := createTransformer(spec, "f1.txt", "color is blue") 187 tr, _ := r.Transform(transformation) 188 content, err := tr.(resource.ContentProvider).Content(context.Background()) 189 c.Assert(err, qt.IsNil) 190 c.Assert(content, qt.Equals, "color is green", qt.Commentf("i=%d", i)) 191 192 assertShouldExist(c, spec, "public/f1.t1.txt", false) 193 } 194 195 assertNoDuplicateWrites(c, spec) 196 }) 197 198 c.Run("File-cached transformation", func(c *qt.C) { 199 c.Parallel() 200 201 fs := afero.NewMemMapFs() 202 203 for i := 0; i < 2; i++ { 204 spec := newTestResourceSpec(specDescriptor{c: c, fs: fs}) 205 206 r := createTransformer(spec, "f1.txt", "color is blue") 207 208 var transformation ResourceTransformation 209 210 if i == 0 { 211 // There is currently a hardcoded list of transformations that we 212 // persist to disk (tocss, postcss). 213 transformation = &testTransformation{ 214 name: "tocss", 215 transform: func(ctx *ResourceTransformationCtx) error { 216 in := helpers.ReaderToString(ctx.From) 217 in = strings.Replace(in, "blue", "green", 1) 218 ctx.AddOutPathIdentifier("." + "cached") 219 ctx.OutMediaType = media.CSVType 220 ctx.Data = map[string]any{ 221 "Hugo": "Rocks!", 222 } 223 fmt.Fprint(ctx.To, in) 224 return nil 225 }, 226 } 227 } else { 228 // Force read from file cache. 229 transformation = &testTransformation{ 230 name: "tocss", 231 transform: func(ctx *ResourceTransformationCtx) error { 232 return herrors.ErrFeatureNotAvailable 233 }, 234 } 235 } 236 237 msg := qt.Commentf("i=%d", i) 238 239 tr, _ := r.Transform(transformation) 240 c.Assert(tr.RelPermalink(), qt.Equals, "/f1.cached.txt", msg) 241 content, err := tr.(resource.ContentProvider).Content(context.Background()) 242 c.Assert(err, qt.IsNil) 243 c.Assert(content, qt.Equals, "color is green", msg) 244 c.Assert(tr.MediaType(), eq, media.CSVType) 245 c.Assert(tr.Data(), qt.DeepEquals, map[string]any{ 246 "Hugo": "Rocks!", 247 }) 248 249 assertNoDuplicateWrites(c, spec) 250 assertShouldExist(c, spec, "public/f1.cached.txt", true) 251 252 } 253 }) 254 255 c.Run("Access RelPermalink first", func(c *qt.C) { 256 c.Parallel() 257 258 spec := newTestResourceSpec(specDescriptor{c: c}) 259 260 t1 := createContentReplacer("t1", "blue", "green") 261 262 r := createTransformer(spec, "f1.txt", "color is blue") 263 264 tr, _ := r.Transform(t1) 265 266 relPermalink := tr.RelPermalink() 267 268 content, err := tr.(resource.ContentProvider).Content(context.Background()) 269 c.Assert(err, qt.IsNil) 270 271 c.Assert(relPermalink, qt.Equals, "/f1.t1.txt") 272 c.Assert(content, qt.Equals, "color is green") 273 c.Assert(tr.MediaType(), eq, media.TextType) 274 275 assertNoDuplicateWrites(c, spec) 276 assertShouldExist(c, spec, "public/f1.t1.txt", true) 277 }) 278 279 c.Run("Content two", func(c *qt.C) { 280 c.Parallel() 281 282 spec := newTestResourceSpec(specDescriptor{c: c}) 283 284 t1 := createContentReplacer("t1", "blue", "green") 285 t2 := createContentReplacer("t1", "color", "car") 286 287 r := createTransformer(spec, "f1.txt", "color is blue") 288 289 tr, _ := r.Transform(t1, t2) 290 content, err := tr.(resource.ContentProvider).Content(context.Background()) 291 c.Assert(err, qt.IsNil) 292 293 c.Assert(content, qt.Equals, "car is green") 294 c.Assert(tr.MediaType(), eq, media.TextType) 295 296 assertNoDuplicateWrites(c, spec) 297 }) 298 299 c.Run("Content two chained", func(c *qt.C) { 300 c.Parallel() 301 302 spec := newTestResourceSpec(specDescriptor{c: c}) 303 304 t1 := createContentReplacer("t1", "blue", "green") 305 t2 := createContentReplacer("t2", "color", "car") 306 307 r := createTransformer(spec, "f1.txt", "color is blue") 308 309 tr1, _ := r.Transform(t1) 310 tr2, _ := tr1.Transform(t2) 311 312 content1, err := tr1.(resource.ContentProvider).Content(context.Background()) 313 c.Assert(err, qt.IsNil) 314 content2, err := tr2.(resource.ContentProvider).Content(context.Background()) 315 c.Assert(err, qt.IsNil) 316 317 c.Assert(content1, qt.Equals, "color is green") 318 c.Assert(content2, qt.Equals, "car is green") 319 320 assertNoDuplicateWrites(c, spec) 321 }) 322 323 c.Run("Content many", func(c *qt.C) { 324 c.Parallel() 325 326 spec := newTestResourceSpec(specDescriptor{c: c}) 327 328 const count = 26 // A-Z 329 330 transformations := make([]ResourceTransformation, count) 331 for i := 0; i < count; i++ { 332 transformations[i] = createContentReplacer(fmt.Sprintf("t%d", i), fmt.Sprint(i), string(rune(i+65))) 333 } 334 335 var countstr strings.Builder 336 for i := 0; i < count; i++ { 337 countstr.WriteString(fmt.Sprint(i)) 338 } 339 340 r := createTransformer(spec, "f1.txt", countstr.String()) 341 342 tr, _ := r.Transform(transformations...) 343 content, err := tr.(resource.ContentProvider).Content(context.Background()) 344 c.Assert(err, qt.IsNil) 345 346 c.Assert(content, qt.Equals, "ABCDEFGHIJKLMNOPQRSTUVWXYZ") 347 348 assertNoDuplicateWrites(c, spec) 349 }) 350 351 c.Run("Image", func(c *qt.C) { 352 c.Parallel() 353 354 spec := newTestResourceSpec(specDescriptor{c: c}) 355 356 transformation := &testTransformation{ 357 name: "test", 358 transform: func(ctx *ResourceTransformationCtx) error { 359 ctx.AddOutPathIdentifier(".changed") 360 return nil 361 }, 362 } 363 364 r := createTransformer(spec, "gopher.png", helpers.ReaderToString(gopherPNG())) 365 366 tr, err := r.Transform(transformation) 367 c.Assert(err, qt.IsNil) 368 c.Assert(tr.MediaType(), eq, media.PNGType) 369 370 img, ok := tr.(images.ImageResource) 371 c.Assert(ok, qt.Equals, true) 372 373 c.Assert(img.Width(), qt.Equals, 75) 374 c.Assert(img.Height(), qt.Equals, 60) 375 376 // RelPermalink called. 377 resizedPublished1, err := img.Resize("40x40") 378 c.Assert(err, qt.IsNil) 379 c.Assert(resizedPublished1.Height(), qt.Equals, 40) 380 c.Assert(resizedPublished1.RelPermalink(), qt.Equals, "/gopher.changed_hu2e827f5a78333ebc04166dd643235dea_1462_40x40_resize_linear_3.png") 381 assertShouldExist(c, spec, "public/gopher.changed_hu2e827f5a78333ebc04166dd643235dea_1462_40x40_resize_linear_3.png", true) 382 383 // Permalink called. 384 resizedPublished2, err := img.Resize("30x30") 385 c.Assert(err, qt.IsNil) 386 c.Assert(resizedPublished2.Height(), qt.Equals, 30) 387 c.Assert(resizedPublished2.Permalink(), qt.Equals, "https://example.com/gopher.changed_hu2e827f5a78333ebc04166dd643235dea_1462_30x30_resize_linear_3.png") 388 assertShouldExist(c, spec, "public/gopher.changed_hu2e827f5a78333ebc04166dd643235dea_1462_30x30_resize_linear_3.png", true) 389 390 // Not published because none of RelPermalink or Permalink was called. 391 resizedNotPublished, err := img.Resize("50x50") 392 c.Assert(err, qt.IsNil) 393 c.Assert(resizedNotPublished.Height(), qt.Equals, 50) 394 // c.Assert(resized.RelPermalink(), qt.Equals, "/gopher.changed_hu2e827f5a78333ebc04166dd643235dea_1462_50x50_resize_linear_2.png") 395 assertShouldExist(c, spec, "public/gopher.changed_hu2e827f5a78333ebc04166dd643235dea_1462_50x50_resize_linear_3.png", false) 396 397 assertNoDuplicateWrites(c, spec) 398 }) 399 400 c.Run("Concurrent", func(c *qt.C) { 401 spec := newTestResourceSpec(specDescriptor{c: c}) 402 403 transformers := make([]Transformer, 10) 404 transformations := make([]ResourceTransformation, 10) 405 406 for i := 0; i < 10; i++ { 407 transformers[i] = createTransformer(spec, fmt.Sprintf("f%d.txt", i), fmt.Sprintf("color is %d", i)) 408 transformations[i] = createContentReplacer("test", strconv.Itoa(i), "blue") 409 } 410 411 var wg sync.WaitGroup 412 413 for i := 0; i < 13; i++ { 414 wg.Add(1) 415 go func(i int) { 416 defer wg.Done() 417 for j := 0; j < 23; j++ { 418 id := (i + j) % 10 419 tr, err := transformers[id].Transform(transformations[id]) 420 c.Assert(err, qt.IsNil) 421 content, err := tr.(resource.ContentProvider).Content(context.Background()) 422 c.Assert(err, qt.IsNil) 423 c.Assert(content, qt.Equals, "color is blue") 424 c.Assert(tr.RelPermalink(), qt.Equals, fmt.Sprintf("/f%d.test.txt", id)) 425 } 426 }(i) 427 } 428 wg.Wait() 429 430 assertNoDuplicateWrites(c, spec) 431 }) 432 } 433 434 type testTransformation struct { 435 name string 436 transform func(ctx *ResourceTransformationCtx) error 437 } 438 439 func (t *testTransformation) Key() internal.ResourceTransformationKey { 440 return internal.NewResourceTransformationKey(t.name) 441 } 442 443 func (t *testTransformation) Transform(ctx *ResourceTransformationCtx) error { 444 return t.transform(ctx) 445 }