github.com/shohhei1126/hugo@v0.42.2-0.20180623210752-3d5928889ad7/resource/resource.go (about) 1 // Copyright 2017-present The Hugo Authors. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // http://www.apache.org/licenses/LICENSE-2.0 7 // 8 // Unless required by applicable law or agreed to in writing, software 9 // distributed under the License is distributed on an "AS IS" BASIS, 10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 package resource 15 16 import ( 17 "fmt" 18 "mime" 19 "os" 20 "path" 21 "path/filepath" 22 "strconv" 23 "strings" 24 "sync" 25 26 "github.com/gohugoio/hugo/common/maps" 27 28 "github.com/spf13/afero" 29 30 "github.com/spf13/cast" 31 32 "github.com/gobwas/glob" 33 "github.com/gohugoio/hugo/helpers" 34 "github.com/gohugoio/hugo/media" 35 "github.com/gohugoio/hugo/source" 36 ) 37 38 var ( 39 _ Resource = (*genericResource)(nil) 40 _ metaAssigner = (*genericResource)(nil) 41 _ Source = (*genericResource)(nil) 42 _ Cloner = (*genericResource)(nil) 43 _ ResourcesLanguageMerger = (*Resources)(nil) 44 ) 45 46 const DefaultResourceType = "unknown" 47 48 // Source is an internal template and not meant for use in the templates. It 49 // may change without notice. 50 type Source interface { 51 AbsSourceFilename() string 52 Publish() error 53 } 54 55 // Cloner is an internal template and not meant for use in the templates. It 56 // may change without notice. 57 type Cloner interface { 58 WithNewBase(base string) Resource 59 } 60 61 type metaAssigner interface { 62 setTitle(title string) 63 setName(name string) 64 updateParams(params map[string]interface{}) 65 } 66 67 // Resource represents a linkable resource, i.e. a content page, image etc. 68 type Resource interface { 69 // Permalink represents the absolute link to this resource. 70 Permalink() string 71 72 // RelPermalink represents the host relative link to this resource. 73 RelPermalink() string 74 75 // ResourceType is the resource type. For most file types, this is the main 76 // part of the MIME type, e.g. "image", "application", "text" etc. 77 // For content pages, this value is "page". 78 ResourceType() string 79 80 // Name is the logical name of this resource. This can be set in the front matter 81 // metadata for this resource. If not set, Hugo will assign a value. 82 // This will in most cases be the base filename. 83 // So, for the image "/some/path/sunset.jpg" this will be "sunset.jpg". 84 // The value returned by this method will be used in the GetByPrefix and ByPrefix methods 85 // on Resources. 86 Name() string 87 88 // Title returns the title if set in front matter. For content pages, this will be the expected value. 89 Title() string 90 91 // Params set in front matter for this resource. 92 Params() map[string]interface{} 93 94 // Content returns this resource's content. It will be equivalent to reading the content 95 // that RelPermalink points to in the published folder. 96 // The return type will be contextual, and should be what you would expect: 97 // * Page: template.HTML 98 // * JSON: String 99 // * Etc. 100 Content() (interface{}, error) 101 } 102 103 type ResourcesLanguageMerger interface { 104 MergeByLanguage(other Resources) Resources 105 // Needed for integration with the tpl package. 106 MergeByLanguageInterface(other interface{}) (interface{}, error) 107 } 108 109 type translatedResource interface { 110 TranslationKey() string 111 } 112 113 // Resources represents a slice of resources, which can be a mix of different types. 114 // I.e. both pages and images etc. 115 type Resources []Resource 116 117 func (r Resources) ByType(tp string) Resources { 118 var filtered Resources 119 120 for _, resource := range r { 121 if resource.ResourceType() == tp { 122 filtered = append(filtered, resource) 123 } 124 } 125 return filtered 126 } 127 128 const prefixDeprecatedMsg = `We have added the more flexible Resources.GetMatch (find one) and Resources.Match (many) to replace the "prefix" methods. 129 130 These matches by a given globbing pattern, e.g. "*.jpg". 131 132 Some examples: 133 134 * To find all resources by its prefix in the root dir of the bundle: .Match image* 135 * To find one resource by its prefix in the root dir of the bundle: .GetMatch image* 136 * To find all JPEG images anywhere in the bundle: .Match **.jpg` 137 138 // GetByPrefix gets the first resource matching the given filename prefix, e.g 139 // "logo" will match logo.png. It returns nil of none found. 140 // In potential ambiguous situations, combine it with ByType. 141 func (r Resources) GetByPrefix(prefix string) Resource { 142 helpers.Deprecated("Resources", "GetByPrefix", prefixDeprecatedMsg, true) 143 prefix = strings.ToLower(prefix) 144 for _, resource := range r { 145 if matchesPrefix(resource, prefix) { 146 return resource 147 } 148 } 149 return nil 150 } 151 152 // ByPrefix gets all resources matching the given base filename prefix, e.g 153 // "logo" will match logo.png. 154 func (r Resources) ByPrefix(prefix string) Resources { 155 helpers.Deprecated("Resources", "ByPrefix", prefixDeprecatedMsg, true) 156 var matches Resources 157 prefix = strings.ToLower(prefix) 158 for _, resource := range r { 159 if matchesPrefix(resource, prefix) { 160 matches = append(matches, resource) 161 } 162 } 163 return matches 164 } 165 166 // GetMatch finds the first Resource matching the given pattern, or nil if none found. 167 // See Match for a more complete explanation about the rules used. 168 func (r Resources) GetMatch(pattern string) Resource { 169 g, err := getGlob(pattern) 170 if err != nil { 171 return nil 172 } 173 174 for _, resource := range r { 175 if g.Match(strings.ToLower(resource.Name())) { 176 return resource 177 } 178 } 179 180 return nil 181 } 182 183 // Match gets all resources matching the given base filename prefix, e.g 184 // "*.png" will match all png files. The "*" does not match path delimiters (/), 185 // so if you organize your resources in sub-folders, you need to be explicit about it, e.g.: 186 // "images/*.png". To match any PNG image anywhere in the bundle you can do "**.png", and 187 // to match all PNG images below the images folder, use "images/**.jpg". 188 // The matching is case insensitive. 189 // Match matches by using the value of Resource.Name, which, by default, is a filename with 190 // path relative to the bundle root with Unix style slashes (/) and no leading slash, e.g. "images/logo.png". 191 // See https://github.com/gobwas/glob for the full rules set. 192 func (r Resources) Match(pattern string) Resources { 193 g, err := getGlob(pattern) 194 if err != nil { 195 return nil 196 } 197 198 var matches Resources 199 for _, resource := range r { 200 if g.Match(strings.ToLower(resource.Name())) { 201 matches = append(matches, resource) 202 } 203 } 204 return matches 205 } 206 207 func matchesPrefix(r Resource, prefix string) bool { 208 return strings.HasPrefix(strings.ToLower(r.Name()), prefix) 209 } 210 211 var ( 212 globCache = make(map[string]glob.Glob) 213 globMu sync.RWMutex 214 ) 215 216 func getGlob(pattern string) (glob.Glob, error) { 217 var g glob.Glob 218 219 globMu.RLock() 220 g, found := globCache[pattern] 221 globMu.RUnlock() 222 if !found { 223 var err error 224 g, err = glob.Compile(strings.ToLower(pattern), '/') 225 if err != nil { 226 return nil, err 227 } 228 229 globMu.Lock() 230 globCache[pattern] = g 231 globMu.Unlock() 232 } 233 234 return g, nil 235 236 } 237 238 // MergeByLanguage adds missing translations in r1 from r2. 239 func (r1 Resources) MergeByLanguage(r2 Resources) Resources { 240 result := append(Resources(nil), r1...) 241 m := make(map[string]bool) 242 for _, r := range r1 { 243 if translated, ok := r.(translatedResource); ok { 244 m[translated.TranslationKey()] = true 245 } 246 } 247 248 for _, r := range r2 { 249 if translated, ok := r.(translatedResource); ok { 250 if _, found := m[translated.TranslationKey()]; !found { 251 result = append(result, r) 252 } 253 } 254 } 255 return result 256 } 257 258 // MergeByLanguageInterface is the generic version of MergeByLanguage. It 259 // is here just so it can be called from the tpl package. 260 func (r1 Resources) MergeByLanguageInterface(in interface{}) (interface{}, error) { 261 r2, ok := in.(Resources) 262 if !ok { 263 return nil, fmt.Errorf("%T cannot be merged by language", in) 264 } 265 return r1.MergeByLanguage(r2), nil 266 } 267 268 type Spec struct { 269 *helpers.PathSpec 270 271 mimeTypes media.Types 272 273 // Holds default filter settings etc. 274 imaging *Imaging 275 276 imageCache *imageCache 277 278 GenImagePath string 279 } 280 281 func NewSpec(s *helpers.PathSpec, mimeTypes media.Types) (*Spec, error) { 282 283 imaging, err := decodeImaging(s.Cfg.GetStringMap("imaging")) 284 if err != nil { 285 return nil, err 286 } 287 288 genImagePath := filepath.FromSlash("_gen/images") 289 290 return &Spec{PathSpec: s, 291 GenImagePath: genImagePath, 292 imaging: &imaging, mimeTypes: mimeTypes, imageCache: newImageCache( 293 s, 294 // We're going to write a cache pruning routine later, so make it extremely 295 // unlikely that the user shoots him or herself in the foot 296 // and this is set to a value that represents data he/she 297 // cares about. This should be set in stone once released. 298 genImagePath, 299 )}, nil 300 } 301 302 func (r *Spec) NewResourceFromFile( 303 targetPathBuilder func(base string) string, 304 file source.File, relTargetFilename string) (Resource, error) { 305 306 return r.newResource(targetPathBuilder, file.Filename(), file.FileInfo(), relTargetFilename) 307 } 308 309 func (r *Spec) NewResourceFromFilename( 310 targetPathBuilder func(base string) string, 311 absSourceFilename, relTargetFilename string) (Resource, error) { 312 313 fi, err := r.sourceFs().Stat(absSourceFilename) 314 if err != nil { 315 return nil, err 316 } 317 return r.newResource(targetPathBuilder, absSourceFilename, fi, relTargetFilename) 318 } 319 320 func (r *Spec) sourceFs() afero.Fs { 321 return r.PathSpec.BaseFs.ContentFs 322 } 323 324 func (r *Spec) newResource( 325 targetPathBuilder func(base string) string, 326 absSourceFilename string, fi os.FileInfo, relTargetFilename string) (Resource, error) { 327 328 var mimeType string 329 ext := filepath.Ext(relTargetFilename) 330 m, found := r.mimeTypes.GetBySuffix(strings.TrimPrefix(ext, ".")) 331 if found { 332 mimeType = m.SubType 333 } else { 334 mimeType = mime.TypeByExtension(ext) 335 if mimeType == "" { 336 mimeType = DefaultResourceType 337 } else { 338 mimeType = mimeType[:strings.Index(mimeType, "/")] 339 } 340 } 341 342 gr := r.newGenericResource(targetPathBuilder, fi, absSourceFilename, relTargetFilename, mimeType) 343 344 if mimeType == "image" { 345 ext := strings.ToLower(helpers.Ext(absSourceFilename)) 346 347 imgFormat, ok := imageFormats[ext] 348 if !ok { 349 // This allows SVG etc. to be used as resources. They will not have the methods of the Image, but 350 // that would not (currently) have worked. 351 return gr, nil 352 } 353 354 f, err := gr.sourceFs().Open(absSourceFilename) 355 if err != nil { 356 return nil, fmt.Errorf("failed to open image source file: %s", err) 357 } 358 defer f.Close() 359 360 hash, err := helpers.MD5FromFileFast(f) 361 if err != nil { 362 return nil, err 363 } 364 365 return &Image{ 366 hash: hash, 367 format: imgFormat, 368 imaging: r.imaging, 369 genericResource: gr}, nil 370 } 371 return gr, nil 372 } 373 374 func (r *Spec) IsInCache(key string) bool { 375 // This is used for cache pruning. We currently only have images, but we could 376 // imagine expanding on this. 377 return r.imageCache.isInCache(key) 378 } 379 380 func (r *Spec) DeleteCacheByPrefix(prefix string) { 381 r.imageCache.deleteByPrefix(prefix) 382 } 383 384 func (r *Spec) CacheStats() string { 385 r.imageCache.mu.RLock() 386 defer r.imageCache.mu.RUnlock() 387 388 s := fmt.Sprintf("Cache entries: %d", len(r.imageCache.store)) 389 390 count := 0 391 for k := range r.imageCache.store { 392 if count > 5 { 393 break 394 } 395 s += "\n" + k 396 count++ 397 } 398 399 return s 400 } 401 402 type dirFile struct { 403 // This is the directory component with Unix-style slashes. 404 dir string 405 // This is the file component. 406 file string 407 } 408 409 func (d dirFile) path() string { 410 return path.Join(d.dir, d.file) 411 } 412 413 type resourceContent struct { 414 content string 415 contentInit sync.Once 416 } 417 418 // genericResource represents a generic linkable resource. 419 type genericResource struct { 420 // The relative path to this resource. 421 relTargetPath dirFile 422 423 // Base is set when the output format's path has a offset, e.g. for AMP. 424 base string 425 426 title string 427 name string 428 params map[string]interface{} 429 430 // Absolute filename to the source, including any content folder path. 431 // Note that this is absolute in relation to the filesystem it is stored in. 432 // It can be a base path filesystem, and then this filename will not match 433 // the path to the file on the real filesystem. 434 sourceFilename string 435 436 // This may be set to tell us to look in another filesystem for this resource. 437 // We, by default, use the sourceFs filesystem in the spec below. 438 overriddenSourceFs afero.Fs 439 440 spec *Spec 441 442 resourceType string 443 osFileInfo os.FileInfo 444 445 targetPathBuilder func(rel string) string 446 447 // We create copies of this struct, so this needs to be a pointer. 448 *resourceContent 449 } 450 451 func (l *genericResource) Content() (interface{}, error) { 452 var err error 453 l.contentInit.Do(func() { 454 var b []byte 455 456 b, err := afero.ReadFile(l.sourceFs(), l.AbsSourceFilename()) 457 if err != nil { 458 return 459 } 460 461 l.content = string(b) 462 463 }) 464 465 return l.content, err 466 } 467 468 func (l *genericResource) sourceFs() afero.Fs { 469 if l.overriddenSourceFs != nil { 470 return l.overriddenSourceFs 471 } 472 return l.spec.sourceFs() 473 } 474 475 func (l *genericResource) Permalink() string { 476 return l.spec.PermalinkForBaseURL(l.relPermalinkForRel(l.relTargetPath.path(), false), l.spec.BaseURL.String()) 477 } 478 479 func (l *genericResource) RelPermalink() string { 480 return l.relPermalinkForRel(l.relTargetPath.path(), true) 481 } 482 483 func (l *genericResource) Name() string { 484 return l.name 485 } 486 487 func (l *genericResource) Title() string { 488 return l.title 489 } 490 491 func (l *genericResource) Params() map[string]interface{} { 492 return l.params 493 } 494 495 func (l *genericResource) setTitle(title string) { 496 l.title = title 497 } 498 499 func (l *genericResource) setName(name string) { 500 l.name = name 501 } 502 503 func (l *genericResource) updateParams(params map[string]interface{}) { 504 if l.params == nil { 505 l.params = params 506 return 507 } 508 509 // Sets the params not already set 510 for k, v := range params { 511 if _, found := l.params[k]; !found { 512 l.params[k] = v 513 } 514 } 515 } 516 517 // Implement the Cloner interface. 518 func (l genericResource) WithNewBase(base string) Resource { 519 l.base = base 520 l.resourceContent = &resourceContent{} 521 return &l 522 } 523 524 func (l *genericResource) relPermalinkForRel(rel string, addBasePath bool) string { 525 return l.spec.PathSpec.URLizeFilename(l.relTargetPathForRel(rel, addBasePath)) 526 } 527 528 func (l *genericResource) relTargetPathForRel(rel string, addBasePath bool) string { 529 if l.targetPathBuilder != nil { 530 rel = l.targetPathBuilder(rel) 531 } 532 533 if l.base != "" { 534 rel = path.Join(l.base, rel) 535 } 536 537 if addBasePath && l.spec.PathSpec.BasePath != "" { 538 rel = path.Join(l.spec.PathSpec.BasePath, rel) 539 } 540 541 if rel[0] != '/' { 542 rel = "/" + rel 543 } 544 545 return rel 546 } 547 548 func (l *genericResource) ResourceType() string { 549 return l.resourceType 550 } 551 552 func (l *genericResource) AbsSourceFilename() string { 553 return l.sourceFilename 554 } 555 556 func (l *genericResource) String() string { 557 return fmt.Sprintf("Resource(%s: %s)", l.resourceType, l.name) 558 } 559 560 func (l *genericResource) Publish() error { 561 f, err := l.sourceFs().Open(l.AbsSourceFilename()) 562 if err != nil { 563 return err 564 } 565 defer f.Close() 566 return helpers.WriteToDisk(l.target(), f, l.spec.BaseFs.PublishFs) 567 } 568 569 const counterPlaceHolder = ":counter" 570 571 // AssignMetadata assigns the given metadata to those resources that supports updates 572 // and matching by wildcard given in `src` using `filepath.Match` with lower cased values. 573 // This assignment is additive, but the most specific match needs to be first. 574 // The `name` and `title` metadata field support shell-matched collection it got a match in. 575 // See https://golang.org/pkg/path/#Match 576 func AssignMetadata(metadata []map[string]interface{}, resources ...Resource) error { 577 578 counters := make(map[string]int) 579 580 for _, r := range resources { 581 if _, ok := r.(metaAssigner); !ok { 582 continue 583 } 584 585 var ( 586 nameSet, titleSet bool 587 nameCounter, titleCounter = 0, 0 588 nameCounterFound, titleCounterFound bool 589 resourceSrcKey = strings.ToLower(r.Name()) 590 ) 591 592 ma := r.(metaAssigner) 593 for _, meta := range metadata { 594 src, found := meta["src"] 595 if !found { 596 return fmt.Errorf("missing 'src' in metadata for resource") 597 } 598 599 srcKey := strings.ToLower(cast.ToString(src)) 600 601 glob, err := getGlob(srcKey) 602 if err != nil { 603 return fmt.Errorf("failed to match resource with metadata: %s", err) 604 } 605 606 match := glob.Match(resourceSrcKey) 607 608 if match { 609 if !nameSet { 610 name, found := meta["name"] 611 if found { 612 name := cast.ToString(name) 613 if !nameCounterFound { 614 nameCounterFound = strings.Contains(name, counterPlaceHolder) 615 } 616 if nameCounterFound && nameCounter == 0 { 617 counterKey := "name_" + srcKey 618 nameCounter = counters[counterKey] + 1 619 counters[counterKey] = nameCounter 620 } 621 622 ma.setName(replaceResourcePlaceholders(name, nameCounter)) 623 nameSet = true 624 } 625 } 626 627 if !titleSet { 628 title, found := meta["title"] 629 if found { 630 title := cast.ToString(title) 631 if !titleCounterFound { 632 titleCounterFound = strings.Contains(title, counterPlaceHolder) 633 } 634 if titleCounterFound && titleCounter == 0 { 635 counterKey := "title_" + srcKey 636 titleCounter = counters[counterKey] + 1 637 counters[counterKey] = titleCounter 638 } 639 ma.setTitle((replaceResourcePlaceholders(title, titleCounter))) 640 titleSet = true 641 } 642 } 643 644 params, found := meta["params"] 645 if found { 646 m := cast.ToStringMap(params) 647 // Needed for case insensitive fetching of params values 648 maps.ToLower(m) 649 ma.updateParams(m) 650 } 651 } 652 } 653 } 654 655 return nil 656 } 657 658 func replaceResourcePlaceholders(in string, counter int) string { 659 return strings.Replace(in, counterPlaceHolder, strconv.Itoa(counter), -1) 660 } 661 662 func (l *genericResource) target() string { 663 target := l.relTargetPathForRel(l.relTargetPath.path(), false) 664 if l.spec.PathSpec.Languages.IsMultihost() { 665 target = path.Join(l.spec.PathSpec.Language.Lang, target) 666 } 667 return filepath.Clean(target) 668 } 669 670 func (r *Spec) newGenericResource( 671 targetPathBuilder func(base string) string, 672 osFileInfo os.FileInfo, 673 sourceFilename, 674 baseFilename, 675 resourceType string) *genericResource { 676 677 // This value is used both to construct URLs and file paths, but start 678 // with a Unix-styled path. 679 baseFilename = filepath.ToSlash(baseFilename) 680 fpath, fname := path.Split(baseFilename) 681 682 return &genericResource{ 683 targetPathBuilder: targetPathBuilder, 684 osFileInfo: osFileInfo, 685 sourceFilename: sourceFilename, 686 relTargetPath: dirFile{dir: fpath, file: fname}, 687 resourceType: resourceType, 688 spec: r, 689 params: make(map[string]interface{}), 690 name: baseFilename, 691 title: baseFilename, 692 resourceContent: &resourceContent{}, 693 } 694 }