github.com/aarzilli/tools@v0.0.0-20151123112009-0d27094f75e0/net/http/repo/helpers.go (about) 1 package repo 2 3 import ( 4 "log" 5 "net/http" 6 "path" 7 8 "github.com/pbberlin/tools/net/http/loghttp" 9 "github.com/pbberlin/tools/runtimepb" 10 "golang.org/x/net/html" 11 ) 12 13 // Some websites have url paths like 14 // .../news/some--title/32168.html 15 // .../news/other-title/82316.html 16 // 17 // We want to condense these to 18 // .../news/some--title-32168.html 19 // .../news/other-title-82316.html 20 func condenseTrailingDir(uri string, n int) (ret string) { 21 22 switch n { 23 case 0: 24 return uri 25 case 1: 26 return uri 27 case 2: 28 base1 := path.Base(uri) 29 rdir1 := path.Dir(uri) // rightest Dir 30 31 base2 := path.Base(rdir1) 32 33 rdir2 := path.Dir(rdir1) 34 35 ret = path.Join(rdir2, base2+"-"+base1) 36 default: 37 runtimepb.StackTrace(4) 38 log.Fatalf("not implemented n > 2 (%v)", n) 39 } 40 41 return 42 43 } 44 45 // Adding preconfigured settings to a fetch command 46 func addDefaults(in FetchCommand) FetchCommand { 47 48 var preset FetchCommand 49 50 h := in.Host 51 if exactPreset, ok := ConfigDefaults[h]; ok { 52 preset = exactPreset 53 } else { 54 preset = ConfigDefaults["unspecified"] 55 } 56 57 in.DepthTolerance = preset.DepthTolerance 58 in.CondenseTrailingDirs = preset.CondenseTrailingDirs 59 if in.DesiredNumber == 0 { 60 in.DesiredNumber = preset.DesiredNumber 61 } 62 63 if in.RssXMLURI == nil || len(in.RssXMLURI) == 0 { 64 in.RssXMLURI = preset.RssXMLURI 65 } 66 67 return in 68 } 69 70 // Each domain might have *several* RSS URLs. 71 // Function matchingRSSURI returns the most fitting RSS URL 72 // for a given SearchPrefix, or empty string. 73 func matchingRSSURI(w http.ResponseWriter, r *http.Request, c FetchCommand) (ret string) { 74 75 lg, lge := loghttp.Logger(w, r) 76 _, _ = lg, lge 77 78 cntr := 0 79 sp := c.SearchPrefix 80 81 MarkX: 82 for { 83 84 // lg("search pref %v", sp) 85 86 if rss, ok := c.RssXMLURI[sp]; ok { 87 ret = rss 88 lg("found rss url %v for %v", ret, sp) 89 break MarkX 90 91 } 92 93 spPrev := sp 94 sp = path.Dir(sp) 95 if spPrev == "." && sp == "." || 96 spPrev == "/" && sp == "/" || 97 spPrev == "" && sp == "." { 98 lg("Did not find a RSS URL for %v %q", c.SearchPrefix, ret) 99 break 100 } 101 102 cntr++ 103 if cntr > 20 { 104 lg("Select RSS Loop did not terminate. %v", c.SearchPrefix) 105 break 106 } 107 } 108 109 return 110 } 111 112 func attrX(attributes []html.Attribute, key string) (s string) { 113 for _, a := range attributes { 114 if key == a.Key { 115 s = a.Val 116 break 117 } 118 } 119 return 120 }