github.com/aarzilli/tools@v0.0.0-20151123112009-0d27094f75e0/net/http/repo/helpers.go (about)

     1  package repo
     2  
     3  import (
     4  	"log"
     5  	"net/http"
     6  	"path"
     7  
     8  	"github.com/pbberlin/tools/net/http/loghttp"
     9  	"github.com/pbberlin/tools/runtimepb"
    10  	"golang.org/x/net/html"
    11  )
    12  
    13  // Some websites have url paths like
    14  // 	.../news/some--title/32168.html
    15  // 	.../news/other-title/82316.html
    16  //
    17  // We want to condense these to
    18  // 	.../news/some--title-32168.html
    19  // 	.../news/other-title-82316.html
    20  func condenseTrailingDir(uri string, n int) (ret string) {
    21  
    22  	switch n {
    23  	case 0:
    24  		return uri
    25  	case 1:
    26  		return uri
    27  	case 2:
    28  		base1 := path.Base(uri)
    29  		rdir1 := path.Dir(uri) // rightest Dir
    30  
    31  		base2 := path.Base(rdir1)
    32  
    33  		rdir2 := path.Dir(rdir1)
    34  
    35  		ret = path.Join(rdir2, base2+"-"+base1)
    36  	default:
    37  		runtimepb.StackTrace(4)
    38  		log.Fatalf("not implemented n > 2 (%v)", n)
    39  	}
    40  
    41  	return
    42  
    43  }
    44  
    45  // Adding preconfigured settings to a fetch command
    46  func addDefaults(in FetchCommand) FetchCommand {
    47  
    48  	var preset FetchCommand
    49  
    50  	h := in.Host
    51  	if exactPreset, ok := ConfigDefaults[h]; ok {
    52  		preset = exactPreset
    53  	} else {
    54  		preset = ConfigDefaults["unspecified"]
    55  	}
    56  
    57  	in.DepthTolerance = preset.DepthTolerance
    58  	in.CondenseTrailingDirs = preset.CondenseTrailingDirs
    59  	if in.DesiredNumber == 0 {
    60  		in.DesiredNumber = preset.DesiredNumber
    61  	}
    62  
    63  	if in.RssXMLURI == nil || len(in.RssXMLURI) == 0 {
    64  		in.RssXMLURI = preset.RssXMLURI
    65  	}
    66  
    67  	return in
    68  }
    69  
    70  // Each domain might have *several* RSS URLs.
    71  // Function matchingRSSURI returns the most fitting RSS URL
    72  // for a given  SearchPrefix, or empty string.
    73  func matchingRSSURI(w http.ResponseWriter, r *http.Request, c FetchCommand) (ret string) {
    74  
    75  	lg, lge := loghttp.Logger(w, r)
    76  	_, _ = lg, lge
    77  
    78  	cntr := 0
    79  	sp := c.SearchPrefix
    80  
    81  MarkX:
    82  	for {
    83  
    84  		// lg("search pref %v", sp)
    85  
    86  		if rss, ok := c.RssXMLURI[sp]; ok {
    87  			ret = rss
    88  			lg("found rss url %v for %v", ret, sp)
    89  			break MarkX
    90  
    91  		}
    92  
    93  		spPrev := sp
    94  		sp = path.Dir(sp)
    95  		if spPrev == "." && sp == "." ||
    96  			spPrev == "/" && sp == "/" ||
    97  			spPrev == "" && sp == "." {
    98  			lg("Did not find a RSS URL for %v %q", c.SearchPrefix, ret)
    99  			break
   100  		}
   101  
   102  		cntr++
   103  		if cntr > 20 {
   104  			lg("Select RSS Loop did not terminate. %v", c.SearchPrefix)
   105  			break
   106  		}
   107  	}
   108  
   109  	return
   110  }
   111  
   112  func attrX(attributes []html.Attribute, key string) (s string) {
   113  	for _, a := range attributes {
   114  		if key == a.Key {
   115  			s = a.Val
   116  			break
   117  		}
   118  	}
   119  	return
   120  }