github.com/aarzilli/tools@v0.0.0-20151123112009-0d27094f75e0/net/http/repo/5_rss_fetch.go (about)

     1  package repo
     2  
     3  import (
     4  	"bytes"
     5  	"encoding/xml"
     6  	"net/http"
     7  	"net/url"
     8  	"path"
     9  	"time"
    10  
    11  	"github.com/pbberlin/tools/net/http/fetch"
    12  	"github.com/pbberlin/tools/net/http/loghttp"
    13  	"github.com/pbberlin/tools/os/fsi"
    14  	"github.com/pbberlin/tools/stringspb"
    15  )
    16  
    17  func rssDoc2DirTree(w http.ResponseWriter, r *http.Request, treeX *DirTree, rssDoc RSS, domain string) {
    18  
    19  	lg, lge := loghttp.Logger(w, r)
    20  	_ = lg
    21  
    22  	if treeX == nil {
    23  		treeX = &DirTree{Name: "root1", Dirs: map[string]DirTree{}, LastFound: time.Now().Truncate(time.Minute)}
    24  	}
    25  
    26  	articleList := []FullArticle{}
    27  
    28  	for _, lpItem := range rssDoc.Items.ItemList {
    29  
    30  		t, err := time.Parse(time.RFC1123Z, lpItem.Published)
    31  		//     := time.Parse("Mon, 2 Jan 2006 15:04:05 -0700", lpItem.Published)
    32  		lge(err)
    33  
    34  		articleList = append(articleList, FullArticle{Url: lpItem.Link, Mod: t})
    35  
    36  	}
    37  
    38  	lg1, _ := loghttp.BuffLoggerUniversal(w, r)
    39  
    40  	path2DirTree(lg1, treeX, articleList, domain, true)
    41  
    42  }
    43  
    44  //
    45  //
    46  // Fetches the RSS.xml file.
    47  func rssXMLFile(w http.ResponseWriter, r *http.Request, fs fsi.FileSystem, rssUrl string) (RSS, *url.URL) {
    48  
    49  	lg, lge := loghttp.Logger(w, r)
    50  
    51  	bts, respInf, err := fetch.UrlGetter(r, fetch.Options{URL: rssUrl})
    52  	lge(err)
    53  
    54  	bts = bytes.Replace(bts, []byte("content:encoded>"), []byte("content-encoded>S"), -1) // hack
    55  
    56  	rssDoc := RSS{}
    57  	err = xml.Unmarshal(bts, &rssDoc)
    58  	lge(err)
    59  
    60  	// save it
    61  	bdmp := stringspb.IndentedDumpBytes(rssDoc)
    62  	err = fs.MkdirAll(path.Join(docRoot, respInf.URL.Host), 0755)
    63  	lge(err)
    64  	err = fs.WriteFile(path.Join(docRoot, respInf.URL.Host, "outp_rss.xml"), bdmp, 0755)
    65  	lge(err)
    66  	lg("RSS resp size %5.2vkB, saved to %v", len(bdmp)/1024, respInf.URL.Host+"/outp_rss.xml")
    67  
    68  	return rssDoc, respInf.URL
    69  }