github.com/aarzilli/tools@v0.0.0-20151123112009-0d27094f75e0/net/http/repo/5_rss_fetch.go (about) 1 package repo 2 3 import ( 4 "bytes" 5 "encoding/xml" 6 "net/http" 7 "net/url" 8 "path" 9 "time" 10 11 "github.com/pbberlin/tools/net/http/fetch" 12 "github.com/pbberlin/tools/net/http/loghttp" 13 "github.com/pbberlin/tools/os/fsi" 14 "github.com/pbberlin/tools/stringspb" 15 ) 16 17 func rssDoc2DirTree(w http.ResponseWriter, r *http.Request, treeX *DirTree, rssDoc RSS, domain string) { 18 19 lg, lge := loghttp.Logger(w, r) 20 _ = lg 21 22 if treeX == nil { 23 treeX = &DirTree{Name: "root1", Dirs: map[string]DirTree{}, LastFound: time.Now().Truncate(time.Minute)} 24 } 25 26 articleList := []FullArticle{} 27 28 for _, lpItem := range rssDoc.Items.ItemList { 29 30 t, err := time.Parse(time.RFC1123Z, lpItem.Published) 31 // := time.Parse("Mon, 2 Jan 2006 15:04:05 -0700", lpItem.Published) 32 lge(err) 33 34 articleList = append(articleList, FullArticle{Url: lpItem.Link, Mod: t}) 35 36 } 37 38 lg1, _ := loghttp.BuffLoggerUniversal(w, r) 39 40 path2DirTree(lg1, treeX, articleList, domain, true) 41 42 } 43 44 // 45 // 46 // Fetches the RSS.xml file. 47 func rssXMLFile(w http.ResponseWriter, r *http.Request, fs fsi.FileSystem, rssUrl string) (RSS, *url.URL) { 48 49 lg, lge := loghttp.Logger(w, r) 50 51 bts, respInf, err := fetch.UrlGetter(r, fetch.Options{URL: rssUrl}) 52 lge(err) 53 54 bts = bytes.Replace(bts, []byte("content:encoded>"), []byte("content-encoded>S"), -1) // hack 55 56 rssDoc := RSS{} 57 err = xml.Unmarshal(bts, &rssDoc) 58 lge(err) 59 60 // save it 61 bdmp := stringspb.IndentedDumpBytes(rssDoc) 62 err = fs.MkdirAll(path.Join(docRoot, respInf.URL.Host), 0755) 63 lge(err) 64 err = fs.WriteFile(path.Join(docRoot, respInf.URL.Host, "outp_rss.xml"), bdmp, 0755) 65 lge(err) 66 lg("RSS resp size %5.2vkB, saved to %v", len(bdmp)/1024, respInf.URL.Host+"/outp_rss.xml") 67 68 return rssDoc, respInf.URL 69 }