github.com/aarzilli/tools@v0.0.0-20151123112009-0d27094f75e0/net/http/repo/2_config.go (about) 1 package repo 2 3 import ( 4 "net/http" 5 6 "github.com/pbberlin/tools/net/http/routes" 7 "github.com/pbberlin/tools/os/fsi" 8 "github.com/pbberlin/tools/os/fsi/httpfs" 9 "github.com/pbberlin/tools/os/fsi/memfs" 10 ) 11 12 // parallel fetchers routines 13 const numWorkers = 3 14 15 var docRoot = "" // no relative path, 'cause working dir too flippant 16 var whichType = 0 // which type of filesystem, default is dsfs 17 18 var memMapFileSys = memfs.New(memfs.DirSort("byDateDesc")) // package variable required as "persistence" 19 var httpFSys = &httpfs.HttpFs{SourceFs: fsi.FileSystem(memMapFileSys)} // memMap is always ready 20 var fileserver1 = http.FileServer(httpFSys.Dir(docRoot)) 21 22 const mountName = "mntftch" 23 24 const uriSetType = "/fetch/set-fs-type" 25 const UriMountNameY = "/" + mountName + "/serve-file/" 26 27 const uriFetchCommandReceiver = "/fetch/command-receive" 28 const uriFetchCommandSender = "/fetch/command-send" 29 30 var RepoURL = routes.AppHost() + UriMountNameY 31 32 var msg = []byte(`<p>This is an embedded static http server.</p> 33 <p> 34 It serves previously downloaded pages<br> 35 i.e. from handelsblatt or economist. 36 </p>`) 37 38 var testCommands = []FetchCommand{ 39 FetchCommand{ 40 Host: "www.handelsblatt.com", 41 SearchPrefix: "/politik/deutschland/aa/bb", 42 }, 43 FetchCommand{ 44 Host: "www.handelsblatt.com", 45 SearchPrefix: "/politik/international/aa/bb", 46 }, 47 FetchCommand{ 48 Host: "www.economist.com", 49 SearchPrefix: "/news/europe/aa", 50 }, 51 } 52 53 // ConfigDefaults are default values for FetchCommands 54 var ConfigDefaults = map[string]FetchCommand{ 55 "unspecified": FetchCommand{ 56 RssXMLURI: map[string]string{}, 57 CondenseTrailingDirs: 0, 58 DepthTolerance: 1, 59 DesiredNumber: 5, 60 }, 61 // "www.handelsblatt.com": FetchCommand{ 62 // RssXMLURI: map[string]string{ 63 // "/": "/contentexport/feed/schlagzeilen", 64 // "/politik": "/contentexport/feed/schlagzeilen", 65 // "/politik/international": "/contentexport/feed/schlagzeilen", 66 // "/politik/deutschland": "/contentexport/feed/schlagzeilen", 67 // }, 68 // CondenseTrailingDirs: 2, 69 // DepthTolerance: 1, 70 // DesiredNumber: 5, 71 // }, 72 "www.economist.com": FetchCommand{ 73 RssXMLURI: map[string]string{ 74 "/news/europe": "/sections/europe/rss.xml", 75 "/news/business-and-finance": "/sections/business-finance/rss.xml", 76 }, 77 CondenseTrailingDirs: 0, 78 DepthTolerance: 2, 79 DesiredNumber: 5, 80 }, 81 "test.economist.com": FetchCommand{ 82 RssXMLURI: map[string]string{ 83 "/news/business-and-finance": "/sections/business-finance/rss.xml", 84 }, 85 CondenseTrailingDirs: 0, 86 DepthTolerance: 2, 87 DesiredNumber: 5, 88 }, 89 "www.welt.de": FetchCommand{ 90 RssXMLURI: map[string]string{ 91 "/wirtschaft/deutschland": "/wirtschaft/?service=Rss", 92 "/wirtschaft/international": "/wirtschaft/?service=Rss", 93 }, 94 95 // CondenseTrailingDirs: 2, 96 CondenseTrailingDirs: 0, 97 DepthTolerance: 1, 98 DesiredNumber: 5, 99 }, 100 }