github.com/aarzilli/tools@v0.0.0-20151123112009-0d27094f75e0/net/http/repo/2_config.go (about)

     1  package repo
     2  
     3  import (
     4  	"net/http"
     5  
     6  	"github.com/pbberlin/tools/net/http/routes"
     7  	"github.com/pbberlin/tools/os/fsi"
     8  	"github.com/pbberlin/tools/os/fsi/httpfs"
     9  	"github.com/pbberlin/tools/os/fsi/memfs"
    10  )
    11  
    12  // parallel fetchers routines
    13  const numWorkers = 3
    14  
    15  var docRoot = ""  // no relative path, 'cause working dir too flippant
    16  var whichType = 0 // which type of filesystem, default is dsfs
    17  
    18  var memMapFileSys = memfs.New(memfs.DirSort("byDateDesc"))             // package variable required as "persistence"
    19  var httpFSys = &httpfs.HttpFs{SourceFs: fsi.FileSystem(memMapFileSys)} // memMap is always ready
    20  var fileserver1 = http.FileServer(httpFSys.Dir(docRoot))
    21  
    22  const mountName = "mntftch"
    23  
    24  const uriSetType = "/fetch/set-fs-type"
    25  const UriMountNameY = "/" + mountName + "/serve-file/"
    26  
    27  const uriFetchCommandReceiver = "/fetch/command-receive"
    28  const uriFetchCommandSender = "/fetch/command-send"
    29  
    30  var RepoURL = routes.AppHost() + UriMountNameY
    31  
    32  var msg = []byte(`<p>This is an embedded static http server.</p>
    33  <p>
    34  It serves previously downloaded pages<br>
    35   i.e. from handelsblatt or economist.
    36  </p>`)
    37  
    38  var testCommands = []FetchCommand{
    39  	FetchCommand{
    40  		Host:         "www.handelsblatt.com",
    41  		SearchPrefix: "/politik/deutschland/aa/bb",
    42  	},
    43  	FetchCommand{
    44  		Host:         "www.handelsblatt.com",
    45  		SearchPrefix: "/politik/international/aa/bb",
    46  	},
    47  	FetchCommand{
    48  		Host:         "www.economist.com",
    49  		SearchPrefix: "/news/europe/aa",
    50  	},
    51  }
    52  
    53  // ConfigDefaults are default values for FetchCommands
    54  var ConfigDefaults = map[string]FetchCommand{
    55  	"unspecified": FetchCommand{
    56  		RssXMLURI:            map[string]string{},
    57  		CondenseTrailingDirs: 0,
    58  		DepthTolerance:       1,
    59  		DesiredNumber:        5,
    60  	},
    61  	// "www.handelsblatt.com": FetchCommand{
    62  	// 	RssXMLURI: map[string]string{
    63  	// 		"/":                      "/contentexport/feed/schlagzeilen",
    64  	// 		"/politik":               "/contentexport/feed/schlagzeilen",
    65  	// 		"/politik/international": "/contentexport/feed/schlagzeilen",
    66  	// 		"/politik/deutschland":   "/contentexport/feed/schlagzeilen",
    67  	// 	},
    68  	// 	CondenseTrailingDirs: 2,
    69  	// 	DepthTolerance:       1,
    70  	// 	DesiredNumber:        5,
    71  	// },
    72  	"www.economist.com": FetchCommand{
    73  		RssXMLURI: map[string]string{
    74  			"/news/europe":               "/sections/europe/rss.xml",
    75  			"/news/business-and-finance": "/sections/business-finance/rss.xml",
    76  		},
    77  		CondenseTrailingDirs: 0,
    78  		DepthTolerance:       2,
    79  		DesiredNumber:        5,
    80  	},
    81  	"test.economist.com": FetchCommand{
    82  		RssXMLURI: map[string]string{
    83  			"/news/business-and-finance": "/sections/business-finance/rss.xml",
    84  		},
    85  		CondenseTrailingDirs: 0,
    86  		DepthTolerance:       2,
    87  		DesiredNumber:        5,
    88  	},
    89  	"www.welt.de": FetchCommand{
    90  		RssXMLURI: map[string]string{
    91  			"/wirtschaft/deutschland":   "/wirtschaft/?service=Rss",
    92  			"/wirtschaft/international": "/wirtschaft/?service=Rss",
    93  		},
    94  
    95  		// CondenseTrailingDirs: 2,
    96  		CondenseTrailingDirs: 0,
    97  		DepthTolerance:       1,
    98  		DesiredNumber:        5,
    99  	},
   100  }