github.com/slspeek/camlistore_namedsearch@v0.0.0-20140519202248-ed6f70f7721a/dev/devcam/server.go (about) 1 /* 2 Copyright 2013 The Camlistore Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 // This file adds the "server" subcommand to devcam, to run camlistored. 18 19 package main 20 21 import ( 22 "errors" 23 "flag" 24 "fmt" 25 "log" 26 "os" 27 "os/exec" 28 "path/filepath" 29 "strconv" 30 "strings" 31 32 "camlistore.org/pkg/cmdmain" 33 "camlistore.org/pkg/osutil" 34 ) 35 36 type serverCmd struct { 37 // start of flag vars 38 all bool 39 hostname string 40 port string 41 tls bool 42 wipe bool 43 debug bool 44 45 mongo bool 46 mysql bool 47 postgres bool 48 sqlite bool 49 50 slow bool 51 throttle int 52 latency int 53 54 fullIndexSync bool 55 56 fullClosure bool 57 mini bool 58 publish bool 59 60 openBrowser bool 61 flickrAPIKey string 62 foursquareAPIKey string 63 picasaAPIKey string 64 twitterAPIKey string 65 extraArgs string // passed to camlistored 66 // end of flag vars 67 68 listen string // address + port to listen on 69 root string // the temp dir where blobs are stored 70 env *Env 71 } 72 73 func init() { 74 cmdmain.RegisterCommand("server", func(flags *flag.FlagSet) cmdmain.CommandRunner { 75 cmd := &serverCmd{ 76 env: NewCopyEnv(), 77 } 78 flags.BoolVar(&cmd.all, "all", false, "Listen on all interfaces.") 79 flags.StringVar(&cmd.hostname, "hostname", "", "Hostname to advertise, defaults to the hostname reported by the kernel.") 80 flags.StringVar(&cmd.port, "port", "3179", "Port to listen on.") 81 flags.BoolVar(&cmd.tls, "tls", false, "Use TLS.") 82 flags.BoolVar(&cmd.wipe, "wipe", false, "Wipe the blobs on disk and the indexer.") 83 flags.BoolVar(&cmd.debug, "debug", false, "Enable http debugging.") 84 flags.BoolVar(&cmd.publish, "publish", true, "Enable publish handlers") 85 flags.BoolVar(&cmd.mini, "mini", false, "Enable minimal mode, where all optional features are disabled. (Currently just publishing)") 86 87 flags.BoolVar(&cmd.mongo, "mongo", false, "Use mongodb as the indexer. Excludes -mysql, -postgres, -sqlite.") 88 flags.BoolVar(&cmd.mysql, "mysql", false, "Use mysql as the indexer. Excludes -mongo, -postgres, -sqlite.") 89 flags.BoolVar(&cmd.postgres, "postgres", false, "Use postgres as the indexer. Excludes -mongo, -mysql, -sqlite.") 90 flags.BoolVar(&cmd.sqlite, "sqlite", false, "Use sqlite as the indexer. Excludes -mongo, -mysql, -postgres.") 91 92 flags.BoolVar(&cmd.slow, "slow", false, "Add artificial latency.") 93 flags.IntVar(&cmd.throttle, "throttle", 150, "If -slow, this is the rate in kBps, to which we should throttle.") 94 flags.IntVar(&cmd.latency, "latency", 90, "If -slow, this is the added latency, in ms.") 95 96 flags.BoolVar(&cmd.fullIndexSync, "fullindexsync", false, "Perform full sync to indexer on startup.") 97 98 flags.BoolVar(&cmd.fullClosure, "fullclosure", false, "Use the ondisk closure library.") 99 100 flags.BoolVar(&cmd.openBrowser, "openbrowser", false, "Open the start page on startup.") 101 flags.StringVar(&cmd.flickrAPIKey, "flickrapikey", "", "The key and secret to use with the Flickr importer. Formatted as '<key>:<secret>'.") 102 flags.StringVar(&cmd.foursquareAPIKey, "foursquareapikey", "", "The key and secret to use with the Foursquare importer. Formatted as '<clientID>:<clientSecret>'.") 103 flags.StringVar(&cmd.picasaAPIKey, "picasakey", "", "The username and password to use with the Picasa importer. Formatted as '<username>:<password>'.") 104 flags.StringVar(&cmd.twitterAPIKey, "twitterapikey", "", "The key and secret to use with the Twitter importer. Formatted as '<APIkey>:<APIsecret>'.") 105 flags.StringVar(&cmd.root, "root", "", "A directory to store data in. Defaults to a location in the OS temp directory.") 106 flags.StringVar(&cmd.extraArgs, "extraargs", "", 107 "List of comma separated options that will be passed to camlistored") 108 return cmd 109 }) 110 } 111 112 func (c *serverCmd) Usage() { 113 fmt.Fprintf(cmdmain.Stderr, "Usage: devcam [globalopts] server [serveropts]\n") 114 } 115 116 func (c *serverCmd) Examples() []string { 117 return []string{ 118 "-wipe -mysql -fullclosure", 119 } 120 } 121 122 func (c *serverCmd) Describe() string { 123 return "run the stand-alone camlistored in dev mode." 124 } 125 126 func (c *serverCmd) checkFlags(args []string) error { 127 if len(args) != 0 { 128 c.Usage() 129 } 130 nindex := 0 131 for _, v := range []bool{c.mongo, c.mysql, c.postgres, c.sqlite} { 132 if v { 133 nindex++ 134 } 135 } 136 if nindex > 1 { 137 return fmt.Errorf("Only one index option allowed") 138 } 139 140 if _, err := strconv.ParseInt(c.port, 0, 0); err != nil { 141 return fmt.Errorf("Invalid -port value: %q", c.port) 142 } 143 return nil 144 } 145 146 func (c *serverCmd) setRoot() error { 147 if c.root == "" { 148 user := osutil.Username() 149 if user == "" { 150 return errors.New("Could not get username from environment") 151 } 152 c.root = filepath.Join(os.TempDir(), "camliroot-"+user, "port"+c.port) 153 } 154 log.Printf("Temp dir root is %v", c.root) 155 if c.wipe { 156 log.Printf("Wiping %v", c.root) 157 if err := os.RemoveAll(c.root); err != nil { 158 return fmt.Errorf("Could not wipe %v: %v", c.root, err) 159 } 160 } 161 return nil 162 } 163 164 func (c *serverCmd) makeSuffixdir(fullpath string) { 165 if err := os.MkdirAll(fullpath, 0755); err != nil { 166 log.Fatalf("Could not create %v: %v", fullpath, err) 167 } 168 } 169 170 func (c *serverCmd) setEnvVars() error { 171 c.env.SetCamdevVars(false) 172 setenv := func(k, v string) { 173 c.env.Set(k, v) 174 } 175 if c.slow { 176 setenv("DEV_THROTTLE_KBPS", fmt.Sprintf("%d", c.throttle)) 177 setenv("DEV_THROTTLE_LATENCY_MS", fmt.Sprintf("%d", c.latency)) 178 } 179 if c.debug { 180 setenv("CAMLI_HTTP_DEBUG", "1") 181 } 182 user := osutil.Username() 183 if user == "" { 184 return errors.New("Could not get username from environment") 185 } 186 setenv("CAMLI_FULL_INDEX_SYNC_ON_START", "false") 187 if c.fullIndexSync { 188 setenv("CAMLI_FULL_INDEX_SYNC_ON_START", "true") 189 } 190 setenv("CAMLI_DBNAME", "devcamli"+user) 191 setenv("CAMLI_MYSQL_ENABLED", "false") 192 setenv("CAMLI_MONGO_ENABLED", "false") 193 setenv("CAMLI_POSTGRES_ENABLED", "false") 194 setenv("CAMLI_SQLITE_ENABLED", "false") 195 setenv("CAMLI_KVINDEX_ENABLED", "false") 196 197 setenv("CAMLI_PUBLISH_ENABLED", strconv.FormatBool(c.publish)) 198 switch { 199 case c.mongo: 200 setenv("CAMLI_MONGO_ENABLED", "true") 201 setenv("CAMLI_INDEXER_PATH", "/index-mongo/") 202 case c.postgres: 203 setenv("CAMLI_POSTGRES_ENABLED", "true") 204 setenv("CAMLI_INDEXER_PATH", "/index-postgres/") 205 case c.mysql: 206 setenv("CAMLI_MYSQL_ENABLED", "true") 207 setenv("CAMLI_INDEXER_PATH", "/index-mysql/") 208 case c.sqlite: 209 setenv("CAMLI_SQLITE_ENABLED", "true") 210 setenv("CAMLI_INDEXER_PATH", "/index-sqlite/") 211 if c.root == "" { 212 panic("no root set") 213 } 214 setenv("CAMLI_DBNAME", filepath.Join(c.root, "sqliteindex.db")) 215 default: 216 setenv("CAMLI_KVINDEX_ENABLED", "true") 217 setenv("CAMLI_INDEXER_PATH", "/index-kv/") 218 if c.root == "" { 219 panic("no root set") 220 } 221 setenv("CAMLI_DBNAME", filepath.Join(c.root, "kvindex.db")) 222 } 223 224 base := "http://localhost:" + c.port 225 c.listen = "127.0.0.1:" + c.port 226 if c.all { 227 c.listen = "0.0.0.0:" + c.port 228 if c.hostname == "" { 229 hostname, err := os.Hostname() 230 if err != nil { 231 return fmt.Errorf("Could not get system hostname: %v", err) 232 } 233 base = "http://" + hostname + ":" + c.port 234 } else { 235 base = "http://" + c.hostname + ":" + c.port 236 } 237 } 238 setenv("CAMLI_TLS", "false") 239 if c.tls { 240 base = strings.Replace(base, "http://", "https://", 1) 241 setenv("CAMLI_TLS", "true") 242 } 243 setenv("CAMLI_BASEURL", base) 244 245 setenv("CAMLI_DEV_CAMLI_ROOT", camliSrcRoot) 246 setenv("CAMLI_AUTH", "devauth:pass3179") 247 fullSuffix := func(name string) string { 248 return filepath.Join(c.root, name) 249 } 250 suffixes := map[string]string{ 251 "CAMLI_ROOT": fullSuffix("bs"), 252 "CAMLI_ROOT_SHARD1": fullSuffix("s1"), 253 "CAMLI_ROOT_SHARD2": fullSuffix("s2"), 254 "CAMLI_ROOT_REPLICA1": fullSuffix("r1"), 255 "CAMLI_ROOT_REPLICA2": fullSuffix("r2"), 256 "CAMLI_ROOT_REPLICA3": fullSuffix("r3"), 257 "CAMLI_ROOT_CACHE": fullSuffix("cache"), 258 "CAMLI_ROOT_ENCMETA": fullSuffix("encmeta"), 259 "CAMLI_ROOT_ENCBLOB": fullSuffix("encblob"), 260 } 261 for k, v := range suffixes { 262 c.makeSuffixdir(v) 263 setenv(k, v) 264 } 265 setenv("CAMLI_PORT", c.port) 266 if c.flickrAPIKey != "" { 267 setenv("CAMLI_FLICKR_ENABLED", "true") 268 setenv("CAMLI_FLICKR_API_KEY", c.flickrAPIKey) 269 } 270 if c.foursquareAPIKey != "" { 271 setenv("CAMLI_FOURSQUARE_ENABLED", "true") 272 setenv("CAMLI_FOURSQUARE_API_KEY", c.foursquareAPIKey) 273 } 274 if c.picasaAPIKey != "" { 275 setenv("CAMLI_PICASA_ENABLED", "true") 276 setenv("CAMLI_PICASA_API_KEY", c.picasaAPIKey) 277 } 278 if c.twitterAPIKey != "" { 279 setenv("CAMLI_TWITTER_ENABLED", "true") 280 setenv("CAMLI_TWITTER_API_KEY", c.twitterAPIKey) 281 } 282 setenv("CAMLI_CONFIG_DIR", "config") 283 return nil 284 } 285 286 func (c *serverCmd) setupIndexer() error { 287 args := []string{"dbinit"} 288 switch { 289 case c.postgres: 290 args = append(args, 291 "-dbtype=postgres", 292 "-user=postgres", 293 "-password=postgres", 294 "-host=localhost", 295 "-dbname="+c.env.m["CAMLI_DBNAME"]) 296 case c.mysql: 297 args = append(args, 298 "-user=root", 299 "-password=root", 300 "-host=localhost", 301 "-dbname="+c.env.m["CAMLI_DBNAME"]) 302 case c.sqlite: 303 args = append(args, 304 "-dbtype=sqlite", 305 "-dbname="+c.env.m["CAMLI_DBNAME"]) 306 case c.mongo: 307 args = append(args, 308 "-dbtype=mongo", 309 "-host=localhost", 310 "-dbname="+c.env.m["CAMLI_DBNAME"]) 311 default: 312 return nil 313 } 314 if c.wipe { 315 args = append(args, "-wipe") 316 } else { 317 args = append(args, "-ignoreexists") 318 } 319 binPath := filepath.Join("bin", "camtool") 320 cmd := exec.Command(binPath, args...) 321 cmd.Stdout = os.Stdout 322 cmd.Stderr = os.Stderr 323 if err := cmd.Run(); err != nil { 324 return fmt.Errorf("Could not run camtool dbinit: %v", err) 325 } 326 return nil 327 } 328 329 func (c *serverCmd) syncTemplateBlobs() error { 330 if c.wipe { 331 templateDir := "dev-server-template" 332 if _, err := os.Stat(templateDir); err != nil { 333 if os.IsNotExist(err) { 334 return nil 335 } 336 return err 337 } 338 blobsDir := filepath.Join(c.root, "sha1") 339 if err := cpDir(templateDir, blobsDir, nil); err != nil { 340 return fmt.Errorf("Could not cp template blobs: %v", err) 341 } 342 } 343 return nil 344 } 345 346 func (c *serverCmd) setFullClosure() error { 347 if c.fullClosure { 348 oldsvn := filepath.Join(c.root, filepath.FromSlash("tmp/closure-lib/.svn")) 349 if err := os.RemoveAll(oldsvn); err != nil { 350 return fmt.Errorf("Could not remove svn checkout of closure-lib %v: %v", 351 oldsvn, err) 352 } 353 log.Println("Updating closure library...") 354 args := []string{"run", "third_party/closure/updatelibrary.go", "-verbose"} 355 cmd := exec.Command("go", args...) 356 cmd.Stdout = os.Stdout 357 cmd.Stderr = os.Stderr 358 if err := cmd.Run(); err != nil { 359 return fmt.Errorf("Could not run updatelibrary.go: %v", err) 360 } 361 c.env.Set("CAMLI_DEV_CLOSURE_DIR", "third_party/closure/lib/closure") 362 } 363 return nil 364 } 365 366 func (c *serverCmd) RunCommand(args []string) error { 367 if c.mini { 368 c.publish = false 369 } 370 err := c.checkFlags(args) 371 if err != nil { 372 return cmdmain.UsageError(fmt.Sprint(err)) 373 } 374 if !*noBuild { 375 withSqlite = c.sqlite 376 for _, name := range []string{ 377 filepath.Join("server", "camlistored"), 378 filepath.Join("cmd", "camtool"), 379 } { 380 err := build(name) 381 if err != nil { 382 return fmt.Errorf("Could not build %v: %v", name, err) 383 } 384 } 385 } 386 if err := c.setRoot(); err != nil { 387 return fmt.Errorf("Could not setup the camli root: %v", err) 388 } 389 if err := c.setEnvVars(); err != nil { 390 return fmt.Errorf("Could not setup the env vars: %v", err) 391 } 392 if err := c.setupIndexer(); err != nil { 393 return fmt.Errorf("Could not setup the indexer: %v", err) 394 } 395 if err := c.syncTemplateBlobs(); err != nil { 396 return fmt.Errorf("Could not copy the template blobs: %v", err) 397 } 398 if err := c.setFullClosure(); err != nil { 399 return fmt.Errorf("Could not setup the closure lib: %v", err) 400 } 401 402 log.Printf("Starting dev server on %v/ui/ with password \"pass3179\"\n", 403 c.env.m["CAMLI_BASEURL"]) 404 405 camliBin := filepath.Join("bin", "camlistored") 406 cmdArgs := []string{ 407 "-configfile=" + filepath.Join(camliSrcRoot, "config", "dev-server-config.json"), 408 "-listen=" + c.listen, 409 "-openbrowser=" + strconv.FormatBool(c.openBrowser), 410 } 411 if c.extraArgs != "" { 412 cmdArgs = append(cmdArgs, strings.Split(c.extraArgs, ",")...) 413 } 414 return runExec(camliBin, cmdArgs, c.env) 415 }