github.com/slspeek/camlistore_namedsearch@v0.0.0-20140519202248-ed6f70f7721a/dev/devcam/server.go (about)

     1  /*
     2  Copyright 2013 The Camlistore Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8       http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  // This file adds the "server" subcommand to devcam, to run camlistored.
    18  
    19  package main
    20  
    21  import (
    22  	"errors"
    23  	"flag"
    24  	"fmt"
    25  	"log"
    26  	"os"
    27  	"os/exec"
    28  	"path/filepath"
    29  	"strconv"
    30  	"strings"
    31  
    32  	"camlistore.org/pkg/cmdmain"
    33  	"camlistore.org/pkg/osutil"
    34  )
    35  
    36  type serverCmd struct {
    37  	// start of flag vars
    38  	all      bool
    39  	hostname string
    40  	port     string
    41  	tls      bool
    42  	wipe     bool
    43  	debug    bool
    44  
    45  	mongo    bool
    46  	mysql    bool
    47  	postgres bool
    48  	sqlite   bool
    49  
    50  	slow     bool
    51  	throttle int
    52  	latency  int
    53  
    54  	fullIndexSync bool
    55  
    56  	fullClosure bool
    57  	mini        bool
    58  	publish     bool
    59  
    60  	openBrowser      bool
    61  	flickrAPIKey     string
    62  	foursquareAPIKey string
    63  	picasaAPIKey     string
    64  	twitterAPIKey    string
    65  	extraArgs        string // passed to camlistored
    66  	// end of flag vars
    67  
    68  	listen string // address + port to listen on
    69  	root   string // the temp dir where blobs are stored
    70  	env    *Env
    71  }
    72  
    73  func init() {
    74  	cmdmain.RegisterCommand("server", func(flags *flag.FlagSet) cmdmain.CommandRunner {
    75  		cmd := &serverCmd{
    76  			env: NewCopyEnv(),
    77  		}
    78  		flags.BoolVar(&cmd.all, "all", false, "Listen on all interfaces.")
    79  		flags.StringVar(&cmd.hostname, "hostname", "", "Hostname to advertise, defaults to the hostname reported by the kernel.")
    80  		flags.StringVar(&cmd.port, "port", "3179", "Port to listen on.")
    81  		flags.BoolVar(&cmd.tls, "tls", false, "Use TLS.")
    82  		flags.BoolVar(&cmd.wipe, "wipe", false, "Wipe the blobs on disk and the indexer.")
    83  		flags.BoolVar(&cmd.debug, "debug", false, "Enable http debugging.")
    84  		flags.BoolVar(&cmd.publish, "publish", true, "Enable publish handlers")
    85  		flags.BoolVar(&cmd.mini, "mini", false, "Enable minimal mode, where all optional features are disabled. (Currently just publishing)")
    86  
    87  		flags.BoolVar(&cmd.mongo, "mongo", false, "Use mongodb as the indexer. Excludes -mysql, -postgres, -sqlite.")
    88  		flags.BoolVar(&cmd.mysql, "mysql", false, "Use mysql as the indexer. Excludes -mongo, -postgres, -sqlite.")
    89  		flags.BoolVar(&cmd.postgres, "postgres", false, "Use postgres as the indexer. Excludes -mongo, -mysql, -sqlite.")
    90  		flags.BoolVar(&cmd.sqlite, "sqlite", false, "Use sqlite as the indexer. Excludes -mongo, -mysql, -postgres.")
    91  
    92  		flags.BoolVar(&cmd.slow, "slow", false, "Add artificial latency.")
    93  		flags.IntVar(&cmd.throttle, "throttle", 150, "If -slow, this is the rate in kBps, to which we should throttle.")
    94  		flags.IntVar(&cmd.latency, "latency", 90, "If -slow, this is the added latency, in ms.")
    95  
    96  		flags.BoolVar(&cmd.fullIndexSync, "fullindexsync", false, "Perform full sync to indexer on startup.")
    97  
    98  		flags.BoolVar(&cmd.fullClosure, "fullclosure", false, "Use the ondisk closure library.")
    99  
   100  		flags.BoolVar(&cmd.openBrowser, "openbrowser", false, "Open the start page on startup.")
   101  		flags.StringVar(&cmd.flickrAPIKey, "flickrapikey", "", "The key and secret to use with the Flickr importer. Formatted as '<key>:<secret>'.")
   102  		flags.StringVar(&cmd.foursquareAPIKey, "foursquareapikey", "", "The key and secret to use with the Foursquare importer. Formatted as '<clientID>:<clientSecret>'.")
   103  		flags.StringVar(&cmd.picasaAPIKey, "picasakey", "", "The username and password to use with the Picasa importer. Formatted as '<username>:<password>'.")
   104  		flags.StringVar(&cmd.twitterAPIKey, "twitterapikey", "", "The key and secret to use with the Twitter importer. Formatted as '<APIkey>:<APIsecret>'.")
   105  		flags.StringVar(&cmd.root, "root", "", "A directory to store data in. Defaults to a location in the OS temp directory.")
   106  		flags.StringVar(&cmd.extraArgs, "extraargs", "",
   107  			"List of comma separated options that will be passed to camlistored")
   108  		return cmd
   109  	})
   110  }
   111  
   112  func (c *serverCmd) Usage() {
   113  	fmt.Fprintf(cmdmain.Stderr, "Usage: devcam [globalopts] server [serveropts]\n")
   114  }
   115  
   116  func (c *serverCmd) Examples() []string {
   117  	return []string{
   118  		"-wipe -mysql -fullclosure",
   119  	}
   120  }
   121  
   122  func (c *serverCmd) Describe() string {
   123  	return "run the stand-alone camlistored in dev mode."
   124  }
   125  
   126  func (c *serverCmd) checkFlags(args []string) error {
   127  	if len(args) != 0 {
   128  		c.Usage()
   129  	}
   130  	nindex := 0
   131  	for _, v := range []bool{c.mongo, c.mysql, c.postgres, c.sqlite} {
   132  		if v {
   133  			nindex++
   134  		}
   135  	}
   136  	if nindex > 1 {
   137  		return fmt.Errorf("Only one index option allowed")
   138  	}
   139  
   140  	if _, err := strconv.ParseInt(c.port, 0, 0); err != nil {
   141  		return fmt.Errorf("Invalid -port value: %q", c.port)
   142  	}
   143  	return nil
   144  }
   145  
   146  func (c *serverCmd) setRoot() error {
   147  	if c.root == "" {
   148  		user := osutil.Username()
   149  		if user == "" {
   150  			return errors.New("Could not get username from environment")
   151  		}
   152  		c.root = filepath.Join(os.TempDir(), "camliroot-"+user, "port"+c.port)
   153  	}
   154  	log.Printf("Temp dir root is %v", c.root)
   155  	if c.wipe {
   156  		log.Printf("Wiping %v", c.root)
   157  		if err := os.RemoveAll(c.root); err != nil {
   158  			return fmt.Errorf("Could not wipe %v: %v", c.root, err)
   159  		}
   160  	}
   161  	return nil
   162  }
   163  
   164  func (c *serverCmd) makeSuffixdir(fullpath string) {
   165  	if err := os.MkdirAll(fullpath, 0755); err != nil {
   166  		log.Fatalf("Could not create %v: %v", fullpath, err)
   167  	}
   168  }
   169  
   170  func (c *serverCmd) setEnvVars() error {
   171  	c.env.SetCamdevVars(false)
   172  	setenv := func(k, v string) {
   173  		c.env.Set(k, v)
   174  	}
   175  	if c.slow {
   176  		setenv("DEV_THROTTLE_KBPS", fmt.Sprintf("%d", c.throttle))
   177  		setenv("DEV_THROTTLE_LATENCY_MS", fmt.Sprintf("%d", c.latency))
   178  	}
   179  	if c.debug {
   180  		setenv("CAMLI_HTTP_DEBUG", "1")
   181  	}
   182  	user := osutil.Username()
   183  	if user == "" {
   184  		return errors.New("Could not get username from environment")
   185  	}
   186  	setenv("CAMLI_FULL_INDEX_SYNC_ON_START", "false")
   187  	if c.fullIndexSync {
   188  		setenv("CAMLI_FULL_INDEX_SYNC_ON_START", "true")
   189  	}
   190  	setenv("CAMLI_DBNAME", "devcamli"+user)
   191  	setenv("CAMLI_MYSQL_ENABLED", "false")
   192  	setenv("CAMLI_MONGO_ENABLED", "false")
   193  	setenv("CAMLI_POSTGRES_ENABLED", "false")
   194  	setenv("CAMLI_SQLITE_ENABLED", "false")
   195  	setenv("CAMLI_KVINDEX_ENABLED", "false")
   196  
   197  	setenv("CAMLI_PUBLISH_ENABLED", strconv.FormatBool(c.publish))
   198  	switch {
   199  	case c.mongo:
   200  		setenv("CAMLI_MONGO_ENABLED", "true")
   201  		setenv("CAMLI_INDEXER_PATH", "/index-mongo/")
   202  	case c.postgres:
   203  		setenv("CAMLI_POSTGRES_ENABLED", "true")
   204  		setenv("CAMLI_INDEXER_PATH", "/index-postgres/")
   205  	case c.mysql:
   206  		setenv("CAMLI_MYSQL_ENABLED", "true")
   207  		setenv("CAMLI_INDEXER_PATH", "/index-mysql/")
   208  	case c.sqlite:
   209  		setenv("CAMLI_SQLITE_ENABLED", "true")
   210  		setenv("CAMLI_INDEXER_PATH", "/index-sqlite/")
   211  		if c.root == "" {
   212  			panic("no root set")
   213  		}
   214  		setenv("CAMLI_DBNAME", filepath.Join(c.root, "sqliteindex.db"))
   215  	default:
   216  		setenv("CAMLI_KVINDEX_ENABLED", "true")
   217  		setenv("CAMLI_INDEXER_PATH", "/index-kv/")
   218  		if c.root == "" {
   219  			panic("no root set")
   220  		}
   221  		setenv("CAMLI_DBNAME", filepath.Join(c.root, "kvindex.db"))
   222  	}
   223  
   224  	base := "http://localhost:" + c.port
   225  	c.listen = "127.0.0.1:" + c.port
   226  	if c.all {
   227  		c.listen = "0.0.0.0:" + c.port
   228  		if c.hostname == "" {
   229  			hostname, err := os.Hostname()
   230  			if err != nil {
   231  				return fmt.Errorf("Could not get system hostname: %v", err)
   232  			}
   233  			base = "http://" + hostname + ":" + c.port
   234  		} else {
   235  			base = "http://" + c.hostname + ":" + c.port
   236  		}
   237  	}
   238  	setenv("CAMLI_TLS", "false")
   239  	if c.tls {
   240  		base = strings.Replace(base, "http://", "https://", 1)
   241  		setenv("CAMLI_TLS", "true")
   242  	}
   243  	setenv("CAMLI_BASEURL", base)
   244  
   245  	setenv("CAMLI_DEV_CAMLI_ROOT", camliSrcRoot)
   246  	setenv("CAMLI_AUTH", "devauth:pass3179")
   247  	fullSuffix := func(name string) string {
   248  		return filepath.Join(c.root, name)
   249  	}
   250  	suffixes := map[string]string{
   251  		"CAMLI_ROOT":          fullSuffix("bs"),
   252  		"CAMLI_ROOT_SHARD1":   fullSuffix("s1"),
   253  		"CAMLI_ROOT_SHARD2":   fullSuffix("s2"),
   254  		"CAMLI_ROOT_REPLICA1": fullSuffix("r1"),
   255  		"CAMLI_ROOT_REPLICA2": fullSuffix("r2"),
   256  		"CAMLI_ROOT_REPLICA3": fullSuffix("r3"),
   257  		"CAMLI_ROOT_CACHE":    fullSuffix("cache"),
   258  		"CAMLI_ROOT_ENCMETA":  fullSuffix("encmeta"),
   259  		"CAMLI_ROOT_ENCBLOB":  fullSuffix("encblob"),
   260  	}
   261  	for k, v := range suffixes {
   262  		c.makeSuffixdir(v)
   263  		setenv(k, v)
   264  	}
   265  	setenv("CAMLI_PORT", c.port)
   266  	if c.flickrAPIKey != "" {
   267  		setenv("CAMLI_FLICKR_ENABLED", "true")
   268  		setenv("CAMLI_FLICKR_API_KEY", c.flickrAPIKey)
   269  	}
   270  	if c.foursquareAPIKey != "" {
   271  		setenv("CAMLI_FOURSQUARE_ENABLED", "true")
   272  		setenv("CAMLI_FOURSQUARE_API_KEY", c.foursquareAPIKey)
   273  	}
   274  	if c.picasaAPIKey != "" {
   275  		setenv("CAMLI_PICASA_ENABLED", "true")
   276  		setenv("CAMLI_PICASA_API_KEY", c.picasaAPIKey)
   277  	}
   278  	if c.twitterAPIKey != "" {
   279  		setenv("CAMLI_TWITTER_ENABLED", "true")
   280  		setenv("CAMLI_TWITTER_API_KEY", c.twitterAPIKey)
   281  	}
   282  	setenv("CAMLI_CONFIG_DIR", "config")
   283  	return nil
   284  }
   285  
   286  func (c *serverCmd) setupIndexer() error {
   287  	args := []string{"dbinit"}
   288  	switch {
   289  	case c.postgres:
   290  		args = append(args,
   291  			"-dbtype=postgres",
   292  			"-user=postgres",
   293  			"-password=postgres",
   294  			"-host=localhost",
   295  			"-dbname="+c.env.m["CAMLI_DBNAME"])
   296  	case c.mysql:
   297  		args = append(args,
   298  			"-user=root",
   299  			"-password=root",
   300  			"-host=localhost",
   301  			"-dbname="+c.env.m["CAMLI_DBNAME"])
   302  	case c.sqlite:
   303  		args = append(args,
   304  			"-dbtype=sqlite",
   305  			"-dbname="+c.env.m["CAMLI_DBNAME"])
   306  	case c.mongo:
   307  		args = append(args,
   308  			"-dbtype=mongo",
   309  			"-host=localhost",
   310  			"-dbname="+c.env.m["CAMLI_DBNAME"])
   311  	default:
   312  		return nil
   313  	}
   314  	if c.wipe {
   315  		args = append(args, "-wipe")
   316  	} else {
   317  		args = append(args, "-ignoreexists")
   318  	}
   319  	binPath := filepath.Join("bin", "camtool")
   320  	cmd := exec.Command(binPath, args...)
   321  	cmd.Stdout = os.Stdout
   322  	cmd.Stderr = os.Stderr
   323  	if err := cmd.Run(); err != nil {
   324  		return fmt.Errorf("Could not run camtool dbinit: %v", err)
   325  	}
   326  	return nil
   327  }
   328  
   329  func (c *serverCmd) syncTemplateBlobs() error {
   330  	if c.wipe {
   331  		templateDir := "dev-server-template"
   332  		if _, err := os.Stat(templateDir); err != nil {
   333  			if os.IsNotExist(err) {
   334  				return nil
   335  			}
   336  			return err
   337  		}
   338  		blobsDir := filepath.Join(c.root, "sha1")
   339  		if err := cpDir(templateDir, blobsDir, nil); err != nil {
   340  			return fmt.Errorf("Could not cp template blobs: %v", err)
   341  		}
   342  	}
   343  	return nil
   344  }
   345  
   346  func (c *serverCmd) setFullClosure() error {
   347  	if c.fullClosure {
   348  		oldsvn := filepath.Join(c.root, filepath.FromSlash("tmp/closure-lib/.svn"))
   349  		if err := os.RemoveAll(oldsvn); err != nil {
   350  			return fmt.Errorf("Could not remove svn checkout of closure-lib %v: %v",
   351  				oldsvn, err)
   352  		}
   353  		log.Println("Updating closure library...")
   354  		args := []string{"run", "third_party/closure/updatelibrary.go", "-verbose"}
   355  		cmd := exec.Command("go", args...)
   356  		cmd.Stdout = os.Stdout
   357  		cmd.Stderr = os.Stderr
   358  		if err := cmd.Run(); err != nil {
   359  			return fmt.Errorf("Could not run updatelibrary.go: %v", err)
   360  		}
   361  		c.env.Set("CAMLI_DEV_CLOSURE_DIR", "third_party/closure/lib/closure")
   362  	}
   363  	return nil
   364  }
   365  
   366  func (c *serverCmd) RunCommand(args []string) error {
   367  	if c.mini {
   368  		c.publish = false
   369  	}
   370  	err := c.checkFlags(args)
   371  	if err != nil {
   372  		return cmdmain.UsageError(fmt.Sprint(err))
   373  	}
   374  	if !*noBuild {
   375  		withSqlite = c.sqlite
   376  		for _, name := range []string{
   377  			filepath.Join("server", "camlistored"),
   378  			filepath.Join("cmd", "camtool"),
   379  		} {
   380  			err := build(name)
   381  			if err != nil {
   382  				return fmt.Errorf("Could not build %v: %v", name, err)
   383  			}
   384  		}
   385  	}
   386  	if err := c.setRoot(); err != nil {
   387  		return fmt.Errorf("Could not setup the camli root: %v", err)
   388  	}
   389  	if err := c.setEnvVars(); err != nil {
   390  		return fmt.Errorf("Could not setup the env vars: %v", err)
   391  	}
   392  	if err := c.setupIndexer(); err != nil {
   393  		return fmt.Errorf("Could not setup the indexer: %v", err)
   394  	}
   395  	if err := c.syncTemplateBlobs(); err != nil {
   396  		return fmt.Errorf("Could not copy the template blobs: %v", err)
   397  	}
   398  	if err := c.setFullClosure(); err != nil {
   399  		return fmt.Errorf("Could not setup the closure lib: %v", err)
   400  	}
   401  
   402  	log.Printf("Starting dev server on %v/ui/ with password \"pass3179\"\n",
   403  		c.env.m["CAMLI_BASEURL"])
   404  
   405  	camliBin := filepath.Join("bin", "camlistored")
   406  	cmdArgs := []string{
   407  		"-configfile=" + filepath.Join(camliSrcRoot, "config", "dev-server-config.json"),
   408  		"-listen=" + c.listen,
   409  		"-openbrowser=" + strconv.FormatBool(c.openBrowser),
   410  	}
   411  	if c.extraArgs != "" {
   412  		cmdArgs = append(cmdArgs, strings.Split(c.extraArgs, ",")...)
   413  	}
   414  	return runExec(camliBin, cmdArgs, c.env)
   415  }