github.com/olivere/camlistore@v0.0.0-20140121221811-1b7ac2da0199/cmd/camput/camput.go (about)

     1  /*
     2  Copyright 2011 Google Inc.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8       http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package main
    18  
    19  import (
    20  	"flag"
    21  	"fmt"
    22  	"log"
    23  	"net/http"
    24  	"net/url"
    25  	"os"
    26  	"strconv"
    27  	"strings"
    28  	"sync"
    29  
    30  	"camlistore.org/pkg/client"
    31  	"camlistore.org/pkg/cmdmain"
    32  	"camlistore.org/pkg/httputil"
    33  	"camlistore.org/pkg/syncutil"
    34  )
    35  
    36  const buffered = 16 // arbitrary
    37  
    38  var (
    39  	flagProxyLocal = false
    40  	flagHTTP       = flag.Bool("verbose_http", false, "show HTTP request summaries")
    41  	flagHaveCache  = true
    42  )
    43  
    44  var (
    45  	uploaderOnce sync.Once
    46  	uploader     *Uploader // initialized by getUploader
    47  )
    48  
    49  func init() {
    50  	if debug, _ := strconv.ParseBool(os.Getenv("CAMLI_DEBUG")); debug {
    51  		flag.BoolVar(&flagProxyLocal, "proxy_local", false, "If true, the HTTP_PROXY environment is also used for localhost requests. This can be helpful during debugging.")
    52  		flag.BoolVar(&flagHaveCache, "havecache", true, "Use the 'have cache', a cache keeping track of what blobs the remote server should already have from previous uploads.")
    53  	}
    54  	cmdmain.ExtraFlagRegistration = func() {
    55  		client.AddFlags()
    56  	}
    57  	cmdmain.PreExit = func() {
    58  		up := getUploader()
    59  		if up.haveCache != nil {
    60  			up.haveCache.Close()
    61  		}
    62  		stats := up.Stats()
    63  		log.Printf("Client stats: %s", stats.String())
    64  		log.Printf("  #HTTP reqs: %d", up.transport.Requests())
    65  	}
    66  }
    67  
    68  func getUploader() *Uploader {
    69  	uploaderOnce.Do(initUploader)
    70  	return uploader
    71  }
    72  
    73  func initUploader() {
    74  	up := newUploader()
    75  	if flagHaveCache {
    76  		gen, err := up.StorageGeneration()
    77  		if err != nil {
    78  			log.Printf("WARNING: not using local server inventory cache; failed to retrieve server's storage generation: %v", err)
    79  		} else {
    80  			up.haveCache = NewKvHaveCache(gen)
    81  			up.Client.SetHaveCache(up.haveCache)
    82  		}
    83  	}
    84  	uploader = up
    85  }
    86  
    87  func handleResult(what string, pr *client.PutResult, err error) error {
    88  	if err != nil {
    89  		log.Printf("Error putting %s: %s", what, err)
    90  		cmdmain.ExitWithFailure = true
    91  		return err
    92  	}
    93  	fmt.Println(pr.BlobRef.String())
    94  	return nil
    95  }
    96  
    97  func getenvEitherCase(k string) string {
    98  	if v := os.Getenv(strings.ToUpper(k)); v != "" {
    99  		return v
   100  	}
   101  	return os.Getenv(strings.ToLower(k))
   102  }
   103  
   104  // proxyFromEnvironment is similar to http.ProxyFromEnvironment but it skips
   105  // $NO_PROXY blacklist so it proxies every requests, including localhost
   106  // requests.
   107  func proxyFromEnvironment(req *http.Request) (*url.URL, error) {
   108  	proxy := getenvEitherCase("HTTP_PROXY")
   109  	if proxy == "" {
   110  		return nil, nil
   111  	}
   112  	proxyURL, err := url.Parse(proxy)
   113  	if err != nil || proxyURL.Scheme == "" {
   114  		if u, err := url.Parse("http://" + proxy); err == nil {
   115  			proxyURL = u
   116  			err = nil
   117  		}
   118  	}
   119  	if err != nil {
   120  		return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err)
   121  	}
   122  	return proxyURL, nil
   123  }
   124  
   125  func newUploader() *Uploader {
   126  	cc := client.NewOrFail()
   127  	if !*cmdmain.FlagVerbose {
   128  		cc.SetLogger(nil)
   129  	}
   130  
   131  	proxy := http.ProxyFromEnvironment
   132  	if flagProxyLocal {
   133  		proxy = proxyFromEnvironment
   134  	}
   135  	tr := cc.TransportForConfig(
   136  		&client.TransportConfig{
   137  			Proxy:   proxy,
   138  			Verbose: *flagHTTP,
   139  		})
   140  	httpStats, _ := tr.(*httputil.StatsTransport)
   141  	cc.SetHTTPClient(&http.Client{Transport: tr})
   142  
   143  	pwd, err := os.Getwd()
   144  	if err != nil {
   145  		log.Fatalf("os.Getwd: %v", err)
   146  	}
   147  
   148  	return &Uploader{
   149  		Client:    cc,
   150  		transport: httpStats,
   151  		pwd:       pwd,
   152  		fdGate:    syncutil.NewGate(100), // gate things that waste fds, assuming a low system limit
   153  	}
   154  }
   155  
   156  func main() {
   157  	cmdmain.Main()
   158  }