github.com/slspeek/camlistore_namedsearch@v0.0.0-20140519202248-ed6f70f7721a/pkg/client/client.go (about) 1 /* 2 Copyright 2011 Google Inc. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 // Package client implements a Camlistore client. 18 package client 19 20 import ( 21 "bytes" 22 "crypto/tls" 23 "encoding/json" 24 "errors" 25 "fmt" 26 "io" 27 "io/ioutil" 28 "log" 29 "net" 30 "net/http" 31 "net/url" 32 "os" 33 "regexp" 34 "strings" 35 "sync" 36 "time" 37 38 "camlistore.org/pkg/auth" 39 "camlistore.org/pkg/blob" 40 "camlistore.org/pkg/blobserver" 41 "camlistore.org/pkg/client/android" 42 "camlistore.org/pkg/httputil" 43 "camlistore.org/pkg/misc" 44 "camlistore.org/pkg/osutil" 45 "camlistore.org/pkg/schema" 46 "camlistore.org/pkg/search" 47 "camlistore.org/pkg/syncutil" 48 "camlistore.org/pkg/types/camtypes" 49 ) 50 51 // A Client provides access to a Camlistore server. 52 type Client struct { 53 // server is the input from user, pre-discovery. 54 // For example "http://foo.com" or "foo.com:1234". 55 // It is the responsibility of initPrefix to parse 56 // server and set prefix, including doing discovery 57 // to figure out what the proper server-declared 58 // prefix is. 59 server string 60 61 prefixOnce syncutil.Once // guards init of following 2 fields 62 prefixv string // URL prefix before "/camli/" 63 isSharePrefix bool // URL is a request for a share blob 64 65 discoOnce syncutil.Once 66 searchRoot string // Handler prefix, or "" if none 67 downloadHelper string // or "" if none 68 storageGen string // storage generation, or "" if not reported 69 syncHandlers []*SyncInfo // "from" and "to" url prefix for each syncHandler 70 serverKeyID string // Server's GPG public key ID. 71 72 signerOnce sync.Once 73 signer *schema.Signer 74 signerErr error 75 76 authMode auth.AuthMode 77 // authErr is set when no auth config is found but we want to defer warning 78 // until discovery fails. 79 authErr error 80 81 httpClient *http.Client 82 haveCache HaveCache 83 84 // If sto is set, it's used before the httpClient or other network operations. 85 sto blobserver.Storage 86 87 initTrustedCertsOnce sync.Once 88 // We define a certificate fingerprint as the 20 digits lowercase prefix 89 // of the SHA256 of the complete certificate (in ASN.1 DER encoding). 90 // trustedCerts contains the fingerprints of the self-signed 91 // certificates we trust. 92 // If not empty, (and if using TLS) the full x509 verification is 93 // disabled, and we instead check the server's certificate against 94 // that list. 95 // The camlistore server prints the fingerprint to add to the config 96 // when starting. 97 trustedCerts []string 98 // if set, we also skip the check against trustedCerts 99 InsecureTLS bool // TODO: hide this. add accessor? 100 101 initIgnoredFilesOnce sync.Once 102 // list of files that camput should ignore. 103 // Defaults to empty, but camput init creates a config with a non 104 // empty list. 105 // See IsIgnoredFile for the matching rules. 106 ignoredFiles []string 107 ignoreChecker func(path string) bool 108 109 pendStatMu sync.Mutex // guards pendStat 110 pendStat map[blob.Ref][]statReq // blobref -> reqs; for next batch(es) 111 112 initSignerPublicKeyBlobrefOnce sync.Once 113 signerPublicKeyRef blob.Ref 114 publicKeyArmored string 115 116 statsMutex sync.Mutex 117 stats Stats 118 119 // via maps the access path from a share root to a desired target. 120 // It is non-nil when in "sharing" mode, where the Client is fetching 121 // a share. 122 via map[string]string // target => via (target is referenced from via) 123 124 log *log.Logger // not nil 125 httpGate *syncutil.Gate 126 127 paramsOnly bool // config file and env vars are ignored. 128 } 129 130 const maxParallelHTTP = 5 131 132 // New returns a new Camlistore Client. 133 // The provided server is either "host:port" (assumed http, not https) or a URL prefix, with or without a path, or a server alias from the client configuration file. A server alias should not be confused with a hostname, therefore it cannot contain any colon or period. 134 // Errors are not returned until subsequent operations. 135 func New(server string) *Client { 136 if !isURLOrHostPort(server) { 137 configOnce.Do(parseConfig) 138 serverConf, ok := config.Servers[server] 139 if !ok { 140 log.Fatalf("%q looks like a server alias, but no such alias found in config at %v", server, osutil.UserClientConfigPath()) 141 } 142 server = serverConf.Server 143 } 144 return newFromParams(server, auth.None{}) 145 } 146 147 func NewOrFail() *Client { 148 c := New(serverOrDie()) 149 err := c.SetupAuth() 150 if err != nil { 151 log.Fatal(err) 152 } 153 return c 154 } 155 156 // NewStorageClient returns a Client that doesn't use HTTP, but uses s 157 // directly. This exists mainly so all the convenience methods on 158 // Client (e.g. the Upload variants) are available against storage 159 // directly. 160 // When using NewStorageClient, callers should call Close when done, 161 // in case the storage wishes to do a cleaner shutdown. 162 func NewStorageClient(s blobserver.Storage) *Client { 163 return &Client{ 164 sto: s, 165 log: log.New(os.Stderr, "", log.Ldate|log.Ltime), 166 haveCache: noHaveCache{}, 167 } 168 } 169 170 // TransportConfig contains options for SetupTransport. 171 type TransportConfig struct { 172 // Proxy optionally specifies the Proxy for the transport. Useful with 173 // camput for debugging even localhost requests. 174 Proxy func(*http.Request) (*url.URL, error) 175 Verbose bool // Verbose enables verbose logging of HTTP requests. 176 } 177 178 // TransportForConfig returns a transport for the client, setting the correct 179 // Proxy, Dial, and TLSClientConfig if needed. It does not mutate c. 180 // It is the caller's responsibility to then use that transport to set 181 // the client's httpClient with SetHTTPClient. 182 func (c *Client) TransportForConfig(tc *TransportConfig) http.RoundTripper { 183 if c == nil { 184 return nil 185 } 186 tlsConfig, err := c.TLSConfig() 187 if err != nil { 188 log.Fatalf("Error while configuring TLS for client: %v", err) 189 } 190 var transport http.RoundTripper 191 proxy := http.ProxyFromEnvironment 192 if tc != nil && tc.Proxy != nil { 193 proxy = tc.Proxy 194 } 195 transport = &http.Transport{ 196 Dial: c.DialFunc(), 197 TLSClientConfig: tlsConfig, 198 Proxy: proxy, 199 MaxIdleConnsPerHost: maxParallelHTTP, 200 } 201 httpStats := &httputil.StatsTransport{ 202 Transport: transport, 203 } 204 if tc != nil { 205 httpStats.VerboseLog = tc.Verbose 206 } 207 transport = httpStats 208 if android.IsChild() { 209 transport = &android.StatsTransport{transport} 210 } 211 return transport 212 } 213 214 type ClientOption interface { 215 modifyClient(*Client) 216 } 217 218 func OptionInsecure(v bool) ClientOption { 219 return optionInsecure(v) 220 } 221 222 type optionInsecure bool 223 224 func (o optionInsecure) modifyClient(c *Client) { 225 c.InsecureTLS = bool(o) 226 } 227 228 func OptionTrustedCert(cert string) ClientOption { 229 return optionTrustedCert(cert) 230 } 231 232 type optionTrustedCert string 233 234 func (o optionTrustedCert) modifyClient(c *Client) { 235 cert := string(o) 236 if cert != "" { 237 c.initTrustedCertsOnce.Do(func() {}) 238 c.trustedCerts = []string{string(o)} 239 } 240 } 241 242 // noop is for use with syncutil.Onces. 243 func noop() error { return nil } 244 245 var shareURLRx = regexp.MustCompile(`^(.+)/(` + blob.Pattern + ")$") 246 247 // NewFromShareRoot uses shareBlobURL to set up and return a client that 248 // will be used to fetch shared blobs. 249 func NewFromShareRoot(shareBlobURL string, opts ...ClientOption) (c *Client, target blob.Ref, err error) { 250 var root string 251 m := shareURLRx.FindStringSubmatch(shareBlobURL) 252 if m == nil { 253 return nil, blob.Ref{}, fmt.Errorf("Unkown share URL base") 254 } 255 c = New(m[1]) 256 c.discoOnce.Do(noop) 257 c.prefixOnce.Do(noop) 258 c.prefixv = m[1] 259 c.isSharePrefix = true 260 c.authMode = auth.None{} 261 c.via = make(map[string]string) 262 root = m[2] 263 264 for _, v := range opts { 265 v.modifyClient(c) 266 } 267 c.SetHTTPClient(&http.Client{Transport: c.TransportForConfig(nil)}) 268 269 req := c.newRequest("GET", shareBlobURL, nil) 270 res, err := c.expect2XX(req) 271 if err != nil { 272 return nil, blob.Ref{}, fmt.Errorf("Error fetching %s: %v", shareBlobURL, err) 273 } 274 defer res.Body.Close() 275 b, err := schema.BlobFromReader(blob.ParseOrZero(root), res.Body) 276 if err != nil { 277 return nil, blob.Ref{}, fmt.Errorf("Error parsing JSON from %s: %v", shareBlobURL, err) 278 } 279 if b.ShareAuthType() != schema.ShareHaveRef { 280 return nil, blob.Ref{}, fmt.Errorf("Unknown share authType of %q", b.ShareAuthType()) 281 } 282 target = b.ShareTarget() 283 if !target.Valid() { 284 return nil, blob.Ref{}, fmt.Errorf("No target.") 285 } 286 c.via[target.String()] = root 287 return c, target, nil 288 } 289 290 // SetHTTPClient sets the Camlistore client's HTTP client. 291 // If nil, the default HTTP client is used. 292 func (c *Client) SetHTTPClient(client *http.Client) { 293 if client == nil { 294 client = http.DefaultClient 295 } 296 c.httpClient = client 297 } 298 299 // HTTPClient returns the Client's underlying http.Client. 300 func (c *Client) HTTPClient() *http.Client { 301 return c.httpClient 302 } 303 304 // A HaveCache caches whether a remote blobserver has a blob. 305 type HaveCache interface { 306 StatBlobCache(br blob.Ref) (size uint32, ok bool) 307 NoteBlobExists(br blob.Ref, size uint32) 308 } 309 310 type noHaveCache struct{} 311 312 func (noHaveCache) StatBlobCache(blob.Ref) (uint32, bool) { return 0, false } 313 func (noHaveCache) NoteBlobExists(blob.Ref, uint32) {} 314 315 func (c *Client) SetHaveCache(cache HaveCache) { 316 if cache == nil { 317 cache = noHaveCache{} 318 } 319 c.haveCache = cache 320 } 321 322 func (c *Client) SetLogger(logger *log.Logger) { 323 if logger == nil { 324 c.log = log.New(ioutil.Discard, "", 0) 325 } else { 326 c.log = logger 327 } 328 } 329 330 func (c *Client) Stats() Stats { 331 c.statsMutex.Lock() 332 defer c.statsMutex.Unlock() 333 return c.stats // copy 334 } 335 336 // ErrNoSearchRoot is returned by SearchRoot if the server doesn't support search. 337 var ErrNoSearchRoot = errors.New("client: server doesn't support search") 338 339 // ErrNoSigning is returned by ServerKeyID if the server doesn't support signing. 340 var ErrNoSigning = fmt.Errorf("client: server doesn't support signing") 341 342 // ErrNoStorageGeneration is returned by StorageGeneration if the 343 // server doesn't report a storage generation value. 344 var ErrNoStorageGeneration = errors.New("client: server doesn't report a storage generation") 345 346 // ErrNoSync is returned by SyncHandlers if the server does not advertise syncs. 347 var ErrNoSync = errors.New("client: server has no sync handlers") 348 349 // BlobRoot returns the server's blobroot URL prefix. 350 // If the client was constructed with an explicit path, 351 // that path is used. Otherwise the server's 352 // default advertised blobRoot is used. 353 func (c *Client) BlobRoot() (string, error) { 354 prefix, err := c.prefix() 355 if err != nil { 356 return "", err 357 } 358 return prefix + "/", nil 359 } 360 361 // ServerKeyID returns the server's GPG public key ID. 362 // If the server isn't running a sign handler, the error will be ErrNoSigning. 363 func (c *Client) ServerKeyID() (string, error) { 364 if err := c.condDiscovery(); err != nil { 365 return "", err 366 } 367 if c.serverKeyID == "" { 368 return "", ErrNoSigning 369 } 370 return c.serverKeyID, nil 371 } 372 373 // SearchRoot returns the server's search handler. 374 // If the server isn't running an index and search handler, the error 375 // will be ErrNoSearchRoot. 376 func (c *Client) SearchRoot() (string, error) { 377 if err := c.condDiscovery(); err != nil { 378 return "", err 379 } 380 if c.searchRoot == "" { 381 return "", ErrNoSearchRoot 382 } 383 return c.searchRoot, nil 384 } 385 386 // StorageGeneration returns the server's unique ID for its storage 387 // generation, reset whenever storage is reset, moved, or partially 388 // lost. 389 // 390 // This is a value that can be used in client cache keys to add 391 // certainty that they're talking to the same instance as previously. 392 // 393 // If the server doesn't return such a value, the error will be 394 // ErrNoStorageGeneration. 395 func (c *Client) StorageGeneration() (string, error) { 396 if err := c.condDiscovery(); err != nil { 397 return "", err 398 } 399 if c.storageGen == "" { 400 return "", ErrNoStorageGeneration 401 } 402 return c.storageGen, nil 403 } 404 405 // SyncInfo holds the data that were acquired with a discovery 406 // and that are relevant to a syncHandler. 407 type SyncInfo struct { 408 From string 409 To string 410 ToIndex bool // whether this sync is from a blob storage to an index 411 } 412 413 // SyncHandlers returns the server's sync handlers "from" and 414 // "to" prefix URLs. 415 // If the server isn't running any sync handler, the error 416 // will be ErrNoSync. 417 func (c *Client) SyncHandlers() ([]*SyncInfo, error) { 418 if err := c.condDiscovery(); err != nil { 419 return nil, err 420 } 421 if c.syncHandlers == nil { 422 return nil, ErrNoSync 423 } 424 return c.syncHandlers, nil 425 } 426 427 var _ search.IGetRecentPermanodes = (*Client)(nil) 428 429 // GetRecentPermanodes implements search.IGetRecentPermanodes against a remote server over HTTP. 430 func (c *Client) GetRecentPermanodes(req *search.RecentRequest) (*search.RecentResponse, error) { 431 sr, err := c.SearchRoot() 432 if err != nil { 433 return nil, err 434 } 435 url := sr + req.URLSuffix() 436 hreq := c.newRequest("GET", url) 437 hres, err := c.expect2XX(hreq) 438 if err != nil { 439 return nil, err 440 } 441 res := new(search.RecentResponse) 442 if err := httputil.DecodeJSON(hres, res); err != nil { 443 return nil, err 444 } 445 if err := res.Err(); err != nil { 446 return nil, err 447 } 448 return res, nil 449 } 450 451 func (c *Client) GetPermanodesWithAttr(req *search.WithAttrRequest) (*search.WithAttrResponse, error) { 452 sr, err := c.SearchRoot() 453 if err != nil { 454 return nil, err 455 } 456 url := sr + req.URLSuffix() 457 hreq := c.newRequest("GET", url) 458 hres, err := c.expect2XX(hreq) 459 if err != nil { 460 return nil, err 461 } 462 res := new(search.WithAttrResponse) 463 if err := httputil.DecodeJSON(hres, res); err != nil { 464 return nil, err 465 } 466 if err := res.Err(); err != nil { 467 return nil, err 468 } 469 return res, nil 470 } 471 472 func (c *Client) Describe(req *search.DescribeRequest) (*search.DescribeResponse, error) { 473 sr, err := c.SearchRoot() 474 if err != nil { 475 return nil, err 476 } 477 url := sr + req.URLSuffix() 478 hreq := c.newRequest("GET", url) 479 hres, err := c.expect2XX(hreq) 480 if err != nil { 481 return nil, err 482 } 483 res := new(search.DescribeResponse) 484 if err := httputil.DecodeJSON(hres, res); err != nil { 485 return nil, err 486 } 487 return res, nil 488 } 489 490 func (c *Client) GetClaims(req *search.ClaimsRequest) (*search.ClaimsResponse, error) { 491 sr, err := c.SearchRoot() 492 if err != nil { 493 return nil, err 494 } 495 url := sr + req.URLSuffix() 496 hreq := c.newRequest("GET", url) 497 hres, err := c.expect2XX(hreq) 498 if err != nil { 499 return nil, err 500 } 501 res := new(search.ClaimsResponse) 502 if err := httputil.DecodeJSON(hres, res); err != nil { 503 return nil, err 504 } 505 return res, nil 506 } 507 508 func (c *Client) Search(req *search.SearchQuery) (*search.SearchResult, error) { 509 sr, err := c.SearchRoot() 510 if err != nil { 511 return nil, err 512 } 513 url := sr + req.URLSuffix() 514 body, err := json.MarshalIndent(req, "", "\t") 515 if err != nil { 516 return nil, err 517 } 518 hreq := c.newRequest("POST", url, bytes.NewReader(body)) 519 hres, err := c.expect2XX(hreq) 520 if err != nil { 521 return nil, err 522 } 523 res := new(search.SearchResult) 524 if err := httputil.DecodeJSON(hres, res); err != nil { 525 return nil, err 526 } 527 return res, nil 528 } 529 530 // SearchExistingFileSchema does a search query looking for an 531 // existing file with entire contents of wholeRef, then does a HEAD 532 // request to verify the file still exists on the server. If so, 533 // it returns that file schema's blobref. 534 // 535 // May return (zero, nil) on ENOENT. A non-nil error is only returned 536 // if there were problems searching. 537 func (c *Client) SearchExistingFileSchema(wholeRef blob.Ref) (blob.Ref, error) { 538 sr, err := c.SearchRoot() 539 if err != nil { 540 return blob.Ref{}, err 541 } 542 url := sr + "camli/search/files?wholedigest=" + wholeRef.String() 543 req := c.newRequest("GET", url) 544 res, err := c.doReqGated(req) 545 if err != nil { 546 return blob.Ref{}, err 547 } 548 if res.StatusCode != 200 { 549 body, _ := ioutil.ReadAll(io.LimitReader(res.Body, 1<<20)) 550 res.Body.Close() 551 return blob.Ref{}, fmt.Errorf("client: got status code %d from URL %s; body %s", res.StatusCode, url, body) 552 } 553 var ress struct { 554 Files []blob.Ref `json:"files"` 555 } 556 if err := httputil.DecodeJSON(res, &ress); err != nil { 557 return blob.Ref{}, fmt.Errorf("client: error parsing JSON from URL %s: %v", url, err) 558 } 559 if len(ress.Files) == 0 { 560 return blob.Ref{}, nil 561 } 562 for _, f := range ress.Files { 563 if c.FileHasContents(f, wholeRef) { 564 return f, nil 565 } 566 } 567 return blob.Ref{}, nil 568 } 569 570 // SetNamedSearch creates or modifies a search expression alias. 571 func (c *Client) SetNamedSearch(n *search.SetNamedRequest) (*search.SetNamedResponse, error) { 572 sr, err := c.SearchRoot() 573 if err != nil { 574 return nil, err 575 } 576 url := sr + "camli/search/setnamed?named=" + n.Named + "&substitute=" + n.Substitute 577 req := c.newRequest("GET", url) 578 res, err := c.doReqGated(req) 579 if err != nil { 580 return nil, err 581 } 582 if res.StatusCode != 200 { 583 body, _ := ioutil.ReadAll(io.LimitReader(res.Body, 1<<20)) 584 res.Body.Close() 585 return nil, fmt.Errorf("client: got status code %d from URL %s; body %s", res.StatusCode, url, body) 586 } 587 var ress search.SetNamedResponse 588 if err := httputil.DecodeJSON(res, &ress); err != nil { 589 return nil, fmt.Errorf("client: error parsing JSON from URL %s: %v", url, err) 590 } 591 return &ress, nil 592 } 593 594 // GetNamedSearch returns the substitute for a given search alias. 595 func (c *Client) GetNamedSearch(n *search.GetNamedRequest) (*search.GetNamedResponse, error) { 596 sr, err := c.SearchRoot() 597 if err != nil { 598 return nil, err 599 } 600 url := sr + "camli/search/getnamed?named=" + n.Named 601 req := c.newRequest("GET", url) 602 res, err := c.doReqGated(req) 603 if err != nil { 604 return nil, err 605 } 606 if res.StatusCode != 200 { 607 body, _ := ioutil.ReadAll(io.LimitReader(res.Body, 1<<20)) 608 res.Body.Close() 609 return nil, fmt.Errorf("client: got status code %d from URL %s; body %s", res.StatusCode, url, body) 610 } 611 var ress search.GetNamedResponse 612 if err := httputil.DecodeJSON(res, &ress); err != nil { 613 return nil, fmt.Errorf("client: error parsing JSON from URL %s: %v", url, err) 614 } 615 return &ress, nil 616 } 617 618 // FileHasContents returns true iff f refers to a "file" or "bytes" schema blob, 619 // the server is configured with a "download helper", and the server responds 620 // that all chunks of 'f' are available and match the digest of wholeRef. 621 func (c *Client) FileHasContents(f, wholeRef blob.Ref) bool { 622 if err := c.condDiscovery(); err != nil { 623 return false 624 } 625 if c.downloadHelper == "" { 626 return false 627 } 628 req := c.newRequest("HEAD", c.downloadHelper+f.String()+"/?verifycontents="+wholeRef.String()) 629 res, err := c.expect2XX(req) 630 if err != nil { 631 log.Printf("download helper HEAD error: %v", err) 632 return false 633 } 634 defer res.Body.Close() 635 return res.Header.Get("X-Camli-Contents") == wholeRef.String() 636 } 637 638 // prefix returns the URL prefix before "/camli/", or before 639 // the blobref hash in case of a share URL. 640 // Examples: http://foo.com:3179/bs or http://foo.com:3179/share 641 func (c *Client) prefix() (string, error) { 642 if err := c.prefixOnce.Do(c.initPrefix); err != nil { 643 return "", err 644 } 645 return c.prefixv, nil 646 } 647 648 // blobPrefix returns the URL prefix before the blobref hash. 649 // Example: http://foo.com:3179/bs/camli or http://foo.com:3179/share 650 func (c *Client) blobPrefix() (string, error) { 651 pfx, err := c.prefix() 652 if err != nil { 653 return "", err 654 } 655 if !c.isSharePrefix { 656 pfx += "/camli" 657 } 658 return pfx, nil 659 } 660 661 // discoRoot returns the user defined server for this client. It prepends "https://" if no scheme was specified. 662 func (c *Client) discoRoot() string { 663 s := c.server 664 if !strings.HasPrefix(s, "http") { 665 s = "https://" + s 666 } 667 return s 668 } 669 670 // initPrefix uses the user provided server URL to define the URL 671 // prefix to the blobserver root. If the server URL has a path 672 // component then it is directly used, otherwise the blobRoot 673 // from the discovery is used as the path. 674 func (c *Client) initPrefix() error { 675 c.isSharePrefix = false 676 root := c.discoRoot() 677 u, err := url.Parse(root) 678 if err != nil { 679 return err 680 } 681 if len(u.Path) > 1 { 682 c.prefixv = strings.TrimRight(root, "/") 683 return nil 684 } 685 return c.condDiscovery() 686 } 687 688 func (c *Client) condDiscovery() error { 689 if c.sto != nil { 690 return errors.New("client not using HTTP") 691 } 692 return c.discoOnce.Do(c.doDiscovery) 693 } 694 695 // DiscoveryDoc returns the server's JSON discovery document. 696 // This method exists purely for the "camtool discovery" command. 697 // Clients shouldn't have to parse this themselves. 698 func (c *Client) DiscoveryDoc() (io.Reader, error) { 699 res, err := c.discoveryResp() 700 if err != nil { 701 return nil, err 702 } 703 defer res.Body.Close() 704 const maxSize = 1 << 20 705 all, err := ioutil.ReadAll(io.LimitReader(res.Body, maxSize+1)) 706 if err != nil { 707 return nil, err 708 } 709 if len(all) > maxSize { 710 return nil, errors.New("discovery document oddly large") 711 } 712 if len(all) > 0 && all[len(all)-1] != '\n' { 713 all = append(all, '\n') 714 } 715 return bytes.NewReader(all), err 716 } 717 718 func (c *Client) discoveryResp() (*http.Response, error) { 719 // If the path is just "" or "/", do discovery against 720 // the URL to see which path we should actually use. 721 req := c.newRequest("GET", c.discoRoot(), nil) 722 req.Header.Set("Accept", "text/x-camli-configuration") 723 res, err := c.doReqGated(req) 724 if err != nil { 725 return nil, err 726 } 727 if res.StatusCode != 200 { 728 res.Body.Close() 729 errMsg := fmt.Sprintf("got status %q from blobserver URL %q during configuration discovery", res.Status, c.discoRoot()) 730 if res.StatusCode == 401 && c.authErr != nil { 731 errMsg = fmt.Sprintf("%v. %v", c.authErr, errMsg) 732 } 733 return nil, errors.New(errMsg) 734 } 735 // TODO(bradfitz): little weird in retrospect that we request 736 // text/x-camli-configuration and expect to get back 737 // text/javascript. Make them consistent. 738 if ct := res.Header.Get("Content-Type"); ct != "text/javascript" { 739 res.Body.Close() 740 return nil, fmt.Errorf("Blobserver returned unexpected type %q from discovery", ct) 741 } 742 return res, nil 743 } 744 745 func (c *Client) doDiscovery() error { 746 root, err := url.Parse(c.discoRoot()) 747 if err != nil { 748 return err 749 } 750 751 res, err := c.discoveryResp() 752 if err != nil { 753 return err 754 } 755 756 // TODO: make a proper struct type for this in another package somewhere: 757 m := make(map[string]interface{}) 758 if err := httputil.DecodeJSON(res, &m); err != nil { 759 return err 760 } 761 762 searchRoot, ok := m["searchRoot"].(string) 763 if ok { 764 u, err := root.Parse(searchRoot) 765 if err != nil { 766 return fmt.Errorf("client: invalid searchRoot %q; failed to resolve", searchRoot) 767 } 768 c.searchRoot = u.String() 769 } 770 771 downloadHelper, ok := m["downloadHelper"].(string) 772 if ok { 773 u, err := root.Parse(downloadHelper) 774 if err != nil { 775 return fmt.Errorf("client: invalid downloadHelper %q; failed to resolve", downloadHelper) 776 } 777 c.downloadHelper = u.String() 778 } 779 780 c.storageGen, _ = m["storageGeneration"].(string) 781 782 blobRoot, ok := m["blobRoot"].(string) 783 if !ok { 784 return fmt.Errorf("No blobRoot in config discovery response") 785 } 786 u, err := root.Parse(blobRoot) 787 if err != nil { 788 return fmt.Errorf("client: error resolving blobRoot: %v", err) 789 } 790 c.prefixv = strings.TrimRight(u.String(), "/") 791 792 syncHandlers, ok := m["syncHandlers"].([]interface{}) 793 if ok { 794 for _, v := range syncHandlers { 795 vmap := v.(map[string]interface{}) 796 from := vmap["from"].(string) 797 ufrom, err := root.Parse(from) 798 if err != nil { 799 return fmt.Errorf("client: invalid %q \"from\" sync; failed to resolve", from) 800 } 801 to := vmap["to"].(string) 802 uto, err := root.Parse(to) 803 if err != nil { 804 return fmt.Errorf("client: invalid %q \"to\" sync; failed to resolve", to) 805 } 806 toIndex, _ := vmap["toIndex"].(bool) 807 c.syncHandlers = append(c.syncHandlers, &SyncInfo{ 808 From: ufrom.String(), 809 To: uto.String(), 810 ToIndex: toIndex, 811 }) 812 } 813 } 814 serverSigning, ok := m["signing"].(map[string]interface{}) 815 if ok { 816 c.serverKeyID = serverSigning["publicKeyId"].(string) 817 } 818 return nil 819 } 820 821 // GetJSON sends a GET request to url, and unmarshals the returned 822 // JSON response into data. The URL's host must match the client's 823 // configured server. 824 func (c *Client) GetJSON(url string, data interface{}) error { 825 if !strings.HasPrefix(url, c.discoRoot()) { 826 return fmt.Errorf("wrong URL (%q) for this server", url) 827 } 828 hreq := c.newRequest("GET", url) 829 resp, err := c.expect2XX(hreq) 830 if err != nil { 831 return err 832 } 833 return httputil.DecodeJSON(resp, data) 834 } 835 836 func (c *Client) newRequest(method, url string, body ...io.Reader) *http.Request { 837 var bodyR io.Reader 838 if len(body) > 0 { 839 bodyR = body[0] 840 } 841 if len(body) > 1 { 842 panic("too many body arguments") 843 } 844 req, err := http.NewRequest(method, c.condRewriteURL(url), bodyR) 845 if err != nil { 846 panic(err.Error()) 847 } 848 // not done by http.NewRequest in Go 1.0: 849 if br, ok := bodyR.(*bytes.Reader); ok { 850 req.ContentLength = int64(br.Len()) 851 } 852 c.authMode.AddAuthHeader(req) 853 return req 854 } 855 856 // expect2XX will doReqGated and promote HTTP response codes outside of 857 // the 200-299 range to a non-nil error containing the response body. 858 func (c *Client) expect2XX(req *http.Request) (*http.Response, error) { 859 res, err := c.doReqGated(req) 860 if err == nil && (res.StatusCode < 200 || res.StatusCode > 299) { 861 buf := new(bytes.Buffer) 862 io.CopyN(buf, res.Body, 1<<20) 863 res.Body.Close() 864 return res, fmt.Errorf("client: got status code %d from URL %s; body %s", res.StatusCode, req.URL.String(), buf.String()) 865 } 866 return res, err 867 } 868 869 func (c *Client) doReqGated(req *http.Request) (*http.Response, error) { 870 c.httpGate.Start() 871 defer c.httpGate.Done() 872 return c.httpClient.Do(req) 873 } 874 875 // insecureTLS returns whether the client is using TLS without any 876 // verification of the server's cert. 877 func (c *Client) insecureTLS() bool { 878 return c.useTLS() && c.InsecureTLS 879 } 880 881 // selfVerifiedSSL returns whether the client config has fingerprints for 882 // (self-signed) trusted certificates. 883 // When true, we run with InsecureSkipVerify and it is our responsibility 884 // to check the server's cert against our trusted certs. 885 func (c *Client) selfVerifiedSSL() bool { 886 return c.useTLS() && len(c.getTrustedCerts()) > 0 887 } 888 889 // condRewriteURL changes "https://" to "http://" if we are in 890 // selfVerifiedSSL mode. We need to do that because we do the TLS 891 // dialing ourselves, and we do not want the http transport layer 892 // to redo it. 893 func (c *Client) condRewriteURL(url string) string { 894 if c.selfVerifiedSSL() || c.insecureTLS() { 895 return strings.Replace(url, "https://", "http://", 1) 896 } 897 return url 898 } 899 900 // TLSConfig returns the correct tls.Config depending on whether 901 // SSL is required, the client's config has some trusted certs, 902 // and we're on android. 903 func (c *Client) TLSConfig() (*tls.Config, error) { 904 if !c.useTLS() { 905 return nil, nil 906 } 907 trustedCerts := c.getTrustedCerts() 908 if len(trustedCerts) > 0 { 909 return &tls.Config{InsecureSkipVerify: true}, nil 910 } 911 if !android.OnAndroid() { 912 return nil, nil 913 } 914 return android.TLSConfig() 915 } 916 917 // DialFunc returns the adequate dial function, depending on 918 // whether SSL is required, the client's config has some trusted 919 // certs, and we're on android. 920 // If the client's config has some trusted certs, the server's 921 // certificate will be checked against those in the config after 922 // the TLS handshake. 923 func (c *Client) DialFunc() func(network, addr string) (net.Conn, error) { 924 trustedCerts := c.getTrustedCerts() 925 if !c.useTLS() || (!c.InsecureTLS && len(trustedCerts) == 0) { 926 // No TLS, or TLS with normal/full verification 927 if android.IsChild() { 928 return func(network, addr string) (net.Conn, error) { 929 return android.Dial(network, addr) 930 } 931 } 932 return nil 933 } 934 935 return func(network, addr string) (net.Conn, error) { 936 var conn *tls.Conn 937 var err error 938 if android.IsChild() { 939 con, err := android.Dial(network, addr) 940 if err != nil { 941 return nil, err 942 } 943 conn = tls.Client(con, &tls.Config{InsecureSkipVerify: true}) 944 if err = conn.Handshake(); err != nil { 945 return nil, err 946 } 947 } else { 948 conn, err = tls.Dial(network, addr, &tls.Config{InsecureSkipVerify: true}) 949 if err != nil { 950 return nil, err 951 } 952 } 953 if c.InsecureTLS { 954 return conn, nil 955 } 956 certs := conn.ConnectionState().PeerCertificates 957 if certs == nil || len(certs) < 1 { 958 return nil, errors.New("Could not get server's certificate from the TLS connection.") 959 } 960 sig := misc.SHA256Prefix(certs[0].Raw) 961 for _, v := range trustedCerts { 962 if v == sig { 963 return conn, nil 964 } 965 } 966 return nil, fmt.Errorf("Server's certificate %v is not in the trusted list", sig) 967 } 968 } 969 970 func (c *Client) Signer() (*schema.Signer, error) { 971 c.signerOnce.Do(c.signerInit) 972 return c.signer, c.signerErr 973 } 974 975 func (c *Client) signerInit() { 976 c.signer, c.signerErr = c.buildSigner() 977 } 978 979 func (c *Client) buildSigner() (*schema.Signer, error) { 980 c.initSignerPublicKeyBlobrefOnce.Do(c.initSignerPublicKeyBlobref) 981 if !c.signerPublicKeyRef.Valid() { 982 return nil, camtypes.Err("client-no-public-key") 983 } 984 return schema.NewSigner(c.signerPublicKeyRef, strings.NewReader(c.publicKeyArmored), c.SecretRingFile()) 985 } 986 987 // sigTime optionally specifies the signature time. 988 // If zero, the current time is used. 989 func (c *Client) signBlob(bb schema.Buildable, sigTime time.Time) (string, error) { 990 signer, err := c.Signer() 991 if err != nil { 992 return "", err 993 } 994 return bb.Builder().SignAt(signer, sigTime) 995 } 996 997 // uploadPublicKey uploads the public key (if one is defined), so 998 // subsequent (likely synchronous) indexing of uploaded signed blobs 999 // will have access to the public key to verify it. In the normal 1000 // case, the stat cache prevents this from doing anything anyway. 1001 func (c *Client) uploadPublicKey() error { 1002 sigRef := c.SignerPublicKeyBlobref() 1003 if !sigRef.Valid() { 1004 return nil 1005 } 1006 var err error 1007 if _, keyUploaded := c.haveCache.StatBlobCache(sigRef); !keyUploaded { 1008 _, err = c.uploadString(c.publicKeyArmored, false) 1009 } 1010 return err 1011 } 1012 1013 func (c *Client) UploadAndSignBlob(b schema.AnyBlob) (*PutResult, error) { 1014 signed, err := c.signBlob(b.Blob(), time.Time{}) 1015 if err != nil { 1016 return nil, err 1017 } 1018 if err := c.uploadPublicKey(); err != nil { 1019 return nil, err 1020 } 1021 return c.uploadString(signed, false) 1022 } 1023 1024 func (c *Client) UploadBlob(b schema.AnyBlob) (*PutResult, error) { 1025 // TODO(bradfitz): ask the blob for its own blobref, rather 1026 // than changing the hash function with uploadString? 1027 return c.uploadString(b.Blob().JSON(), true) 1028 } 1029 1030 func (c *Client) uploadString(s string, stat bool) (*PutResult, error) { 1031 uh := NewUploadHandleFromString(s) 1032 uh.SkipStat = !stat 1033 return c.Upload(uh) 1034 } 1035 1036 func (c *Client) UploadNewPermanode() (*PutResult, error) { 1037 unsigned := schema.NewUnsignedPermanode() 1038 return c.UploadAndSignBlob(unsigned) 1039 } 1040 1041 func (c *Client) UploadPlannedPermanode(key string, sigTime time.Time) (*PutResult, error) { 1042 unsigned := schema.NewPlannedPermanode(key) 1043 signed, err := c.signBlob(unsigned, sigTime) 1044 if err != nil { 1045 return nil, err 1046 } 1047 if err := c.uploadPublicKey(); err != nil { 1048 return nil, err 1049 } 1050 return c.uploadString(signed, true) 1051 } 1052 1053 // IsIgnoredFile returns whether the file at fullpath should be ignored by camput. 1054 // The fullpath is checked against the ignoredFiles list, trying the following rules in this order: 1055 // 1) star-suffix style matching (.e.g *.jpg). 1056 // 2) Shell pattern match as done by http://golang.org/pkg/path/filepath/#Match 1057 // 3) If the pattern is an absolute path to a directory, fullpath matches if it is that directory or a child of it. 1058 // 4) If the pattern is a relative path, fullpath matches if it has pattern as a path component (i.e the pattern is a part of fullpath that fits exactly between two path separators). 1059 func (c *Client) IsIgnoredFile(fullpath string) bool { 1060 c.initIgnoredFilesOnce.Do(c.initIgnoredFiles) 1061 return c.ignoreChecker(fullpath) 1062 } 1063 1064 // Close closes the client. In most cases, it's not necessary to close a Client. 1065 // The exception is for Clients created using NewStorageClient, where the Storage 1066 // might implement io.Closer. 1067 func (c *Client) Close() error { 1068 if cl, ok := c.sto.(io.Closer); ok { 1069 return cl.Close() 1070 } 1071 return nil 1072 } 1073 1074 // NewFromParams returns a Client that uses the specified server base URL 1075 // and auth but does not use any on-disk config files or environment variables 1076 // for its configuration. It may still use the disk for caches. 1077 func NewFromParams(server string, mode auth.AuthMode) *Client { 1078 cl := newFromParams(server, mode) 1079 cl.paramsOnly = true 1080 return cl 1081 } 1082 1083 func newFromParams(server string, mode auth.AuthMode) *Client { 1084 httpClient := &http.Client{ 1085 Transport: &http.Transport{ 1086 MaxIdleConnsPerHost: maxParallelHTTP, 1087 }, 1088 } 1089 return &Client{ 1090 server: server, 1091 httpClient: httpClient, 1092 httpGate: syncutil.NewGate(maxParallelHTTP), 1093 haveCache: noHaveCache{}, 1094 log: log.New(os.Stderr, "", log.Ldate|log.Ltime), 1095 authMode: mode, 1096 } 1097 }