github.com/letsencrypt/trillian@v1.1.2-0.20180615153820-ae375a99d36a/server/cloudspanner_storage_provider.go (about)

     1  // Copyright 2018 Google Inc. All Rights Reserved.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package server
    16  
    17  import (
    18  	"bytes"
    19  	"compress/gzip"
    20  	"context"
    21  	"encoding/base64"
    22  	"flag"
    23  	"io/ioutil"
    24  	"sync"
    25  	"time"
    26  
    27  	"cloud.google.com/go/spanner"
    28  	"github.com/golang/glog"
    29  	"github.com/google/trillian/monitoring"
    30  	"github.com/google/trillian/storage"
    31  	"github.com/google/trillian/storage/cloudspanner"
    32  )
    33  
    34  var (
    35  	csURI                                = flag.String("cloudspanner_uri", "", "Connection URI for CloudSpanner database")
    36  	csNumChannels                        = flag.Int("cloudspanner_num_channels", 0, "Number of gRPC channels to use to talk to CloudSpanner.")
    37  	csSessionMaxOpened                   = flag.Uint64("cloudspanner_max_open_sessions", 0, "Max open sessions.")
    38  	csSessionMinOpened                   = flag.Uint64("cloudspanner_min_open_sessions", 0, "Min open sessions.")
    39  	csSessionMaxIdle                     = flag.Uint64("cloudspanner_max_idle_sessions", 0, "Max idle sessions.")
    40  	csSessionMaxBurst                    = flag.Uint64("cloudspanner_max_burst_sessions", 0, "Max concurrent create session requests.")
    41  	csSessionWriteSessions               = flag.Float64("cloudspanner_write_sessions", 0, "Fraction of write capable sessions to maintain.")
    42  	csSessionHCWorkers                   = flag.Int("cloudspanner_num_healthcheckers", 0, "Number of health check workers for Spanner session pool.")
    43  	csSessionHCInterval                  = flag.Duration("cloudspanner_healthcheck_interval", 0, "Interval betweek pinging sessions.")
    44  	csDequeueAcrossMerkleBucketsFraction = flag.Float64("cloudspanner_dequeue_bucket_fraction", 0.75, "Fraction of merkle keyspace to dequeue from, set to zero to disable.")
    45  	csReadOnlyStaleness                  = flag.Duration("cloudspanner_readonly_staleness", time.Minute, "How far in the past to perform readonly operations. Within limits, raising this should help to increase performance/reduce latency.")
    46  
    47  	csMu              sync.RWMutex
    48  	csStorageInstance *cloudSpannerProvider
    49  )
    50  
    51  func init() {
    52  	if err := RegisterStorageProvider("cloud_spanner", newCloudSpannerStorageProvider); err != nil {
    53  		panic(err)
    54  	}
    55  }
    56  
    57  func warn() {
    58  	w := `H4sIAAAAAAAA/4xUsW7rMAzc8xUE2lE41B2sWlzsZ3TwoKEIgkQZ64mLsxga8vUPlG3FrZ2Hd1Ng3onHE5UDPQOEmVnwjCGhjyLC8RLcPgfhIvmwot8/CaHF9CMdOthdGmKvdSQQET85TqxJtKzjgnd4mYaFilDIlmhsnKql977mZSqzYcLy5K/zCUX66sbtNOAwteTXiVph5m4nigGzzUH7e3+a3XIRf5PFyhQQEV6UXLeY8nL292gyujlMIlIbdcUwet9ieBx/snWIOXkievPenyOMiDjnjOHj+MMJhjZfFBFpHF+AcQkGpr9f1nz02YoKcPXed5nvjHG2DGtB//7gGwHCq69HIPMBGa7hIYi3mPlOBOhf/Z8eMBmAdNVjZKlCFuiQgK19Y1YKrXDT5KWX7ohVC+cArnUKwGAF/rwvk6CrVhJ1DuDDF9igfVtEuFf8U2M0MXW4wf28pBy/4yOuOaLZw2+Qa76m5PpSFy+5N0usbnyr66+AjY7cx3eKz5VHrZpFlqL6nJa82+gI/H3Vh+TKm9Fmib7I5GXSvcStTQrndxwIw4dQvpak00DGpKvbnVgIxXk4kD31oLnTSkgkxchmJ01Vnj7lQLZFXrV532bpfqLJbTzqfygXrLHkh/CoP5Hq13DXJYuV3fD/DcRbm+5f7s1tvNj/RLBD9T6vNbi9dYpT05QTKsV1B+Ut4m8AAAD///IJ0vhIBgAA`
    59  
    60  	wd, _ := base64.StdEncoding.DecodeString(w)
    61  	b := bytes.NewReader(wd)
    62  	r, _ := gzip.NewReader(b)
    63  	r.Close()
    64  	t, _ := ioutil.ReadAll(r)
    65  	glog.Warningf("WARNING\n%s\nCloudspanner is an experimental storage implementation, and only supports Logs currently.", string(t))
    66  }
    67  
    68  type cloudSpannerProvider struct {
    69  	client *spanner.Client
    70  }
    71  
    72  func configFromFlags() spanner.ClientConfig {
    73  	r := spanner.ClientConfig{}
    74  	setIntIfNotDefault(&r.NumChannels, *csNumChannels)
    75  	setUint64IfNotDefault(&r.SessionPoolConfig.MaxOpened, *csSessionMaxOpened)
    76  	setUint64IfNotDefault(&r.SessionPoolConfig.MinOpened, *csSessionMinOpened)
    77  	setUint64IfNotDefault(&r.SessionPoolConfig.MaxIdle, *csSessionMaxIdle)
    78  	setUint64IfNotDefault(&r.SessionPoolConfig.MaxBurst, *csSessionMaxBurst)
    79  	setFloat64IfNotDefault(&r.SessionPoolConfig.WriteSessions, *csSessionWriteSessions)
    80  	setIntIfNotDefault(&r.SessionPoolConfig.HealthCheckWorkers, *csSessionHCWorkers)
    81  	r.SessionPoolConfig.HealthCheckInterval = *csSessionHCInterval
    82  	return r
    83  }
    84  
    85  func newCloudSpannerStorageProvider(mf monitoring.MetricFactory) (StorageProvider, error) {
    86  	csMu.Lock()
    87  	defer csMu.Unlock()
    88  
    89  	if csStorageInstance != nil {
    90  		return csStorageInstance, nil
    91  	}
    92  
    93  	client, err := spanner.NewClientWithConfig(context.TODO(), *csURI, configFromFlags())
    94  	if err != nil {
    95  		return nil, err
    96  	}
    97  	csStorageInstance = &cloudSpannerProvider{
    98  		client: client,
    99  	}
   100  	return csStorageInstance, nil
   101  }
   102  
   103  func (s *cloudSpannerProvider) LogStorage() storage.LogStorage {
   104  	warn()
   105  	opts := cloudspanner.LogStorageOptions{}
   106  	frac := *csDequeueAcrossMerkleBucketsFraction
   107  	if frac > 1.0 {
   108  		frac = 1.0
   109  	}
   110  	if frac > 0 {
   111  		opts.DequeueAcrossMerkleBuckets = true
   112  		opts.DequeueAcrossMerkleBucketsRangeFraction = frac
   113  	}
   114  	if *csReadOnlyStaleness > 0 {
   115  		opts.ReadOnlyStaleness = *csReadOnlyStaleness
   116  	}
   117  	return cloudspanner.NewLogStorageWithOpts(s.client, opts)
   118  }
   119  
   120  func (s *cloudSpannerProvider) MapStorage() storage.MapStorage {
   121  	warn()
   122  	return nil
   123  }
   124  
   125  func (s *cloudSpannerProvider) AdminStorage() storage.AdminStorage {
   126  	warn()
   127  	return cloudspanner.NewAdminStorage(s.client)
   128  }
   129  
   130  func (s *cloudSpannerProvider) Close() error {
   131  	s.client.Close()
   132  	return nil
   133  }
   134  
   135  func setIntIfNotDefault(t *int, v int) {
   136  	if v != 0 {
   137  		*t = v
   138  	}
   139  }
   140  
   141  func setUint64IfNotDefault(t *uint64, v uint64) {
   142  	if v != 0 {
   143  		*t = v
   144  	}
   145  }
   146  
   147  func setFloat64IfNotDefault(t *float64, v float64) {
   148  	if v != 0 {
   149  		*t = v
   150  	}
   151  }