github.com/hasnat/dolt/go@v0.0.0-20210628190320-9eb5d843fbb7/store/cmd/noms/noms_sync.go (about)

     1  // Copyright 2019 Dolthub, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  //
    15  // This file incorporates work covered by the following copyright and
    16  // permission notice:
    17  //
    18  // Copyright 2016 Attic Labs, Inc. All rights reserved.
    19  // Licensed under the Apache License, version 2.0:
    20  // http://www.apache.org/licenses/LICENSE-2.0
    21  
    22  package main
    23  
    24  import (
    25  	"context"
    26  	"fmt"
    27  	"log"
    28  	"time"
    29  
    30  	"github.com/dustin/go-humanize"
    31  	flag "github.com/juju/gnuflag"
    32  
    33  	"github.com/dolthub/dolt/go/store/cmd/noms/util"
    34  	"github.com/dolthub/dolt/go/store/config"
    35  	"github.com/dolthub/dolt/go/store/datas"
    36  	"github.com/dolthub/dolt/go/store/types"
    37  	"github.com/dolthub/dolt/go/store/util/profile"
    38  	"github.com/dolthub/dolt/go/store/util/status"
    39  	"github.com/dolthub/dolt/go/store/util/verbose"
    40  )
    41  
    42  var (
    43  	p int
    44  )
    45  
    46  var nomsSync = &util.Command{
    47  	Run:       runSync,
    48  	UsageLine: "sync [options] <source-object> <dest-dataset>",
    49  	Short:     "Moves datasets between or within databases",
    50  	Long:      "See Spelling Objects at https://github.com/attic-labs/noms/blob/master/doc/spelling.md for details on the object and dataset arguments.",
    51  	Flags:     setupSyncFlags,
    52  	Nargs:     2,
    53  }
    54  
    55  func setupSyncFlags() *flag.FlagSet {
    56  	syncFlagSet := flag.NewFlagSet("sync", flag.ExitOnError)
    57  	syncFlagSet.IntVar(&p, "p", 512, "parallelism")
    58  	verbose.RegisterVerboseFlags(syncFlagSet)
    59  	profile.RegisterProfileFlags(syncFlagSet)
    60  	return syncFlagSet
    61  }
    62  
    63  func runSync(ctx context.Context, args []string) int {
    64  	cfg := config.NewResolver()
    65  	sourceStore, sourceObj, err := cfg.GetPath(ctx, args[0])
    66  	util.CheckError(err)
    67  	defer sourceStore.Close()
    68  
    69  	if sourceObj == nil {
    70  		util.CheckErrorNoUsage(fmt.Errorf("Object not found: %s", args[0]))
    71  	}
    72  
    73  	sinkDB, sinkDataset, err := cfg.GetDataset(ctx, args[1])
    74  	util.CheckError(err)
    75  	defer sinkDB.Close()
    76  
    77  	start := time.Now()
    78  	progressCh := make(chan datas.PullProgress)
    79  	lastProgressCh := make(chan datas.PullProgress)
    80  
    81  	go func() {
    82  		var last datas.PullProgress
    83  
    84  		for info := range progressCh {
    85  			last = info
    86  			if info.KnownCount == 1 {
    87  				// It's better to print "up to date" than "0% (0/1); 100% (1/1)".
    88  				continue
    89  			}
    90  
    91  			if status.WillPrint() {
    92  				pct := 100.0 * float64(info.DoneCount) / float64(info.KnownCount)
    93  				status.Printf("Syncing - %.2f%% (%s/s)", pct, bytesPerSec(info.ApproxWrittenBytes, start))
    94  			}
    95  		}
    96  		lastProgressCh <- last
    97  	}()
    98  
    99  	sourceRef, err := types.NewRef(sourceObj, sourceStore.Format())
   100  	util.CheckError(err)
   101  	sinkRef, sinkExists, err := sinkDataset.MaybeHeadRef()
   102  	util.CheckError(err)
   103  	nonFF := false
   104  	f := func() error {
   105  		defer profile.MaybeStartProfile().Stop()
   106  		err := datas.Pull(ctx, sourceStore, sinkDB, sourceRef, progressCh)
   107  
   108  		if err != nil {
   109  			return err
   110  		}
   111  
   112  		var tempDS datas.Dataset
   113  		tempDS, err = sinkDB.FastForward(ctx, sinkDataset, sourceRef)
   114  		if err == datas.ErrMergeNeeded {
   115  			sinkDataset, err = sinkDB.SetHead(ctx, sinkDataset, sourceRef)
   116  			nonFF = true
   117  		} else if err == nil {
   118  			sinkDataset = tempDS
   119  		}
   120  
   121  		return err
   122  	}
   123  
   124  	err = f()
   125  
   126  	if err != nil {
   127  		log.Fatal(err)
   128  	}
   129  
   130  	close(progressCh)
   131  	if last := <-lastProgressCh; last.DoneCount > 0 {
   132  		status.Printf("Done - Synced %s in %s (%s/s)",
   133  			humanize.Bytes(last.ApproxWrittenBytes), since(start), bytesPerSec(last.ApproxWrittenBytes, start))
   134  		status.Done()
   135  	} else if !sinkExists {
   136  		fmt.Printf("All chunks already exist at destination! Created new dataset %s.\n", args[1])
   137  	} else if nonFF && !sourceRef.Equals(sinkRef) {
   138  		fmt.Printf("Abandoning %s; new head is %s\n", sinkRef.TargetHash(), sourceRef.TargetHash())
   139  	} else {
   140  		fmt.Printf("Dataset %s is already up to date.\n", args[1])
   141  	}
   142  
   143  	return 0
   144  }
   145  
   146  func bytesPerSec(bytes uint64, start time.Time) string {
   147  	bps := float64(bytes) / float64(time.Since(start).Seconds())
   148  	return humanize.Bytes(uint64(bps))
   149  }
   150  
   151  func since(start time.Time) string {
   152  	round := time.Second / 100
   153  	now := time.Now().Round(round)
   154  	return now.Sub(start.Round(round)).String()
   155  }