github.com/coreos/mantle@v0.13.0/cmd/ore/gcloud/sync.go (about)

     1  // Copyright 2016 CoreOS, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package gcloud
    16  
    17  import (
    18  	"fmt"
    19  	"os"
    20  
    21  	"github.com/spf13/cobra"
    22  	"golang.org/x/net/context"
    23  
    24  	"github.com/coreos/mantle/lang/worker"
    25  	"github.com/coreos/mantle/storage"
    26  	"github.com/coreos/mantle/storage/index"
    27  )
    28  
    29  var (
    30  	syncDryRun     bool
    31  	syncForce      bool
    32  	syncDelete     bool
    33  	syncRecursive  bool
    34  	syncIndexDirs  bool
    35  	syncIndexPages bool
    36  	syncIndexTitle string
    37  	cmdSync        = &cobra.Command{
    38  		Use:   "sync gs://src/foo gs://dst/bar",
    39  		Short: "Copy objects between GS buckets",
    40  		Run:   runSync,
    41  	}
    42  )
    43  
    44  func init() {
    45  	cmdSync.Flags().BoolVarP(&syncDryRun, "dry-run", "n", false,
    46  		"perform a trial run, do not make changes")
    47  	cmdSync.Flags().BoolVarP(&syncForce, "force", "f", false,
    48  		"write everything, even when already up-to-date")
    49  	cmdSync.Flags().BoolVar(&syncDelete, "delete", false,
    50  		"delete extra objects and indexes")
    51  	cmdSync.Flags().BoolVarP(&syncRecursive, "recursive", "r", false,
    52  		"sync nested prefixes")
    53  	cmdSync.Flags().BoolVarP(&syncIndexDirs, "index-dirs", "D", false,
    54  		"generate HTML pages to mimic a directory tree")
    55  	cmdSync.Flags().BoolVarP(&syncIndexPages, "index-html", "I", false,
    56  		"generate index.html pages for each directory")
    57  	cmdSync.Flags().StringVarP(&syncIndexTitle, "html-title", "T", "",
    58  		"use the given title instead of bucket name in index pages")
    59  	GCloud.AddCommand(cmdSync)
    60  }
    61  
    62  func runSync(cmd *cobra.Command, args []string) {
    63  	if len(args) != 2 {
    64  		fmt.Fprintf(os.Stderr, "Expected exactly two gs:// URLs. Got: %v\n", args)
    65  		os.Exit(2)
    66  	}
    67  
    68  	ctx := context.Background()
    69  	src, err := storage.NewBucket(api.Client(), args[0])
    70  	if err != nil {
    71  		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
    72  		os.Exit(1)
    73  	}
    74  	src.WriteDryRun(true) // do not write to src
    75  
    76  	dst, err := storage.NewBucket(api.Client(), args[1])
    77  	if err != nil {
    78  		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
    79  		os.Exit(1)
    80  	}
    81  	dst.WriteDryRun(syncDryRun)
    82  	dst.WriteAlways(syncForce)
    83  
    84  	err = worker.Parallel(ctx,
    85  		func(c context.Context) error {
    86  			return src.FetchPrefix(c, src.Prefix(), syncRecursive)
    87  		},
    88  		func(c context.Context) error {
    89  			return dst.FetchPrefix(c, dst.Prefix(), syncRecursive)
    90  		})
    91  	if err != nil {
    92  		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
    93  		os.Exit(1)
    94  	}
    95  
    96  	job := index.NewSyncIndexJob(src, dst)
    97  	job.DirectoryHTML(syncIndexDirs)
    98  	job.IndexHTML(syncIndexPages)
    99  	job.Delete(syncDelete)
   100  	job.Recursive(syncRecursive)
   101  	if syncIndexTitle != "" {
   102  		job.Name(syncIndexTitle)
   103  	}
   104  	if err := job.Do(ctx); err != nil {
   105  		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
   106  		os.Exit(1)
   107  	}
   108  }