github.com/coreos/mantle@v0.13.0/storage/sync.go (about)

     1  // Copyright 2016 CoreOS, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package storage
    16  
    17  import (
    18  	"strings"
    19  
    20  	"golang.org/x/net/context"
    21  	gs "google.golang.org/api/storage/v1"
    22  
    23  	"github.com/coreos/mantle/lang/worker"
    24  )
    25  
    26  // Filter is a type of function that returns true if an object should be
    27  // included in a given operation or false if it should be excluded/ignored.
    28  type Filter func(*gs.Object) bool
    29  
    30  type SyncJob struct {
    31  	Source      *Bucket
    32  	Destination *Bucket
    33  
    34  	sourcePrefix      *string
    35  	destinationPrefix *string
    36  	sourceFilter      Filter
    37  	deleteFilter      Filter
    38  	enableDelete      bool
    39  	notRecursive      bool // inverted because recursive is default
    40  }
    41  
    42  func Sync(ctx context.Context, src, dst *Bucket) error {
    43  	job := SyncJob{Source: src, Destination: dst}
    44  	return job.Do(ctx)
    45  }
    46  
    47  // SourcePrefix overrides the Source bucket's default prefix.
    48  func (sj *SyncJob) SourcePrefix(p string) {
    49  	p = FixPrefix(p)
    50  	sj.sourcePrefix = &p
    51  }
    52  
    53  // DestinationPrefix overrides the Destination bucket's default prefix.
    54  func (sj *SyncJob) DestinationPrefix(p string) {
    55  	p = FixPrefix(p)
    56  	sj.destinationPrefix = &p
    57  }
    58  
    59  // SourceFilter selects which objects to copy from Source.
    60  func (sj *SyncJob) SourceFilter(f Filter) {
    61  	sj.sourceFilter = f
    62  }
    63  
    64  // DeleteFilter selects which objects may be pruned from Destination.
    65  func (sj *SyncJob) DeleteFilter(f Filter) {
    66  	sj.deleteFilter = f
    67  }
    68  
    69  // Delete toggles deletion of extra objects from Destination.
    70  func (sj *SyncJob) Delete(enable bool) {
    71  	sj.enableDelete = enable
    72  }
    73  
    74  // Recursive toggles copying subdirectories from Source (the default).
    75  func (sj *SyncJob) Recursive(enable bool) {
    76  	sj.notRecursive = !enable
    77  }
    78  
    79  func (sj *SyncJob) Do(ctx context.Context) error {
    80  	if sj.sourcePrefix == nil {
    81  		prefix := sj.Source.Prefix()
    82  		sj.sourcePrefix = &prefix
    83  	}
    84  	if sj.destinationPrefix == nil {
    85  		prefix := sj.Destination.Prefix()
    86  		sj.destinationPrefix = &prefix
    87  	}
    88  
    89  	// Assemble a set of existing objects which may be deleted.
    90  	oldNames := make(map[string]struct{})
    91  	for _, oldObj := range sj.Destination.Objects() {
    92  		if !sj.hasPrefix(oldObj.Name, *sj.destinationPrefix) {
    93  			continue
    94  		}
    95  		if sj.deleteFilter != nil && !sj.deleteFilter(oldObj) {
    96  			continue
    97  		}
    98  		oldNames[oldObj.Name] = struct{}{}
    99  	}
   100  
   101  	wg := worker.NewWorkerGroup(ctx, MaxConcurrentRequests)
   102  	for _, srcObj := range sj.Source.Objects() {
   103  		if !sj.hasPrefix(srcObj.Name, *sj.sourcePrefix) {
   104  			continue
   105  		}
   106  		if sj.sourceFilter != nil && !sj.sourceFilter(srcObj) {
   107  			continue
   108  		}
   109  
   110  		obj := srcObj // for the sake of the closure
   111  		name := sj.newName(srcObj)
   112  
   113  		worker := func(c context.Context) error {
   114  			return sj.Destination.Copy(c, obj, name)
   115  		}
   116  		if err := wg.Start(worker); err != nil {
   117  			return wg.WaitError(err)
   118  		}
   119  
   120  		// Drop from set of deletion candidates.
   121  		delete(oldNames, name)
   122  	}
   123  
   124  	for oldName := range oldNames {
   125  		name := oldName // for the sake of the closure
   126  		worker := func(c context.Context) error {
   127  			return sj.Destination.Delete(c, name)
   128  		}
   129  		if err := wg.Start(worker); err != nil {
   130  			return wg.WaitError(err)
   131  		}
   132  	}
   133  
   134  	return wg.Wait()
   135  }
   136  
   137  func (sj *SyncJob) hasPrefix(name, prefix string) bool {
   138  	if !strings.HasPrefix(name, prefix) {
   139  		return false
   140  	}
   141  	if sj.notRecursive {
   142  		suffix := name[len(prefix):]
   143  		if strings.Contains(suffix, "/") {
   144  			return false
   145  		}
   146  	}
   147  	return true
   148  }
   149  
   150  func (sj *SyncJob) newName(srcObj *gs.Object) string {
   151  	return *sj.destinationPrefix + srcObj.Name[len(*sj.sourcePrefix):]
   152  }