github.com/m3db/m3@v1.5.0/src/cmd/tools/read_ids/main/main.go (about)

     1  // Copyright (c) 2016 Uber Technologies, Inc.
     2  //
     3  // Permission is hereby granted, free of charge, to any person obtaining a copy
     4  // of this software and associated documentation files (the "Software"), to deal
     5  // in the Software without restriction, including without limitation the rights
     6  // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
     7  // copies of the Software, and to permit persons to whom the Software is
     8  // furnished to do so, subject to the following conditions:
     9  //
    10  // The above copyright notice and this permission notice shall be included in
    11  // all copies or substantial portions of the Software.
    12  //
    13  // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    14  // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    15  // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    16  // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    17  // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    18  // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    19  // THE SOFTWARE.
    20  
    21  package main
    22  
    23  import (
    24  	"flag"
    25  	"log"
    26  	"os"
    27  	"strconv"
    28  	"strings"
    29  	"time"
    30  
    31  	"github.com/m3db/m3/src/dbnode/generated/thrift/rpc"
    32  	nchannel "github.com/m3db/m3/src/dbnode/network/server/tchannelthrift/node/channel"
    33  	"github.com/m3db/m3/src/x/ident"
    34  	xretry "github.com/m3db/m3/src/x/retry"
    35  
    36  	tchannel "github.com/uber/tchannel-go"
    37  	"github.com/uber/tchannel-go/thrift"
    38  	"go.uber.org/zap"
    39  )
    40  
    41  func main() {
    42  	var (
    43  		tchannelNodeAddrArg = flag.String("nodetchanneladdr", "127.0.0.1:9003", "Node TChannel server address")
    44  		namespaceArg        = flag.String("namespace", "default", "Namespace to read from")
    45  		shardsArg           = flag.String("shards", "0", "Shards to pull IDs from, comma separated")
    46  		pageLimitArg        = flag.Int64("pagelimit", 4096, "Page limit to pull for a single request")
    47  	)
    48  	flag.Parse()
    49  
    50  	if *tchannelNodeAddrArg == "" ||
    51  		*namespaceArg == "" ||
    52  		*shardsArg == "" ||
    53  		*pageLimitArg < 0 {
    54  		flag.Usage()
    55  		os.Exit(1)
    56  	}
    57  
    58  	tchannelNodeAddr := *tchannelNodeAddrArg
    59  	namespace := *namespaceArg
    60  	shards := []uint32{}
    61  	for _, str := range strings.Split(*shardsArg, ",") {
    62  		value, err := strconv.Atoi(str)
    63  		if err != nil {
    64  			log.Fatalf("could not parse shard '%s': %v", str, err)
    65  		}
    66  		if value < 0 {
    67  			log.Fatalf("could not parse shard '%s': not uint", str)
    68  		}
    69  		shards = append(shards, uint32(value))
    70  	}
    71  	pageLimit := *pageLimitArg
    72  
    73  	rawLogger, err := zap.NewDevelopment()
    74  	if err != nil {
    75  		log.Fatalf("unable to create logger: %+v", err)
    76  	}
    77  	log := rawLogger.Sugar()
    78  
    79  	channel, err := tchannel.NewChannel("Client", nil)
    80  	if err != nil {
    81  		log.Fatalf("could not create new tchannel channel: %v", err)
    82  	}
    83  	endpoint := &thrift.ClientOptions{HostPort: tchannelNodeAddr}
    84  	thriftClient := thrift.NewClient(channel, nchannel.ChannelName, endpoint)
    85  	client := rpc.NewTChanNodeClient(thriftClient)
    86  
    87  	writer := os.Stdout
    88  
    89  	for _, shard := range shards {
    90  		log.Infof("reading ids for shard %d", shard)
    91  		var (
    92  			total     int
    93  			pageToken []byte
    94  			retrier   = xretry.NewRetrier(xretry.NewOptions().
    95  					SetBackoffFactor(2).
    96  					SetMaxRetries(3).
    97  					SetInitialBackoff(time.Second).
    98  					SetJitter(true))
    99  			optionIncludeSizes     = true
   100  			optionIncludeChecksums = true
   101  			moreResults            = true
   102  		)
   103  		// Declare before loop to avoid redeclaring each iteration
   104  		attemptFn := func() error {
   105  			tctx, _ := thrift.NewContext(60 * time.Second)
   106  			req := rpc.NewFetchBlocksMetadataRawV2Request()
   107  			req.NameSpace = ident.StringID(namespace).Bytes()
   108  			req.Shard = int32(shard)
   109  			req.RangeStart = 0
   110  			req.RangeEnd = time.Now().Add(365 * 24 * time.Hour).UnixNano()
   111  			req.Limit = pageLimit
   112  			req.PageToken = pageToken
   113  			req.IncludeSizes = &optionIncludeSizes
   114  			req.IncludeChecksums = &optionIncludeChecksums
   115  
   116  			result, err := client.FetchBlocksMetadataRawV2(tctx, req)
   117  			if err != nil {
   118  				return err
   119  			}
   120  
   121  			if result.NextPageToken != nil {
   122  				// Create space on the heap for the page token and take it's
   123  				// address to avoid having to keep the entire result around just
   124  				// for the page token
   125  				pageToken = append([]byte(nil), result.NextPageToken...)
   126  			} else {
   127  				// No further results
   128  				moreResults = false
   129  			}
   130  
   131  			endLine := []byte("\n")
   132  			for _, elem := range result.Elements {
   133  				writer.Write(elem.ID)
   134  				writer.Write(endLine)
   135  				total++
   136  			}
   137  
   138  			return nil
   139  		}
   140  		for moreResults {
   141  			if err := retrier.Attempt(attemptFn); err != nil {
   142  				log.Fatalf("could not stream metadata: %v", err)
   143  			}
   144  		}
   145  		log.Infof("read %d ids for shard %d", total, shard)
   146  	}
   147  }