kythe.io@v0.0.68-0.20240422202219-7225dbc01741/kythe/go/util/tools/scan_leveldb/scan_leveldb.go (about)

     1  /*
     2   * Copyright 2016 The Kythe Authors. All rights reserved.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *   http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  // Binary scan_leveldb is the cat command for LevelDB.  As well as being able to print each
    18  // key-value pair on its own line (or as a JSON object), scan_leveldb is able to decode common Kythe
    19  // protocol buffers stored in each value (see --proto_value).
    20  package main
    21  
    22  import (
    23  	"context"
    24  	"encoding/base64"
    25  	"encoding/json"
    26  	"flag"
    27  	"fmt"
    28  	"io"
    29  	"os"
    30  	"strconv"
    31  	"strings"
    32  
    33  	"kythe.io/kythe/go/storage/leveldb"
    34  	"kythe.io/kythe/go/util/flagutil"
    35  	"kythe.io/kythe/go/util/log"
    36  
    37  	"google.golang.org/protobuf/proto"
    38  	"google.golang.org/protobuf/reflect/protoreflect"
    39  	"google.golang.org/protobuf/reflect/protoregistry"
    40  
    41  	_ "kythe.io/kythe/proto/serving_go_proto"
    42  	_ "kythe.io/kythe/proto/storage_go_proto"
    43  )
    44  
    45  var (
    46  	emitJSON   = flag.Bool("json", false, "Emit JSON objects instead of a line per key-value")
    47  	lineFormat = flag.String("format", "@key@\t@value@", "Format of each key value line")
    48  	keyPrefix  = flag.String("prefix", "", "Only scan the key range with the given prefix")
    49  
    50  	stringKey   = flag.Bool("string_key", true, "Decode each key as a string rather than raw bytes")
    51  	stringValue = flag.Bool("string_value", false, "Decode each value as a string rather than raw bytes (--proto_value overrides this)")
    52  	protoValue  = flag.String("proto_value", "", `Decode each value as the given protocol buffer type (e.g. "kythe.proto.serving.FileDecorations" or "kythe.proto.serving.PagedEdgeSet")`)
    53  )
    54  
    55  func init() {
    56  	flag.Usage = flagutil.SimpleUsage(
    57  		"Scan/print each key-value in the given LevelDB(s)",
    58  		"[--string_key=false] [--string_value|--proto_value type]\n[--json] [--format f] [--prefix p] <leveldb-path>+")
    59  }
    60  
    61  func main() {
    62  	flag.Parse()
    63  
    64  	if flag.NArg() == 0 {
    65  		flagutil.UsageError("Missing path to LevelDB")
    66  	}
    67  
    68  	var protoValueType protoreflect.MessageType
    69  	if *protoValue != "" {
    70  		var err error
    71  		protoValueType, err = protoregistry.GlobalTypes.FindMessageByName(protoreflect.FullName(*protoValue))
    72  		if err != nil {
    73  			flagutil.UsageErrorf("could not understand protocol buffer type: %q: %v", *protoValue, err)
    74  		}
    75  	}
    76  
    77  	var en *json.Encoder
    78  	if *emitJSON {
    79  		en = json.NewEncoder(os.Stdout)
    80  	}
    81  
    82  	ctx := context.Background()
    83  	for _, path := range flag.Args() {
    84  		func() {
    85  			db, err := leveldb.Open(path, nil)
    86  			if err != nil {
    87  				log.Fatalf("Error opening %q: %v", path, err)
    88  			}
    89  			defer db.Close(ctx)
    90  
    91  			it, err := db.ScanPrefix(ctx, []byte(*keyPrefix), nil)
    92  			if err != nil {
    93  				log.Fatalf("Error creating iterator for %q: %v", path, err)
    94  			}
    95  			defer it.Close()
    96  
    97  			for {
    98  				key, val, err := it.Next()
    99  				if err == io.EOF {
   100  					break
   101  				} else if err != nil {
   102  					log.Fatalf("Error during scan of %q: %v", path, err)
   103  				}
   104  
   105  				var k, v any
   106  
   107  				if protoValueType == nil {
   108  					if *stringKey {
   109  						k = strconv.Quote(string(key))
   110  					} else {
   111  						k = base64.StdEncoding.EncodeToString(key)
   112  					}
   113  					if *stringValue {
   114  						v = strconv.Quote(string(val))
   115  					} else {
   116  						v = base64.StdEncoding.EncodeToString(val)
   117  					}
   118  				} else {
   119  					p := protoValueType.New().Interface()
   120  					if err := proto.Unmarshal(val, p); err != nil {
   121  						log.Fatalf("Error unmarshaling value to %q: %v", *protoValue, err)
   122  					}
   123  
   124  					k, v = string(key), p
   125  				}
   126  
   127  				if en == nil {
   128  					fmt.Println(strings.NewReplacer(
   129  						"@key@", fmt.Sprintf("%s", k),
   130  						"@value@", fmt.Sprintf("%s", v),
   131  					).Replace(*lineFormat))
   132  				} else {
   133  					en.Encode(keyValue{k, v})
   134  				}
   135  			}
   136  		}()
   137  	}
   138  }
   139  
   140  type keyValue struct {
   141  	Key   any `json:"key"`
   142  	Value any `json:"value"`
   143  }