github.com/dolthub/dolt/go@v0.40.5-0.20240520175717-68db7794bea6/store/cmd/noms/noms_walk.go (about)

     1  // Copyright 2022 Dolthub, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package main
    16  
    17  import (
    18  	"context"
    19  	"fmt"
    20  	"io"
    21  	"os"
    22  	"strings"
    23  
    24  	flag "github.com/juju/gnuflag"
    25  
    26  	"github.com/dolthub/dolt/go/gen/fb/serial"
    27  	"github.com/dolthub/dolt/go/store/cmd/noms/util"
    28  	"github.com/dolthub/dolt/go/store/config"
    29  	"github.com/dolthub/dolt/go/store/d"
    30  	"github.com/dolthub/dolt/go/store/hash"
    31  	"github.com/dolthub/dolt/go/store/nbs"
    32  	"github.com/dolthub/dolt/go/store/types"
    33  	"github.com/dolthub/dolt/go/store/util/outputpager"
    34  	"github.com/dolthub/dolt/go/store/util/verbose"
    35  )
    36  
    37  var nomsWalk = &util.Command{
    38  	Run:       runWalk,
    39  	UsageLine: "walk [flags] [<object>]",
    40  	Short:     "Prints a depth-first listing of all paths to leaf data, beginning with the reference provided. If no ref is provided, uses the manifest root.",
    41  	Long:      "See Spelling Objects at https://github.com/attic-labs/noms/blob/master/doc/spelling.md for details on the object argument.",
    42  	Flags:     setupWalkFlags,
    43  	Nargs:     0,
    44  }
    45  
    46  var (
    47  	quiet = false
    48  )
    49  
    50  func setupWalkFlags() *flag.FlagSet {
    51  	walkPathSet := flag.NewFlagSet("walk", flag.ExitOnError)
    52  	outputpager.RegisterOutputpagerFlags(walkPathSet)
    53  	verbose.RegisterVerboseFlags(walkPathSet)
    54  	walkPathSet.BoolVar(&quiet, "quiet", false, "If true do not print all ref paths, only dangling refs")
    55  	return walkPathSet
    56  }
    57  
    58  func runWalk(ctx context.Context, args []string) int {
    59  	cfg := config.NewResolver()
    60  
    61  	var value types.Value
    62  
    63  	var startHash string
    64  	if len(args) < 1 {
    65  		manifestReader, err := os.Open("./.dolt/noms/manifest")
    66  		if err != nil {
    67  			fmt.Fprintln(os.Stderr, "Error reading manifest: ", err)
    68  			return 1
    69  		}
    70  
    71  		manifest, err := nbs.ParseManifest(manifestReader)
    72  		d.PanicIfError(err)
    73  
    74  		startHash = manifest.GetRoot().String()
    75  	} else {
    76  		startHash = args[0]
    77  	}
    78  
    79  	fullPath := startHash
    80  
    81  	if strings.HasPrefix(fullPath, "#") && !strings.HasPrefix(fullPath, ".dolt/noms::#") {
    82  		fullPath = ".dolt/noms::" + fullPath
    83  	} else if !strings.HasPrefix(fullPath, ".dolt/noms::#") {
    84  		fullPath = ".dolt/noms::#" + fullPath
    85  	}
    86  
    87  	database, vrw, value, err := cfg.GetPath(ctx, fullPath)
    88  
    89  	if err != nil {
    90  		util.CheckErrorNoUsage(err)
    91  	} else {
    92  	}
    93  
    94  	defer database.Close()
    95  
    96  	if value == nil {
    97  		fmt.Fprintf(os.Stderr, "Object not found: %s\n", fullPath)
    98  		return 0
    99  	}
   100  
   101  	if showPages {
   102  		pgr := outputpager.Start()
   103  		defer pgr.Stop()
   104  
   105  		err := walkAddrs(ctx, pgr.Writer, startHash, value, vrw)
   106  		if err != nil {
   107  			fmt.Fprintf(pgr.Writer, "error encountered: %s", err.Error())
   108  		}
   109  		fmt.Fprintln(pgr.Writer)
   110  	} else {
   111  		err := walkAddrs(ctx, os.Stdout, startHash, value, vrw)
   112  		if err != nil {
   113  			if err != nil {
   114  				fmt.Fprintf(os.Stdout, "error encountered: %s", err.Error())
   115  			}
   116  		}
   117  		fmt.Fprintln(os.Stdout)
   118  	}
   119  
   120  	return 0
   121  }
   122  
   123  var seenMessages = hash.NewHashSet()
   124  var numProcessed = 0
   125  
   126  func walkAddrs(ctx context.Context, w io.Writer, path string, value types.Value, cfg types.ValueReadWriter) error {
   127  	walk := func(addr hash.Hash) error {
   128  		value, err := cfg.ReadValue(ctx, addr)
   129  
   130  		if err != nil {
   131  			return err
   132  		}
   133  
   134  		if value == nil {
   135  			fmt.Fprintf(w, "Dangling reference: hash %s not found for path %s\n", addr.String(), path)
   136  			return nil
   137  		}
   138  
   139  		numProcessed++
   140  
   141  		newPath := fmt.Sprintf("%s > %s(%s)", path, addr.String(), serialType(value))
   142  		if !quiet {
   143  			fmt.Fprintf(w, "%s\n", newPath)
   144  		}
   145  
   146  		if numProcessed%100_000 == 0 {
   147  			fmt.Fprintf(os.Stderr, "%d refs walked\n", numProcessed)
   148  		}
   149  
   150  		// We only want to recurse on messages we haven't seen before. This means not outputting some possible paths to
   151  		// some chunks, but since there are so very many paths to a typical chunk this is a huge time saver.
   152  		if !seenMessages.Has(addr) {
   153  			seenMessages.Insert(addr)
   154  			return walkAddrs(ctx, w, newPath, value, cfg)
   155  		}
   156  
   157  		return nil
   158  	}
   159  
   160  	switch msg := value.(type) {
   161  	case types.SerialMessage:
   162  		return msg.WalkAddrs(types.Format_Default, walk)
   163  	default:
   164  		// non-serial values can't be walked
   165  		return nil
   166  	}
   167  }
   168  
   169  func serialType(value types.Value) string {
   170  	sm, ok := value.(types.SerialMessage)
   171  	if !ok {
   172  		return typeString(value)
   173  	}
   174  
   175  	return serial.GetFileID(sm)
   176  }