github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/cmd/test/memory/memory.go (about)

     1  // Package memory provides the memory test command.
     2  package memory
     3  
     4  import (
     5  	"context"
     6  	"runtime"
     7  	"sync"
     8  
     9  	"github.com/rclone/rclone/cmd"
    10  	"github.com/rclone/rclone/cmd/test"
    11  	"github.com/rclone/rclone/fs"
    12  	"github.com/rclone/rclone/fs/operations"
    13  	"github.com/spf13/cobra"
    14  )
    15  
    16  func init() {
    17  	test.Command.AddCommand(commandDefinition)
    18  }
    19  
    20  var commandDefinition = &cobra.Command{
    21  	Use:   "memory remote:path",
    22  	Short: `Load all the objects at remote:path into memory and report memory stats.`,
    23  	Annotations: map[string]string{
    24  		"versionIntroduced": "v1.55",
    25  	},
    26  	Run: func(command *cobra.Command, args []string) {
    27  		cmd.CheckArgs(1, 1, command, args)
    28  		fsrc := cmd.NewFsSrc(args)
    29  		cmd.Run(false, false, command, func() error {
    30  			ctx := context.Background()
    31  			ci := fs.GetConfig(context.Background())
    32  			metadata := ci.Metadata && fsrc.Features().ReadMetadata
    33  			objects, _, _, err := operations.Count(ctx, fsrc)
    34  			if err != nil {
    35  				return err
    36  			}
    37  			objs := make([]fs.Object, 0, objects)
    38  			var before, after runtime.MemStats
    39  			runtime.GC()
    40  			runtime.ReadMemStats(&before)
    41  			var mu sync.Mutex
    42  			err = operations.ListFn(ctx, fsrc, func(o fs.Object) {
    43  				// Read the metadata so it gets cached in the object
    44  				if metadata {
    45  					_, err := fs.GetMetadata(ctx, o)
    46  					if err != nil {
    47  						fs.Errorf(o, "Failed to read metadata: %v", err)
    48  					}
    49  				}
    50  				mu.Lock()
    51  				objs = append(objs, o)
    52  				mu.Unlock()
    53  			})
    54  			if err != nil {
    55  				return err
    56  			}
    57  			runtime.GC()
    58  			runtime.ReadMemStats(&after)
    59  			var allocChange int64
    60  			if after.Alloc >= before.Alloc {
    61  				allocChange = int64(after.Alloc - before.Alloc)
    62  			} else {
    63  				allocChange = -int64(before.Alloc - after.Alloc)
    64  			}
    65  			var sysChange int64
    66  			if after.Sys >= before.Sys {
    67  				sysChange = int64(after.Sys - before.Sys)
    68  			} else {
    69  				sysChange = -int64(before.Sys - after.Sys)
    70  			}
    71  			if ci.HumanReadable {
    72  				objString := fs.CountSuffix(int64(len(objs)))
    73  				var usedString string
    74  				if after.Alloc >= before.Alloc {
    75  					usedString = fs.SizeSuffix(int64(after.Alloc - before.Alloc)).ByteUnit()
    76  				} else {
    77  					usedString = "-" + fs.SizeSuffix(int64(before.Alloc-after.Alloc)).ByteUnit()
    78  				}
    79  				avgString := fs.SizeSuffix(allocChange / int64(len(objs))).ByteUnit()
    80  				fs.Logf(nil, "%s objects took %s, %s/object", objString, usedString, avgString)
    81  
    82  				var sysBeforeString string
    83  				if before.Sys <= fs.SizeSuffixMaxValue {
    84  					sysBeforeString = fs.SizeSuffix(int64(before.Sys)).String()
    85  				} else {
    86  					sysBeforeString = ">" + fs.SizeSuffixMax.String()
    87  				}
    88  				var sysAfterString string
    89  				if after.Sys <= fs.SizeSuffixMaxValue {
    90  					sysAfterString = fs.SizeSuffix(int64(after.Sys)).ByteUnit()
    91  				} else {
    92  					sysAfterString = ">" + fs.SizeSuffixMax.ByteUnit()
    93  				}
    94  				var sysUsedString string
    95  				if after.Sys >= before.Sys {
    96  					sysUsedString = fs.SizeSuffix(int64(after.Sys - before.Sys)).ByteUnit()
    97  				} else {
    98  					sysUsedString = "-" + fs.SizeSuffix(int64(before.Sys-after.Sys)).ByteUnit()
    99  				}
   100  				fs.Logf(nil, "System memory changed from %s to %s a change of %s", sysBeforeString, sysAfterString, sysUsedString)
   101  			} else {
   102  				fs.Logf(nil, "%d objects took %d bytes, %.1f bytes/object", len(objs), allocChange, float64(allocChange)/float64(len(objs)))
   103  				fs.Logf(nil, "System memory changed from %d to %d bytes a change of %d bytes", before.Sys, after.Sys, sysChange)
   104  			}
   105  			return nil
   106  		})
   107  	},
   108  }