github.com/psexton/git-lfs@v2.1.1-0.20170517224304-289a18b2bc53+incompatible/test/git-lfs-test-server-api/main.go (about)

     1  package main
     2  
     3  import (
     4  	"bufio"
     5  	"crypto/sha256"
     6  	"encoding/hex"
     7  	"fmt"
     8  	"math/rand"
     9  	"os"
    10  	"strconv"
    11  	"strings"
    12  
    13  	"github.com/git-lfs/git-lfs/config"
    14  	"github.com/git-lfs/git-lfs/errors"
    15  	"github.com/git-lfs/git-lfs/lfs"
    16  	"github.com/git-lfs/git-lfs/lfsapi"
    17  	"github.com/git-lfs/git-lfs/progress"
    18  	"github.com/git-lfs/git-lfs/test"
    19  	"github.com/git-lfs/git-lfs/tq"
    20  	"github.com/spf13/cobra"
    21  )
    22  
    23  type TestObject struct {
    24  	Oid  string
    25  	Size int64
    26  }
    27  
    28  type ServerTest struct {
    29  	Name string
    30  	F    func(m *tq.Manifest, oidsExist, oidsMissing []TestObject) error
    31  }
    32  
    33  var (
    34  	RootCmd = &cobra.Command{
    35  		Use:   "git-lfs-test-server-api [--url=<apiurl> | --clone=<cloneurl>] [<oid-exists-file> <oid-missing-file>]",
    36  		Short: "Test a Git LFS API server for compliance",
    37  		Run:   testServerApi,
    38  	}
    39  	apiUrl     string
    40  	cloneUrl   string
    41  	savePrefix string
    42  
    43  	tests []ServerTest
    44  )
    45  
    46  func main() {
    47  	RootCmd.Execute()
    48  }
    49  
    50  func testServerApi(cmd *cobra.Command, args []string) {
    51  	if (len(apiUrl) == 0 && len(cloneUrl) == 0) ||
    52  		(len(apiUrl) != 0 && len(cloneUrl) != 0) {
    53  		exit("Must supply either --url or --clone (and not both)")
    54  	}
    55  
    56  	if len(args) != 0 && len(args) != 2 {
    57  		exit("Must supply either no file arguments or both the exists AND missing file")
    58  	}
    59  
    60  	if len(args) != 0 && len(savePrefix) > 0 {
    61  		exit("Cannot combine input files and --save option")
    62  	}
    63  
    64  	// Force loading of config before we alter it
    65  	config.Config.Git.All()
    66  
    67  	manifest, err := buildManifest()
    68  	if err != nil {
    69  		exit("error building tq.Manifest: " + err.Error())
    70  	}
    71  
    72  	var oidsExist, oidsMissing []TestObject
    73  	if len(args) >= 2 {
    74  		fmt.Printf("Reading test data from files (no server content changes)\n")
    75  		oidsExist = readTestOids(args[0])
    76  		oidsMissing = readTestOids(args[1])
    77  	} else {
    78  		fmt.Printf("Creating test data (will upload to server)\n")
    79  		var err error
    80  		oidsExist, oidsMissing, err = buildTestData(manifest)
    81  		if err != nil {
    82  			exit("Failed to set up test data, aborting")
    83  		}
    84  		if len(savePrefix) > 0 {
    85  			existFile := savePrefix + "_exists"
    86  			missingFile := savePrefix + "_missing"
    87  			saveTestOids(existFile, oidsExist)
    88  			saveTestOids(missingFile, oidsMissing)
    89  			fmt.Printf("Wrote test to %s, %s for future use\n", existFile, missingFile)
    90  		}
    91  
    92  	}
    93  
    94  	ok := runTests(manifest, oidsExist, oidsMissing)
    95  	if !ok {
    96  		exit("One or more tests failed, see above")
    97  	}
    98  	fmt.Println("All tests passed")
    99  }
   100  
   101  func readTestOids(filename string) []TestObject {
   102  	f, err := os.OpenFile(filename, os.O_RDONLY, 0644)
   103  	if err != nil {
   104  		exit("Error opening file %s", filename)
   105  	}
   106  	defer f.Close()
   107  
   108  	var ret []TestObject
   109  	rdr := bufio.NewReader(f)
   110  	line, err := rdr.ReadString('\n')
   111  	for err == nil {
   112  		fields := strings.Fields(strings.TrimSpace(line))
   113  		if len(fields) == 2 {
   114  			sz, _ := strconv.ParseInt(fields[1], 10, 64)
   115  			ret = append(ret, TestObject{Oid: fields[0], Size: sz})
   116  		}
   117  
   118  		line, err = rdr.ReadString('\n')
   119  	}
   120  
   121  	return ret
   122  }
   123  
   124  type testDataCallback struct{}
   125  
   126  func (*testDataCallback) Fatalf(format string, args ...interface{}) {
   127  	exit(format, args...)
   128  }
   129  func (*testDataCallback) Errorf(format string, args ...interface{}) {
   130  	fmt.Printf(format, args...)
   131  }
   132  
   133  func buildManifest() (*tq.Manifest, error) {
   134  	cfg := config.Config
   135  
   136  	// Configure the endpoint manually
   137  	finder := lfsapi.NewEndpointFinder(config.Config.Git)
   138  
   139  	var endp lfsapi.Endpoint
   140  	if len(cloneUrl) > 0 {
   141  		endp = finder.NewEndpointFromCloneURL(cloneUrl)
   142  	} else {
   143  		endp = finder.NewEndpoint(apiUrl)
   144  	}
   145  
   146  	apiClient, err := lfsapi.NewClient(cfg.Os, cfg.Git)
   147  	apiClient.Endpoints = &constantEndpoint{
   148  		e:              endp,
   149  		EndpointFinder: apiClient.Endpoints,
   150  	}
   151  	if err != nil {
   152  		return nil, err
   153  	}
   154  	return tq.NewManifestWithClient(apiClient), nil
   155  }
   156  
   157  type constantEndpoint struct {
   158  	e lfsapi.Endpoint
   159  
   160  	lfsapi.EndpointFinder
   161  }
   162  
   163  func (c *constantEndpoint) NewEndpointFromCloneURL(rawurl string) lfsapi.Endpoint { return c.e }
   164  
   165  func (c *constantEndpoint) NewEndpoint(rawurl string) lfsapi.Endpoint { return c.e }
   166  
   167  func (c *constantEndpoint) Endpoint(operation, remote string) lfsapi.Endpoint { return c.e }
   168  
   169  func (c *constantEndpoint) RemoteEndpoint(operation, remote string) lfsapi.Endpoint { return c.e }
   170  
   171  func buildTestData(manifest *tq.Manifest) (oidsExist, oidsMissing []TestObject, err error) {
   172  	const oidCount = 50
   173  	oidsExist = make([]TestObject, 0, oidCount)
   174  	oidsMissing = make([]TestObject, 0, oidCount)
   175  	meter := progress.NewMeter(progress.WithOSEnv(config.Config.Os))
   176  
   177  	// Build test data for existing files & upload
   178  	// Use test repo for this to simplify the process of making sure data matches oid
   179  	// We're not performing a real test at this point (although an upload fail will break it)
   180  	var callback testDataCallback
   181  	repo := test.NewRepo(&callback)
   182  	repo.Pushd()
   183  	defer repo.Cleanup()
   184  	// just one commit
   185  	commit := test.CommitInput{CommitterName: "A N Other", CommitterEmail: "noone@somewhere.com"}
   186  	for i := 0; i < oidCount; i++ {
   187  		filename := fmt.Sprintf("file%d.dat", i)
   188  		sz := int64(rand.Intn(200)) + 50
   189  		commit.Files = append(commit.Files, &test.FileInput{Filename: filename, Size: sz})
   190  		meter.Add(sz)
   191  	}
   192  	outputs := repo.AddCommits([]*test.CommitInput{&commit})
   193  
   194  	// now upload
   195  	uploadQueue := tq.NewTransferQueue(tq.Upload, manifest, "origin", tq.WithProgress(meter))
   196  	for _, f := range outputs[0].Files {
   197  		oidsExist = append(oidsExist, TestObject{Oid: f.Oid, Size: f.Size})
   198  
   199  		t, err := uploadTransfer(f.Oid, "Test file")
   200  		if err != nil {
   201  			return nil, nil, err
   202  		}
   203  		uploadQueue.Add(t.Name, t.Path, t.Oid, t.Size)
   204  	}
   205  	uploadQueue.Wait()
   206  
   207  	for _, err := range uploadQueue.Errors() {
   208  		if errors.IsFatalError(err) {
   209  			exit("Fatal error setting up test data: %s", err)
   210  		}
   211  	}
   212  
   213  	// Generate SHAs for missing files, random but repeatable
   214  	// No actual file content needed for these
   215  	rand.Seed(int64(oidCount))
   216  	runningSha := sha256.New()
   217  	for i := 0; i < oidCount; i++ {
   218  		runningSha.Write([]byte{byte(rand.Intn(256))})
   219  		oid := hex.EncodeToString(runningSha.Sum(nil))
   220  		sz := int64(rand.Intn(200)) + 50
   221  		oidsMissing = append(oidsMissing, TestObject{Oid: oid, Size: sz})
   222  	}
   223  	return oidsExist, oidsMissing, nil
   224  }
   225  
   226  func saveTestOids(filename string, objs []TestObject) {
   227  	f, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
   228  	if err != nil {
   229  		exit("Error opening file %s", filename)
   230  	}
   231  	defer f.Close()
   232  
   233  	for _, o := range objs {
   234  		f.WriteString(fmt.Sprintf("%s %d\n", o.Oid, o.Size))
   235  	}
   236  
   237  }
   238  
   239  func runTests(manifest *tq.Manifest, oidsExist, oidsMissing []TestObject) bool {
   240  	ok := true
   241  	fmt.Printf("Running %d tests...\n", len(tests))
   242  	for _, t := range tests {
   243  		err := runTest(t, manifest, oidsExist, oidsMissing)
   244  		if err != nil {
   245  			ok = false
   246  		}
   247  	}
   248  	return ok
   249  }
   250  
   251  func runTest(t ServerTest, manifest *tq.Manifest, oidsExist, oidsMissing []TestObject) error {
   252  	const linelen = 70
   253  	line := t.Name
   254  	if len(line) > linelen {
   255  		line = line[:linelen]
   256  	} else if len(line) < linelen {
   257  		line = fmt.Sprintf("%s%s", line, strings.Repeat(" ", linelen-len(line)))
   258  	}
   259  	fmt.Printf("%s...\r", line)
   260  
   261  	err := t.F(manifest, oidsExist, oidsMissing)
   262  	if err != nil {
   263  		fmt.Printf("%s FAILED\n", line)
   264  		fmt.Println(err.Error())
   265  	} else {
   266  		fmt.Printf("%s OK\n", line)
   267  	}
   268  	return err
   269  }
   270  
   271  // Exit prints a formatted message and exits.
   272  func exit(format string, args ...interface{}) {
   273  	fmt.Fprintf(os.Stderr, format, args...)
   274  	os.Exit(2)
   275  }
   276  
   277  func addTest(name string, f func(manifest *tq.Manifest, oidsExist, oidsMissing []TestObject) error) {
   278  	tests = append(tests, ServerTest{Name: name, F: f})
   279  }
   280  
   281  func callBatchApi(manifest *tq.Manifest, dir tq.Direction, objs []TestObject) ([]*tq.Transfer, error) {
   282  	apiobjs := make([]*tq.Transfer, 0, len(objs))
   283  	for _, o := range objs {
   284  		apiobjs = append(apiobjs, &tq.Transfer{Oid: o.Oid, Size: o.Size})
   285  	}
   286  
   287  	bres, err := tq.Batch(manifest, dir, "origin", apiobjs)
   288  	if err != nil {
   289  		return nil, err
   290  	}
   291  	return bres.Objects, nil
   292  }
   293  
   294  // Combine 2 slices into one by "randomly" interleaving
   295  // Not actually random, same sequence each time so repeatable
   296  func interleaveTestData(slice1, slice2 []TestObject) []TestObject {
   297  	// Predictable sequence, mixin existing & missing semi-randomly
   298  	rand.Seed(21)
   299  	count := len(slice1) + len(slice2)
   300  	ret := make([]TestObject, 0, count)
   301  	slice1Idx := 0
   302  	slice2Idx := 0
   303  	for left := count; left > 0; {
   304  		for i := rand.Intn(3) + 1; slice1Idx < len(slice1) && i > 0; i-- {
   305  			obj := slice1[slice1Idx]
   306  			ret = append(ret, obj)
   307  			slice1Idx++
   308  			left--
   309  		}
   310  		for i := rand.Intn(3) + 1; slice2Idx < len(slice2) && i > 0; i-- {
   311  			obj := slice2[slice2Idx]
   312  			ret = append(ret, obj)
   313  			slice2Idx++
   314  			left--
   315  		}
   316  	}
   317  	return ret
   318  }
   319  
   320  func uploadTransfer(oid, filename string) (*tq.Transfer, error) {
   321  	localMediaPath, err := lfs.LocalMediaPath(oid)
   322  	if err != nil {
   323  		return nil, errors.Wrapf(err, "Error uploading file %s (%s)", filename, oid)
   324  	}
   325  
   326  	fi, err := os.Stat(localMediaPath)
   327  	if err != nil {
   328  		return nil, errors.Wrapf(err, "Error uploading file %s (%s)", filename, oid)
   329  	}
   330  
   331  	return &tq.Transfer{
   332  		Name: filename,
   333  		Path: localMediaPath,
   334  		Oid:  oid,
   335  		Size: fi.Size(),
   336  	}, nil
   337  }
   338  
   339  func init() {
   340  	RootCmd.Flags().StringVarP(&apiUrl, "url", "u", "", "URL of the API (must supply this or --clone)")
   341  	RootCmd.Flags().StringVarP(&cloneUrl, "clone", "c", "", "Clone URL from which to find API (must supply this or --url)")
   342  	RootCmd.Flags().StringVarP(&savePrefix, "save", "s", "", "Saves generated data to <prefix>_exists|missing for subsequent use")
   343  }