github.com/psexton/git-lfs@v2.1.1-0.20170517224304-289a18b2bc53+incompatible/test/git-lfs-test-server-api/main.go (about) 1 package main 2 3 import ( 4 "bufio" 5 "crypto/sha256" 6 "encoding/hex" 7 "fmt" 8 "math/rand" 9 "os" 10 "strconv" 11 "strings" 12 13 "github.com/git-lfs/git-lfs/config" 14 "github.com/git-lfs/git-lfs/errors" 15 "github.com/git-lfs/git-lfs/lfs" 16 "github.com/git-lfs/git-lfs/lfsapi" 17 "github.com/git-lfs/git-lfs/progress" 18 "github.com/git-lfs/git-lfs/test" 19 "github.com/git-lfs/git-lfs/tq" 20 "github.com/spf13/cobra" 21 ) 22 23 type TestObject struct { 24 Oid string 25 Size int64 26 } 27 28 type ServerTest struct { 29 Name string 30 F func(m *tq.Manifest, oidsExist, oidsMissing []TestObject) error 31 } 32 33 var ( 34 RootCmd = &cobra.Command{ 35 Use: "git-lfs-test-server-api [--url=<apiurl> | --clone=<cloneurl>] [<oid-exists-file> <oid-missing-file>]", 36 Short: "Test a Git LFS API server for compliance", 37 Run: testServerApi, 38 } 39 apiUrl string 40 cloneUrl string 41 savePrefix string 42 43 tests []ServerTest 44 ) 45 46 func main() { 47 RootCmd.Execute() 48 } 49 50 func testServerApi(cmd *cobra.Command, args []string) { 51 if (len(apiUrl) == 0 && len(cloneUrl) == 0) || 52 (len(apiUrl) != 0 && len(cloneUrl) != 0) { 53 exit("Must supply either --url or --clone (and not both)") 54 } 55 56 if len(args) != 0 && len(args) != 2 { 57 exit("Must supply either no file arguments or both the exists AND missing file") 58 } 59 60 if len(args) != 0 && len(savePrefix) > 0 { 61 exit("Cannot combine input files and --save option") 62 } 63 64 // Force loading of config before we alter it 65 config.Config.Git.All() 66 67 manifest, err := buildManifest() 68 if err != nil { 69 exit("error building tq.Manifest: " + err.Error()) 70 } 71 72 var oidsExist, oidsMissing []TestObject 73 if len(args) >= 2 { 74 fmt.Printf("Reading test data from files (no server content changes)\n") 75 oidsExist = readTestOids(args[0]) 76 oidsMissing = readTestOids(args[1]) 77 } else { 78 fmt.Printf("Creating test data (will upload to server)\n") 79 var err error 80 oidsExist, oidsMissing, err = buildTestData(manifest) 81 if err != nil { 82 exit("Failed to set up test data, aborting") 83 } 84 if len(savePrefix) > 0 { 85 existFile := savePrefix + "_exists" 86 missingFile := savePrefix + "_missing" 87 saveTestOids(existFile, oidsExist) 88 saveTestOids(missingFile, oidsMissing) 89 fmt.Printf("Wrote test to %s, %s for future use\n", existFile, missingFile) 90 } 91 92 } 93 94 ok := runTests(manifest, oidsExist, oidsMissing) 95 if !ok { 96 exit("One or more tests failed, see above") 97 } 98 fmt.Println("All tests passed") 99 } 100 101 func readTestOids(filename string) []TestObject { 102 f, err := os.OpenFile(filename, os.O_RDONLY, 0644) 103 if err != nil { 104 exit("Error opening file %s", filename) 105 } 106 defer f.Close() 107 108 var ret []TestObject 109 rdr := bufio.NewReader(f) 110 line, err := rdr.ReadString('\n') 111 for err == nil { 112 fields := strings.Fields(strings.TrimSpace(line)) 113 if len(fields) == 2 { 114 sz, _ := strconv.ParseInt(fields[1], 10, 64) 115 ret = append(ret, TestObject{Oid: fields[0], Size: sz}) 116 } 117 118 line, err = rdr.ReadString('\n') 119 } 120 121 return ret 122 } 123 124 type testDataCallback struct{} 125 126 func (*testDataCallback) Fatalf(format string, args ...interface{}) { 127 exit(format, args...) 128 } 129 func (*testDataCallback) Errorf(format string, args ...interface{}) { 130 fmt.Printf(format, args...) 131 } 132 133 func buildManifest() (*tq.Manifest, error) { 134 cfg := config.Config 135 136 // Configure the endpoint manually 137 finder := lfsapi.NewEndpointFinder(config.Config.Git) 138 139 var endp lfsapi.Endpoint 140 if len(cloneUrl) > 0 { 141 endp = finder.NewEndpointFromCloneURL(cloneUrl) 142 } else { 143 endp = finder.NewEndpoint(apiUrl) 144 } 145 146 apiClient, err := lfsapi.NewClient(cfg.Os, cfg.Git) 147 apiClient.Endpoints = &constantEndpoint{ 148 e: endp, 149 EndpointFinder: apiClient.Endpoints, 150 } 151 if err != nil { 152 return nil, err 153 } 154 return tq.NewManifestWithClient(apiClient), nil 155 } 156 157 type constantEndpoint struct { 158 e lfsapi.Endpoint 159 160 lfsapi.EndpointFinder 161 } 162 163 func (c *constantEndpoint) NewEndpointFromCloneURL(rawurl string) lfsapi.Endpoint { return c.e } 164 165 func (c *constantEndpoint) NewEndpoint(rawurl string) lfsapi.Endpoint { return c.e } 166 167 func (c *constantEndpoint) Endpoint(operation, remote string) lfsapi.Endpoint { return c.e } 168 169 func (c *constantEndpoint) RemoteEndpoint(operation, remote string) lfsapi.Endpoint { return c.e } 170 171 func buildTestData(manifest *tq.Manifest) (oidsExist, oidsMissing []TestObject, err error) { 172 const oidCount = 50 173 oidsExist = make([]TestObject, 0, oidCount) 174 oidsMissing = make([]TestObject, 0, oidCount) 175 meter := progress.NewMeter(progress.WithOSEnv(config.Config.Os)) 176 177 // Build test data for existing files & upload 178 // Use test repo for this to simplify the process of making sure data matches oid 179 // We're not performing a real test at this point (although an upload fail will break it) 180 var callback testDataCallback 181 repo := test.NewRepo(&callback) 182 repo.Pushd() 183 defer repo.Cleanup() 184 // just one commit 185 commit := test.CommitInput{CommitterName: "A N Other", CommitterEmail: "noone@somewhere.com"} 186 for i := 0; i < oidCount; i++ { 187 filename := fmt.Sprintf("file%d.dat", i) 188 sz := int64(rand.Intn(200)) + 50 189 commit.Files = append(commit.Files, &test.FileInput{Filename: filename, Size: sz}) 190 meter.Add(sz) 191 } 192 outputs := repo.AddCommits([]*test.CommitInput{&commit}) 193 194 // now upload 195 uploadQueue := tq.NewTransferQueue(tq.Upload, manifest, "origin", tq.WithProgress(meter)) 196 for _, f := range outputs[0].Files { 197 oidsExist = append(oidsExist, TestObject{Oid: f.Oid, Size: f.Size}) 198 199 t, err := uploadTransfer(f.Oid, "Test file") 200 if err != nil { 201 return nil, nil, err 202 } 203 uploadQueue.Add(t.Name, t.Path, t.Oid, t.Size) 204 } 205 uploadQueue.Wait() 206 207 for _, err := range uploadQueue.Errors() { 208 if errors.IsFatalError(err) { 209 exit("Fatal error setting up test data: %s", err) 210 } 211 } 212 213 // Generate SHAs for missing files, random but repeatable 214 // No actual file content needed for these 215 rand.Seed(int64(oidCount)) 216 runningSha := sha256.New() 217 for i := 0; i < oidCount; i++ { 218 runningSha.Write([]byte{byte(rand.Intn(256))}) 219 oid := hex.EncodeToString(runningSha.Sum(nil)) 220 sz := int64(rand.Intn(200)) + 50 221 oidsMissing = append(oidsMissing, TestObject{Oid: oid, Size: sz}) 222 } 223 return oidsExist, oidsMissing, nil 224 } 225 226 func saveTestOids(filename string, objs []TestObject) { 227 f, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) 228 if err != nil { 229 exit("Error opening file %s", filename) 230 } 231 defer f.Close() 232 233 for _, o := range objs { 234 f.WriteString(fmt.Sprintf("%s %d\n", o.Oid, o.Size)) 235 } 236 237 } 238 239 func runTests(manifest *tq.Manifest, oidsExist, oidsMissing []TestObject) bool { 240 ok := true 241 fmt.Printf("Running %d tests...\n", len(tests)) 242 for _, t := range tests { 243 err := runTest(t, manifest, oidsExist, oidsMissing) 244 if err != nil { 245 ok = false 246 } 247 } 248 return ok 249 } 250 251 func runTest(t ServerTest, manifest *tq.Manifest, oidsExist, oidsMissing []TestObject) error { 252 const linelen = 70 253 line := t.Name 254 if len(line) > linelen { 255 line = line[:linelen] 256 } else if len(line) < linelen { 257 line = fmt.Sprintf("%s%s", line, strings.Repeat(" ", linelen-len(line))) 258 } 259 fmt.Printf("%s...\r", line) 260 261 err := t.F(manifest, oidsExist, oidsMissing) 262 if err != nil { 263 fmt.Printf("%s FAILED\n", line) 264 fmt.Println(err.Error()) 265 } else { 266 fmt.Printf("%s OK\n", line) 267 } 268 return err 269 } 270 271 // Exit prints a formatted message and exits. 272 func exit(format string, args ...interface{}) { 273 fmt.Fprintf(os.Stderr, format, args...) 274 os.Exit(2) 275 } 276 277 func addTest(name string, f func(manifest *tq.Manifest, oidsExist, oidsMissing []TestObject) error) { 278 tests = append(tests, ServerTest{Name: name, F: f}) 279 } 280 281 func callBatchApi(manifest *tq.Manifest, dir tq.Direction, objs []TestObject) ([]*tq.Transfer, error) { 282 apiobjs := make([]*tq.Transfer, 0, len(objs)) 283 for _, o := range objs { 284 apiobjs = append(apiobjs, &tq.Transfer{Oid: o.Oid, Size: o.Size}) 285 } 286 287 bres, err := tq.Batch(manifest, dir, "origin", apiobjs) 288 if err != nil { 289 return nil, err 290 } 291 return bres.Objects, nil 292 } 293 294 // Combine 2 slices into one by "randomly" interleaving 295 // Not actually random, same sequence each time so repeatable 296 func interleaveTestData(slice1, slice2 []TestObject) []TestObject { 297 // Predictable sequence, mixin existing & missing semi-randomly 298 rand.Seed(21) 299 count := len(slice1) + len(slice2) 300 ret := make([]TestObject, 0, count) 301 slice1Idx := 0 302 slice2Idx := 0 303 for left := count; left > 0; { 304 for i := rand.Intn(3) + 1; slice1Idx < len(slice1) && i > 0; i-- { 305 obj := slice1[slice1Idx] 306 ret = append(ret, obj) 307 slice1Idx++ 308 left-- 309 } 310 for i := rand.Intn(3) + 1; slice2Idx < len(slice2) && i > 0; i-- { 311 obj := slice2[slice2Idx] 312 ret = append(ret, obj) 313 slice2Idx++ 314 left-- 315 } 316 } 317 return ret 318 } 319 320 func uploadTransfer(oid, filename string) (*tq.Transfer, error) { 321 localMediaPath, err := lfs.LocalMediaPath(oid) 322 if err != nil { 323 return nil, errors.Wrapf(err, "Error uploading file %s (%s)", filename, oid) 324 } 325 326 fi, err := os.Stat(localMediaPath) 327 if err != nil { 328 return nil, errors.Wrapf(err, "Error uploading file %s (%s)", filename, oid) 329 } 330 331 return &tq.Transfer{ 332 Name: filename, 333 Path: localMediaPath, 334 Oid: oid, 335 Size: fi.Size(), 336 }, nil 337 } 338 339 func init() { 340 RootCmd.Flags().StringVarP(&apiUrl, "url", "u", "", "URL of the API (must supply this or --clone)") 341 RootCmd.Flags().StringVarP(&cloneUrl, "clone", "c", "", "Clone URL from which to find API (must supply this or --url)") 342 RootCmd.Flags().StringVarP(&savePrefix, "save", "s", "", "Saves generated data to <prefix>_exists|missing for subsequent use") 343 }