github.com/stellar/stellar-etl@v1.0.1-0.20240312145900-4874b6bf2b89/cmd/export_ledgers.go (about) 1 package cmd 2 3 import ( 4 "fmt" 5 6 "github.com/sirupsen/logrus" 7 "github.com/spf13/cobra" 8 "github.com/stellar/stellar-etl/internal/input" 9 "github.com/stellar/stellar-etl/internal/transform" 10 "github.com/stellar/stellar-etl/internal/utils" 11 ) 12 13 var ledgersCmd = &cobra.Command{ 14 Use: "export_ledgers", 15 Short: "Exports the ledger data.", 16 Long: `Exports ledger data within the specified range to an output file. Encodes ledgers as JSON objects and exports them to the output file.`, 17 Run: func(cmd *cobra.Command, args []string) { 18 cmdLogger.SetLevel(logrus.InfoLevel) 19 endNum, strictExport, isTest, isFuture, extra := utils.MustCommonFlags(cmd.Flags(), cmdLogger) 20 cmdLogger.StrictExport = strictExport 21 startNum, path, limit := utils.MustArchiveFlags(cmd.Flags(), cmdLogger) 22 cloudStorageBucket, cloudCredentials, cloudProvider := utils.MustCloudStorageFlags(cmd.Flags(), cmdLogger) 23 24 ledgers, err := input.GetLedgers(startNum, endNum, limit, isTest, isFuture) 25 if err != nil { 26 cmdLogger.Fatal("could not read ledgers: ", err) 27 } 28 29 outFile := mustOutFile(path) 30 31 numFailures := 0 32 totalNumBytes := 0 33 for i, lcm := range ledgers { 34 transformed, err := transform.TransformLedger(lcm) 35 if err != nil { 36 cmdLogger.LogError(fmt.Errorf("could not json transform ledger %d: %s", startNum+uint32(i), err)) 37 numFailures += 1 38 continue 39 } 40 41 numBytes, err := exportEntry(transformed, outFile, extra) 42 if err != nil { 43 cmdLogger.LogError(fmt.Errorf("could not export ledger %d: %s", startNum+uint32(i), err)) 44 numFailures += 1 45 continue 46 } 47 totalNumBytes += numBytes 48 } 49 50 outFile.Close() 51 cmdLogger.Info("Number of bytes written: ", totalNumBytes) 52 53 printTransformStats(len(ledgers), numFailures) 54 55 maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path) 56 }, 57 } 58 59 func init() { 60 rootCmd.AddCommand(ledgersCmd) 61 utils.AddCommonFlags(ledgersCmd.Flags()) 62 utils.AddArchiveFlags("ledgers", ledgersCmd.Flags()) 63 utils.AddCloudStorageFlags(ledgersCmd.Flags()) 64 ledgersCmd.MarkFlagRequired("end-ledger") 65 /* 66 Current flags: 67 start-ledger: the ledger sequence number for the beginning of the export period 68 end-ledger: the ledger sequence number for the end of the export range (required) 69 70 limit: maximum number of ledgers to export; default to 60 (1 ledger per 5 seconds over our 5 minute update period) 71 output-file: filename of the output file 72 73 TODO: implement extra flags if possible 74 serialize-method: the method for serialization of the output data (JSON, XDR, etc) 75 start and end time as a replacement for start and end sequence numbers 76 */ 77 }