github.com/stellar/stellar-etl@v1.0.1-0.20240312145900-4874b6bf2b89/cmd/export_operations.go (about) 1 package cmd 2 3 import ( 4 "fmt" 5 6 "github.com/sirupsen/logrus" 7 "github.com/spf13/cobra" 8 "github.com/stellar/stellar-etl/internal/input" 9 "github.com/stellar/stellar-etl/internal/transform" 10 "github.com/stellar/stellar-etl/internal/utils" 11 ) 12 13 var operationsCmd = &cobra.Command{ 14 Use: "export_operations", 15 Short: "Exports the operations data over a specified range", 16 Long: `Exports the operations data over a specified range. Each operation is an individual command that mutates the Stellar ledger.`, 17 Run: func(cmd *cobra.Command, args []string) { 18 cmdLogger.SetLevel(logrus.InfoLevel) 19 endNum, strictExport, isTest, isFuture, extra := utils.MustCommonFlags(cmd.Flags(), cmdLogger) 20 cmdLogger.StrictExport = strictExport 21 startNum, path, limit := utils.MustArchiveFlags(cmd.Flags(), cmdLogger) 22 cloudStorageBucket, cloudCredentials, cloudProvider := utils.MustCloudStorageFlags(cmd.Flags(), cmdLogger) 23 env := utils.GetEnvironmentDetails(isTest, isFuture) 24 25 operations, err := input.GetOperations(startNum, endNum, limit, env) 26 if err != nil { 27 cmdLogger.Fatal("could not read operations: ", err) 28 } 29 30 outFile := mustOutFile(path) 31 numFailures := 0 32 totalNumBytes := 0 33 for _, transformInput := range operations { 34 transformed, err := transform.TransformOperation(transformInput.Operation, transformInput.OperationIndex, transformInput.Transaction, transformInput.LedgerSeqNum, transformInput.LedgerCloseMeta, env.NetworkPassphrase) 35 if err != nil { 36 txIndex := transformInput.Transaction.Index 37 cmdLogger.LogError(fmt.Errorf("could not transform operation %d in transaction %d in ledger %d: %v", transformInput.OperationIndex, txIndex, transformInput.LedgerSeqNum, err)) 38 numFailures += 1 39 continue 40 } 41 42 numBytes, err := exportEntry(transformed, outFile, extra) 43 if err != nil { 44 cmdLogger.LogError(fmt.Errorf("could not export operation: %v", err)) 45 numFailures += 1 46 continue 47 } 48 totalNumBytes += numBytes 49 } 50 51 outFile.Close() 52 cmdLogger.Info("Number of bytes written: ", totalNumBytes) 53 54 printTransformStats(len(operations), numFailures) 55 56 maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path) 57 }, 58 } 59 60 func init() { 61 rootCmd.AddCommand(operationsCmd) 62 utils.AddCommonFlags(operationsCmd.Flags()) 63 utils.AddArchiveFlags("operations", operationsCmd.Flags()) 64 utils.AddCloudStorageFlags(operationsCmd.Flags()) 65 operationsCmd.MarkFlagRequired("end-ledger") 66 67 /* 68 Current flags: 69 start-ledger: the ledger sequence number for the beginning of the export period 70 end-ledger: the ledger sequence number for the end of the export range (required) 71 72 limit: maximum number of operations to export; default to 6,000,000 73 each transaction can have up to 100 operations 74 each ledger can have up to 1000 transactions 75 there are 60 new ledgers in a 5 minute period 76 77 output-file: filename of the output file 78 79 TODO: implement extra flags if possible 80 serialize-method: the method for serialization of the output data (JSON, XDR, etc) 81 start and end time as a replacement for start and end sequence numbers 82 */ 83 }