github.com/stellar/stellar-etl@v1.0.1-0.20240312145900-4874b6bf2b89/cmd/export_effects.go (about) 1 package cmd 2 3 import ( 4 "github.com/sirupsen/logrus" 5 "github.com/spf13/cobra" 6 "github.com/stellar/stellar-etl/internal/input" 7 "github.com/stellar/stellar-etl/internal/transform" 8 "github.com/stellar/stellar-etl/internal/utils" 9 ) 10 11 var effectsCmd = &cobra.Command{ 12 Use: "export_effects", 13 Short: "Exports the effects data over a specified range", 14 Long: "Exports the effects data over a specified range to an output file.", 15 Run: func(cmd *cobra.Command, args []string) { 16 cmdLogger.SetLevel(logrus.InfoLevel) 17 endNum, strictExport, isTest, isFuture, extra := utils.MustCommonFlags(cmd.Flags(), cmdLogger) 18 cmdLogger.StrictExport = strictExport 19 startNum, path, limit := utils.MustArchiveFlags(cmd.Flags(), cmdLogger) 20 cloudStorageBucket, cloudCredentials, cloudProvider := utils.MustCloudStorageFlags(cmd.Flags(), cmdLogger) 21 env := utils.GetEnvironmentDetails(isTest, isFuture) 22 23 transactions, err := input.GetTransactions(startNum, endNum, limit, env) 24 if err != nil { 25 cmdLogger.Fatalf("could not read transactions in [%d, %d] (limit=%d): %v", startNum, endNum, limit, err) 26 } 27 28 outFile := mustOutFile(path) 29 numFailures := 0 30 totalNumBytes := 0 31 for _, transformInput := range transactions { 32 LedgerSeq := uint32(transformInput.LedgerHistory.Header.LedgerSeq) 33 effects, err := transform.TransformEffect(transformInput.Transaction, LedgerSeq, transformInput.LedgerCloseMeta, env.NetworkPassphrase) 34 if err != nil { 35 txIndex := transformInput.Transaction.Index 36 cmdLogger.Errorf("could not transform transaction %d in ledger %d: %v", txIndex, LedgerSeq, err) 37 numFailures += 1 38 continue 39 } 40 41 for _, transformed := range effects { 42 numBytes, err := exportEntry(transformed, outFile, extra) 43 if err != nil { 44 cmdLogger.LogError(err) 45 numFailures += 1 46 continue 47 } 48 totalNumBytes += numBytes 49 } 50 } 51 52 outFile.Close() 53 cmdLogger.Info("Number of bytes written: ", totalNumBytes) 54 55 printTransformStats(len(transactions), numFailures) 56 57 maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path) 58 }, 59 } 60 61 func init() { 62 rootCmd.AddCommand(effectsCmd) 63 utils.AddCommonFlags(effectsCmd.Flags()) 64 utils.AddArchiveFlags("effects", effectsCmd.Flags()) 65 utils.AddCloudStorageFlags(effectsCmd.Flags()) 66 effectsCmd.MarkFlagRequired("end-ledger") 67 68 /* 69 Current flags: 70 start-ledger: the ledger sequence number for the beginning of the export period 71 end-ledger: the ledger sequence number for the end of the export range (required) 72 73 limit: maximum number of effects to export; default to 6,000,000 74 each transaction can have up to 100 effects 75 each ledger can have up to 1000 transactions 76 there are 60 new ledgers in a 5 minute period 77 78 output-file: filename of the output file 79 80 TODO: implement extra flags if possible 81 serialize-method: the method for serialization of the output data (JSON, XDR, etc) 82 start and end time as a replacement for start and end sequence numbers 83 */ 84 }