github.com/stellar/stellar-etl@v1.0.1-0.20240312145900-4874b6bf2b89/cmd/export_assets.go (about) 1 package cmd 2 3 import ( 4 "fmt" 5 6 "github.com/sirupsen/logrus" 7 "github.com/spf13/cobra" 8 "github.com/stellar/stellar-etl/internal/input" 9 "github.com/stellar/stellar-etl/internal/transform" 10 "github.com/stellar/stellar-etl/internal/utils" 11 ) 12 13 var assetsCmd = &cobra.Command{ 14 Use: "export_assets", 15 Short: "Exports the assets data over a specified range", 16 Long: `Exports the assets that are created from payment operations over a specified ledger range`, 17 Run: func(cmd *cobra.Command, args []string) { 18 cmdLogger.SetLevel(logrus.InfoLevel) 19 endNum, strictExport, isTest, isFuture, extra := utils.MustCommonFlags(cmd.Flags(), cmdLogger) 20 cmdLogger.StrictExport = strictExport 21 startNum, path, limit := utils.MustArchiveFlags(cmd.Flags(), cmdLogger) 22 cloudStorageBucket, cloudCredentials, cloudProvider := utils.MustCloudStorageFlags(cmd.Flags(), cmdLogger) 23 24 outFile := mustOutFile(path) 25 26 paymentOps, err := input.GetPaymentOperations(startNum, endNum, limit, isTest, isFuture) 27 if err != nil { 28 cmdLogger.Fatal("could not read asset: ", err) 29 } 30 31 // With seenIDs, the code doesn't export duplicate assets within a single export. Note that across exports, assets may be duplicated 32 seenIDs := map[uint64]bool{} 33 numFailures := 0 34 totalNumBytes := 0 35 for _, transformInput := range paymentOps { 36 transformed, err := transform.TransformAsset(transformInput.Operation, transformInput.OperationIndex, transformInput.TransactionIndex, transformInput.LedgerSeqNum) 37 if err != nil { 38 txIndex := transformInput.TransactionIndex 39 cmdLogger.LogError(fmt.Errorf("could not extract asset from operation %d in transaction %d in ledger %d: ", transformInput.OperationIndex, txIndex, transformInput.LedgerSeqNum)) 40 numFailures += 1 41 continue 42 } 43 44 // if we have seen the asset already, do not export it 45 if _, exists := seenIDs[transformed.AssetID]; exists { 46 continue 47 } 48 49 seenIDs[transformed.AssetID] = true 50 numBytes, err := exportEntry(transformed, outFile, extra) 51 if err != nil { 52 cmdLogger.Error(err) 53 numFailures += 1 54 continue 55 } 56 totalNumBytes += numBytes 57 } 58 59 outFile.Close() 60 cmdLogger.Infof("%d bytes written to %s", totalNumBytes, outFile.Name()) 61 62 printTransformStats(len(paymentOps), numFailures) 63 64 maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path) 65 }, 66 } 67 68 func init() { 69 rootCmd.AddCommand(assetsCmd) 70 utils.AddCommonFlags(assetsCmd.Flags()) 71 utils.AddArchiveFlags("assets", assetsCmd.Flags()) 72 utils.AddCloudStorageFlags(assetsCmd.Flags()) 73 assetsCmd.MarkFlagRequired("end-ledger") 74 75 /* 76 Current flags: 77 start-ledger: the ledger sequence number for the beginning of the export period 78 end-ledger: the ledger sequence number for the end of the export range (required) 79 80 limit: maximum number of operations to export; default to 6,000,000 81 each transaction can have up to 100 operations 82 each ledger can have up to 1000 transactions 83 there are 60 new ledgers in a 5 minute period 84 85 output-file: filename of the output file 86 87 TODO: implement extra flags if possible 88 serialize-method: the method for serialization of the output data (JSON, XDR, etc) 89 start and end time as a replacement for start and end sequence numbers 90 */ 91 }