github.com/stellar/stellar-etl@v1.0.1-0.20240312145900-4874b6bf2b89/cmd/export_assets.go (about)

     1  package cmd
     2  
     3  import (
     4  	"fmt"
     5  
     6  	"github.com/sirupsen/logrus"
     7  	"github.com/spf13/cobra"
     8  	"github.com/stellar/stellar-etl/internal/input"
     9  	"github.com/stellar/stellar-etl/internal/transform"
    10  	"github.com/stellar/stellar-etl/internal/utils"
    11  )
    12  
    13  var assetsCmd = &cobra.Command{
    14  	Use:   "export_assets",
    15  	Short: "Exports the assets data over a specified range",
    16  	Long:  `Exports the assets that are created from payment operations over a specified ledger range`,
    17  	Run: func(cmd *cobra.Command, args []string) {
    18  		cmdLogger.SetLevel(logrus.InfoLevel)
    19  		endNum, strictExport, isTest, isFuture, extra := utils.MustCommonFlags(cmd.Flags(), cmdLogger)
    20  		cmdLogger.StrictExport = strictExport
    21  		startNum, path, limit := utils.MustArchiveFlags(cmd.Flags(), cmdLogger)
    22  		cloudStorageBucket, cloudCredentials, cloudProvider := utils.MustCloudStorageFlags(cmd.Flags(), cmdLogger)
    23  
    24  		outFile := mustOutFile(path)
    25  
    26  		paymentOps, err := input.GetPaymentOperations(startNum, endNum, limit, isTest, isFuture)
    27  		if err != nil {
    28  			cmdLogger.Fatal("could not read asset: ", err)
    29  		}
    30  
    31  		// With seenIDs, the code doesn't export duplicate assets within a single export. Note that across exports, assets may be duplicated
    32  		seenIDs := map[uint64]bool{}
    33  		numFailures := 0
    34  		totalNumBytes := 0
    35  		for _, transformInput := range paymentOps {
    36  			transformed, err := transform.TransformAsset(transformInput.Operation, transformInput.OperationIndex, transformInput.TransactionIndex, transformInput.LedgerSeqNum)
    37  			if err != nil {
    38  				txIndex := transformInput.TransactionIndex
    39  				cmdLogger.LogError(fmt.Errorf("could not extract asset from operation %d in transaction %d in ledger %d: ", transformInput.OperationIndex, txIndex, transformInput.LedgerSeqNum))
    40  				numFailures += 1
    41  				continue
    42  			}
    43  
    44  			// if we have seen the asset already, do not export it
    45  			if _, exists := seenIDs[transformed.AssetID]; exists {
    46  				continue
    47  			}
    48  
    49  			seenIDs[transformed.AssetID] = true
    50  			numBytes, err := exportEntry(transformed, outFile, extra)
    51  			if err != nil {
    52  				cmdLogger.Error(err)
    53  				numFailures += 1
    54  				continue
    55  			}
    56  			totalNumBytes += numBytes
    57  		}
    58  
    59  		outFile.Close()
    60  		cmdLogger.Infof("%d bytes written to %s", totalNumBytes, outFile.Name())
    61  
    62  		printTransformStats(len(paymentOps), numFailures)
    63  
    64  		maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path)
    65  	},
    66  }
    67  
    68  func init() {
    69  	rootCmd.AddCommand(assetsCmd)
    70  	utils.AddCommonFlags(assetsCmd.Flags())
    71  	utils.AddArchiveFlags("assets", assetsCmd.Flags())
    72  	utils.AddCloudStorageFlags(assetsCmd.Flags())
    73  	assetsCmd.MarkFlagRequired("end-ledger")
    74  
    75  	/*
    76  		Current flags:
    77  			start-ledger: the ledger sequence number for the beginning of the export period
    78  			end-ledger: the ledger sequence number for the end of the export range (required)
    79  
    80  			limit: maximum number of operations to export; default to 6,000,000
    81  				each transaction can have up to 100 operations
    82  				each ledger can have up to 1000 transactions
    83  				there are 60 new ledgers in a 5 minute period
    84  
    85  			output-file: filename of the output file
    86  
    87  		TODO: implement extra flags if possible
    88  			serialize-method: the method for serialization of the output data (JSON, XDR, etc)
    89  			start and end time as a replacement for start and end sequence numbers
    90  	*/
    91  }