github.com/stellar/stellar-etl@v1.0.1-0.20240312145900-4874b6bf2b89/cmd/export_transactions.go (about)

     1  package cmd
     2  
     3  import (
     4  	"fmt"
     5  
     6  	"github.com/sirupsen/logrus"
     7  	"github.com/spf13/cobra"
     8  	"github.com/stellar/stellar-etl/internal/input"
     9  	"github.com/stellar/stellar-etl/internal/transform"
    10  	"github.com/stellar/stellar-etl/internal/utils"
    11  )
    12  
    13  var transactionsCmd = &cobra.Command{
    14  	Use:   "export_transactions",
    15  	Short: "Exports the transaction data over a specified range.",
    16  	Long:  `Exports the transaction data over a specified range to an output file.`,
    17  	Run: func(cmd *cobra.Command, args []string) {
    18  		cmdLogger.SetLevel(logrus.InfoLevel)
    19  		endNum, strictExport, isTest, isFuture, extra := utils.MustCommonFlags(cmd.Flags(), cmdLogger)
    20  		cmdLogger.StrictExport = strictExport
    21  		startNum, path, limit := utils.MustArchiveFlags(cmd.Flags(), cmdLogger)
    22  		cloudStorageBucket, cloudCredentials, cloudProvider := utils.MustCloudStorageFlags(cmd.Flags(), cmdLogger)
    23  		env := utils.GetEnvironmentDetails(isTest, isFuture)
    24  
    25  		transactions, err := input.GetTransactions(startNum, endNum, limit, env)
    26  		if err != nil {
    27  			cmdLogger.Fatal("could not read transactions: ", err)
    28  		}
    29  
    30  		outFile := mustOutFile(path)
    31  		numFailures := 0
    32  		totalNumBytes := 0
    33  		for _, transformInput := range transactions {
    34  			transformed, err := transform.TransformTransaction(transformInput.Transaction, transformInput.LedgerHistory)
    35  			if err != nil {
    36  				ledgerSeq := transformInput.LedgerHistory.Header.LedgerSeq
    37  				cmdLogger.LogError(fmt.Errorf("could not transform transaction %d in ledger %d: ", transformInput.Transaction.Index, ledgerSeq))
    38  				numFailures += 1
    39  				continue
    40  			}
    41  
    42  			numBytes, err := exportEntry(transformed, outFile, extra)
    43  			if err != nil {
    44  				cmdLogger.LogError(fmt.Errorf("could not export transaction: %v", err))
    45  				numFailures += 1
    46  				continue
    47  			}
    48  			totalNumBytes += numBytes
    49  		}
    50  
    51  		outFile.Close()
    52  		cmdLogger.Info("Number of bytes written: ", totalNumBytes)
    53  
    54  		printTransformStats(len(transactions), numFailures)
    55  
    56  		maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path)
    57  	},
    58  }
    59  
    60  func init() {
    61  	rootCmd.AddCommand(transactionsCmd)
    62  	utils.AddCommonFlags(transactionsCmd.Flags())
    63  	utils.AddArchiveFlags("transactions", transactionsCmd.Flags())
    64  	utils.AddCloudStorageFlags(transactionsCmd.Flags())
    65  	transactionsCmd.MarkFlagRequired("end-ledger")
    66  
    67  	/*
    68  		Current flags:
    69  			start-ledger: the ledger sequence number for the beginning of the export period
    70  			end-ledger: the ledger sequence number for the end of the export range (*required)
    71  
    72  			limit: maximum number of transactions to export
    73  				TODO: measure a good default value that ensures all transactions within a 5 minute period will be exported with a single call
    74  				The current max_tx_set_size is 1000 and there are 60 new ledgers in a 5 minute period:
    75  					1000*60 = 60000
    76  
    77  			output-file: filename of the output file
    78  
    79  		TODO: implement extra flags if possible
    80  			serialize-method: the method for serialization of the output data (JSON, XDR, etc)
    81  			start and end time as a replacement for start and end sequence numbers
    82  	*/
    83  }