github.com/stellar/stellar-etl@v1.0.1-0.20240312145900-4874b6bf2b89/cmd/export_effects.go (about)

     1  package cmd
     2  
     3  import (
     4  	"github.com/sirupsen/logrus"
     5  	"github.com/spf13/cobra"
     6  	"github.com/stellar/stellar-etl/internal/input"
     7  	"github.com/stellar/stellar-etl/internal/transform"
     8  	"github.com/stellar/stellar-etl/internal/utils"
     9  )
    10  
    11  var effectsCmd = &cobra.Command{
    12  	Use:   "export_effects",
    13  	Short: "Exports the effects data over a specified range",
    14  	Long:  "Exports the effects data over a specified range to an output file.",
    15  	Run: func(cmd *cobra.Command, args []string) {
    16  		cmdLogger.SetLevel(logrus.InfoLevel)
    17  		endNum, strictExport, isTest, isFuture, extra := utils.MustCommonFlags(cmd.Flags(), cmdLogger)
    18  		cmdLogger.StrictExport = strictExport
    19  		startNum, path, limit := utils.MustArchiveFlags(cmd.Flags(), cmdLogger)
    20  		cloudStorageBucket, cloudCredentials, cloudProvider := utils.MustCloudStorageFlags(cmd.Flags(), cmdLogger)
    21  		env := utils.GetEnvironmentDetails(isTest, isFuture)
    22  
    23  		transactions, err := input.GetTransactions(startNum, endNum, limit, env)
    24  		if err != nil {
    25  			cmdLogger.Fatalf("could not read transactions in [%d, %d] (limit=%d): %v", startNum, endNum, limit, err)
    26  		}
    27  
    28  		outFile := mustOutFile(path)
    29  		numFailures := 0
    30  		totalNumBytes := 0
    31  		for _, transformInput := range transactions {
    32  			LedgerSeq := uint32(transformInput.LedgerHistory.Header.LedgerSeq)
    33  			effects, err := transform.TransformEffect(transformInput.Transaction, LedgerSeq, transformInput.LedgerCloseMeta, env.NetworkPassphrase)
    34  			if err != nil {
    35  				txIndex := transformInput.Transaction.Index
    36  				cmdLogger.Errorf("could not transform transaction %d in ledger %d: %v", txIndex, LedgerSeq, err)
    37  				numFailures += 1
    38  				continue
    39  			}
    40  
    41  			for _, transformed := range effects {
    42  				numBytes, err := exportEntry(transformed, outFile, extra)
    43  				if err != nil {
    44  					cmdLogger.LogError(err)
    45  					numFailures += 1
    46  					continue
    47  				}
    48  				totalNumBytes += numBytes
    49  			}
    50  		}
    51  
    52  		outFile.Close()
    53  		cmdLogger.Info("Number of bytes written: ", totalNumBytes)
    54  
    55  		printTransformStats(len(transactions), numFailures)
    56  
    57  		maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path)
    58  	},
    59  }
    60  
    61  func init() {
    62  	rootCmd.AddCommand(effectsCmd)
    63  	utils.AddCommonFlags(effectsCmd.Flags())
    64  	utils.AddArchiveFlags("effects", effectsCmd.Flags())
    65  	utils.AddCloudStorageFlags(effectsCmd.Flags())
    66  	effectsCmd.MarkFlagRequired("end-ledger")
    67  
    68  	/*
    69  		Current flags:
    70  			start-ledger: the ledger sequence number for the beginning of the export period
    71  			end-ledger: the ledger sequence number for the end of the export range (required)
    72  
    73  			limit: maximum number of effects to export; default to 6,000,000
    74  				each transaction can have up to 100 effects
    75  				each ledger can have up to 1000 transactions
    76  				there are 60 new ledgers in a 5 minute period
    77  
    78  			output-file: filename of the output file
    79  
    80  		TODO: implement extra flags if possible
    81  			serialize-method: the method for serialization of the output data (JSON, XDR, etc)
    82  			start and end time as a replacement for start and end sequence numbers
    83  	*/
    84  }