github.com/stellar/stellar-etl@v1.0.1-0.20240312145900-4874b6bf2b89/cmd/export_all_history.go (about)

     1  package cmd
     2  
     3  import (
     4  	"fmt"
     5  
     6  	"github.com/sirupsen/logrus"
     7  	"github.com/spf13/cobra"
     8  
     9  	"github.com/stellar/stellar-etl/internal/input"
    10  	"github.com/stellar/stellar-etl/internal/toid"
    11  	"github.com/stellar/stellar-etl/internal/transform"
    12  	"github.com/stellar/stellar-etl/internal/utils"
    13  )
    14  
    15  var allHistoryCmd = &cobra.Command{
    16  	Use:   "export_all_history",
    17  	Short: "Exports all stellar network history.",
    18  	Long: `Exports historical stellar network data between provided start-ledger/end-ledger to output files. 
    19  This is a temporary command used to reduce the amount of requests to history archives 
    20  in order to mitigate egress costs for the entity hosting history archives.`,
    21  	Run: func(cmd *cobra.Command, args []string) {
    22  		cmdLogger.SetLevel(logrus.InfoLevel)
    23  		endNum, strictExport, isTest, isFuture, extra := utils.MustCommonFlags(cmd.Flags(), cmdLogger)
    24  		cmdLogger.StrictExport = strictExport
    25  		startNum, path, limit := utils.MustArchiveFlags(cmd.Flags(), cmdLogger)
    26  		cloudStorageBucket, cloudCredentials, cloudProvider := utils.MustCloudStorageFlags(cmd.Flags(), cmdLogger)
    27  		env := utils.GetEnvironmentDetails(isTest, isFuture)
    28  
    29  		allHistory, err := input.GetAllHistory(startNum, endNum, limit, env)
    30  		if err != nil {
    31  			cmdLogger.Fatal("could not read all history: ", err)
    32  		}
    33  
    34  		cmdLogger.Info("start doing other exports")
    35  		getOperations(allHistory.Operations, extra, cloudStorageBucket, cloudCredentials, cloudProvider, path+"exported_operations.txt", env)
    36  		getTrades(allHistory.Trades, extra, cloudStorageBucket, cloudCredentials, cloudProvider, path+"exported_trades.txt")
    37  		getEffects(allHistory.Ledgers, extra, cloudStorageBucket, cloudCredentials, cloudProvider, path+"exported_effects.txt", env)
    38  		getTransactions(allHistory.Ledgers, extra, cloudStorageBucket, cloudCredentials, cloudProvider, path+"exported_transactions.txt")
    39  		getDiagnosticEvents(allHistory.Ledgers, extra, cloudStorageBucket, cloudCredentials, cloudProvider, path+"exported_diagnostic_events.txt")
    40  		cmdLogger.Info("done doing other exports")
    41  	},
    42  }
    43  
    44  func getOperations(operations []input.OperationTransformInput, extra map[string]string, cloudStorageBucket string, cloudCredentials string, cloudProvider string, path string, env utils.EnvironmentDetails) {
    45  	outFileOperations := mustOutFile(path)
    46  	numFailures := 0
    47  	totalNumBytes := 0
    48  	for _, transformInput := range operations {
    49  		transformed, err := transform.TransformOperation(transformInput.Operation, transformInput.OperationIndex, transformInput.Transaction, transformInput.LedgerSeqNum, transformInput.LedgerCloseMeta, env.NetworkPassphrase)
    50  		if err != nil {
    51  			txIndex := transformInput.Transaction.Index
    52  			cmdLogger.LogError(fmt.Errorf("could not transform operation %d in transaction %d in ledger %d: %v", transformInput.OperationIndex, txIndex, transformInput.LedgerSeqNum, err))
    53  			numFailures += 1
    54  			continue
    55  		}
    56  
    57  		numBytes, err := exportEntry(transformed, outFileOperations, extra)
    58  		if err != nil {
    59  			cmdLogger.LogError(fmt.Errorf("could not export operation: %v", err))
    60  			numFailures += 1
    61  			continue
    62  		}
    63  		totalNumBytes += numBytes
    64  	}
    65  
    66  	outFileOperations.Close()
    67  	cmdLogger.Info("Number of bytes written: ", totalNumBytes)
    68  
    69  	printTransformStats(len(operations), numFailures)
    70  
    71  	maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path)
    72  }
    73  
    74  func getTrades(trades []input.TradeTransformInput, extra map[string]string, cloudStorageBucket string, cloudCredentials string, cloudProvider string, path string) {
    75  	outFile := mustOutFile(path)
    76  	numFailures := 0
    77  	totalNumBytes := 0
    78  	for _, tradeInput := range trades {
    79  		trades, err := transform.TransformTrade(tradeInput.OperationIndex, tradeInput.OperationHistoryID, tradeInput.Transaction, tradeInput.CloseTime)
    80  		if err != nil {
    81  			parsedID := toid.Parse(tradeInput.OperationHistoryID)
    82  			cmdLogger.LogError(fmt.Errorf("from ledger %d, transaction %d, operation %d: %v", parsedID.LedgerSequence, parsedID.TransactionOrder, parsedID.OperationOrder, err))
    83  			numFailures += 1
    84  			continue
    85  		}
    86  
    87  		for _, transformed := range trades {
    88  			numBytes, err := exportEntry(transformed, outFile, extra)
    89  			if err != nil {
    90  				cmdLogger.LogError(err)
    91  				numFailures += 1
    92  				continue
    93  			}
    94  			totalNumBytes += numBytes
    95  		}
    96  	}
    97  
    98  	outFile.Close()
    99  	cmdLogger.Info("Number of bytes written: ", totalNumBytes)
   100  
   101  	printTransformStats(len(trades), numFailures)
   102  
   103  	maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path)
   104  }
   105  
   106  func getEffects(transactions []input.LedgerTransformInput, extra map[string]string, cloudStorageBucket string, cloudCredentials string, cloudProvider string, path string, env utils.EnvironmentDetails) {
   107  	outFile := mustOutFile(path)
   108  	numFailures := 0
   109  	totalNumBytes := 0
   110  	for _, transformInput := range transactions {
   111  		LedgerSeq := uint32(transformInput.LedgerHistory.Header.LedgerSeq)
   112  		effects, err := transform.TransformEffect(transformInput.Transaction, LedgerSeq, transformInput.LedgerCloseMeta, env.NetworkPassphrase)
   113  		if err != nil {
   114  			txIndex := transformInput.Transaction.Index
   115  			cmdLogger.Errorf("could not transform transaction %d in ledger %d: %v", txIndex, LedgerSeq, err)
   116  			numFailures += 1
   117  			continue
   118  		}
   119  
   120  		for _, transformed := range effects {
   121  			numBytes, err := exportEntry(transformed, outFile, extra)
   122  			if err != nil {
   123  				cmdLogger.LogError(err)
   124  				numFailures += 1
   125  				continue
   126  			}
   127  			totalNumBytes += numBytes
   128  		}
   129  	}
   130  
   131  	outFile.Close()
   132  	cmdLogger.Info("Number of bytes written: ", totalNumBytes)
   133  
   134  	printTransformStats(len(transactions), numFailures)
   135  
   136  	maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path)
   137  }
   138  
   139  func getTransactions(transactions []input.LedgerTransformInput, extra map[string]string, cloudStorageBucket string, cloudCredentials string, cloudProvider string, path string) {
   140  	outFile := mustOutFile(path)
   141  	numFailures := 0
   142  	totalNumBytes := 0
   143  	for _, transformInput := range transactions {
   144  		transformed, err := transform.TransformTransaction(transformInput.Transaction, transformInput.LedgerHistory)
   145  		if err != nil {
   146  			ledgerSeq := transformInput.LedgerHistory.Header.LedgerSeq
   147  			cmdLogger.LogError(fmt.Errorf("could not transform transaction %d in ledger %d: ", transformInput.Transaction.Index, ledgerSeq))
   148  			numFailures += 1
   149  			continue
   150  		}
   151  
   152  		numBytes, err := exportEntry(transformed, outFile, extra)
   153  		if err != nil {
   154  			cmdLogger.LogError(fmt.Errorf("could not export transaction: %v", err))
   155  			numFailures += 1
   156  			continue
   157  		}
   158  		totalNumBytes += numBytes
   159  	}
   160  
   161  	outFile.Close()
   162  	cmdLogger.Info("Number of bytes written: ", totalNumBytes)
   163  
   164  	printTransformStats(len(transactions), numFailures)
   165  
   166  	maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path)
   167  }
   168  
   169  func getDiagnosticEvents(transactions []input.LedgerTransformInput, extra map[string]string, cloudStorageBucket string, cloudCredentials string, cloudProvider string, path string) {
   170  	outFile := mustOutFile(path)
   171  	numFailures := 0
   172  	for _, transformInput := range transactions {
   173  		transformed, err, ok := transform.TransformDiagnosticEvent(transformInput.Transaction, transformInput.LedgerHistory)
   174  		if err != nil {
   175  			ledgerSeq := transformInput.LedgerHistory.Header.LedgerSeq
   176  			cmdLogger.LogError(fmt.Errorf("could not transform diagnostic events in transaction %d in ledger %d: ", transformInput.Transaction.Index, ledgerSeq))
   177  			numFailures += 1
   178  			continue
   179  		}
   180  
   181  		if !ok {
   182  			continue
   183  		}
   184  		for _, diagnosticEvent := range transformed {
   185  			_, err := exportEntry(diagnosticEvent, outFile, extra)
   186  			if err != nil {
   187  				cmdLogger.LogError(fmt.Errorf("could not export diagnostic event: %v", err))
   188  				numFailures += 1
   189  				continue
   190  			}
   191  		}
   192  	}
   193  
   194  	outFile.Close()
   195  
   196  	printTransformStats(len(transactions), numFailures)
   197  
   198  	maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path)
   199  }
   200  
   201  func init() {
   202  	rootCmd.AddCommand(allHistoryCmd)
   203  	utils.AddCommonFlags(allHistoryCmd.Flags())
   204  	utils.AddArchiveFlags("", allHistoryCmd.Flags())
   205  	utils.AddCloudStorageFlags(allHistoryCmd.Flags())
   206  	allHistoryCmd.MarkFlagRequired("end-ledger")
   207  }