github.com/stellar/stellar-etl@v1.0.1-0.20240312145900-4874b6bf2b89/cmd/export_all_history.go (about) 1 package cmd 2 3 import ( 4 "fmt" 5 6 "github.com/sirupsen/logrus" 7 "github.com/spf13/cobra" 8 9 "github.com/stellar/stellar-etl/internal/input" 10 "github.com/stellar/stellar-etl/internal/toid" 11 "github.com/stellar/stellar-etl/internal/transform" 12 "github.com/stellar/stellar-etl/internal/utils" 13 ) 14 15 var allHistoryCmd = &cobra.Command{ 16 Use: "export_all_history", 17 Short: "Exports all stellar network history.", 18 Long: `Exports historical stellar network data between provided start-ledger/end-ledger to output files. 19 This is a temporary command used to reduce the amount of requests to history archives 20 in order to mitigate egress costs for the entity hosting history archives.`, 21 Run: func(cmd *cobra.Command, args []string) { 22 cmdLogger.SetLevel(logrus.InfoLevel) 23 endNum, strictExport, isTest, isFuture, extra := utils.MustCommonFlags(cmd.Flags(), cmdLogger) 24 cmdLogger.StrictExport = strictExport 25 startNum, path, limit := utils.MustArchiveFlags(cmd.Flags(), cmdLogger) 26 cloudStorageBucket, cloudCredentials, cloudProvider := utils.MustCloudStorageFlags(cmd.Flags(), cmdLogger) 27 env := utils.GetEnvironmentDetails(isTest, isFuture) 28 29 allHistory, err := input.GetAllHistory(startNum, endNum, limit, env) 30 if err != nil { 31 cmdLogger.Fatal("could not read all history: ", err) 32 } 33 34 cmdLogger.Info("start doing other exports") 35 getOperations(allHistory.Operations, extra, cloudStorageBucket, cloudCredentials, cloudProvider, path+"exported_operations.txt", env) 36 getTrades(allHistory.Trades, extra, cloudStorageBucket, cloudCredentials, cloudProvider, path+"exported_trades.txt") 37 getEffects(allHistory.Ledgers, extra, cloudStorageBucket, cloudCredentials, cloudProvider, path+"exported_effects.txt", env) 38 getTransactions(allHistory.Ledgers, extra, cloudStorageBucket, cloudCredentials, cloudProvider, path+"exported_transactions.txt") 39 getDiagnosticEvents(allHistory.Ledgers, extra, cloudStorageBucket, cloudCredentials, cloudProvider, path+"exported_diagnostic_events.txt") 40 cmdLogger.Info("done doing other exports") 41 }, 42 } 43 44 func getOperations(operations []input.OperationTransformInput, extra map[string]string, cloudStorageBucket string, cloudCredentials string, cloudProvider string, path string, env utils.EnvironmentDetails) { 45 outFileOperations := mustOutFile(path) 46 numFailures := 0 47 totalNumBytes := 0 48 for _, transformInput := range operations { 49 transformed, err := transform.TransformOperation(transformInput.Operation, transformInput.OperationIndex, transformInput.Transaction, transformInput.LedgerSeqNum, transformInput.LedgerCloseMeta, env.NetworkPassphrase) 50 if err != nil { 51 txIndex := transformInput.Transaction.Index 52 cmdLogger.LogError(fmt.Errorf("could not transform operation %d in transaction %d in ledger %d: %v", transformInput.OperationIndex, txIndex, transformInput.LedgerSeqNum, err)) 53 numFailures += 1 54 continue 55 } 56 57 numBytes, err := exportEntry(transformed, outFileOperations, extra) 58 if err != nil { 59 cmdLogger.LogError(fmt.Errorf("could not export operation: %v", err)) 60 numFailures += 1 61 continue 62 } 63 totalNumBytes += numBytes 64 } 65 66 outFileOperations.Close() 67 cmdLogger.Info("Number of bytes written: ", totalNumBytes) 68 69 printTransformStats(len(operations), numFailures) 70 71 maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path) 72 } 73 74 func getTrades(trades []input.TradeTransformInput, extra map[string]string, cloudStorageBucket string, cloudCredentials string, cloudProvider string, path string) { 75 outFile := mustOutFile(path) 76 numFailures := 0 77 totalNumBytes := 0 78 for _, tradeInput := range trades { 79 trades, err := transform.TransformTrade(tradeInput.OperationIndex, tradeInput.OperationHistoryID, tradeInput.Transaction, tradeInput.CloseTime) 80 if err != nil { 81 parsedID := toid.Parse(tradeInput.OperationHistoryID) 82 cmdLogger.LogError(fmt.Errorf("from ledger %d, transaction %d, operation %d: %v", parsedID.LedgerSequence, parsedID.TransactionOrder, parsedID.OperationOrder, err)) 83 numFailures += 1 84 continue 85 } 86 87 for _, transformed := range trades { 88 numBytes, err := exportEntry(transformed, outFile, extra) 89 if err != nil { 90 cmdLogger.LogError(err) 91 numFailures += 1 92 continue 93 } 94 totalNumBytes += numBytes 95 } 96 } 97 98 outFile.Close() 99 cmdLogger.Info("Number of bytes written: ", totalNumBytes) 100 101 printTransformStats(len(trades), numFailures) 102 103 maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path) 104 } 105 106 func getEffects(transactions []input.LedgerTransformInput, extra map[string]string, cloudStorageBucket string, cloudCredentials string, cloudProvider string, path string, env utils.EnvironmentDetails) { 107 outFile := mustOutFile(path) 108 numFailures := 0 109 totalNumBytes := 0 110 for _, transformInput := range transactions { 111 LedgerSeq := uint32(transformInput.LedgerHistory.Header.LedgerSeq) 112 effects, err := transform.TransformEffect(transformInput.Transaction, LedgerSeq, transformInput.LedgerCloseMeta, env.NetworkPassphrase) 113 if err != nil { 114 txIndex := transformInput.Transaction.Index 115 cmdLogger.Errorf("could not transform transaction %d in ledger %d: %v", txIndex, LedgerSeq, err) 116 numFailures += 1 117 continue 118 } 119 120 for _, transformed := range effects { 121 numBytes, err := exportEntry(transformed, outFile, extra) 122 if err != nil { 123 cmdLogger.LogError(err) 124 numFailures += 1 125 continue 126 } 127 totalNumBytes += numBytes 128 } 129 } 130 131 outFile.Close() 132 cmdLogger.Info("Number of bytes written: ", totalNumBytes) 133 134 printTransformStats(len(transactions), numFailures) 135 136 maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path) 137 } 138 139 func getTransactions(transactions []input.LedgerTransformInput, extra map[string]string, cloudStorageBucket string, cloudCredentials string, cloudProvider string, path string) { 140 outFile := mustOutFile(path) 141 numFailures := 0 142 totalNumBytes := 0 143 for _, transformInput := range transactions { 144 transformed, err := transform.TransformTransaction(transformInput.Transaction, transformInput.LedgerHistory) 145 if err != nil { 146 ledgerSeq := transformInput.LedgerHistory.Header.LedgerSeq 147 cmdLogger.LogError(fmt.Errorf("could not transform transaction %d in ledger %d: ", transformInput.Transaction.Index, ledgerSeq)) 148 numFailures += 1 149 continue 150 } 151 152 numBytes, err := exportEntry(transformed, outFile, extra) 153 if err != nil { 154 cmdLogger.LogError(fmt.Errorf("could not export transaction: %v", err)) 155 numFailures += 1 156 continue 157 } 158 totalNumBytes += numBytes 159 } 160 161 outFile.Close() 162 cmdLogger.Info("Number of bytes written: ", totalNumBytes) 163 164 printTransformStats(len(transactions), numFailures) 165 166 maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path) 167 } 168 169 func getDiagnosticEvents(transactions []input.LedgerTransformInput, extra map[string]string, cloudStorageBucket string, cloudCredentials string, cloudProvider string, path string) { 170 outFile := mustOutFile(path) 171 numFailures := 0 172 for _, transformInput := range transactions { 173 transformed, err, ok := transform.TransformDiagnosticEvent(transformInput.Transaction, transformInput.LedgerHistory) 174 if err != nil { 175 ledgerSeq := transformInput.LedgerHistory.Header.LedgerSeq 176 cmdLogger.LogError(fmt.Errorf("could not transform diagnostic events in transaction %d in ledger %d: ", transformInput.Transaction.Index, ledgerSeq)) 177 numFailures += 1 178 continue 179 } 180 181 if !ok { 182 continue 183 } 184 for _, diagnosticEvent := range transformed { 185 _, err := exportEntry(diagnosticEvent, outFile, extra) 186 if err != nil { 187 cmdLogger.LogError(fmt.Errorf("could not export diagnostic event: %v", err)) 188 numFailures += 1 189 continue 190 } 191 } 192 } 193 194 outFile.Close() 195 196 printTransformStats(len(transactions), numFailures) 197 198 maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path) 199 } 200 201 func init() { 202 rootCmd.AddCommand(allHistoryCmd) 203 utils.AddCommonFlags(allHistoryCmd.Flags()) 204 utils.AddArchiveFlags("", allHistoryCmd.Flags()) 205 utils.AddCloudStorageFlags(allHistoryCmd.Flags()) 206 allHistoryCmd.MarkFlagRequired("end-ledger") 207 }