github.com/stellar/stellar-etl@v1.0.1-0.20240312145900-4874b6bf2b89/internal/transform/ledger.go (about)

     1  package transform
     2  
     3  import (
     4  	"fmt"
     5  	"strconv"
     6  
     7  	"github.com/stellar/stellar-etl/internal/toid"
     8  	"github.com/stellar/stellar-etl/internal/utils"
     9  
    10  	"github.com/stellar/go/historyarchive"
    11  	"github.com/stellar/go/xdr"
    12  )
    13  
    14  // TransformLedger converts a ledger from the history archive ingestion system into a form suitable for BigQuery
    15  func TransformLedger(inputLedger historyarchive.Ledger) (LedgerOutput, error) {
    16  	ledgerHeader := inputLedger.Header.Header
    17  
    18  	outputSequence := uint32(ledgerHeader.LedgerSeq)
    19  
    20  	outputLedgerID := toid.New(int32(outputSequence), 0, 0).ToInt64()
    21  
    22  	outputLedgerHash := utils.HashToHexString(inputLedger.Header.Hash)
    23  	outputPreviousHash := utils.HashToHexString(ledgerHeader.PreviousLedgerHash)
    24  
    25  	outputLedgerHeader, err := xdr.MarshalBase64(ledgerHeader)
    26  	if err != nil {
    27  		return LedgerOutput{}, fmt.Errorf("for ledger %d (ledger id=%d): %v", outputSequence, outputLedgerID, err)
    28  	}
    29  
    30  	outputTransactionCount, outputOperationCount, outputSuccessfulCount, outputFailedCount, outputTxSetOperationCount, err := extractCounts(inputLedger)
    31  	if err != nil {
    32  		return LedgerOutput{}, fmt.Errorf("for ledger %d (ledger id=%d): %v", outputSequence, outputLedgerID, err)
    33  	}
    34  
    35  	outputCloseTime, err := utils.TimePointToUTCTimeStamp(ledgerHeader.ScpValue.CloseTime)
    36  	if err != nil {
    37  		return LedgerOutput{}, err
    38  	}
    39  
    40  	outputTotalCoins := int64(ledgerHeader.TotalCoins)
    41  	if outputTotalCoins < 0 {
    42  		return LedgerOutput{}, fmt.Errorf("the total number of coins (%d) is negative for ledger %d (ledger id=%d)", outputTotalCoins, outputSequence, outputLedgerID)
    43  	}
    44  
    45  	outputFeePool := int64(ledgerHeader.FeePool)
    46  	if outputFeePool < 0 {
    47  		return LedgerOutput{}, fmt.Errorf("the fee pool (%d) is negative for ledger %d (ledger id=%d)", outputFeePool, outputSequence, outputLedgerID)
    48  	}
    49  
    50  	outputBaseFee := uint32(ledgerHeader.BaseFee)
    51  
    52  	outputBaseReserve := uint32(ledgerHeader.BaseReserve)
    53  
    54  	outputMaxTxSetSize := uint32(ledgerHeader.MaxTxSetSize)
    55  
    56  	outputProtocolVersion := uint32(ledgerHeader.LedgerVersion)
    57  
    58  	transformedLedger := LedgerOutput{
    59  		Sequence:                   outputSequence,
    60  		LedgerID:                   outputLedgerID,
    61  		LedgerHash:                 outputLedgerHash,
    62  		PreviousLedgerHash:         outputPreviousHash,
    63  		LedgerHeader:               outputLedgerHeader,
    64  		TransactionCount:           outputTransactionCount,
    65  		OperationCount:             outputOperationCount,
    66  		SuccessfulTransactionCount: outputSuccessfulCount,
    67  		FailedTransactionCount:     outputFailedCount,
    68  		TxSetOperationCount:        outputTxSetOperationCount,
    69  		ClosedAt:                   outputCloseTime,
    70  		TotalCoins:                 outputTotalCoins,
    71  		FeePool:                    outputFeePool,
    72  		BaseFee:                    outputBaseFee,
    73  		BaseReserve:                outputBaseReserve,
    74  		MaxTxSetSize:               outputMaxTxSetSize,
    75  		ProtocolVersion:            outputProtocolVersion,
    76  	}
    77  	return transformedLedger, nil
    78  }
    79  
    80  func TransactionProcessing(l xdr.LedgerCloseMeta) []xdr.TransactionResultMeta {
    81  	switch l.V {
    82  	case 0:
    83  		return l.MustV0().TxProcessing
    84  	case 1:
    85  		return l.MustV1().TxProcessing
    86  	default:
    87  		panic(fmt.Sprintf("Unsupported LedgerCloseMeta.V: %d", l.V))
    88  	}
    89  }
    90  
    91  func extractCounts(ledger historyarchive.Ledger) (transactionCount int32, operationCount int32, successTxCount int32, failedTxCount int32, txSetOperationCount string, err error) {
    92  	transactions := GetTransactionSet(ledger)
    93  	results := ledger.TransactionResult.TxResultSet.Results
    94  	txCount := len(transactions)
    95  	if txCount != len(results) {
    96  		err = fmt.Errorf("The number of transactions and results are different (%d != %d)", txCount, len(results))
    97  		return
    98  	}
    99  
   100  	txSetOperationCounter := int32(0)
   101  	for i := 0; i < txCount; i++ {
   102  		operations := transactions[i].Operations()
   103  		numberOfOps := int32(len(operations))
   104  		txSetOperationCounter += numberOfOps
   105  
   106  		// for successful transactions, the operation count is based on the operations results slice
   107  		if results[i].Result.Successful() {
   108  			operationResults, ok := results[i].Result.OperationResults()
   109  			if !ok {
   110  				err = fmt.Errorf("Could not access operation results for result %d", i)
   111  				return
   112  			}
   113  
   114  			successTxCount++
   115  			operationCount += int32(len(operationResults))
   116  		} else {
   117  			failedTxCount++
   118  		}
   119  
   120  	}
   121  	transactionCount = int32(txCount) - failedTxCount
   122  	txSetOperationCount = strconv.FormatInt(int64(txSetOperationCounter), 10)
   123  	return
   124  }
   125  
   126  func GetTransactionSet(transactionEntry historyarchive.Ledger) (transactionProcessing []xdr.TransactionEnvelope) {
   127  	switch transactionEntry.Transaction.Ext.V {
   128  	case 0:
   129  		return transactionEntry.Transaction.TxSet.Txs
   130  	case 1:
   131  		return getTransactionPhase(transactionEntry.Transaction.Ext.GeneralizedTxSet.V1TxSet.Phases)
   132  	default:
   133  		panic(fmt.Sprintf("Unsupported TransactionHistoryEntry.Ext: %d", transactionEntry.Transaction.Ext.V))
   134  	}
   135  }
   136  
   137  func getTransactionPhase(transactionPhase []xdr.TransactionPhase) (transactionEnvelope []xdr.TransactionEnvelope) {
   138  	transactionSlice := []xdr.TransactionEnvelope{}
   139  	for _, phase := range transactionPhase {
   140  		switch phase.V {
   141  		case 0:
   142  			components := phase.MustV0Components()
   143  			for _, component := range components {
   144  				switch component.Type {
   145  				case 0:
   146  					transactionSlice = append(transactionSlice, component.TxsMaybeDiscountedFee.Txs...)
   147  
   148  				default:
   149  					panic(fmt.Sprintf("Unsupported TxSetComponentType: %d", component.Type))
   150  				}
   151  
   152  			}
   153  		default:
   154  			panic(fmt.Sprintf("Unsupported TransactionPhase.V: %d", phase.V))
   155  		}
   156  	}
   157  	return transactionSlice
   158  
   159  }