github.com/stellar/stellar-etl@v1.0.1-0.20240312145900-4874b6bf2b89/cmd/export_ledgers.go (about)

     1  package cmd
     2  
     3  import (
     4  	"fmt"
     5  
     6  	"github.com/sirupsen/logrus"
     7  	"github.com/spf13/cobra"
     8  	"github.com/stellar/stellar-etl/internal/input"
     9  	"github.com/stellar/stellar-etl/internal/transform"
    10  	"github.com/stellar/stellar-etl/internal/utils"
    11  )
    12  
    13  var ledgersCmd = &cobra.Command{
    14  	Use:   "export_ledgers",
    15  	Short: "Exports the ledger data.",
    16  	Long:  `Exports ledger data within the specified range to an output file. Encodes ledgers as JSON objects and exports them to the output file.`,
    17  	Run: func(cmd *cobra.Command, args []string) {
    18  		cmdLogger.SetLevel(logrus.InfoLevel)
    19  		endNum, strictExport, isTest, isFuture, extra := utils.MustCommonFlags(cmd.Flags(), cmdLogger)
    20  		cmdLogger.StrictExport = strictExport
    21  		startNum, path, limit := utils.MustArchiveFlags(cmd.Flags(), cmdLogger)
    22  		cloudStorageBucket, cloudCredentials, cloudProvider := utils.MustCloudStorageFlags(cmd.Flags(), cmdLogger)
    23  
    24  		ledgers, err := input.GetLedgers(startNum, endNum, limit, isTest, isFuture)
    25  		if err != nil {
    26  			cmdLogger.Fatal("could not read ledgers: ", err)
    27  		}
    28  
    29  		outFile := mustOutFile(path)
    30  
    31  		numFailures := 0
    32  		totalNumBytes := 0
    33  		for i, lcm := range ledgers {
    34  			transformed, err := transform.TransformLedger(lcm)
    35  			if err != nil {
    36  				cmdLogger.LogError(fmt.Errorf("could not json transform ledger %d: %s", startNum+uint32(i), err))
    37  				numFailures += 1
    38  				continue
    39  			}
    40  
    41  			numBytes, err := exportEntry(transformed, outFile, extra)
    42  			if err != nil {
    43  				cmdLogger.LogError(fmt.Errorf("could not export ledger %d: %s", startNum+uint32(i), err))
    44  				numFailures += 1
    45  				continue
    46  			}
    47  			totalNumBytes += numBytes
    48  		}
    49  
    50  		outFile.Close()
    51  		cmdLogger.Info("Number of bytes written: ", totalNumBytes)
    52  
    53  		printTransformStats(len(ledgers), numFailures)
    54  
    55  		maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path)
    56  	},
    57  }
    58  
    59  func init() {
    60  	rootCmd.AddCommand(ledgersCmd)
    61  	utils.AddCommonFlags(ledgersCmd.Flags())
    62  	utils.AddArchiveFlags("ledgers", ledgersCmd.Flags())
    63  	utils.AddCloudStorageFlags(ledgersCmd.Flags())
    64  	ledgersCmd.MarkFlagRequired("end-ledger")
    65  	/*
    66  		Current flags:
    67  			start-ledger: the ledger sequence number for the beginning of the export period
    68  			end-ledger: the ledger sequence number for the end of the export range (required)
    69  
    70  			limit: maximum number of ledgers to export; default to 60 (1 ledger per 5 seconds over our 5 minute update period)
    71  			output-file: filename of the output file
    72  
    73  		TODO: implement extra flags if possible
    74  			serialize-method: the method for serialization of the output data (JSON, XDR, etc)
    75  			start and end time as a replacement for start and end sequence numbers
    76  	*/
    77  }