github.com/ari-anchor/sei-tendermint@v0.0.0-20230519144642-dc826b7b56bb/test/e2e/node/main.go (about) 1 package main 2 3 import ( 4 "context" 5 "errors" 6 "fmt" 7 "net" 8 "net/http" 9 "os" 10 "path/filepath" 11 "strings" 12 "time" 13 14 "github.com/spf13/viper" 15 "go.opentelemetry.io/otel/sdk/trace" 16 "google.golang.org/grpc" 17 18 abciclient "github.com/ari-anchor/sei-tendermint/abci/client" 19 "github.com/ari-anchor/sei-tendermint/abci/server" 20 "github.com/ari-anchor/sei-tendermint/config" 21 "github.com/ari-anchor/sei-tendermint/crypto/ed25519" 22 "github.com/ari-anchor/sei-tendermint/internal/p2p" 23 "github.com/ari-anchor/sei-tendermint/libs/log" 24 tmnet "github.com/ari-anchor/sei-tendermint/libs/net" 25 "github.com/ari-anchor/sei-tendermint/light" 26 lproxy "github.com/ari-anchor/sei-tendermint/light/proxy" 27 lrpc "github.com/ari-anchor/sei-tendermint/light/rpc" 28 dbs "github.com/ari-anchor/sei-tendermint/light/store/db" 29 "github.com/ari-anchor/sei-tendermint/node" 30 "github.com/ari-anchor/sei-tendermint/privval" 31 grpcprivval "github.com/ari-anchor/sei-tendermint/privval/grpc" 32 privvalproto "github.com/ari-anchor/sei-tendermint/proto/tendermint/privval" 33 rpcserver "github.com/ari-anchor/sei-tendermint/rpc/jsonrpc/server" 34 "github.com/ari-anchor/sei-tendermint/test/e2e/app" 35 e2e "github.com/ari-anchor/sei-tendermint/test/e2e/pkg" 36 ) 37 38 // main is the binary entrypoint. 39 func main() { 40 ctx, cancel := context.WithCancel(context.Background()) 41 defer cancel() 42 43 if len(os.Args) != 2 { 44 fmt.Printf("Usage: %v <configfile>", os.Args[0]) 45 return 46 } 47 configFile := "" 48 if len(os.Args) == 2 { 49 configFile = os.Args[1] 50 } 51 52 if err := run(ctx, configFile); err != nil { 53 os.Exit(1) 54 } 55 } 56 57 // run runs the application - basically like main() with error handling. 58 func run(ctx context.Context, configFile string) error { 59 cfg, err := LoadConfig(configFile) 60 if err != nil { 61 return err 62 } 63 64 logger, err := log.NewDefaultLogger(log.LogFormatPlain, log.LogLevelInfo) 65 if err != nil { 66 // have print here because we can't log (yet), use the logger 67 // everywhere else. 68 fmt.Fprintln(os.Stderr, "ERROR:", err) 69 return err 70 } 71 72 if cfg.Mode == string(e2e.ModeLight) { 73 err = startLightNode(ctx, logger, cfg) 74 } else { 75 // Start remote signer (must start before node if running builtin). 76 if cfg.PrivValServer != "" { 77 if err = startSigner(ctx, logger, cfg); err != nil { 78 logger.Error("starting signer", 79 "server", cfg.PrivValServer, 80 "err", err) 81 return err 82 } 83 if cfg.Protocol == "builtin" { 84 time.Sleep(1 * time.Second) 85 } 86 } 87 88 // Start app server. 89 switch cfg.Protocol { 90 case "socket", "grpc": 91 err = startApp(ctx, logger, cfg) 92 case "builtin": 93 if cfg.Mode == string(e2e.ModeSeed) { 94 err = startSeedNode(ctx) 95 } else { 96 err = startNode(ctx, cfg) 97 } 98 default: 99 err = fmt.Errorf("invalid protocol %q", cfg.Protocol) 100 } 101 } 102 103 if err != nil { 104 logger.Error("starting node", 105 "protocol", cfg.Protocol, 106 "mode", cfg.Mode, 107 "err", err) 108 return err 109 } 110 111 // Apparently there's no way to wait for the server, so we just sleep 112 for { 113 time.Sleep(1 * time.Hour) 114 } 115 } 116 117 // startApp starts the application server, listening for connections from Tendermint. 118 func startApp(ctx context.Context, logger log.Logger, cfg *Config) error { 119 app, err := app.NewApplication(cfg.App()) 120 if err != nil { 121 return err 122 } 123 server, err := server.NewServer(logger, cfg.Listen, cfg.Protocol, app) 124 if err != nil { 125 return err 126 } 127 err = server.Start(ctx) 128 if err != nil { 129 return err 130 } 131 logger.Info(fmt.Sprintf("Server listening on %v (%v protocol)", cfg.Listen, cfg.Protocol)) 132 return nil 133 } 134 135 // startNode starts a Tendermint node running the application directly. It assumes the Tendermint 136 // configuration is in $TMHOME/config/tendermint.toml. 137 // 138 // FIXME There is no way to simply load the configuration from a file, so we need to pull in Viper. 139 func startNode(ctx context.Context, cfg *Config) error { 140 app, err := app.NewApplication(cfg.App()) 141 if err != nil { 142 return err 143 } 144 145 tmcfg, nodeLogger, err := setupNode() 146 if err != nil { 147 return fmt.Errorf("failed to setup config: %w", err) 148 } 149 150 n, err := node.New( 151 ctx, 152 tmcfg, 153 nodeLogger, 154 make(chan struct{}), 155 abciclient.NewLocalClient(nodeLogger, app), 156 nil, 157 []trace.TracerProviderOption{}, 158 ) 159 if err != nil { 160 return err 161 } 162 return n.Start(ctx) 163 } 164 165 func startSeedNode(ctx context.Context) error { 166 tmcfg, nodeLogger, err := setupNode() 167 if err != nil { 168 return fmt.Errorf("failed to setup config: %w", err) 169 } 170 171 tmcfg.Mode = config.ModeSeed 172 173 n, err := node.New(ctx, tmcfg, nodeLogger, make(chan struct{}), nil, nil, []trace.TracerProviderOption{}) 174 if err != nil { 175 return err 176 } 177 return n.Start(ctx) 178 } 179 180 func startLightNode(ctx context.Context, logger log.Logger, cfg *Config) error { 181 tmcfg, nodeLogger, err := setupNode() 182 if err != nil { 183 return err 184 } 185 186 dbContext := &config.DBContext{ID: "light", Config: tmcfg} 187 lightDB, err := config.DefaultDBProvider(dbContext) 188 if err != nil { 189 return err 190 } 191 192 providers := rpcEndpoints(tmcfg.P2P.PersistentPeers) 193 194 c, err := light.NewHTTPClient( 195 ctx, 196 cfg.ChainID, 197 light.TrustOptions{ 198 Period: tmcfg.StateSync.TrustPeriod, 199 Height: tmcfg.StateSync.TrustHeight, 200 Hash: tmcfg.StateSync.TrustHashBytes(), 201 }, 202 providers[0], 203 providers[1:], 204 dbs.New(lightDB), 205 light.Logger(nodeLogger), 206 ) 207 if err != nil { 208 return err 209 } 210 211 rpccfg := rpcserver.DefaultConfig() 212 rpccfg.MaxBodyBytes = tmcfg.RPC.MaxBodyBytes 213 rpccfg.MaxHeaderBytes = tmcfg.RPC.MaxHeaderBytes 214 rpccfg.MaxOpenConnections = tmcfg.RPC.MaxOpenConnections 215 // If necessary adjust global WriteTimeout to ensure it's greater than 216 // TimeoutBroadcastTxCommit. 217 // See https://github.com/ari-anchor/sei-tendermint/issues/3435 218 // Note we don't need to adjust anything if the timeout is already unlimited. 219 if rpccfg.WriteTimeout > 0 && rpccfg.WriteTimeout <= tmcfg.RPC.TimeoutBroadcastTxCommit { 220 rpccfg.WriteTimeout = tmcfg.RPC.TimeoutBroadcastTxCommit + 1*time.Second 221 } 222 223 p, err := lproxy.NewProxy(c, tmcfg.RPC.ListenAddress, providers[0], rpccfg, nodeLogger, 224 lrpc.KeyPathFn(lrpc.DefaultMerkleKeyPathFn())) 225 if err != nil { 226 return err 227 } 228 229 logger.Info("Starting proxy...", "laddr", tmcfg.RPC.ListenAddress) 230 if err := p.ListenAndServe(ctx); err != http.ErrServerClosed { 231 // Error starting or closing listener: 232 logger.Error("proxy ListenAndServe", "err", err) 233 } 234 235 return nil 236 } 237 238 // startSigner starts a signer server connecting to the given endpoint. 239 func startSigner(ctx context.Context, logger log.Logger, cfg *Config) error { 240 filePV, err := privval.LoadFilePV(cfg.PrivValKey, cfg.PrivValState) 241 if err != nil { 242 return err 243 } 244 245 protocol, address := tmnet.ProtocolAndAddress(cfg.PrivValServer) 246 var dialFn privval.SocketDialer 247 switch protocol { 248 case "tcp": 249 dialFn = privval.DialTCPFn(address, 3*time.Second, ed25519.GenPrivKey()) 250 case "unix": 251 dialFn = privval.DialUnixFn(address) 252 case "grpc": 253 lis, err := net.Listen("tcp", address) 254 if err != nil { 255 return err 256 } 257 ss := grpcprivval.NewSignerServer(logger, cfg.ChainID, filePV) 258 259 s := grpc.NewServer() 260 261 privvalproto.RegisterPrivValidatorAPIServer(s, ss) 262 263 go func() { // no need to clean up since we remove docker containers 264 if err := s.Serve(lis); err != nil { 265 panic(err) 266 } 267 go func() { 268 <-ctx.Done() 269 s.GracefulStop() 270 }() 271 }() 272 273 return nil 274 default: 275 return fmt.Errorf("invalid privval protocol %q", protocol) 276 } 277 278 endpoint := privval.NewSignerDialerEndpoint(logger, dialFn, 279 privval.SignerDialerEndpointRetryWaitInterval(1*time.Second), 280 privval.SignerDialerEndpointConnRetries(100)) 281 282 err = privval.NewSignerServer(endpoint, cfg.ChainID, filePV).Start(ctx) 283 if err != nil { 284 return err 285 } 286 287 logger.Info(fmt.Sprintf("Remote signer connecting to %v", cfg.PrivValServer)) 288 return nil 289 } 290 291 func setupNode() (*config.Config, log.Logger, error) { 292 var tmcfg *config.Config 293 294 home := os.Getenv("TMHOME") 295 if home == "" { 296 return nil, nil, errors.New("TMHOME not set") 297 } 298 299 viper.AddConfigPath(filepath.Join(home, "config")) 300 viper.SetConfigName("config") 301 302 if err := viper.ReadInConfig(); err != nil { 303 return nil, nil, err 304 } 305 306 tmcfg = config.DefaultConfig() 307 308 if err := viper.Unmarshal(tmcfg); err != nil { 309 return nil, nil, err 310 } 311 312 tmcfg.SetRoot(home) 313 314 if err := tmcfg.ValidateBasic(); err != nil { 315 return nil, nil, fmt.Errorf("error in config file: %w", err) 316 } 317 318 nodeLogger, err := log.NewDefaultLogger(tmcfg.LogFormat, tmcfg.LogLevel) 319 if err != nil { 320 return nil, nil, err 321 } 322 323 return tmcfg, nodeLogger.With("module", "main"), nil 324 } 325 326 // rpcEndpoints takes a list of persistent peers and splits them into a list of rpc endpoints 327 // using 26657 as the port number 328 func rpcEndpoints(peers string) []string { 329 arr := strings.Split(peers, ",") 330 endpoints := make([]string, len(arr)) 331 for i, v := range arr { 332 addr, err := p2p.ParseNodeAddress(v) 333 if err != nil { 334 panic(err) 335 } 336 // use RPC port instead 337 addr.Port = 26657 338 var rpcEndpoint string 339 // for ipv6 addresses 340 if strings.Contains(addr.Hostname, ":") { 341 rpcEndpoint = "http://[" + addr.Hostname + "]:" + fmt.Sprint(addr.Port) 342 } else { // for ipv4 addresses 343 rpcEndpoint = "http://" + addr.Hostname + ":" + fmt.Sprint(addr.Port) 344 } 345 endpoints[i] = rpcEndpoint 346 } 347 return endpoints 348 }