github.com/zhiqiangxu/go-ethereum@v1.9.16-0.20210824055606-be91cfdebc48/p2p/simulations/adapters/exec.go (about) 1 // Copyright 2017 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package adapters 18 19 import ( 20 "bytes" 21 "context" 22 "encoding/json" 23 "errors" 24 "fmt" 25 "io" 26 "net" 27 "net/http" 28 "os" 29 "os/exec" 30 "os/signal" 31 "path/filepath" 32 "strings" 33 "sync" 34 "syscall" 35 "time" 36 37 "github.com/docker/docker/pkg/reexec" 38 "github.com/gorilla/websocket" 39 "github.com/zhiqiangxu/go-ethereum/log" 40 "github.com/zhiqiangxu/go-ethereum/node" 41 "github.com/zhiqiangxu/go-ethereum/p2p" 42 "github.com/zhiqiangxu/go-ethereum/p2p/enode" 43 "github.com/zhiqiangxu/go-ethereum/rpc" 44 ) 45 46 func init() { 47 // Register a reexec function to start a simulation node when the current binary is 48 // executed as "p2p-node" (rather than whatever the main() function would normally do). 49 reexec.Register("p2p-node", execP2PNode) 50 } 51 52 // ExecAdapter is a NodeAdapter which runs simulation nodes by executing the current binary 53 // as a child process. 54 type ExecAdapter struct { 55 // BaseDir is the directory under which the data directories for each 56 // simulation node are created. 57 BaseDir string 58 59 nodes map[enode.ID]*ExecNode 60 } 61 62 // NewExecAdapter returns an ExecAdapter which stores node data in 63 // subdirectories of the given base directory 64 func NewExecAdapter(baseDir string) *ExecAdapter { 65 return &ExecAdapter{ 66 BaseDir: baseDir, 67 nodes: make(map[enode.ID]*ExecNode), 68 } 69 } 70 71 // Name returns the name of the adapter for logging purposes 72 func (e *ExecAdapter) Name() string { 73 return "exec-adapter" 74 } 75 76 // NewNode returns a new ExecNode using the given config 77 func (e *ExecAdapter) NewNode(config *NodeConfig) (Node, error) { 78 if len(config.Services) == 0 { 79 return nil, errors.New("node must have at least one service") 80 } 81 for _, service := range config.Services { 82 if _, exists := serviceFuncs[service]; !exists { 83 return nil, fmt.Errorf("unknown node service %q", service) 84 } 85 } 86 87 // create the node directory using the first 12 characters of the ID 88 // as Unix socket paths cannot be longer than 256 characters 89 dir := filepath.Join(e.BaseDir, config.ID.String()[:12]) 90 if err := os.Mkdir(dir, 0755); err != nil { 91 return nil, fmt.Errorf("error creating node directory: %s", err) 92 } 93 94 err := config.initDummyEnode() 95 if err != nil { 96 return nil, err 97 } 98 99 // generate the config 100 conf := &execNodeConfig{ 101 Stack: node.DefaultConfig, 102 Node: config, 103 } 104 if config.DataDir != "" { 105 conf.Stack.DataDir = config.DataDir 106 } else { 107 conf.Stack.DataDir = filepath.Join(dir, "data") 108 } 109 110 // these parameters are crucial for execadapter node to run correctly 111 conf.Stack.WSHost = "127.0.0.1" 112 conf.Stack.WSPort = 0 113 conf.Stack.WSOrigins = []string{"*"} 114 conf.Stack.WSExposeAll = true 115 conf.Stack.P2P.EnableMsgEvents = config.EnableMsgEvents 116 conf.Stack.P2P.NoDiscovery = true 117 conf.Stack.P2P.NAT = nil 118 conf.Stack.NoUSB = true 119 120 // Listen on a localhost port, which we set when we 121 // initialise NodeConfig (usually a random port) 122 conf.Stack.P2P.ListenAddr = fmt.Sprintf(":%d", config.Port) 123 124 node := &ExecNode{ 125 ID: config.ID, 126 Dir: dir, 127 Config: conf, 128 adapter: e, 129 } 130 node.newCmd = node.execCommand 131 e.nodes[node.ID] = node 132 return node, nil 133 } 134 135 // ExecNode starts a simulation node by exec'ing the current binary and 136 // running the configured services 137 type ExecNode struct { 138 ID enode.ID 139 Dir string 140 Config *execNodeConfig 141 Cmd *exec.Cmd 142 Info *p2p.NodeInfo 143 144 adapter *ExecAdapter 145 client *rpc.Client 146 wsAddr string 147 newCmd func() *exec.Cmd 148 } 149 150 // Addr returns the node's enode URL 151 func (n *ExecNode) Addr() []byte { 152 if n.Info == nil { 153 return nil 154 } 155 return []byte(n.Info.Enode) 156 } 157 158 // Client returns an rpc.Client which can be used to communicate with the 159 // underlying services (it is set once the node has started) 160 func (n *ExecNode) Client() (*rpc.Client, error) { 161 return n.client, nil 162 } 163 164 // Start exec's the node passing the ID and service as command line arguments 165 // and the node config encoded as JSON in an environment variable. 166 func (n *ExecNode) Start(snapshots map[string][]byte) (err error) { 167 if n.Cmd != nil { 168 return errors.New("already started") 169 } 170 defer func() { 171 if err != nil { 172 n.Stop() 173 } 174 }() 175 176 // encode a copy of the config containing the snapshot 177 confCopy := *n.Config 178 confCopy.Snapshots = snapshots 179 confCopy.PeerAddrs = make(map[string]string) 180 for id, node := range n.adapter.nodes { 181 confCopy.PeerAddrs[id.String()] = node.wsAddr 182 } 183 confData, err := json.Marshal(confCopy) 184 if err != nil { 185 return fmt.Errorf("error generating node config: %s", err) 186 } 187 188 // start the one-shot server that waits for startup information 189 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 190 defer cancel() 191 statusURL, statusC := n.waitForStartupJSON(ctx) 192 193 // start the node 194 cmd := n.newCmd() 195 cmd.Stdout = os.Stdout 196 cmd.Stderr = os.Stderr 197 cmd.Env = append(os.Environ(), 198 envStatusURL+"="+statusURL, 199 envNodeConfig+"="+string(confData), 200 ) 201 if err := cmd.Start(); err != nil { 202 return fmt.Errorf("error starting node: %s", err) 203 } 204 n.Cmd = cmd 205 206 // Wait for the node to start. 207 status := <-statusC 208 if status.Err != "" { 209 return errors.New(status.Err) 210 } 211 client, err := rpc.DialWebsocket(ctx, status.WSEndpoint, "") 212 if err != nil { 213 return fmt.Errorf("can't connect to RPC server: %v", err) 214 } 215 216 // Node ready :) 217 n.client = client 218 n.wsAddr = status.WSEndpoint 219 n.Info = status.NodeInfo 220 return nil 221 } 222 223 // waitForStartupJSON runs a one-shot HTTP server to receive a startup report. 224 func (n *ExecNode) waitForStartupJSON(ctx context.Context) (string, chan nodeStartupJSON) { 225 var ( 226 ch = make(chan nodeStartupJSON, 1) 227 quitOnce sync.Once 228 srv http.Server 229 ) 230 l, err := net.Listen("tcp", "127.0.0.1:0") 231 if err != nil { 232 ch <- nodeStartupJSON{Err: err.Error()} 233 return "", ch 234 } 235 quit := func(status nodeStartupJSON) { 236 quitOnce.Do(func() { 237 l.Close() 238 ch <- status 239 }) 240 } 241 srv.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 242 var status nodeStartupJSON 243 if err := json.NewDecoder(r.Body).Decode(&status); err != nil { 244 status.Err = fmt.Sprintf("can't decode startup report: %v", err) 245 } 246 quit(status) 247 }) 248 // Run the HTTP server, but don't wait forever and shut it down 249 // if the context is canceled. 250 go srv.Serve(l) 251 go func() { 252 <-ctx.Done() 253 quit(nodeStartupJSON{Err: "didn't get startup report"}) 254 }() 255 256 url := "http://" + l.Addr().String() 257 return url, ch 258 } 259 260 // execCommand returns a command which runs the node locally by exec'ing 261 // the current binary but setting argv[0] to "p2p-node" so that the child 262 // runs execP2PNode 263 func (n *ExecNode) execCommand() *exec.Cmd { 264 return &exec.Cmd{ 265 Path: reexec.Self(), 266 Args: []string{"p2p-node", strings.Join(n.Config.Node.Services, ","), n.ID.String()}, 267 } 268 } 269 270 // Stop stops the node by first sending SIGTERM and then SIGKILL if the node 271 // doesn't stop within 5s 272 func (n *ExecNode) Stop() error { 273 if n.Cmd == nil { 274 return nil 275 } 276 defer func() { 277 n.Cmd = nil 278 }() 279 280 if n.client != nil { 281 n.client.Close() 282 n.client = nil 283 n.wsAddr = "" 284 n.Info = nil 285 } 286 287 if err := n.Cmd.Process.Signal(syscall.SIGTERM); err != nil { 288 return n.Cmd.Process.Kill() 289 } 290 waitErr := make(chan error, 1) 291 go func() { 292 waitErr <- n.Cmd.Wait() 293 }() 294 select { 295 case err := <-waitErr: 296 return err 297 case <-time.After(5 * time.Second): 298 return n.Cmd.Process.Kill() 299 } 300 } 301 302 // NodeInfo returns information about the node 303 func (n *ExecNode) NodeInfo() *p2p.NodeInfo { 304 info := &p2p.NodeInfo{ 305 ID: n.ID.String(), 306 } 307 if n.client != nil { 308 n.client.Call(&info, "admin_nodeInfo") 309 } 310 return info 311 } 312 313 // ServeRPC serves RPC requests over the given connection by dialling the 314 // node's WebSocket address and joining the two connections 315 func (n *ExecNode) ServeRPC(clientConn *websocket.Conn) error { 316 conn, _, err := websocket.DefaultDialer.Dial(n.wsAddr, nil) 317 if err != nil { 318 return err 319 } 320 var wg sync.WaitGroup 321 wg.Add(2) 322 go wsCopy(&wg, conn, clientConn) 323 go wsCopy(&wg, clientConn, conn) 324 wg.Wait() 325 conn.Close() 326 return nil 327 } 328 329 func wsCopy(wg *sync.WaitGroup, src, dst *websocket.Conn) { 330 defer wg.Done() 331 for { 332 msgType, r, err := src.NextReader() 333 if err != nil { 334 return 335 } 336 w, err := dst.NextWriter(msgType) 337 if err != nil { 338 return 339 } 340 if _, err = io.Copy(w, r); err != nil { 341 return 342 } 343 } 344 } 345 346 // Snapshots creates snapshots of the services by calling the 347 // simulation_snapshot RPC method 348 func (n *ExecNode) Snapshots() (map[string][]byte, error) { 349 if n.client == nil { 350 return nil, errors.New("RPC not started") 351 } 352 var snapshots map[string][]byte 353 return snapshots, n.client.Call(&snapshots, "simulation_snapshot") 354 } 355 356 // execNodeConfig is used to serialize the node configuration so it can be 357 // passed to the child process as a JSON encoded environment variable 358 type execNodeConfig struct { 359 Stack node.Config `json:"stack"` 360 Node *NodeConfig `json:"node"` 361 Snapshots map[string][]byte `json:"snapshots,omitempty"` 362 PeerAddrs map[string]string `json:"peer_addrs,omitempty"` 363 } 364 365 // execP2PNode starts a simulation node when the current binary is executed with 366 // argv[0] being "p2p-node", reading the service / ID from argv[1] / argv[2] 367 // and the node config from an environment variable. 368 func execP2PNode() { 369 glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.LogfmtFormat())) 370 glogger.Verbosity(log.LvlInfo) 371 log.Root().SetHandler(glogger) 372 statusURL := os.Getenv(envStatusURL) 373 if statusURL == "" { 374 log.Crit("missing " + envStatusURL) 375 } 376 377 // Start the node and gather startup report. 378 var status nodeStartupJSON 379 stack, stackErr := startExecNodeStack() 380 if stackErr != nil { 381 status.Err = stackErr.Error() 382 } else { 383 status.WSEndpoint = "ws://" + stack.WSEndpoint() 384 status.NodeInfo = stack.Server().NodeInfo() 385 } 386 387 // Send status to the host. 388 statusJSON, _ := json.Marshal(status) 389 if _, err := http.Post(statusURL, "application/json", bytes.NewReader(statusJSON)); err != nil { 390 log.Crit("Can't post startup info", "url", statusURL, "err", err) 391 } 392 if stackErr != nil { 393 os.Exit(1) 394 } 395 396 // Stop the stack if we get a SIGTERM signal. 397 go func() { 398 sigc := make(chan os.Signal, 1) 399 signal.Notify(sigc, syscall.SIGTERM) 400 defer signal.Stop(sigc) 401 <-sigc 402 log.Info("Received SIGTERM, shutting down...") 403 stack.Stop() 404 }() 405 stack.Wait() // Wait for the stack to exit. 406 } 407 408 func startExecNodeStack() (*node.Node, error) { 409 // read the services from argv 410 serviceNames := strings.Split(os.Args[1], ",") 411 412 // decode the config 413 confEnv := os.Getenv(envNodeConfig) 414 if confEnv == "" { 415 return nil, fmt.Errorf("missing " + envNodeConfig) 416 } 417 var conf execNodeConfig 418 if err := json.Unmarshal([]byte(confEnv), &conf); err != nil { 419 return nil, fmt.Errorf("error decoding %s: %v", envNodeConfig, err) 420 } 421 422 // create enode record 423 nodeTcpConn, _ := net.ResolveTCPAddr("tcp", conf.Stack.P2P.ListenAddr) 424 if nodeTcpConn.IP == nil { 425 nodeTcpConn.IP = net.IPv4(127, 0, 0, 1) 426 } 427 conf.Node.initEnode(nodeTcpConn.IP, nodeTcpConn.Port, nodeTcpConn.Port) 428 conf.Stack.P2P.PrivateKey = conf.Node.PrivateKey 429 conf.Stack.Logger = log.New("node.id", conf.Node.ID.String()) 430 431 // initialize the devp2p stack 432 stack, err := node.New(&conf.Stack) 433 if err != nil { 434 return nil, fmt.Errorf("error creating node stack: %v", err) 435 } 436 437 // register the services, collecting them into a map so we can wrap 438 // them in a snapshot service 439 services := make(map[string]node.Service, len(serviceNames)) 440 for _, name := range serviceNames { 441 serviceFunc, exists := serviceFuncs[name] 442 if !exists { 443 return nil, fmt.Errorf("unknown node service %q", err) 444 } 445 constructor := func(nodeCtx *node.ServiceContext) (node.Service, error) { 446 ctx := &ServiceContext{ 447 RPCDialer: &wsRPCDialer{addrs: conf.PeerAddrs}, 448 NodeContext: nodeCtx, 449 Config: conf.Node, 450 } 451 if conf.Snapshots != nil { 452 ctx.Snapshot = conf.Snapshots[name] 453 } 454 service, err := serviceFunc(ctx) 455 if err != nil { 456 return nil, err 457 } 458 services[name] = service 459 return service, nil 460 } 461 if err := stack.Register(constructor); err != nil { 462 return stack, fmt.Errorf("error registering service %q: %v", name, err) 463 } 464 } 465 466 // register the snapshot service 467 err = stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { 468 return &snapshotService{services}, nil 469 }) 470 if err != nil { 471 return stack, fmt.Errorf("error starting snapshot service: %v", err) 472 } 473 474 // start the stack 475 if err = stack.Start(); err != nil { 476 err = fmt.Errorf("error starting stack: %v", err) 477 } 478 return stack, err 479 } 480 481 const ( 482 envStatusURL = "_P2P_STATUS_URL" 483 envNodeConfig = "_P2P_NODE_CONFIG" 484 ) 485 486 // nodeStartupJSON is sent to the simulation host after startup. 487 type nodeStartupJSON struct { 488 Err string 489 WSEndpoint string 490 NodeInfo *p2p.NodeInfo 491 } 492 493 // snapshotService is a node.Service which wraps a list of services and 494 // exposes an API to generate a snapshot of those services 495 type snapshotService struct { 496 services map[string]node.Service 497 } 498 499 func (s *snapshotService) APIs() []rpc.API { 500 return []rpc.API{{ 501 Namespace: "simulation", 502 Version: "1.0", 503 Service: SnapshotAPI{s.services}, 504 }} 505 } 506 507 func (s *snapshotService) Protocols() []p2p.Protocol { 508 return nil 509 } 510 511 func (s *snapshotService) Start(*p2p.Server) error { 512 return nil 513 } 514 515 func (s *snapshotService) Stop() error { 516 return nil 517 } 518 519 // SnapshotAPI provides an RPC method to create snapshots of services 520 type SnapshotAPI struct { 521 services map[string]node.Service 522 } 523 524 func (api SnapshotAPI) Snapshot() (map[string][]byte, error) { 525 snapshots := make(map[string][]byte) 526 for name, service := range api.services { 527 if s, ok := service.(interface { 528 Snapshot() ([]byte, error) 529 }); ok { 530 snap, err := s.Snapshot() 531 if err != nil { 532 return nil, err 533 } 534 snapshots[name] = snap 535 } 536 } 537 return snapshots, nil 538 } 539 540 type wsRPCDialer struct { 541 addrs map[string]string 542 } 543 544 // DialRPC implements the RPCDialer interface by creating a WebSocket RPC 545 // client of the given node 546 func (w *wsRPCDialer) DialRPC(id enode.ID) (*rpc.Client, error) { 547 addr, ok := w.addrs[id.String()] 548 if !ok { 549 return nil, fmt.Errorf("unknown node: %s", id) 550 } 551 return rpc.DialWebsocket(context.Background(), addr, "http://localhost") 552 }