github.com/hxx258456/ccgo@v0.0.5-0.20230213014102-48b35f46f66f/go-control-plane/pkg/server/sotw/v3/server.go (about)

     1  // Copyright 2020 Envoyproxy Authors
     2  //
     3  //   Licensed under the Apache License, Version 2.0 (the "License");
     4  //   you may not use this file except in compliance with the License.
     5  //   You may obtain a copy of the License at
     6  //
     7  //       http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  //   Unless required by applicable law or agreed to in writing, software
    10  //   distributed under the License is distributed on an "AS IS" BASIS,
    11  //   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  //   See the License for the specific language governing permissions and
    13  //   limitations under the License.
    14  
    15  // Package sotw provides an implementation of GRPC SoTW (State of The World) part of XDS server
    16  package sotw
    17  
    18  import (
    19  	"context"
    20  	"errors"
    21  	"strconv"
    22  	"sync/atomic"
    23  
    24  	"github.com/hxx258456/ccgo/grpc"
    25  	"github.com/hxx258456/ccgo/grpc/codes"
    26  	"github.com/hxx258456/ccgo/grpc/status"
    27  
    28  	core "github.com/hxx258456/ccgo/go-control-plane/envoy/config/core/v3"
    29  	discovery "github.com/hxx258456/ccgo/go-control-plane/envoy/service/discovery/v3"
    30  	"github.com/hxx258456/ccgo/go-control-plane/pkg/cache/v3"
    31  	"github.com/hxx258456/ccgo/go-control-plane/pkg/resource/v3"
    32  )
    33  
    34  type Server interface {
    35  	StreamHandler(stream Stream, typeURL string) error
    36  }
    37  
    38  type Callbacks interface {
    39  	// OnStreamOpen is called once an xDS stream is open with a stream ID and the type URL (or "" for ADS).
    40  	// Returning an error will end processing and close the stream. OnStreamClosed will still be called.
    41  	OnStreamOpen(context.Context, int64, string) error
    42  	// OnStreamClosed is called immediately prior to closing an xDS stream with a stream ID.
    43  	OnStreamClosed(int64)
    44  	// OnStreamRequest is called once a request is received on a stream.
    45  	// Returning an error will end processing and close the stream. OnStreamClosed will still be called.
    46  	OnStreamRequest(int64, *discovery.DiscoveryRequest) error
    47  	// OnStreamResponse is called immediately prior to sending a response on a stream.
    48  	OnStreamResponse(context.Context, int64, *discovery.DiscoveryRequest, *discovery.DiscoveryResponse)
    49  }
    50  
    51  // NewServer creates handlers from a config watcher and callbacks.
    52  func NewServer(ctx context.Context, config cache.ConfigWatcher, callbacks Callbacks) Server {
    53  	return &server{cache: config, callbacks: callbacks, ctx: ctx}
    54  }
    55  
    56  type server struct {
    57  	cache     cache.ConfigWatcher
    58  	callbacks Callbacks
    59  	ctx       context.Context
    60  
    61  	// streamCount for counting bi-di streams
    62  	streamCount int64
    63  }
    64  
    65  // Generic RPC stream.
    66  type Stream interface {
    67  	grpc.ServerStream
    68  
    69  	Send(*discovery.DiscoveryResponse) error
    70  	Recv() (*discovery.DiscoveryRequest, error)
    71  }
    72  
    73  // watches for all xDS resource types
    74  type watches struct {
    75  	endpoints        chan cache.Response
    76  	clusters         chan cache.Response
    77  	routes           chan cache.Response
    78  	listeners        chan cache.Response
    79  	secrets          chan cache.Response
    80  	runtimes         chan cache.Response
    81  	extensionConfigs chan cache.Response
    82  
    83  	endpointCancel        func()
    84  	clusterCancel         func()
    85  	routeCancel           func()
    86  	listenerCancel        func()
    87  	secretCancel          func()
    88  	runtimeCancel         func()
    89  	extensionConfigCancel func()
    90  
    91  	endpointNonce        string
    92  	clusterNonce         string
    93  	routeNonce           string
    94  	listenerNonce        string
    95  	secretNonce          string
    96  	runtimeNonce         string
    97  	extensionConfigNonce string
    98  
    99  	// Opaque resources share a muxed channel. Nonces and watch cancellations are indexed by type URL.
   100  	responses     chan cache.Response
   101  	cancellations map[string]func()
   102  	nonces        map[string]string
   103  }
   104  
   105  // Initialize all watches
   106  func (values *watches) Init() {
   107  	// muxed channel needs a buffer to release go-routines populating it
   108  	values.responses = make(chan cache.Response, 5)
   109  	values.cancellations = make(map[string]func())
   110  	values.nonces = make(map[string]string)
   111  }
   112  
   113  // Token response value used to signal a watch failure in muxed watches.
   114  var errorResponse = &cache.RawResponse{}
   115  
   116  // Cancel all watches
   117  func (values *watches) Cancel() {
   118  	if values.endpointCancel != nil {
   119  		values.endpointCancel()
   120  	}
   121  	if values.clusterCancel != nil {
   122  		values.clusterCancel()
   123  	}
   124  	if values.routeCancel != nil {
   125  		values.routeCancel()
   126  	}
   127  	if values.listenerCancel != nil {
   128  		values.listenerCancel()
   129  	}
   130  	if values.secretCancel != nil {
   131  		values.secretCancel()
   132  	}
   133  	if values.runtimeCancel != nil {
   134  		values.runtimeCancel()
   135  	}
   136  	if values.extensionConfigCancel != nil {
   137  		values.extensionConfigCancel()
   138  	}
   139  	for _, cancel := range values.cancellations {
   140  		if cancel != nil {
   141  			cancel()
   142  		}
   143  	}
   144  }
   145  
   146  // process handles a bi-di stream request
   147  func (s *server) process(stream Stream, reqCh <-chan *discovery.DiscoveryRequest, defaultTypeURL string) error {
   148  	// increment stream count
   149  	streamID := atomic.AddInt64(&s.streamCount, 1)
   150  
   151  	// unique nonce generator for req-resp pairs per xDS stream; the server
   152  	// ignores stale nonces. nonce is only modified within send() function.
   153  	var streamNonce int64
   154  
   155  	// a collection of stack allocated watches per request type
   156  	var values watches
   157  	values.Init()
   158  	defer func() {
   159  		values.Cancel()
   160  		if s.callbacks != nil {
   161  			s.callbacks.OnStreamClosed(streamID)
   162  		}
   163  	}()
   164  
   165  	// sends a response by serializing to protobuf Any
   166  	send := func(resp cache.Response) (string, error) {
   167  		if resp == nil {
   168  			return "", errors.New("missing response")
   169  		}
   170  
   171  		out, err := resp.GetDiscoveryResponse()
   172  		if err != nil {
   173  			return "", err
   174  		}
   175  
   176  		// increment nonce
   177  		streamNonce = streamNonce + 1
   178  		out.Nonce = strconv.FormatInt(streamNonce, 10)
   179  		if s.callbacks != nil {
   180  			s.callbacks.OnStreamResponse(resp.GetContext(), streamID, resp.GetRequest(), out)
   181  		}
   182  		return out.Nonce, stream.Send(out)
   183  	}
   184  
   185  	if s.callbacks != nil {
   186  		if err := s.callbacks.OnStreamOpen(stream.Context(), streamID, defaultTypeURL); err != nil {
   187  			return err
   188  		}
   189  	}
   190  
   191  	// node may only be set on the first discovery request
   192  	var node = &core.Node{}
   193  
   194  	for {
   195  		select {
   196  		case <-s.ctx.Done():
   197  			return nil
   198  		// config watcher can send the requested resources types in any order
   199  		case resp, more := <-values.endpoints:
   200  			if !more {
   201  				return status.Errorf(codes.Unavailable, "endpoints watch failed")
   202  			}
   203  			nonce, err := send(resp)
   204  			if err != nil {
   205  				return err
   206  			}
   207  			values.endpointNonce = nonce
   208  
   209  		case resp, more := <-values.clusters:
   210  			if !more {
   211  				return status.Errorf(codes.Unavailable, "clusters watch failed")
   212  			}
   213  			nonce, err := send(resp)
   214  			if err != nil {
   215  				return err
   216  			}
   217  			values.clusterNonce = nonce
   218  
   219  		case resp, more := <-values.routes:
   220  			if !more {
   221  				return status.Errorf(codes.Unavailable, "routes watch failed")
   222  			}
   223  			nonce, err := send(resp)
   224  			if err != nil {
   225  				return err
   226  			}
   227  			values.routeNonce = nonce
   228  
   229  		case resp, more := <-values.listeners:
   230  			if !more {
   231  				return status.Errorf(codes.Unavailable, "listeners watch failed")
   232  			}
   233  			nonce, err := send(resp)
   234  			if err != nil {
   235  				return err
   236  			}
   237  			values.listenerNonce = nonce
   238  
   239  		case resp, more := <-values.secrets:
   240  			if !more {
   241  				return status.Errorf(codes.Unavailable, "secrets watch failed")
   242  			}
   243  			nonce, err := send(resp)
   244  			if err != nil {
   245  				return err
   246  			}
   247  			values.secretNonce = nonce
   248  
   249  		case resp, more := <-values.runtimes:
   250  			if !more {
   251  				return status.Errorf(codes.Unavailable, "runtimes watch failed")
   252  			}
   253  			nonce, err := send(resp)
   254  			if err != nil {
   255  				return err
   256  			}
   257  			values.runtimeNonce = nonce
   258  
   259  		case resp, more := <-values.extensionConfigs:
   260  			if !more {
   261  				return status.Errorf(codes.Unavailable, "extensionConfigs watch failed")
   262  			}
   263  			nonce, err := send(resp)
   264  			if err != nil {
   265  				return err
   266  			}
   267  			values.extensionConfigNonce = nonce
   268  
   269  		case resp, more := <-values.responses:
   270  			if more {
   271  				if resp == errorResponse {
   272  					return status.Errorf(codes.Unavailable, "resource watch failed")
   273  				}
   274  				typeURL := resp.GetRequest().TypeUrl
   275  				nonce, err := send(resp)
   276  				if err != nil {
   277  					return err
   278  				}
   279  				values.nonces[typeURL] = nonce
   280  			}
   281  
   282  		case req, more := <-reqCh:
   283  			// input stream ended or errored out
   284  			if !more {
   285  				return nil
   286  			}
   287  			if req == nil {
   288  				return status.Errorf(codes.Unavailable, "empty request")
   289  			}
   290  
   291  			// node field in discovery request is delta-compressed
   292  			if req.Node != nil {
   293  				node = req.Node
   294  			} else {
   295  				req.Node = node
   296  			}
   297  
   298  			// nonces can be reused across streams; we verify nonce only if nonce is not initialized
   299  			nonce := req.GetResponseNonce()
   300  
   301  			// type URL is required for ADS but is implicit for xDS
   302  			if defaultTypeURL == resource.AnyType {
   303  				if req.TypeUrl == "" {
   304  					return status.Errorf(codes.InvalidArgument, "type URL is required for ADS")
   305  				}
   306  			} else if req.TypeUrl == "" {
   307  				req.TypeUrl = defaultTypeURL
   308  			}
   309  
   310  			if s.callbacks != nil {
   311  				if err := s.callbacks.OnStreamRequest(streamID, req); err != nil {
   312  					return err
   313  				}
   314  			}
   315  
   316  			// cancel existing watches to (re-)request a newer version
   317  			switch {
   318  			case req.TypeUrl == resource.EndpointType:
   319  				if values.endpointNonce == "" || values.endpointNonce == nonce {
   320  					if values.endpointCancel != nil {
   321  						values.endpointCancel()
   322  					}
   323  					values.endpoints = make(chan cache.Response, 1)
   324  					values.endpointCancel = s.cache.CreateWatch(req, values.endpoints)
   325  				}
   326  			case req.TypeUrl == resource.ClusterType:
   327  				if values.clusterNonce == "" || values.clusterNonce == nonce {
   328  					if values.clusterCancel != nil {
   329  						values.clusterCancel()
   330  					}
   331  					values.clusters = make(chan cache.Response, 1)
   332  					values.clusterCancel = s.cache.CreateWatch(req, values.clusters)
   333  				}
   334  			case req.TypeUrl == resource.RouteType:
   335  				if values.routeNonce == "" || values.routeNonce == nonce {
   336  					if values.routeCancel != nil {
   337  						values.routeCancel()
   338  					}
   339  					values.routes = make(chan cache.Response, 1)
   340  					values.routeCancel = s.cache.CreateWatch(req, values.routes)
   341  				}
   342  			case req.TypeUrl == resource.ListenerType:
   343  				if values.listenerNonce == "" || values.listenerNonce == nonce {
   344  					if values.listenerCancel != nil {
   345  						values.listenerCancel()
   346  					}
   347  					values.listeners = make(chan cache.Response, 1)
   348  					values.listenerCancel = s.cache.CreateWatch(req, values.listeners)
   349  				}
   350  			case req.TypeUrl == resource.SecretType:
   351  				if values.secretNonce == "" || values.secretNonce == nonce {
   352  					if values.secretCancel != nil {
   353  						values.secretCancel()
   354  					}
   355  					values.secrets = make(chan cache.Response, 1)
   356  					values.secretCancel = s.cache.CreateWatch(req, values.secrets)
   357  				}
   358  			case req.TypeUrl == resource.RuntimeType:
   359  				if values.runtimeNonce == "" || values.runtimeNonce == nonce {
   360  					if values.runtimeCancel != nil {
   361  						values.runtimeCancel()
   362  					}
   363  					values.runtimes = make(chan cache.Response, 1)
   364  					values.runtimeCancel = s.cache.CreateWatch(req, values.runtimes)
   365  				}
   366  			case req.TypeUrl == resource.ExtensionConfigType:
   367  				if values.extensionConfigNonce == "" || values.extensionConfigNonce == nonce {
   368  					if values.extensionConfigCancel != nil {
   369  						values.extensionConfigCancel()
   370  					}
   371  					values.extensionConfigs = make(chan cache.Response, 1)
   372  					values.extensionConfigCancel = s.cache.CreateWatch(req, values.extensionConfigs)
   373  				}
   374  			default:
   375  				typeURL := req.TypeUrl
   376  				responseNonce, seen := values.nonces[typeURL]
   377  				if !seen || responseNonce == nonce {
   378  					if cancel, seen := values.cancellations[typeURL]; seen && cancel != nil {
   379  						cancel()
   380  					}
   381  					values.cancellations[typeURL] = s.cache.CreateWatch(req, values.responses)
   382  				}
   383  			}
   384  		}
   385  	}
   386  }
   387  
   388  // StreamHandler converts a blocking read call to channels and initiates stream processing
   389  func (s *server) StreamHandler(stream Stream, typeURL string) error {
   390  	// a channel for receiving incoming requests
   391  	reqCh := make(chan *discovery.DiscoveryRequest)
   392  	go func() {
   393  		defer close(reqCh)
   394  		for {
   395  			req, err := stream.Recv()
   396  			if err != nil {
   397  				return
   398  			}
   399  			select {
   400  			case reqCh <- req:
   401  			case <-stream.Context().Done():
   402  				return
   403  			case <-s.ctx.Done():
   404  				return
   405  			}
   406  		}
   407  	}()
   408  
   409  	return s.process(stream, reqCh, typeURL)
   410  }