github.com/hxx258456/ccgo@v0.0.5-0.20230213014102-48b35f46f66f/go-control-plane/pkg/server/sotw/v3/server.go (about) 1 // Copyright 2020 Envoyproxy Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 // Package sotw provides an implementation of GRPC SoTW (State of The World) part of XDS server 16 package sotw 17 18 import ( 19 "context" 20 "errors" 21 "strconv" 22 "sync/atomic" 23 24 "github.com/hxx258456/ccgo/grpc" 25 "github.com/hxx258456/ccgo/grpc/codes" 26 "github.com/hxx258456/ccgo/grpc/status" 27 28 core "github.com/hxx258456/ccgo/go-control-plane/envoy/config/core/v3" 29 discovery "github.com/hxx258456/ccgo/go-control-plane/envoy/service/discovery/v3" 30 "github.com/hxx258456/ccgo/go-control-plane/pkg/cache/v3" 31 "github.com/hxx258456/ccgo/go-control-plane/pkg/resource/v3" 32 ) 33 34 type Server interface { 35 StreamHandler(stream Stream, typeURL string) error 36 } 37 38 type Callbacks interface { 39 // OnStreamOpen is called once an xDS stream is open with a stream ID and the type URL (or "" for ADS). 40 // Returning an error will end processing and close the stream. OnStreamClosed will still be called. 41 OnStreamOpen(context.Context, int64, string) error 42 // OnStreamClosed is called immediately prior to closing an xDS stream with a stream ID. 43 OnStreamClosed(int64) 44 // OnStreamRequest is called once a request is received on a stream. 45 // Returning an error will end processing and close the stream. OnStreamClosed will still be called. 46 OnStreamRequest(int64, *discovery.DiscoveryRequest) error 47 // OnStreamResponse is called immediately prior to sending a response on a stream. 48 OnStreamResponse(context.Context, int64, *discovery.DiscoveryRequest, *discovery.DiscoveryResponse) 49 } 50 51 // NewServer creates handlers from a config watcher and callbacks. 52 func NewServer(ctx context.Context, config cache.ConfigWatcher, callbacks Callbacks) Server { 53 return &server{cache: config, callbacks: callbacks, ctx: ctx} 54 } 55 56 type server struct { 57 cache cache.ConfigWatcher 58 callbacks Callbacks 59 ctx context.Context 60 61 // streamCount for counting bi-di streams 62 streamCount int64 63 } 64 65 // Generic RPC stream. 66 type Stream interface { 67 grpc.ServerStream 68 69 Send(*discovery.DiscoveryResponse) error 70 Recv() (*discovery.DiscoveryRequest, error) 71 } 72 73 // watches for all xDS resource types 74 type watches struct { 75 endpoints chan cache.Response 76 clusters chan cache.Response 77 routes chan cache.Response 78 listeners chan cache.Response 79 secrets chan cache.Response 80 runtimes chan cache.Response 81 extensionConfigs chan cache.Response 82 83 endpointCancel func() 84 clusterCancel func() 85 routeCancel func() 86 listenerCancel func() 87 secretCancel func() 88 runtimeCancel func() 89 extensionConfigCancel func() 90 91 endpointNonce string 92 clusterNonce string 93 routeNonce string 94 listenerNonce string 95 secretNonce string 96 runtimeNonce string 97 extensionConfigNonce string 98 99 // Opaque resources share a muxed channel. Nonces and watch cancellations are indexed by type URL. 100 responses chan cache.Response 101 cancellations map[string]func() 102 nonces map[string]string 103 } 104 105 // Initialize all watches 106 func (values *watches) Init() { 107 // muxed channel needs a buffer to release go-routines populating it 108 values.responses = make(chan cache.Response, 5) 109 values.cancellations = make(map[string]func()) 110 values.nonces = make(map[string]string) 111 } 112 113 // Token response value used to signal a watch failure in muxed watches. 114 var errorResponse = &cache.RawResponse{} 115 116 // Cancel all watches 117 func (values *watches) Cancel() { 118 if values.endpointCancel != nil { 119 values.endpointCancel() 120 } 121 if values.clusterCancel != nil { 122 values.clusterCancel() 123 } 124 if values.routeCancel != nil { 125 values.routeCancel() 126 } 127 if values.listenerCancel != nil { 128 values.listenerCancel() 129 } 130 if values.secretCancel != nil { 131 values.secretCancel() 132 } 133 if values.runtimeCancel != nil { 134 values.runtimeCancel() 135 } 136 if values.extensionConfigCancel != nil { 137 values.extensionConfigCancel() 138 } 139 for _, cancel := range values.cancellations { 140 if cancel != nil { 141 cancel() 142 } 143 } 144 } 145 146 // process handles a bi-di stream request 147 func (s *server) process(stream Stream, reqCh <-chan *discovery.DiscoveryRequest, defaultTypeURL string) error { 148 // increment stream count 149 streamID := atomic.AddInt64(&s.streamCount, 1) 150 151 // unique nonce generator for req-resp pairs per xDS stream; the server 152 // ignores stale nonces. nonce is only modified within send() function. 153 var streamNonce int64 154 155 // a collection of stack allocated watches per request type 156 var values watches 157 values.Init() 158 defer func() { 159 values.Cancel() 160 if s.callbacks != nil { 161 s.callbacks.OnStreamClosed(streamID) 162 } 163 }() 164 165 // sends a response by serializing to protobuf Any 166 send := func(resp cache.Response) (string, error) { 167 if resp == nil { 168 return "", errors.New("missing response") 169 } 170 171 out, err := resp.GetDiscoveryResponse() 172 if err != nil { 173 return "", err 174 } 175 176 // increment nonce 177 streamNonce = streamNonce + 1 178 out.Nonce = strconv.FormatInt(streamNonce, 10) 179 if s.callbacks != nil { 180 s.callbacks.OnStreamResponse(resp.GetContext(), streamID, resp.GetRequest(), out) 181 } 182 return out.Nonce, stream.Send(out) 183 } 184 185 if s.callbacks != nil { 186 if err := s.callbacks.OnStreamOpen(stream.Context(), streamID, defaultTypeURL); err != nil { 187 return err 188 } 189 } 190 191 // node may only be set on the first discovery request 192 var node = &core.Node{} 193 194 for { 195 select { 196 case <-s.ctx.Done(): 197 return nil 198 // config watcher can send the requested resources types in any order 199 case resp, more := <-values.endpoints: 200 if !more { 201 return status.Errorf(codes.Unavailable, "endpoints watch failed") 202 } 203 nonce, err := send(resp) 204 if err != nil { 205 return err 206 } 207 values.endpointNonce = nonce 208 209 case resp, more := <-values.clusters: 210 if !more { 211 return status.Errorf(codes.Unavailable, "clusters watch failed") 212 } 213 nonce, err := send(resp) 214 if err != nil { 215 return err 216 } 217 values.clusterNonce = nonce 218 219 case resp, more := <-values.routes: 220 if !more { 221 return status.Errorf(codes.Unavailable, "routes watch failed") 222 } 223 nonce, err := send(resp) 224 if err != nil { 225 return err 226 } 227 values.routeNonce = nonce 228 229 case resp, more := <-values.listeners: 230 if !more { 231 return status.Errorf(codes.Unavailable, "listeners watch failed") 232 } 233 nonce, err := send(resp) 234 if err != nil { 235 return err 236 } 237 values.listenerNonce = nonce 238 239 case resp, more := <-values.secrets: 240 if !more { 241 return status.Errorf(codes.Unavailable, "secrets watch failed") 242 } 243 nonce, err := send(resp) 244 if err != nil { 245 return err 246 } 247 values.secretNonce = nonce 248 249 case resp, more := <-values.runtimes: 250 if !more { 251 return status.Errorf(codes.Unavailable, "runtimes watch failed") 252 } 253 nonce, err := send(resp) 254 if err != nil { 255 return err 256 } 257 values.runtimeNonce = nonce 258 259 case resp, more := <-values.extensionConfigs: 260 if !more { 261 return status.Errorf(codes.Unavailable, "extensionConfigs watch failed") 262 } 263 nonce, err := send(resp) 264 if err != nil { 265 return err 266 } 267 values.extensionConfigNonce = nonce 268 269 case resp, more := <-values.responses: 270 if more { 271 if resp == errorResponse { 272 return status.Errorf(codes.Unavailable, "resource watch failed") 273 } 274 typeURL := resp.GetRequest().TypeUrl 275 nonce, err := send(resp) 276 if err != nil { 277 return err 278 } 279 values.nonces[typeURL] = nonce 280 } 281 282 case req, more := <-reqCh: 283 // input stream ended or errored out 284 if !more { 285 return nil 286 } 287 if req == nil { 288 return status.Errorf(codes.Unavailable, "empty request") 289 } 290 291 // node field in discovery request is delta-compressed 292 if req.Node != nil { 293 node = req.Node 294 } else { 295 req.Node = node 296 } 297 298 // nonces can be reused across streams; we verify nonce only if nonce is not initialized 299 nonce := req.GetResponseNonce() 300 301 // type URL is required for ADS but is implicit for xDS 302 if defaultTypeURL == resource.AnyType { 303 if req.TypeUrl == "" { 304 return status.Errorf(codes.InvalidArgument, "type URL is required for ADS") 305 } 306 } else if req.TypeUrl == "" { 307 req.TypeUrl = defaultTypeURL 308 } 309 310 if s.callbacks != nil { 311 if err := s.callbacks.OnStreamRequest(streamID, req); err != nil { 312 return err 313 } 314 } 315 316 // cancel existing watches to (re-)request a newer version 317 switch { 318 case req.TypeUrl == resource.EndpointType: 319 if values.endpointNonce == "" || values.endpointNonce == nonce { 320 if values.endpointCancel != nil { 321 values.endpointCancel() 322 } 323 values.endpoints = make(chan cache.Response, 1) 324 values.endpointCancel = s.cache.CreateWatch(req, values.endpoints) 325 } 326 case req.TypeUrl == resource.ClusterType: 327 if values.clusterNonce == "" || values.clusterNonce == nonce { 328 if values.clusterCancel != nil { 329 values.clusterCancel() 330 } 331 values.clusters = make(chan cache.Response, 1) 332 values.clusterCancel = s.cache.CreateWatch(req, values.clusters) 333 } 334 case req.TypeUrl == resource.RouteType: 335 if values.routeNonce == "" || values.routeNonce == nonce { 336 if values.routeCancel != nil { 337 values.routeCancel() 338 } 339 values.routes = make(chan cache.Response, 1) 340 values.routeCancel = s.cache.CreateWatch(req, values.routes) 341 } 342 case req.TypeUrl == resource.ListenerType: 343 if values.listenerNonce == "" || values.listenerNonce == nonce { 344 if values.listenerCancel != nil { 345 values.listenerCancel() 346 } 347 values.listeners = make(chan cache.Response, 1) 348 values.listenerCancel = s.cache.CreateWatch(req, values.listeners) 349 } 350 case req.TypeUrl == resource.SecretType: 351 if values.secretNonce == "" || values.secretNonce == nonce { 352 if values.secretCancel != nil { 353 values.secretCancel() 354 } 355 values.secrets = make(chan cache.Response, 1) 356 values.secretCancel = s.cache.CreateWatch(req, values.secrets) 357 } 358 case req.TypeUrl == resource.RuntimeType: 359 if values.runtimeNonce == "" || values.runtimeNonce == nonce { 360 if values.runtimeCancel != nil { 361 values.runtimeCancel() 362 } 363 values.runtimes = make(chan cache.Response, 1) 364 values.runtimeCancel = s.cache.CreateWatch(req, values.runtimes) 365 } 366 case req.TypeUrl == resource.ExtensionConfigType: 367 if values.extensionConfigNonce == "" || values.extensionConfigNonce == nonce { 368 if values.extensionConfigCancel != nil { 369 values.extensionConfigCancel() 370 } 371 values.extensionConfigs = make(chan cache.Response, 1) 372 values.extensionConfigCancel = s.cache.CreateWatch(req, values.extensionConfigs) 373 } 374 default: 375 typeURL := req.TypeUrl 376 responseNonce, seen := values.nonces[typeURL] 377 if !seen || responseNonce == nonce { 378 if cancel, seen := values.cancellations[typeURL]; seen && cancel != nil { 379 cancel() 380 } 381 values.cancellations[typeURL] = s.cache.CreateWatch(req, values.responses) 382 } 383 } 384 } 385 } 386 } 387 388 // StreamHandler converts a blocking read call to channels and initiates stream processing 389 func (s *server) StreamHandler(stream Stream, typeURL string) error { 390 // a channel for receiving incoming requests 391 reqCh := make(chan *discovery.DiscoveryRequest) 392 go func() { 393 defer close(reqCh) 394 for { 395 req, err := stream.Recv() 396 if err != nil { 397 return 398 } 399 select { 400 case reqCh <- req: 401 case <-stream.Context().Done(): 402 return 403 case <-s.ctx.Done(): 404 return 405 } 406 } 407 }() 408 409 return s.process(stream, reqCh, typeURL) 410 }