github.com/clubpay/ronykit/kit@v0.14.4-0.20240515065620-d0dace45cbc7/bridge_south.go (about) 1 package kit 2 3 import ( 4 "context" 5 "fmt" 6 "reflect" 7 "sync" 8 9 "github.com/clubpay/ronykit/kit/errors" 10 "github.com/clubpay/ronykit/kit/utils" 11 ) 12 13 type Cluster interface { 14 // Start starts the gateway to accept connections. 15 Start(ctx context.Context) error 16 // Shutdown shuts down the gateway gracefully. 17 Shutdown(ctx context.Context) error 18 // Subscribe registers the southBridge as the delegate of the backend. 19 Subscribe(id string, d ClusterDelegate) 20 // Publish sends the data to the other instance identified by id. 21 Publish(id string, data []byte) error 22 // Subscribers return the list of the instance ids. 23 Subscribers() ([]string, error) 24 } 25 26 type ClusterWithStore interface { 27 // Store returns a shared key-value store between all instances. 28 Store() ClusterStore 29 } 30 31 type ClusterDelegate interface { 32 OnMessage(data []byte) 33 } 34 35 // southBridge is a container component that connects EdgeServers using Cluster. 36 type southBridge struct { 37 ctxPool 38 id string 39 wg *sync.WaitGroup 40 eh ErrHandlerFunc 41 c map[string]Contract 42 cb Cluster 43 tp TracePropagator 44 l Logger 45 46 inProgressMtx utils.SpinLock 47 inProgress map[string]chan *envelopeCarrier 48 msgFactories map[string]MessageFactoryFunc 49 } 50 51 var _ ClusterDelegate = (*southBridge)(nil) 52 53 func (sb *southBridge) registerContract(input, output Message) { 54 sb.msgFactories[reflect.TypeOf(input).String()] = CreateMessageFactory(input) 55 sb.msgFactories[reflect.TypeOf(output).String()] = CreateMessageFactory(output) 56 } 57 58 func (sb *southBridge) Start(ctx context.Context) error { 59 return sb.cb.Start(ctx) 60 } 61 62 func (sb *southBridge) Shutdown(ctx context.Context) error { 63 return sb.cb.Shutdown(ctx) 64 } 65 66 func (sb *southBridge) OnMessage(data []byte) { 67 carrier := &envelopeCarrier{} 68 if err := carrier.FromJSON(data); err != nil { 69 sb.eh(nil, fmt.Errorf("failed to decode envelope carrier: %w", err)) 70 71 return 72 } 73 74 sb.wg.Add(1) 75 conn := &clusterConn{ 76 cb: sb.cb, 77 originID: carrier.OriginID, 78 sessionID: carrier.SessionID, 79 serverID: sb.id, 80 kv: map[string]string{}, 81 wf: sb.writeFunc, 82 } 83 ctx := sb.acquireCtx(conn) 84 ctx.sb = sb 85 86 switch carrier.Kind { 87 case incomingCarrier: 88 sb.onIncomingMessage(ctx, carrier) 89 case outgoingCarrier: 90 sb.onOutgoingMessage(ctx, carrier) 91 case eofCarrier: 92 sb.onEOF(carrier) 93 } 94 95 sb.releaseCtx(ctx) 96 sb.wg.Done() 97 } 98 99 func (sb *southBridge) onIncomingMessage(ctx *Context, carrier *envelopeCarrier) { 100 ctx.forwarded = true 101 102 msg := sb.msgFactories[carrier.Data.MsgType]() 103 switch v := msg.(type) { 104 case RawMessage: 105 v.CopyFrom(carrier.Data.Msg) 106 msg = v 107 default: 108 unmarshalEnvelopeCarrier(carrier.Data.Msg, msg) 109 } 110 111 if sb.tp != nil { 112 ctx.SetUserContext(sb.tp.Extract(ctx.Context(), carrier.Data)) 113 } 114 115 for k, v := range carrier.Data.ConnHdr { 116 ctx.Conn().Set(k, v) 117 } 118 119 ctx.in. 120 SetID(carrier.Data.EnvelopeID). 121 SetHdrMap(carrier.Data.Hdr). 122 SetMsg(msg) 123 124 ctx.execute( 125 ExecuteArg{ 126 ServiceName: carrier.Data.ServiceName, 127 ContractID: carrier.Data.ContractID, 128 Route: carrier.Data.Route, 129 }, 130 sb.c[carrier.Data.ContractID], 131 ) 132 133 err := sb.cb.Publish( 134 carrier.OriginID, 135 newEnvelopeCarrier( 136 eofCarrier, 137 carrier.SessionID, 138 sb.id, 139 carrier.OriginID, 140 ).ToJSON(), 141 ) 142 if err != nil { 143 sb.eh(ctx, err) 144 } 145 } 146 147 func (sb *southBridge) onOutgoingMessage(ctx *Context, carrier *envelopeCarrier) { 148 sb.inProgressMtx.Lock() 149 ch, ok := sb.inProgress[carrier.SessionID] 150 sb.inProgressMtx.Unlock() 151 152 if ok { 153 select { 154 case ch <- carrier: 155 default: 156 sb.eh(ctx, ErrWritingToClusterConnection) 157 } 158 } 159 } 160 161 func (sb *southBridge) onEOF(carrier *envelopeCarrier) { 162 sb.inProgressMtx.Lock() 163 ch, ok := sb.inProgress[carrier.SessionID] 164 delete(sb.inProgress, carrier.SessionID) 165 sb.inProgressMtx.Unlock() 166 if ok { 167 close(ch) 168 } 169 } 170 171 func (sb *southBridge) sendMessage(sessionID string, targetID string, data []byte) (<-chan *envelopeCarrier, error) { 172 ch := make(chan *envelopeCarrier, 4) 173 sb.inProgressMtx.Lock() 174 sb.inProgress[sessionID] = ch 175 sb.inProgressMtx.Unlock() 176 177 err := sb.cb.Publish(targetID, data) 178 if err != nil { 179 sb.inProgressMtx.Lock() 180 delete(sb.inProgress, sessionID) 181 sb.inProgressMtx.Unlock() 182 183 return nil, err 184 } 185 186 return ch, nil 187 } 188 189 func (sb *southBridge) wrapWithCoordinator(c Contract) Contract { 190 if c.EdgeSelector() == nil || sb == nil { 191 return c 192 } 193 194 cw := &contractWrap{ 195 Contract: c, 196 h: []HandlerFunc{ 197 sb.genForwarderHandler(c.EdgeSelector()), 198 }, 199 } 200 201 return cw 202 } 203 204 func (sb *southBridge) genForwarderHandler(sel EdgeSelectorFunc) HandlerFunc { 205 return func(ctx *Context) { 206 if ctx.forwarded { 207 return 208 } 209 210 target, err := sel(ctx.Limited()) 211 if ctx.Error(err) { 212 return 213 } 214 215 if target == "" || target == sb.id { 216 return 217 } 218 219 err = ctx.executeRemote( 220 executeRemoteArg{ 221 Target: target, 222 In: newEnvelopeCarrier( 223 incomingCarrier, 224 utils.RandomID(32), 225 ctx.sb.id, 226 target, 227 ).FillWithContext(ctx), 228 OutCallback: func(carrier *envelopeCarrier) { 229 if carrier.Data == nil { 230 return 231 } 232 f, ok := sb.msgFactories[carrier.Data.MsgType] 233 if !ok { 234 return 235 } 236 237 msg := f() 238 switch msg.(type) { 239 case RawMessage: 240 msg = RawMessage(carrier.Data.Msg) 241 default: 242 unmarshalEnvelopeCarrier(carrier.Data.Msg, msg) 243 } 244 245 for k, v := range carrier.Data.ConnHdr { 246 ctx.Conn().Set(k, v) 247 } 248 249 ctx.Out(). 250 SetID(carrier.Data.EnvelopeID). 251 SetHdrMap(carrier.Data.Hdr). 252 SetMsg(msg). 253 Send() 254 }, 255 }, 256 ) 257 258 ctx.Error(err) 259 260 // We should stop executing next handlers, since our request has been executed on 261 // a remote machine 262 ctx.StopExecution() 263 } 264 } 265 266 func (sb *southBridge) writeFunc(c *clusterConn, e *Envelope) error { 267 ec := newEnvelopeCarrier( 268 outgoingCarrier, 269 c.sessionID, 270 c.serverID, 271 c.originID, 272 ). 273 FillWithEnvelope(e) 274 275 if sb.tp != nil { 276 sb.tp.Inject(e.ctx.ctx, ec.Data) 277 } 278 279 return c.cb.Publish(c.originID, ec.ToJSON()) 280 } 281 282 type clusterConn struct { 283 sessionID string 284 originID string 285 serverID string 286 cb Cluster 287 288 id uint64 289 clientIP string 290 stream bool 291 292 kvMtx sync.Mutex 293 kv map[string]string 294 wf func(c *clusterConn, e *Envelope) error 295 } 296 297 var _ Conn = (*clusterConn)(nil) 298 299 func (c *clusterConn) ConnID() uint64 { 300 return c.id 301 } 302 303 func (c *clusterConn) ClientIP() string { 304 return c.clientIP 305 } 306 307 func (c *clusterConn) Write(_ []byte) (int, error) { 308 return 0, ErrWritingToClusterConnection 309 } 310 311 func (c *clusterConn) WriteEnvelope(e *Envelope) error { 312 return c.wf(c, e) 313 } 314 315 func (c *clusterConn) Stream() bool { 316 return c.stream 317 } 318 319 func (c *clusterConn) Walk(f func(key string, val string) bool) { 320 c.kvMtx.Lock() 321 defer c.kvMtx.Unlock() 322 323 for k, v := range c.kv { 324 if !f(k, v) { 325 return 326 } 327 } 328 } 329 330 func (c *clusterConn) Get(key string) string { 331 c.kvMtx.Lock() 332 v := c.kv[key] 333 c.kvMtx.Unlock() 334 335 return v 336 } 337 338 func (c *clusterConn) Set(key string, val string) { 339 c.kvMtx.Lock() 340 c.kv[key] = val 341 c.kvMtx.Unlock() 342 } 343 344 func (c *clusterConn) Keys() []string { 345 keys := make([]string, 0, len(c.kv)) 346 c.kvMtx.Lock() 347 for k := range c.kv { 348 keys = append(keys, k) 349 } 350 c.kvMtx.Unlock() 351 352 return keys 353 } 354 355 var ( 356 ErrSouthBridgeDisabled = errors.New("south bridge is disabled") 357 ErrWritingToClusterConnection = errors.New("writing to cluster connection is not possible") 358 )