open-match.dev/open-match@v1.8.1/examples/scale/frontend/frontend.go (about) 1 // Copyright 2019 Google LLC 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package frontend 16 17 import ( 18 "context" 19 "math/rand" 20 "sync" 21 "time" 22 23 "github.com/sirupsen/logrus" 24 "go.opencensus.io/stats" 25 "go.opencensus.io/trace" 26 "open-match.dev/open-match/examples/scale/scenarios" 27 "open-match.dev/open-match/internal/appmain" 28 "open-match.dev/open-match/internal/config" 29 "open-match.dev/open-match/internal/rpc" 30 "open-match.dev/open-match/internal/telemetry" 31 "open-match.dev/open-match/pkg/pb" 32 ) 33 34 var ( 35 logger = logrus.WithFields(logrus.Fields{ 36 "app": "openmatch", 37 "component": "scale.frontend", 38 }) 39 activeScenario = scenarios.ActiveScenario 40 41 mTicketsCreated = telemetry.Counter("scale_frontend_tickets_created", "tickets created") 42 mTicketCreationsFailed = telemetry.Counter("scale_frontend_ticket_creations_failed", "tickets created") 43 mRunnersWaiting = concurrentGauge(telemetry.Gauge("scale_frontend_runners_waiting", "runners waiting")) 44 mRunnersCreating = concurrentGauge(telemetry.Gauge("scale_frontend_runners_creating", "runners creating")) 45 mTicketsDeleted = telemetry.Counter("scale_frontend_tickets_deleted", "tickets deleted") 46 mTicketDeletesFailed = telemetry.Counter("scale_frontend_ticket_deletes_failed", "ticket deletes failed") 47 mBackfillsCreated = telemetry.Counter("scale_frontend_backfills_created", "backfills_created") 48 mBackfillCreationsFailed = telemetry.Counter("scale_frontend_backfill_creations_failed", "backfill creations failed") 49 mTicketsTimeToAssignment = telemetry.HistogramWithBounds("scale_frontend_tickets_time_to_assignment", "tickets time to assignment", stats.UnitMilliseconds, []float64{0.01, 0.05, 0.1, 0.3, 0.6, 0.8, 1, 2, 3, 4, 5, 6, 8, 10, 13, 16, 20, 25, 30, 40, 50, 65, 80, 100, 130, 160, 200, 250, 300, 400, 500, 650, 800, 1000, 2000, 5000, 10000, 20000, 50000, 100000, 200000, 500000, 1000000}) 50 ) 51 52 type ticketToWatch struct { 53 id string 54 createdAt time.Time 55 } 56 57 // Run triggers execution of the scale frontend component that creates 58 // tickets at scale in Open Match. 59 func BindService(p *appmain.Params, b *appmain.Bindings) error { 60 go run(p.Config()) 61 62 return nil 63 } 64 65 func run(cfg config.View) { 66 conn, err := rpc.GRPCClientFromConfig(cfg, "api.frontend") 67 if err != nil { 68 logger.WithFields(logrus.Fields{ 69 "error": err.Error(), 70 }).Fatal("failed to get Frontend connection") 71 } 72 fe := pb.NewFrontendServiceClient(conn) 73 74 if activeScenario.FrontendCreatesBackfillsOnStart { 75 createBackfills(fe, activeScenario.FrontendTotalBackfillsToCreate) 76 } 77 78 ticketQPS := int(activeScenario.FrontendTicketCreatedQPS) 79 ticketTotal := activeScenario.FrontendTotalTicketsToCreate 80 totalCreated := 0 81 82 for range time.Tick(time.Second) { 83 for i := 0; i < ticketQPS; i++ { 84 if ticketTotal == -1 || totalCreated < ticketTotal { 85 go runner(fe) 86 } 87 } 88 } 89 } 90 91 func runner(fe pb.FrontendServiceClient) { 92 ctx, cancel := context.WithCancel(context.Background()) 93 defer cancel() 94 95 g := stateGauge{} 96 defer g.stop() 97 98 g.start(mRunnersWaiting) 99 // A random sleep at the start of the worker evens calls out over the second 100 // period, and makes timing between ticket creation calls a more realistic 101 // poisson distribution. 102 time.Sleep(time.Duration(rand.Int63n(int64(time.Second)))) 103 104 g.start(mRunnersCreating) 105 createdAt := time.Now() 106 id, err := createTicket(ctx, fe) 107 if err != nil { 108 logger.WithError(err).Error("failed to create a ticket") 109 return 110 } 111 112 err = watchAssignments(ctx, fe, ticketToWatch{id: id, createdAt: createdAt}) 113 if err != nil { 114 logger.WithError(err).Errorf("failed to get ticket assignment: %s", id) 115 } else { 116 ms := time.Since(createdAt).Nanoseconds() / 1e6 117 stats.Record(ctx, mTicketsTimeToAssignment.M(ms)) 118 } 119 120 if activeScenario.FrontendDeletesTickets { 121 err = deleteTicket(ctx, fe, id) 122 if err != nil { 123 logger.WithError(err).Errorf("failed to delete ticket: %s", id) 124 } 125 } 126 } 127 128 func createTicket(ctx context.Context, fe pb.FrontendServiceClient) (string, error) { 129 ctx, span := trace.StartSpan(ctx, "scale.frontend/CreateTicket") 130 defer span.End() 131 132 req := &pb.CreateTicketRequest{ 133 Ticket: activeScenario.Ticket(), 134 } 135 136 resp, err := fe.CreateTicket(ctx, req) 137 if err != nil { 138 telemetry.RecordUnitMeasurement(ctx, mTicketCreationsFailed) 139 return "", err 140 } 141 142 telemetry.RecordUnitMeasurement(ctx, mTicketsCreated) 143 return resp.Id, nil 144 } 145 146 func watchAssignments(ctx context.Context, fe pb.FrontendServiceClient, ticket ticketToWatch) error { 147 ctx, cancel := context.WithCancel(ctx) 148 defer cancel() 149 stream, err := fe.WatchAssignments(ctx, &pb.WatchAssignmentsRequest{TicketId: ticket.id}) 150 if err != nil { 151 return err 152 } 153 154 var a *pb.Assignment 155 for a.GetConnection() == "" { 156 resp, err := stream.Recv() 157 if err != nil { 158 return err 159 } 160 161 a = resp.Assignment 162 } 163 164 return nil 165 } 166 167 func createBackfills(fe pb.FrontendServiceClient, numBackfillsToCreate int) error { 168 for i := 0; i < numBackfillsToCreate; i++ { 169 err := createBackfill(fe) 170 if err != nil { 171 return err 172 } 173 } 174 175 return nil 176 } 177 178 func createBackfill(fe pb.FrontendServiceClient) error { 179 ctx, span := trace.StartSpan(context.Background(), "scale.frontend/CreateBackfill") 180 defer span.End() 181 182 req := pb.CreateBackfillRequest{ 183 Backfill: activeScenario.Backfill(), 184 } 185 186 _, err := fe.CreateBackfill(ctx, &req) 187 if err != nil { 188 telemetry.RecordUnitMeasurement(ctx, mBackfillCreationsFailed) 189 logger.WithError(err).Error("failed to create backfill") 190 return err 191 } 192 193 telemetry.RecordUnitMeasurement(ctx, mBackfillsCreated) 194 return nil 195 } 196 197 func deleteTicket(ctx context.Context, fe pb.FrontendServiceClient, ticketId string) error { 198 _, err := fe.DeleteTicket(ctx, &pb.DeleteTicketRequest{TicketId: ticketId}) 199 if err != nil { 200 telemetry.RecordUnitMeasurement(ctx, mTicketDeletesFailed) 201 } else { 202 telemetry.RecordUnitMeasurement(ctx, mTicketsDeleted) 203 } 204 205 return err 206 } 207 208 // Allows concurrent moficiation of a gauge value by modifying the concurrent 209 // value with a delta. 210 func concurrentGauge(s *stats.Int64Measure) func(delta int64) { 211 m := sync.Mutex{} 212 v := int64(0) 213 return func(delta int64) { 214 m.Lock() 215 defer m.Unlock() 216 217 v += delta 218 telemetry.SetGauge(context.Background(), s, v) 219 } 220 } 221 222 // stateGauge will have a single value be applied to one gauge at a time. 223 type stateGauge struct { 224 f func(int64) 225 } 226 227 // start begins a stage measured in a gauge, stopping any previously started 228 // stage. 229 func (g *stateGauge) start(f func(int64)) { 230 g.stop() 231 g.f = f 232 f(1) 233 } 234 235 // stop finishes the current stage by decrementing the gauge. 236 func (g *stateGauge) stop() { 237 if g.f != nil { 238 g.f(-1) 239 g.f = nil 240 } 241 }