open-match.dev/open-match@v1.8.1/examples/scale/frontend/frontend.go (about)

     1  // Copyright 2019 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package frontend
    16  
    17  import (
    18  	"context"
    19  	"math/rand"
    20  	"sync"
    21  	"time"
    22  
    23  	"github.com/sirupsen/logrus"
    24  	"go.opencensus.io/stats"
    25  	"go.opencensus.io/trace"
    26  	"open-match.dev/open-match/examples/scale/scenarios"
    27  	"open-match.dev/open-match/internal/appmain"
    28  	"open-match.dev/open-match/internal/config"
    29  	"open-match.dev/open-match/internal/rpc"
    30  	"open-match.dev/open-match/internal/telemetry"
    31  	"open-match.dev/open-match/pkg/pb"
    32  )
    33  
    34  var (
    35  	logger = logrus.WithFields(logrus.Fields{
    36  		"app":       "openmatch",
    37  		"component": "scale.frontend",
    38  	})
    39  	activeScenario = scenarios.ActiveScenario
    40  
    41  	mTicketsCreated          = telemetry.Counter("scale_frontend_tickets_created", "tickets created")
    42  	mTicketCreationsFailed   = telemetry.Counter("scale_frontend_ticket_creations_failed", "tickets created")
    43  	mRunnersWaiting          = concurrentGauge(telemetry.Gauge("scale_frontend_runners_waiting", "runners waiting"))
    44  	mRunnersCreating         = concurrentGauge(telemetry.Gauge("scale_frontend_runners_creating", "runners creating"))
    45  	mTicketsDeleted          = telemetry.Counter("scale_frontend_tickets_deleted", "tickets deleted")
    46  	mTicketDeletesFailed     = telemetry.Counter("scale_frontend_ticket_deletes_failed", "ticket deletes failed")
    47  	mBackfillsCreated        = telemetry.Counter("scale_frontend_backfills_created", "backfills_created")
    48  	mBackfillCreationsFailed = telemetry.Counter("scale_frontend_backfill_creations_failed", "backfill creations failed")
    49  	mTicketsTimeToAssignment = telemetry.HistogramWithBounds("scale_frontend_tickets_time_to_assignment", "tickets time to assignment", stats.UnitMilliseconds, []float64{0.01, 0.05, 0.1, 0.3, 0.6, 0.8, 1, 2, 3, 4, 5, 6, 8, 10, 13, 16, 20, 25, 30, 40, 50, 65, 80, 100, 130, 160, 200, 250, 300, 400, 500, 650, 800, 1000, 2000, 5000, 10000, 20000, 50000, 100000, 200000, 500000, 1000000})
    50  )
    51  
    52  type ticketToWatch struct {
    53  	id        string
    54  	createdAt time.Time
    55  }
    56  
    57  // Run triggers execution of the scale frontend component that creates
    58  // tickets at scale in Open Match.
    59  func BindService(p *appmain.Params, b *appmain.Bindings) error {
    60  	go run(p.Config())
    61  
    62  	return nil
    63  }
    64  
    65  func run(cfg config.View) {
    66  	conn, err := rpc.GRPCClientFromConfig(cfg, "api.frontend")
    67  	if err != nil {
    68  		logger.WithFields(logrus.Fields{
    69  			"error": err.Error(),
    70  		}).Fatal("failed to get Frontend connection")
    71  	}
    72  	fe := pb.NewFrontendServiceClient(conn)
    73  
    74  	if activeScenario.FrontendCreatesBackfillsOnStart {
    75  		createBackfills(fe, activeScenario.FrontendTotalBackfillsToCreate)
    76  	}
    77  
    78  	ticketQPS := int(activeScenario.FrontendTicketCreatedQPS)
    79  	ticketTotal := activeScenario.FrontendTotalTicketsToCreate
    80  	totalCreated := 0
    81  
    82  	for range time.Tick(time.Second) {
    83  		for i := 0; i < ticketQPS; i++ {
    84  			if ticketTotal == -1 || totalCreated < ticketTotal {
    85  				go runner(fe)
    86  			}
    87  		}
    88  	}
    89  }
    90  
    91  func runner(fe pb.FrontendServiceClient) {
    92  	ctx, cancel := context.WithCancel(context.Background())
    93  	defer cancel()
    94  
    95  	g := stateGauge{}
    96  	defer g.stop()
    97  
    98  	g.start(mRunnersWaiting)
    99  	// A random sleep at the start of the worker evens calls out over the second
   100  	// period, and makes timing between ticket creation calls a more realistic
   101  	// poisson distribution.
   102  	time.Sleep(time.Duration(rand.Int63n(int64(time.Second))))
   103  
   104  	g.start(mRunnersCreating)
   105  	createdAt := time.Now()
   106  	id, err := createTicket(ctx, fe)
   107  	if err != nil {
   108  		logger.WithError(err).Error("failed to create a ticket")
   109  		return
   110  	}
   111  
   112  	err = watchAssignments(ctx, fe, ticketToWatch{id: id, createdAt: createdAt})
   113  	if err != nil {
   114  		logger.WithError(err).Errorf("failed to get ticket assignment: %s", id)
   115  	} else {
   116  		ms := time.Since(createdAt).Nanoseconds() / 1e6
   117  		stats.Record(ctx, mTicketsTimeToAssignment.M(ms))
   118  	}
   119  
   120  	if activeScenario.FrontendDeletesTickets {
   121  		err = deleteTicket(ctx, fe, id)
   122  		if err != nil {
   123  			logger.WithError(err).Errorf("failed to delete ticket: %s", id)
   124  		}
   125  	}
   126  }
   127  
   128  func createTicket(ctx context.Context, fe pb.FrontendServiceClient) (string, error) {
   129  	ctx, span := trace.StartSpan(ctx, "scale.frontend/CreateTicket")
   130  	defer span.End()
   131  
   132  	req := &pb.CreateTicketRequest{
   133  		Ticket: activeScenario.Ticket(),
   134  	}
   135  
   136  	resp, err := fe.CreateTicket(ctx, req)
   137  	if err != nil {
   138  		telemetry.RecordUnitMeasurement(ctx, mTicketCreationsFailed)
   139  		return "", err
   140  	}
   141  
   142  	telemetry.RecordUnitMeasurement(ctx, mTicketsCreated)
   143  	return resp.Id, nil
   144  }
   145  
   146  func watchAssignments(ctx context.Context, fe pb.FrontendServiceClient, ticket ticketToWatch) error {
   147  	ctx, cancel := context.WithCancel(ctx)
   148  	defer cancel()
   149  	stream, err := fe.WatchAssignments(ctx, &pb.WatchAssignmentsRequest{TicketId: ticket.id})
   150  	if err != nil {
   151  		return err
   152  	}
   153  
   154  	var a *pb.Assignment
   155  	for a.GetConnection() == "" {
   156  		resp, err := stream.Recv()
   157  		if err != nil {
   158  			return err
   159  		}
   160  
   161  		a = resp.Assignment
   162  	}
   163  
   164  	return nil
   165  }
   166  
   167  func createBackfills(fe pb.FrontendServiceClient, numBackfillsToCreate int) error {
   168  	for i := 0; i < numBackfillsToCreate; i++ {
   169  		err := createBackfill(fe)
   170  		if err != nil {
   171  			return err
   172  		}
   173  	}
   174  
   175  	return nil
   176  }
   177  
   178  func createBackfill(fe pb.FrontendServiceClient) error {
   179  	ctx, span := trace.StartSpan(context.Background(), "scale.frontend/CreateBackfill")
   180  	defer span.End()
   181  
   182  	req := pb.CreateBackfillRequest{
   183  		Backfill: activeScenario.Backfill(),
   184  	}
   185  
   186  	_, err := fe.CreateBackfill(ctx, &req)
   187  	if err != nil {
   188  		telemetry.RecordUnitMeasurement(ctx, mBackfillCreationsFailed)
   189  		logger.WithError(err).Error("failed to create backfill")
   190  		return err
   191  	}
   192  
   193  	telemetry.RecordUnitMeasurement(ctx, mBackfillsCreated)
   194  	return nil
   195  }
   196  
   197  func deleteTicket(ctx context.Context, fe pb.FrontendServiceClient, ticketId string) error {
   198  	_, err := fe.DeleteTicket(ctx, &pb.DeleteTicketRequest{TicketId: ticketId})
   199  	if err != nil {
   200  		telemetry.RecordUnitMeasurement(ctx, mTicketDeletesFailed)
   201  	} else {
   202  		telemetry.RecordUnitMeasurement(ctx, mTicketsDeleted)
   203  	}
   204  
   205  	return err
   206  }
   207  
   208  // Allows concurrent moficiation of a gauge value by modifying the concurrent
   209  // value with a delta.
   210  func concurrentGauge(s *stats.Int64Measure) func(delta int64) {
   211  	m := sync.Mutex{}
   212  	v := int64(0)
   213  	return func(delta int64) {
   214  		m.Lock()
   215  		defer m.Unlock()
   216  
   217  		v += delta
   218  		telemetry.SetGauge(context.Background(), s, v)
   219  	}
   220  }
   221  
   222  // stateGauge will have a single value be applied to one gauge at a time.
   223  type stateGauge struct {
   224  	f func(int64)
   225  }
   226  
   227  // start begins a stage measured in a gauge, stopping any previously started
   228  // stage.
   229  func (g *stateGauge) start(f func(int64)) {
   230  	g.stop()
   231  	g.f = f
   232  	f(1)
   233  }
   234  
   235  // stop finishes the current stage by decrementing the gauge.
   236  func (g *stateGauge) stop() {
   237  	if g.f != nil {
   238  		g.f(-1)
   239  		g.f = nil
   240  	}
   241  }