go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/server/dsmapper/module.go (about)

     1  // Copyright 2021 The LUCI Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package dsmapper
    16  
    17  import (
    18  	"context"
    19  	"flag"
    20  
    21  	"go.chromium.org/luci/server/gaeemulation"
    22  	"go.chromium.org/luci/server/module"
    23  	"go.chromium.org/luci/server/tq"
    24  )
    25  
    26  // ModuleName can be used to refer to this module when declaring dependencies.
    27  var ModuleName = module.RegisterName("go.chromium.org/luci/server/dsmapper")
    28  
    29  // ModuleOptions contain configuration of the dsmapper server module.
    30  type ModuleOptions struct {
    31  	// MapperQueue is a name of the Cloud Tasks queue to use for mapping jobs.
    32  	//
    33  	// This queue will perform all "heavy" tasks. It should be configured
    34  	// appropriately to allow desired number of shards to run in parallel.
    35  	//
    36  	// For example, if the largest submitted job is expected to have 128 shards,
    37  	// max_concurrent_requests setting of the mapper queue should be at least 128,
    38  	// otherwise some shards will be stalled waiting for others to finish
    39  	// (defeating the purpose of having large number of shards).
    40  	//
    41  	// If empty, "default" is used.
    42  	MapperQueue string
    43  
    44  	// ControlQueue is a name of the Cloud Tasks queue to use for control signals.
    45  	//
    46  	// This queue is used very lightly when starting and stopping jobs (roughly
    47  	// 2*Shards tasks overall per job). A default queue.yaml settings for such
    48  	// queue should be sufficient (unless you run a lot of different jobs at
    49  	// once).
    50  	//
    51  	// If empty, "default" is used.
    52  	ControlQueue string
    53  }
    54  
    55  // Register registers the command line flags.
    56  func (o *ModuleOptions) Register(f *flag.FlagSet) {
    57  	if o.MapperQueue == "" {
    58  		o.MapperQueue = "default"
    59  	}
    60  	if o.ControlQueue == "" {
    61  		o.ControlQueue = "default"
    62  	}
    63  	f.StringVar(
    64  		&o.MapperQueue,
    65  		"dsmapper-mapper-queue",
    66  		o.MapperQueue,
    67  		`Cloud Tasks queue to use for mapping jobs.`,
    68  	)
    69  	f.StringVar(
    70  		&o.ControlQueue,
    71  		"dsmapper-control-queue",
    72  		o.ControlQueue,
    73  		`Cloud Tasks queue to use for control signals.`,
    74  	)
    75  }
    76  
    77  // NewModule returns a server module that initializes Default controller.
    78  func NewModule(opts *ModuleOptions) module.Module {
    79  	if opts == nil {
    80  		opts = &ModuleOptions{}
    81  	}
    82  	return &serverModule{opts: opts}
    83  }
    84  
    85  // NewModuleFromFlags is a variant of NewModule that initializes options through
    86  // command line flags.
    87  //
    88  // Calling this function registers flags in flag.CommandLine. They are usually
    89  // parsed in server.Main(...).
    90  func NewModuleFromFlags() module.Module {
    91  	opts := &ModuleOptions{}
    92  	opts.Register(flag.CommandLine)
    93  	return NewModule(opts)
    94  }
    95  
    96  // serverModule implements module.Module.
    97  type serverModule struct {
    98  	opts *ModuleOptions
    99  }
   100  
   101  // Name is part of module.Module interface.
   102  func (*serverModule) Name() module.Name {
   103  	return ModuleName
   104  }
   105  
   106  // Dependencies is part of module.Module interface.
   107  func (*serverModule) Dependencies() []module.Dependency {
   108  	return []module.Dependency{
   109  		module.RequiredDependency(gaeemulation.ModuleName),
   110  		module.RequiredDependency(tq.ModuleName),
   111  	}
   112  }
   113  
   114  // Initialize is part of module.Module interface.
   115  func (m *serverModule) Initialize(ctx context.Context, host module.Host, opts module.HostOptions) (context.Context, error) {
   116  	Default.ControlQueue = m.opts.ControlQueue
   117  	Default.MapperQueue = m.opts.MapperQueue
   118  	Default.Install(&tq.Default)
   119  	return nil, nil
   120  }