github.com/google/syzkaller@v0.0.0-20251211124644-a066d2bc4b02/dashboard/app/batch_main.go (about)

     1  // Copyright 2017 syzkaller project authors. All rights reserved.
     2  // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
     3  
     4  package main
     5  
     6  import (
     7  	"context"
     8  	"fmt"
     9  	"net/http"
    10  
    11  	"cloud.google.com/go/batch/apiv1"
    12  	"cloud.google.com/go/batch/apiv1/batchpb"
    13  	"github.com/google/uuid"
    14  	"google.golang.org/appengine/v2/log"
    15  	"google.golang.org/protobuf/types/known/durationpb"
    16  )
    17  
    18  func initBatchProcessors() {
    19  	http.HandleFunc("/cron/batch_coverage", handleBatchCoverage)
    20  	http.HandleFunc("/cron/batch_db_export", handleBatchDBExport)
    21  	http.HandleFunc("/cron/batch_coverage_clean", handleBatchCoverageClean)
    22  }
    23  
    24  // from https://cloud.google.com/batch/docs/samples/batch-create-script-job
    25  func createScriptJob(ctx context.Context, projectID, jobNamePrefix, script string,
    26  	timeout int64, sa *batchpb.ServiceAccount) error {
    27  	region := "us-central1"
    28  	jobName := fmt.Sprintf("%s-%s", jobNamePrefix, uuid.New().String())
    29  
    30  	batchClient, err := batch.NewClient(ctx)
    31  	if err != nil {
    32  		return fmt.Errorf("failed NewClient: %w", err)
    33  	}
    34  	defer batchClient.Close()
    35  
    36  	taskGroups := []*batchpb.TaskGroup{
    37  		{
    38  			TaskSpec: &batchpb.TaskSpec{
    39  				Runnables: []*batchpb.Runnable{{
    40  					Executable: &batchpb.Runnable_Script_{
    41  						Script: &batchpb.Runnable_Script{Command: &batchpb.Runnable_Script_Text{
    42  							Text: script,
    43  						}},
    44  					},
    45  				}},
    46  				ComputeResource: &batchpb.ComputeResource{
    47  					// CpuMilli is milliseconds per cpu-second. This means the task requires 8 whole CPUs.
    48  					CpuMilli:  8000,
    49  					MemoryMib: 30 * 1024,
    50  				},
    51  				MaxRunDuration: &durationpb.Duration{
    52  					Seconds: timeout,
    53  				},
    54  			},
    55  		},
    56  	}
    57  
    58  	// Policies are used to define on what kind of virtual machines the tasks will run on.
    59  	// In this case, we tell the system to use "e2-standard-4" machine type.
    60  	// Read more about machine types here: https://cloud.google.com/compute/docs/machine-types
    61  	allocationPolicy := &batchpb.AllocationPolicy{
    62  		Instances: []*batchpb.AllocationPolicy_InstancePolicyOrTemplate{{
    63  			PolicyTemplate: &batchpb.AllocationPolicy_InstancePolicyOrTemplate_Policy{
    64  				Policy: &batchpb.AllocationPolicy_InstancePolicy{
    65  					// Spot machines may also be used here.
    66  					// The problem with spot machines is the high probability of preemption on long runs.
    67  					// Quarter long coverage merge costs 4 hours on 8 cores and we trigger it only once/week.
    68  					// TODO(tarasmadan): switch back to the spot machines in combination with higher retry count
    69  					ProvisioningModel: batchpb.AllocationPolicy_STANDARD,
    70  					// Quarter long aggregation OOMs on machine with 16 gigs and 8 cores.
    71  					// It is important to use machine with min 32 gigs for 8 cores.
    72  					MachineType: "e2-standard-8",
    73  				},
    74  			},
    75  			InstallOpsAgent: true,
    76  		}},
    77  		ServiceAccount: sa,
    78  	}
    79  
    80  	logsPolicy := &batchpb.LogsPolicy{
    81  		Destination: batchpb.LogsPolicy_CLOUD_LOGGING,
    82  	}
    83  
    84  	// The job's parent is the region in which the job will run.
    85  	parent := fmt.Sprintf("projects/%s/locations/%s", projectID, region)
    86  
    87  	job := batchpb.Job{
    88  		TaskGroups:       taskGroups,
    89  		AllocationPolicy: allocationPolicy,
    90  		LogsPolicy:       logsPolicy,
    91  	}
    92  
    93  	req := &batchpb.CreateJobRequest{
    94  		Parent: parent,
    95  		JobId:  jobName,
    96  		Job:    &job,
    97  	}
    98  
    99  	createdJob, err := batchClient.CreateJob(ctx, req)
   100  	if err != nil {
   101  		return fmt.Errorf("unable to create job: %w", err)
   102  	}
   103  
   104  	log.Infof(ctx, "job created: %v\n", createdJob)
   105  
   106  	return nil
   107  }