github.com/apache/beam/sdks/v2@v2.48.2/go/test/load/cogbk/cogbk.go (about)

     1  // Licensed to the Apache Software Foundation (ASF) under one or more
     2  // contributor license agreements.  See the NOTICE file distributed with
     3  // this work for additional information regarding copyright ownership.
     4  // The ASF licenses this file to You under the Apache License, Version 2.0
     5  // (the "License"); you may not use this file except in compliance with
     6  // the License.  You may obtain a copy of the License at
     7  //
     8  //    http://www.apache.org/licenses/LICENSE-2.0
     9  //
    10  // Unless required by applicable law or agreed to in writing, software
    11  // distributed under the License is distributed on an "AS IS" BASIS,
    12  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  // See the License for the specific language governing permissions and
    14  // limitations under the License.
    15  
    16  package main
    17  
    18  import (
    19  	"context"
    20  	"flag"
    21  
    22  	"github.com/apache/beam/sdks/v2/go/pkg/beam"
    23  	"github.com/apache/beam/sdks/v2/go/pkg/beam/io/synthetic"
    24  	"github.com/apache/beam/sdks/v2/go/pkg/beam/log"
    25  	"github.com/apache/beam/sdks/v2/go/pkg/beam/register"
    26  	"github.com/apache/beam/sdks/v2/go/pkg/beam/x/beamx"
    27  	"github.com/apache/beam/sdks/v2/go/test/load"
    28  )
    29  
    30  var (
    31  	iterations = flag.Int(
    32  		"iterations",
    33  		1,
    34  		"A number of reiterations over per-key-grouped values to be performed.")
    35  	syntheticConfig = flag.String(
    36  		"input_options",
    37  		"",
    38  		"A JSON object that describes the configuration for the first synthetic source.")
    39  	coSyntheticConfig = flag.String(
    40  		"co_input_options",
    41  		"",
    42  		"A JSON object that describes the configuration for the second synthetic source.")
    43  )
    44  
    45  func init() {
    46  	register.DoFn4x0[[]byte, func(*[]byte) bool, func(*[]byte) bool, func([]byte, []byte)]((*ungroupAndReiterateFn)(nil))
    47  	register.Emitter2[[]byte, []byte]()
    48  	register.Iter1[[]byte]()
    49  }
    50  
    51  // ungroupAndReiterateFn reiterates given number of times over CoGBK's output.
    52  type ungroupAndReiterateFn struct {
    53  	Iterations int
    54  }
    55  
    56  // TODO use re-iterators once supported.
    57  
    58  func (fn *ungroupAndReiterateFn) ProcessElement(key []byte, p1values, p2values func(*[]byte) bool, emit func([]byte, []byte)) {
    59  	var value []byte
    60  	for i := 0; i < fn.Iterations; i++ {
    61  		for p1values(&value) {
    62  			// emit output only once
    63  			if i == fn.Iterations-1 {
    64  				emit(key, value)
    65  			}
    66  		}
    67  		for p2values(&value) {
    68  			if i == fn.Iterations-1 {
    69  				emit(key, value)
    70  			}
    71  		}
    72  	}
    73  }
    74  
    75  func parseSyntheticConfig(config string) synthetic.SourceConfig {
    76  	if config == "" {
    77  		panic("--input_options and --co_input_options not provided")
    78  	} else {
    79  		encoded := []byte(config)
    80  		return synthetic.DefaultSourceConfig().BuildFromJSON(encoded)
    81  	}
    82  }
    83  
    84  func main() {
    85  	flag.Parse()
    86  	beam.Init()
    87  	ctx := context.Background()
    88  
    89  	p, s := beam.NewPipelineWithRoot()
    90  
    91  	src1 := synthetic.SourceSingle(s, parseSyntheticConfig(*syntheticConfig))
    92  	pc1 := beam.ParDo(s, &load.RuntimeMonitor{}, src1)
    93  
    94  	src2 := synthetic.SourceSingle(s, parseSyntheticConfig(*coSyntheticConfig))
    95  	pc2 := beam.ParDo(s, &load.RuntimeMonitor{}, src2)
    96  
    97  	joined := beam.CoGroupByKey(s, pc1, pc2)
    98  	pc := beam.ParDo(s, &ungroupAndReiterateFn{Iterations: *iterations}, joined)
    99  	beam.ParDo(s, &load.RuntimeMonitor{}, pc)
   100  
   101  	presult, err := beamx.RunWithMetrics(ctx, p)
   102  	if err != nil {
   103  		log.Fatalf(ctx, "Failed to execute job: %v", err)
   104  	}
   105  
   106  	if presult != nil {
   107  		metrics := presult.Metrics().AllMetrics()
   108  		load.PublishMetrics(metrics)
   109  	}
   110  }