github.com/apache/beam/sdks/v2@v2.48.2/go/examples/snippets/01_03intro.go (about)

     1  // Licensed to the Apache Software Foundation (ASF) under one or more
     2  // contributor license agreements.  See the NOTICE file distributed with
     3  // this work for additional information regarding copyright ownership.
     4  // The ASF licenses this file to You under the Apache License, Version 2.0
     5  // (the "License"); you may not use this file except in compliance with
     6  // the License.  You may obtain a copy of the License at
     7  //
     8  //    http://www.apache.org/licenses/LICENSE-2.0
     9  //
    10  // Unless required by applicable law or agreed to in writing, software
    11  // distributed under the License is distributed on an "AS IS" BASIS,
    12  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  // See the License for the specific language governing permissions and
    14  // limitations under the License.
    15  
    16  package snippets
    17  
    18  import (
    19  	"flag"
    20  
    21  	"github.com/apache/beam/sdks/v2/go/pkg/beam"
    22  	"github.com/apache/beam/sdks/v2/go/pkg/beam/io/textio"
    23  )
    24  
    25  // PipelineConstruction contains snippets for the initial sections of
    26  // the Beam Programming Guide, from initializing to submitting a
    27  // pipeline.
    28  func PipelineConstruction() {
    29  	// [START pipeline_options]
    30  	// If beamx or Go flags are used, flags must be parsed first,
    31  	// before beam.Init() is called.
    32  	flag.Parse()
    33  	// [END pipeline_options]
    34  
    35  	// [START pipelines_constructing_creating]
    36  	// beam.Init() is an initialization hook that must be called
    37  	// near the beginning of main(), before creating a pipeline.
    38  	beam.Init()
    39  
    40  	// Create the Pipeline object and root scope.
    41  	pipeline, scope := beam.NewPipelineWithRoot()
    42  	// [END pipelines_constructing_creating]
    43  
    44  	// [START pipelines_constructing_reading]
    45  	// Read the file at the URI 'gs://some/inputData.txt' and return
    46  	// the lines as a PCollection<string>.
    47  	// Notice the scope as the first variable when calling
    48  	// the method as is needed when calling all transforms.
    49  	lines := textio.Read(scope, "gs://some/inputData.txt")
    50  
    51  	// [END pipelines_constructing_reading]
    52  
    53  	_ = []any{pipeline, scope, lines}
    54  }
    55  
    56  // Create demonstrates using beam.CreateList.
    57  func Create() {
    58  	// [START model_pcollection]
    59  	lines := []string{
    60  		"To be, or not to be: that is the question: ",
    61  		"Whether 'tis nobler in the mind to suffer ",
    62  		"The slings and arrows of outrageous fortune, ",
    63  		"Or to take arms against a sea of troubles, ",
    64  	}
    65  
    66  	// Create the Pipeline object and root scope.
    67  	// It's conventional to use p as the Pipeline variable and
    68  	// s as the scope variable.
    69  	p, s := beam.NewPipelineWithRoot()
    70  
    71  	// Pass the slice to beam.CreateList, to create the pcollection.
    72  	// The scope variable s is used to add the CreateList transform
    73  	// to the pipeline.
    74  	linesPCol := beam.CreateList(s, lines)
    75  	// [END model_pcollection]
    76  	_ = []any{p, linesPCol}
    77  }
    78  
    79  // PipelineOptions shows basic pipeline options using flags.
    80  func PipelineOptions() {
    81  	// [START pipeline_options_define_custom]
    82  	// Use standard Go flags to define pipeline options.
    83  	var (
    84  		input  = flag.String("input", "", "")
    85  		output = flag.String("output", "", "")
    86  	)
    87  	// [END pipeline_options_define_custom]
    88  
    89  	_ = []any{input, output}
    90  }
    91  
    92  // PipelineOptionsCustom shows slightly less basic pipeline options using flags.
    93  func PipelineOptionsCustom() {
    94  	// [START pipeline_options_define_custom_with_help_and_default]
    95  	var (
    96  		input  = flag.String("input", "gs://my-bucket/input", "Input for the pipeline")
    97  		output = flag.String("output", "gs://my-bucket/output", "Output for the pipeline")
    98  	)
    99  	// [END pipeline_options_define_custom_with_help_and_default]
   100  
   101  	_ = []any{input, output}
   102  }