github.com/apache/beam/sdks/v2@v2.48.2/go/examples/snippets/08windowing.go (about)

     1  // Licensed to the Apache Software Foundation (ASF) under one or more
     2  // contributor license agreements.  See the NOTICE file distributed with
     3  // this work for additional information regarding copyright ownership.
     4  // The ASF licenses this file to You under the Apache License, Version 2.0
     5  // (the "License"); you may not use this file except in compliance with
     6  // the License.  You may obtain a copy of the License at
     7  //
     8  //    http://www.apache.org/licenses/LICENSE-2.0
     9  //
    10  // Unless required by applicable law or agreed to in writing, software
    11  // distributed under the License is distributed on an "AS IS" BASIS,
    12  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  // See the License for the specific language governing permissions and
    14  // limitations under the License.
    15  
    16  package snippets
    17  
    18  import (
    19  	"time"
    20  
    21  	"github.com/apache/beam/sdks/v2/go/pkg/beam"
    22  	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/mtime"
    23  	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/window"
    24  )
    25  
    26  func settingWindows(s beam.Scope, items beam.PCollection) {
    27  	// [START setting_fixed_windows]
    28  	fixedWindowedItems := beam.WindowInto(s,
    29  		window.NewFixedWindows(60*time.Second),
    30  		items)
    31  	// [END setting_fixed_windows]
    32  
    33  	// [START setting_sliding_windows]
    34  	slidingWindowedItems := beam.WindowInto(s,
    35  		window.NewSlidingWindows(5*time.Second, 30*time.Second),
    36  		items)
    37  	// [END setting_sliding_windows]
    38  
    39  	// [START setting_session_windows]
    40  	sessionWindowedItems := beam.WindowInto(s,
    41  		window.NewSessions(600*time.Second),
    42  		items)
    43  	// [END setting_session_windows]
    44  
    45  	// [START setting_global_window]
    46  	globalWindowedItems := beam.WindowInto(s,
    47  		window.NewGlobalWindows(),
    48  		items)
    49  	// [END setting_global_window]
    50  
    51  	// [START setting_allowed_lateness]
    52  	windowedItems := beam.WindowInto(s,
    53  		window.NewFixedWindows(1*time.Minute), items,
    54  		beam.AllowedLateness(2*24*time.Hour), // 2 days
    55  	)
    56  	// [END setting_allowed_lateness]
    57  
    58  	_ = []beam.PCollection{
    59  		fixedWindowedItems,
    60  		slidingWindowedItems,
    61  		sessionWindowedItems,
    62  		globalWindowedItems,
    63  		windowedItems,
    64  	}
    65  }
    66  
    67  // LogEntry is a dummy type for documentation purposes.
    68  type LogEntry int
    69  
    70  func extractEventTime(LogEntry) time.Time {
    71  	// Note: Returning time.Now() is always going to be processing time
    72  	// not EventTime. For true event time, one needs to extract the
    73  	// time from the element itself.
    74  	return time.Now()
    75  }
    76  
    77  // [START setting_timestamp]
    78  
    79  // AddTimestampDoFn extracts an event time from a LogEntry.
    80  func AddTimestampDoFn(element LogEntry, emit func(beam.EventTime, LogEntry)) {
    81  	et := extractEventTime(element)
    82  	// Defining an emitter with beam.EventTime as the first parameter
    83  	// allows the DoFn to set the event time for the emitted element.
    84  	emit(mtime.FromTime(et), element)
    85  }
    86  
    87  // [END setting_timestamp]
    88  
    89  func timestampedCollection(s beam.Scope, unstampedLogs beam.PCollection) {
    90  	// [START setting_timestamp_pipeline]
    91  	stampedLogs := beam.ParDo(s, AddTimestampDoFn, unstampedLogs)
    92  	// [END setting_timestamp_pipeline]
    93  	_ = stampedLogs
    94  }