github.com/apache/beam/sdks/v2@v2.48.2/go/examples/fhirio/import/import.go (about)

     1  // Licensed to the Apache Software Foundation (ASF) under one or more
     2  // contributor license agreements.  See the NOTICE file distributed with
     3  // this work for additional information regarding copyright ownership.
     4  // The ASF licenses this file to You under the Apache License, Version 2.0
     5  // (the "License"); you may not use this file except in compliance with
     6  // the License.  You may obtain a copy of the License at
     7  //
     8  //    http://www.apache.org/licenses/LICENSE-2.0
     9  //
    10  // Unless required by applicable law or agreed to in writing, software
    11  // distributed under the License is distributed on an "AS IS" BASIS,
    12  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  // See the License for the specific language governing permissions and
    14  // limitations under the License.
    15  
    16  // import is a pipeline example using the fhirio connector to bulk import FHIR
    17  // resources from GCS into a given FHIR store.
    18  //
    19  // Pre-requisites:
    20  // 1. NDJSON-encoded FHIR resources stored in GCS.
    21  // 2. Dataflow Runner enabled: https://cloud.google.com/dataflow/docs/quickstarts.
    22  // 3. A Google Cloud FHIR store.
    23  //
    24  // Running this pipeline requires providing a fully qualified GCS address
    25  // (potentially containing wildcards) to where your FHIR resources are stored, a
    26  // path to the FHIR store where the resources should be written to, in addition
    27  // to the usual flags for the Dataflow runner.
    28  //
    29  // An example command for executing this pipeline on GCP is as follows:
    30  //
    31  //	export PROJECT="$(gcloud config get-value project)"
    32  //	export TEMP_LOCATION="gs://MY-BUCKET/temp"
    33  //	export STAGING_LOCATION="gs://MY-BUCKET/staging"
    34  //	export REGION="us-central1"
    35  //	export SOURCE_GCS_LOCATION="gs://MY_BUCKET/path/to/resources/**"
    36  //	export FHIR_STORE_PATH="MY_FHIR_STORE_PATH"
    37  //	cd ./sdks/go
    38  //	go run ./examples/fhirio/import/import.go \
    39  //	  --runner=dataflow \
    40  //	  --temp_location=$TEMP_LOCATION \
    41  //	  --staging_location=$STAGING_LOCATION \
    42  //	  --project=$PROJECT \
    43  //	  --region=$REGION \
    44  //	  --worker_harness_container_image=apache/beam_go_sdk:latest \
    45  //	  --sourceGcsLocation=$SOURCE_GCS_LOCATION \
    46  //	  --fhirStore=$FHIR_STORE_PATH
    47  package main
    48  
    49  import (
    50  	"context"
    51  	"flag"
    52  
    53  	"github.com/apache/beam/sdks/v2/go/pkg/beam"
    54  	"github.com/apache/beam/sdks/v2/go/pkg/beam/io/fhirio"
    55  	"github.com/apache/beam/sdks/v2/go/pkg/beam/io/textio"
    56  	"github.com/apache/beam/sdks/v2/go/pkg/beam/log"
    57  	"github.com/apache/beam/sdks/v2/go/pkg/beam/x/beamx"
    58  )
    59  
    60  var (
    61  	// Required flag with the source directory for GCS files to read, including
    62  	// wildcards. Directory should contain the resources files in NDJSON format.
    63  	sourceGcsLocation = flag.String("sourceGcsLocation", "", "The source directory for GCS files to read, including wildcards.")
    64  
    65  	// Required flag with target FHIR store to write data to, must be of the full format:
    66  	// "projects/project_id/locations/location/datasets/DATASET_ID/fhirStores/FHIR_STORE_ID"
    67  	fhirStore = flag.String("fhirStore", "", "The target FHIR Store to write data to, must be of the full format.")
    68  )
    69  
    70  func main() {
    71  	flag.Parse()
    72  	beam.Init()
    73  
    74  	p, s := beam.NewPipelineWithRoot()
    75  
    76  	// Read resources from GCS.
    77  	resourcesInGcs := textio.Read(s, *sourceGcsLocation)
    78  
    79  	// Import the read resources to the provided FHIR store.
    80  	fhirio.Import(s, *fhirStore, "", "", fhirio.ContentStructureResource, resourcesInGcs)
    81  
    82  	ctx := context.Background()
    83  	if err := beamx.Run(ctx, p); err != nil {
    84  		log.Fatalf(ctx, "Failed to execute job: %v", err)
    85  	}
    86  }