github.com/apache/beam/sdks/v2@v2.48.2/python/apache_beam/runners/job/manager.py (about)

     1  #
     2  # Licensed to the Apache Software Foundation (ASF) under one or more
     3  # contributor license agreements.  See the NOTICE file distributed with
     4  # this work for additional information regarding copyright ownership.
     5  # The ASF licenses this file to You under the Apache License, Version 2.0
     6  # (the "License"); you may not use this file except in compliance with
     7  # the License.  You may obtain a copy of the License at
     8  #
     9  #    http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  # See the License for the specific language governing permissions and
    15  # limitations under the License.
    16  #
    17  
    18  """A object to control to the Job API Co-Process
    19  """
    20  
    21  # pytype: skip-file
    22  
    23  import logging
    24  import subprocess
    25  import time
    26  
    27  import grpc
    28  
    29  from apache_beam.portability.api import beam_job_api_pb2_grpc
    30  
    31  _LOGGER = logging.getLogger(__name__)
    32  
    33  
    34  class DockerRPCManager(object):
    35    """A native co-process to start a contianer that speaks the JobApi
    36    """
    37    def __init__(self, run_command=None):
    38      # TODO(BEAM-2431): Change this to a docker container from a command.
    39      self.process = subprocess.Popen([
    40          'python',
    41          '-m',
    42          'apache_beam.runners.experimental.python_rpc_direct.server'
    43      ])
    44  
    45      self.channel = grpc.insecure_channel('localhost:50051')
    46      self.service = beam_job_api_pb2_grpc.JobServiceStub(self.channel)
    47  
    48      # Sleep for 2 seconds for process to start completely
    49      # This is just for the co-process and would be removed
    50      # once we migrate to docker.
    51      time.sleep(2)
    52  
    53    def __del__(self):
    54      """Terminate the co-process when the manager is GC'ed
    55      """
    56      _LOGGER.info('Shutting the co-process')
    57      self.process.terminate()