github.com/pachyderm/pachyderm@v1.13.4/examples/spouts/spout-marker/spout-marker-example.py (about)

     1  #!/usr/bin/env python3
     2  import time
     3  import tarfile
     4  import os
     5  import stat
     6  import io
     7  import argparse
     8  
     9  def open_pipe(path_to_file, attempts=0, timeout=3, sleep_int=5):
    10      if attempts < timeout :
    11          flags = os.O_WRONLY  # Refer to "man 2 open".
    12          mode = stat.S_IWUSR  # This is 0o400.
    13          umask = 0o777 ^ mode  # Prevents always downgrading umask to 0.
    14          umask_original = os.umask(umask)
    15          try:
    16              file = os.open(path_to_file, flags, mode)
    17              # you must open the pipe as binary to prevent line-buffering problems.
    18              return os.fdopen(file, "wb")
    19          except OSError as oe:
    20              print ('{0} attempt of {1}; error opening spout: {2}'.format(attempts + 1, timeout, oe))
    21              os.umask(umask_original)
    22              time.sleep(sleep_int)
    23              return open_pipe(path_to_file, attempts + 1)
    24          finally:
    25              os.umask(umask_original)
    26      return None
    27  
    28  def open_marker(path_to_file, attempts=0, timeout=3, sleep_int=5):
    29      if attempts < timeout :
    30          try:
    31              file = open(path_to_file, "r")
    32              return file.read().split("\n")
    33          except Exception as oe:
    34              print ('{0} attempt of {1}; error opening marker: {2}'.format(attempts + 1, timeout, oe))
    35              time.sleep(sleep_int)
    36              return open_marker(path_to_file, attempts + 1)
    37      print ('no marker file, using empty array')
    38      return []
    39  
    40  def main():
    41      output_character = os.getenv('OUTPUT_CHARACTER', '.')
    42      parser = argparse.ArgumentParser(description='Add a single character every 10 seconds to a file called output.')
    43      parser.add_argument('-c', '--output_character', required=False,
    44                          help="""Character to output.""",
    45                          default=output_character)
    46      args = parser.parse_args()
    47  
    48      # Set the character we're going to output.
    49  
    50      print ('going to output: {0}'.format(args.output_character))
    51          
    52      # Open the named pipe to start the spout's output.
    53      print ('opening output pipe /pfs/out')
    54      spout = open_pipe('/pfs/out')
    55  
    56      # Read in what we've output so far from the marker.
    57      # Each line is a history of the output.
    58      # The last line is the most recent.
    59      print ('opening marker file /pfs/mymark')
    60      lines = open_marker('/pfs/mymark')
    61  
    62      if spout != None:
    63          while True:
    64              lines.append((lines[-1] if len(lines) > 0 else "") + args.output_character)
    65              marker_contents = "\n".join(lines).encode("utf-8")
    66              output_contents=lines[-1].encode("utf-8")
    67              print ('marker contains: {0}'.format(marker_contents))
    68              print ('output contains: {0}'.format(output_contents))
    69              time.sleep(10)
    70              with tarfile.open(fileobj=spout, mode="w|", encoding="utf-8") as tar_stream:
    71                  # This is our marker.  It'll be hidden from the output repo, but
    72                  # will be visible to us in /pfs/mymark when the spout starts.
    73                  tar_info_marker = tarfile.TarInfo("mymark")
    74                  tar_info_marker.size = len(marker_contents)
    75                  tar_info_marker.mode = 0o600
    76                  tar_stream.addfile(tarinfo=tar_info_marker, fileobj=io.BytesIO(marker_contents))
    77                  # This is the current output.  This file is named "output".
    78                  # It will be visible to any pipelines which use this spout as their input.
    79                  # It will contain the last line of contents, above.
    80                  tar_info_output = tarfile.TarInfo("output");
    81                  tar_info_output.size = len(output_contents)
    82                  tar_info_output.mode = 0o600
    83                  tar_stream.addfile(tarinfo=tar_info_output, fileobj=io.BytesIO(output_contents))
    84                  tar_stream.close()
    85          else:
    86              print ("couldn't open /pfs/out")
    87  
    88  if __name__== "__main__":
    89    main()