github.com/pachyderm/pachyderm@v1.13.4/src/server/worker/pipeline/transform/transform.proto (about)

     1  syntax = "proto3";
     2  
     3  package pachyderm.worker.pipeline.transform;
     4  option go_package = "github.com/pachyderm/pachyderm/src/server/worker/pipeline/transform";
     5  
     6  import "gogoproto/gogo.proto";
     7  
     8  import "client/pfs/pfs.proto";
     9  import "client/pps/pps.proto";
    10  import "server/worker/common/common.proto";
    11  
    12  // DatumInputs is the message contained in the object pointed to by the
    13  // DatumData.datums field.
    14  message DatumInputs {
    15    repeated common.Input inputs = 1;
    16    int64 index = 2;
    17  }
    18  
    19  message DatumInputsList {
    20    repeated DatumInputs datums = 1;
    21  }
    22  
    23  // HashtreeObjects is the message contained in the object generated by the
    24  // registry when a job moves from the 'running' state to the 'merging' state. It
    25  // contains references to all generated hashtree chunks from the job, which must
    26  // be merged into the final hashtree.
    27  message HashtreeObjects {
    28    repeated string chunk_objects = 1;
    29    repeated string stats_objects = 2;
    30  }
    31  
    32  message RecoveredDatums {
    33    repeated string hashes = 1;
    34  }
    35  
    36  message RecoveredDatumObjects {
    37    repeated string objects = 1;
    38  }
    39  
    40  message HashtreeInfo {
    41    // Address used for fetching a cached version directly from the worker
    42    string address = 1;
    43  
    44    // The subtask ID can be used to fetch the cached hashtree directly from the worker
    45    string subtask_id = 2 [(gogoproto.customname) = "SubtaskID"];
    46  
    47    // The object can be used to fetch the hashtree from object storage if the
    48    // worker cannot be reached or does not have it cached.
    49    string object = 3;
    50  }
    51  
    52  message DatumStats {
    53    pps.ProcessStats process_stats = 1;
    54    int64 datums_processed = 2;
    55    int64 datums_skipped = 3;
    56    int64 datums_failed = 5;
    57    int64 datums_recovered = 6;
    58    string failed_datum_id = 8 [(gogoproto.customname) = "FailedDatumID"];
    59  }
    60  
    61  message DatumData {
    62    // Inputs
    63    string job_id = 1 [(gogoproto.customname) = "JobID"];
    64    string datums_object = 8;
    65    pfs.Commit output_commit = 3;
    66  
    67    // Outputs
    68    DatumStats stats = 4;
    69    HashtreeInfo chunk_hashtree = 5;
    70    HashtreeInfo stats_hashtree = 6;
    71    string recovered_datums_object = 7;
    72  }
    73  
    74  message MergeData {
    75    // Inputs
    76    string job_id = 1 [(gogoproto.customname) = "JobID"];
    77    repeated HashtreeInfo hashtrees = 2;
    78    pfs.Object parent = 3;
    79    int64 shard = 4;
    80    bool stats = 5;
    81  
    82    // Outputs
    83    pfs.Object tree = 6;
    84    uint64 tree_size = 7;
    85  }