github.com/pachyderm/pachyderm@v1.13.4/src/client/admin/v1_9/pps/pps.proto

syntax = "proto3";

package pps_1_9;
option go_package = "github.com/pachyderm/pachyderm/src/client/admin/v1_9/pps";

import "google/protobuf/empty.proto";
import "google/protobuf/timestamp.proto";
import "google/protobuf/duration.proto";

import "gogoproto/gogo.proto";

import "client/admin/v1_9/pfs/pfs.proto";

message Secret {
  // Name must be the name of the secret in kubernetes.
  string name = 1;
  // Key of the secret to load into env_var, this field only has meaning if EnvVar != "".
  string key = 4;
  string mount_path = 2;
  string env_var = 3;
}

message Transform {
  string image = 1;
  repeated string cmd = 2;
  repeated string err_cmd = 13;
  map<string, string> env = 3;
  repeated Secret secrets = 4;
  repeated string image_pull_secrets = 9;
  repeated string stdin = 5;
  repeated string err_stdin = 14;
  repeated int64 accept_return_code = 6;
  bool debug = 7;
  string user = 10;
  string working_dir = 11;
  string dockerfile = 12;
}

message TFJob {
  // tf_job  is a serialized Kubeflow TFJob spec. Pachyderm sends this directly
  // to a kubernetes cluster on which kubeflow has been installed, instead of
  // creating a pipeline ReplicationController as it normally would.
  string tf_job = 1 [(gogoproto.customname) = "TFJob"];
}

message Egress {
  string URL = 1;
}

message Job {
  string id = 1 [(gogoproto.customname) = "ID"];
}

enum JobState {
  JOB_STARTING = 0;
  JOB_RUNNING = 1;
  JOB_FAILURE = 2;
  JOB_SUCCESS = 3;
  JOB_KILLED = 4;
  JOB_MERGING = 5;
}

message Service {
  int32 internal_port = 1;
  int32 external_port = 2;
  string ip = 3 [(gogoproto.customname) = "IP"];
  string type = 4;
  map<string, string> annotations = 5;
}

message Spout {
  bool overwrite = 1;
  Service service = 2;
  string marker = 3;
}

message PFSInput {
  string name = 1;
  string repo = 2;
  string branch = 3;
  string commit = 4;
  string glob = 5;
  string join_on = 8;
  bool lazy = 6;
  // EmptyFiles, if true, will cause files from this PFS input to be
  // presented as empty files. This is useful in shuffle pipelines where you
  // want to read the names of files and reorganize them using symlinks.
  bool empty_files = 7;
}

message CronInput {
  string name = 1;
  string repo = 2;
  string commit = 3;
  string spec = 4;
  // Overwrite, if true, will expose a single datum that gets overwritten each
  // tick. If false, it will create a new datum for each tick.
  bool overwrite = 6;
  google.protobuf.Timestamp start = 5;
}

message GitInput {
  string name = 1;
  string url = 2 [(gogoproto.customname) = "URL"];
  string branch = 3;
  string commit = 4;
}

message Input {
  PFSInput pfs = 6;
  repeated Input join = 7;
  repeated Input cross = 2;
  repeated Input union = 3;
  CronInput cron = 4;
  GitInput git = 5;
}

message JobInput {
  string name = 4;
  pfs_1_9.Commit commit = 1;
  string glob = 2;
  bool lazy = 3;
}

message ParallelismSpec {
  reserved 1;

  // Starts the pipeline/job with a 'constant' workers, unless 'constant' is
  // zero. If 'constant' is zero (which is the zero value of ParallelismSpec),
  // then Pachyderm will choose the number of workers that is started,
  // (currently it chooses the number of workers in the cluster)
  uint64 constant = 2;

  // Starts the pipeline/job with number of workers equal to 'coefficient' * N,
  // where N is the number of nodes in the kubernetes cluster.
  //
  // For example, if each Kubernetes node has four CPUs, you might set
  // 'coefficient' to four, so that there are four Pachyderm workers per
  // Kubernetes node, and each Pachyderm worker gets one CPU. If you want to
  // reserve half the nodes in your cluster for other tasks, you might set
  // 'coefficient' to 0.5.
  double coefficient = 3;
}

// HashTreeSpec sets the number of shards into which pps splits a pipeline's
// output commits (sharded commits are implemented in Pachyderm 1.8+ only)
message HashtreeSpec {
  uint64 constant = 1;
}

message InputFile {
  // This file's absolute path within its pfs repo.
  string path = 4;

  // This file's hash
  bytes hash = 5;
}

message Datum {
  // ID is the hash computed from all the files
  string id = 1 [(gogoproto.customname) = "ID"];
  Job job = 2;
}

enum DatumState {
    FAILED = 0;
    SUCCESS = 1;
    SKIPPED = 2;
    STARTING = 3;
    RECOVERED = 4;
}

message DatumInfo {
  Datum datum = 1;
  DatumState state = 2;
  ProcessStats stats = 3;
  pfs_1_9.File pfs_state = 4;
  repeated pfs_1_9.FileInfo data = 5;
}

message Aggregate {
  int64 count = 1;
  double mean = 2;
  double stddev = 3;
  double fifth_percentile = 4;
  double ninety_fifth_percentile = 5;
}

message ProcessStats {
  google.protobuf.Duration download_time = 1;
  google.protobuf.Duration process_time = 2;
  google.protobuf.Duration upload_time = 3;
  uint64 download_bytes = 4;
  uint64 upload_bytes = 5;
}

message AggregateProcessStats {
  Aggregate download_time = 1;
  Aggregate process_time = 2;
  Aggregate upload_time = 3;
  Aggregate download_bytes = 4;
  Aggregate upload_bytes = 5;
}

message WorkerStatus {
  string worker_id = 1 [(gogoproto.customname) = "WorkerID"];
  string job_id = 2 [(gogoproto.customname) = "JobID"];
  repeated pps_1_9.InputFile data = 3;
  // Started is the time processing on the current datum began.
  google.protobuf.Timestamp started = 4;
  ProcessStats stats = 5;
  int64 queue_size = 6;
}

// ResourceSpec describes the amount of resources that pipeline pods should
// request from kubernetes, for scheduling.
message ResourceSpec {
  reserved 3;

  // The number of CPUs each worker needs (partial values are allowed, and
  // encouraged)
  float cpu = 1;

  // The amount of memory each worker needs (in bytes, with allowed
  // SI suffixes (M, K, G, Mi, Ki, Gi, etc).
  string memory = 2;

  // The spec for GPU resources.
  GPUSpec gpu = 5;

  // The amount of ephemeral storage each worker needs (in bytes, with allowed
  // SI suffixes (M, K, G, Mi, Ki, Gi, etc).
  string disk = 4;
}

message GPUSpec {
  // The type of GPU (nvidia.com/gpu or amd.com/gpu for example).
  string type = 1;
  // The number of GPUs to request.
  int64 number = 2;
}

// EtcdJobInfo is the portion of the JobInfo that gets stored in etcd during
// job execution. It contains fields which change over the lifetime of the job
// but aren't used in the execution of the job.
message EtcdJobInfo {
  Job job = 1;
  Pipeline pipeline = 2;
  pfs_1_9.Commit output_commit = 3;
  // Job restart count (e.g. due to datum failure)
  uint64 restart = 4;

  // Counts of how many times we processed or skipped a datum
  int64 data_processed = 5;
  int64 data_skipped = 6;
  int64 data_total = 7;
  int64 data_failed = 8;
  int64 data_recovered = 15;

  // Download/process/upload time and download/upload bytes
  ProcessStats stats = 9;

  pfs_1_9.Commit stats_commit = 10;
  JobState state = 11;
  string reason = 12;
  google.protobuf.Timestamp started = 13;
  google.protobuf.Timestamp finished = 14;
}

message JobInfo {
  reserved 4, 5, 28, 34;
  Job job = 1;
  Transform transform = 2;                     // requires ListJobRequest.Full
  Pipeline pipeline = 3;
  uint64 pipeline_version = 13;                // requires ListJobRequest.Full
  pfs_1_9.Commit spec_commit = 47;
  ParallelismSpec parallelism_spec = 12;       // requires ListJobRequest.Full
  Egress egress = 15;                          // requires ListJobRequest.Full
  Job parent_job = 6;
  google.protobuf.Timestamp started = 7;
  google.protobuf.Timestamp finished = 8;
  pfs_1_9.Commit output_commit = 9;
  JobState state = 10;
  string reason = 35;  // reason explains why the job is in the current state
  Service service = 14;                        // requires ListJobRequest.Full
  Spout spout = 45;                            // requires ListJobRequest.Full
  pfs_1_9.Repo output_repo = 18;
  string output_branch = 17;                   // requires ListJobRequest.Full
  uint64 restart = 20;
  int64 data_processed = 22;
  int64 data_skipped = 30;
  int64 data_failed = 40;
  int64 data_recovered = 46;
  int64 data_total = 23;
  ProcessStats stats = 31;
  repeated WorkerStatus worker_status = 24;
  ResourceSpec resource_requests = 25;         // requires ListJobRequest.Full
  ResourceSpec resource_limits = 36;           // requires ListJobRequest.Full
  Input input = 26;                            // requires ListJobRequest.Full
  pfs_1_9.BranchInfo new_branch = 27;
  pfs_1_9.Commit stats_commit = 29;
  bool enable_stats = 32;                      // requires ListJobRequest.Full
  string salt = 33;                            // requires ListJobRequest.Full
  ChunkSpec chunk_spec = 37;                   // requires ListJobRequest.Full
  google.protobuf.Duration datum_timeout = 38; // requires ListJobRequest.Full
  google.protobuf.Duration job_timeout = 39;   // requires ListJobRequest.Full
  int64 datum_tries = 41;                      // requires ListJobRequest.Full
  SchedulingSpec scheduling_spec = 42;         // requires ListJobRequest.Full
  string pod_spec = 43;                        // requires ListJobRequest.Full
  string pod_patch = 44;                       // requires ListJobRequest.Full
}

enum WorkerState {
  POD_RUNNING = 0;
  POD_SUCCESS = 1;
  POD_FAILED = 2;
}

message Worker {
  string name = 1;
  WorkerState state = 2;
}

message JobInfos {
  repeated JobInfo job_info = 1;
}

message Pipeline {
  string name = 1;
}

enum PipelineState {
  // There is an EtcdPipelineInfo + spec commit, but no RC
  // This happens when a pipeline has been created but not yet picked up by a
  // PPS server.
  PIPELINE_STARTING = 0;
  // A pipeline has a spec commit and a service + RC
  // This is the normal state of a pipeline.
  PIPELINE_RUNNING = 1;
  // Equivalent to STARTING (there is an EtcdPipelineInfo + commit, but no RC)
  // After some error caused runPipeline to exit, but before the pipeline is
  // re-run. This is when the exponential backoff is in effect.
  PIPELINE_RESTARTING = 2;
  // We have retried too many times and we have given up on this pipeline (or
  // the pipeline image doesn't exist)
  PIPELINE_FAILURE = 3;
  // The pipeline has been explicitly paused by the user (the pipeline spec's
  // Stopped field should be true if the pipeline is in this state)
  PIPELINE_PAUSED = 4;
  // The pipeline is fully functional, but there are no commits to process.
  PIPELINE_STANDBY = 5;
}

// EtcdPipelineInfo is proto that Pachd stores in etcd for each pipeline. It
// tracks the state of the pipeline, and points to its metadata in PFS (and,
// by pointing to a PFS commit, de facto tracks the pipeline's version)
message EtcdPipelineInfo {
  PipelineState state = 1;
  string reason = 4;
  pfs_1_9.Commit spec_commit = 2;
  map<int32, int32> job_counts = 3;
  string auth_token = 5;
  JobState last_job_state = 6;
  uint64 parallelism = 7;
}

message PipelineInfo {
  reserved 3, 4, 22, 26, 27, 18;
  string id = 17 [(gogoproto.customname) = "ID"];
  Pipeline pipeline = 1;
  uint64 version = 11;
  Transform transform = 2;
  // tf_job encodes a Kubeflow TFJob spec. Pachyderm uses this to create TFJobs
  // when running in a kubernetes cluster on which kubeflow has been installed.
  // Exactly one of 'tf_job' and 'transform' should be set
  TFJob tf_job = 46 [(gogoproto.customname) = "TFJob"];
  ParallelismSpec parallelism_spec = 10;
  HashtreeSpec hashtree_spec = 42;
  Egress egress = 15;
  google.protobuf.Timestamp created_at = 6;

  // state indicates the current state of the pipeline. This is not stored in
  // PFS along with the rest of this data structure--PPS.InspectPipeline fills
  // it in
  PipelineState state = 7;
  // same for stopped field
  bool stopped = 38;
  string recent_error = 8;

  // job_counts and last_job_state indicates the number of jobs within this
  // pipeline in a given state and the state of the most recently created job,
  // respectively. This is not stored in PFS along with the rest of this data
  // structure--PPS.InspectPipeline fills it in from the EtcdPipelineInfo.
  map<int32, int32> job_counts = 9;
  JobState last_job_state = 43;

  string output_branch = 16;
  ResourceSpec resource_requests = 19;
  ResourceSpec resource_limits = 31;
  Input input = 20;
  string description = 21;
  string cache_size = 23;
  bool enable_stats = 24;
  string salt = 25;

  // reason includes any error messages associated with a failed pipeline
  string reason = 28;
  int64 max_queue_size = 29;
  Service service = 30;
  Spout spout = 45;
  ChunkSpec chunk_spec = 32;
  google.protobuf.Duration datum_timeout = 33;
  google.protobuf.Duration job_timeout = 34;
  string githook_url = 35 [(gogoproto.customname) = "GithookURL"];
  pfs_1_9.Commit spec_commit = 36;
  bool standby = 37;
  int64 datum_tries = 39;
  SchedulingSpec scheduling_spec = 40;
  string pod_spec = 41;
  string pod_patch = 44;
}

message PipelineInfos {
  repeated PipelineInfo pipeline_info = 1;
}

message CreateJobRequest {
  reserved 3, 4, 1, 10, 7, 9, 8, 12, 11, 13, 14, 21, 15, 16, 17, 18, 19, 20, 22, 23, 24;
  Pipeline pipeline = 2;
  pfs_1_9.Commit output_commit = 25;

  // Fields below should only be set when restoring an extracted job.
  uint64 restart = 26;

  // Counts of how many times we processed or skipped a datum
  int64 data_processed = 27;
  int64 data_skipped = 28;
  int64 data_total = 29;
  int64 data_failed = 30;
  int64 data_recovered = 31;

  // Download/process/upload time and download/upload bytes
  ProcessStats stats = 32;

  pfs_1_9.Commit stats_commit = 33;
  JobState state = 34;
  string reason = 35;
  google.protobuf.Timestamp started = 36;
  google.protobuf.Timestamp finished = 37;
}

message InspectJobRequest {
  // Callers should set either Job or OutputCommit, not both.
  Job job = 1;
  pfs_1_9.Commit output_commit = 3;
  bool block_state = 2; // block until state is either JOB_STATE_FAILURE or JOB_STATE_SUCCESS
}

message ListJobRequest {
  Pipeline pipeline = 1;                // nil means all pipelines
  repeated pfs_1_9.Commit input_commit = 2; // nil means all inputs
  pfs_1_9.Commit output_commit = 3;         // nil means all outputs

  // History indicates return jobs from historical versions of pipelines
  // semantics are:
  // 0: Return jobs from the current version of the pipeline or pipelines.
  // 1: Return the above and jobs from the next most recent version
  // 2: etc.
  //-1: Return jobs from all historical versions.
  int64 history = 4;

  // Full indicates whether the result should include all pipeline details in
  // each JobInfo, or limited information including name and status, but
  // excluding information in the pipeline spec. Leaving this "false" can make
  // the call significantly faster in clusters with a large number of pipelines
  // and jobs.
  // Note that if 'input_commit' is set, this field is coerced to "true"
  bool full = 5;
}

message FlushJobRequest {
  repeated pfs_1_9.Commit commits = 1;
  repeated Pipeline to_pipelines = 2;
}

message DeleteJobRequest {
  Job job = 1;
}

message StopJobRequest {
  Job job = 1;
}

message UpdateJobStateRequest {
  Job job = 1;
  JobState state = 2;
  string reason = 3;
}

message GetLogsRequest {
  reserved 4;
  // The pipeline from which we want to get logs (required if the job in 'job'
  // was created as part of a pipeline. To get logs from a non-orphan job
  // without the pipeline that created it, you need to use ElasticSearch).
  Pipeline pipeline = 2;

  // The job from which we want to get logs.
  Job job = 1;

  // Names of input files from which we want processing logs. This may contain
  // multiple files, to query pipelines that contain multiple inputs. Each
  // filter may be an absolute path of a file within a pps repo, or it may be
  // a hash for that file (to search for files at specific versions)
  repeated string data_filters = 3;

  Datum datum = 6;

  // If true get logs from the master process
  bool master = 5;

  // Continue to follow new logs as they become available.
  bool follow = 7;

  // If nonzero, the number of lines from the end of the logs to return.  Note:
  // tail applies per container, so you will get tail * <number of pods> total
  // lines back.
  int64 tail = 8;
}

// LogMessage is a log line from a PPS worker, annotated with metadata
// indicating when and why the line was logged.
message LogMessage {
  // The job and pipeline for which a PFS file is being processed (if the job
  // is an orphan job, pipeline name and ID will be unset)
  string pipeline_name = 1;
  string job_id = 3 [(gogoproto.customname) = "JobID"];
  string worker_id = 7 [(gogoproto.customname) = "WorkerID"];
  string datum_id = 9 [(gogoproto.customname) = "DatumID"];
  bool master = 10;

  // The PFS files being processed (one per pipeline/job input)
  repeated InputFile data = 4;

  // User is true if log message comes from the users code.
  bool user = 8;

  // The message logged, and the time at which it was logged
  google.protobuf.Timestamp ts = 5;
  string message = 6;
}

message RestartDatumRequest {
  Job job = 1;
  repeated string data_filters = 2;
}

message InspectDatumRequest {
  Datum datum = 1;
}

message ListDatumRequest {
  Job job = 1;
  int64 page_size = 2;
  int64 page = 3;
}

message ListDatumResponse {
  repeated DatumInfo datum_infos = 1;
  int64 total_pages = 2;
  int64 page = 3;
}

// ListDatumStreamResponse is identical to ListDatumResponse, except that only
// one DatumInfo is present (as these responses are streamed)
message ListDatumStreamResponse {
  DatumInfo datum_info = 1;
  // total_pages is only set in the first response (and set to 0 in all other
  // responses)
  int64 total_pages = 2;
  // page is only set in the first response (and set to 0 in all other
  // responses)
  int64 page = 3;
}

// ChunkSpec specifies how a pipeline should chunk its datums.
message ChunkSpec {
  // number, if nonzero, specifies that each chunk should contain `number`
  // datums. Chunks may contain fewer if the total number of datums don't
  // divide evenly.
  int64 number = 1;
  // size_bytes, if nonzero, specifies a target size for each chunk of datums.
  // Chunks may be larger or smaller than size_bytes, but will usually be
  // pretty close to size_bytes in size.
  int64 size_bytes = 2;
}

message SchedulingSpec {
  map<string, string> node_selector = 1;
  string priority_class_name = 2;
}

message CreatePipelineRequest {
  reserved 3, 4, 11, 15, 19;
  Pipeline pipeline = 1;
  // tf_job encodes a Kubeflow TFJob spec. Pachyderm uses this to create TFJobs
  // when running in a kubernetes cluster on which kubeflow has been installed.
  // Exactly one of 'tf_job' and 'transform' should be set
  TFJob tf_job = 35 [(gogoproto.customname) = "TFJob"];
  Transform transform = 2;
  ParallelismSpec parallelism_spec = 7;
  HashtreeSpec hashtree_spec = 31;
  Egress egress = 9;
  bool update = 5;
  string output_branch = 10;
  ResourceSpec resource_requests = 12;
  ResourceSpec resource_limits = 22;
  Input input = 13;
  string description = 14;
  string cache_size = 16;
  bool enable_stats = 17;
  // Reprocess forces the pipeline to reprocess all datums.
  // It only has meaning if Update is true
  bool reprocess = 18;
  int64 max_queue_size = 20;
  Service service = 21;
  Spout spout = 33;
  ChunkSpec chunk_spec = 23;
  google.protobuf.Duration datum_timeout = 24;
  google.protobuf.Duration job_timeout = 25;
  string salt = 26;
  bool standby = 27;
  int64 datum_tries = 28;
  SchedulingSpec scheduling_spec = 29;
  string pod_spec = 30; // deprecated, use pod_patch below
  string pod_patch = 32; // a json patch will be applied to the pipeline's pod_spec before it's created;
  pfs_1_9.Commit spec_commit = 34;
}

message InspectPipelineRequest {
  Pipeline pipeline = 1;
}

message ListPipelineRequest {
  // If non-nil, only return info about a single pipeline, this is redundant
  // with InspectPipeline unless history is non-zero.
  Pipeline pipeline = 1;
  // History indicates how many historical versions you want returned. Its
  // semantics are:
  // 0: Return the current version of the pipeline or pipelines.
  // 1: Return the above and the next most recent version
  // 2: etc.
  //-1: Return all historical versions.
  int64 history = 2;
}

message DeletePipelineRequest {
  reserved 2, 3;
  Pipeline pipeline = 1;
  bool all = 4;
  bool force = 5;
}

message StartPipelineRequest {
  Pipeline pipeline = 1;
}

message StopPipelineRequest {
  Pipeline pipeline = 1;
}

message RunPipelineRequest {
  reserved 3;
  Pipeline pipeline = 1;
  repeated pfs_1_9.CommitProvenance provenance = 2;
  string job_id = 4 [(gogoproto.customname) = "JobID"];
}

message RunCronRequest {
  Pipeline pipeline = 1;
}


message GarbageCollectRequest {
    // Memory is how much memory to use in computing which objects are alive. A
    // larger number will result in more precise garbage collection (at the
    // cost of more memory usage).
    int64 memory_bytes = 1;
}
message GarbageCollectResponse {}

message ActivateAuthRequest {}
message ActivateAuthResponse {}

service API {
  rpc CreateJob(CreateJobRequest) returns (Job) {}
  rpc InspectJob(InspectJobRequest) returns (JobInfo) {}
  // ListJob returns information about current and past Pachyderm jobs. This is
  // deprecated in favor of ListJobStream
  rpc ListJob(ListJobRequest) returns (JobInfos) {}
  // ListJobStream returns information about current and past Pachyderm jobs.
  rpc ListJobStream(ListJobRequest) returns (stream JobInfo) {}
  rpc FlushJob(FlushJobRequest) returns (stream JobInfo) {}
  rpc DeleteJob(DeleteJobRequest) returns (google.protobuf.Empty) {}
  rpc StopJob(StopJobRequest) returns (google.protobuf.Empty) {}
  rpc InspectDatum(InspectDatumRequest) returns (DatumInfo) {}
  // ListDatum returns information about each datum fed to a Pachyderm job. This
  // is deprecated in favor of ListDatumStream
  rpc ListDatum(ListDatumRequest) returns (ListDatumResponse) {}
  // ListDatumStream returns information about each datum fed to a Pachyderm job
  rpc ListDatumStream(ListDatumRequest) returns (stream ListDatumStreamResponse) {}
  rpc RestartDatum(RestartDatumRequest) returns (google.protobuf.Empty) {}

  rpc CreatePipeline(CreatePipelineRequest) returns (google.protobuf.Empty) {}
  rpc InspectPipeline(InspectPipelineRequest) returns (PipelineInfo) {}
  rpc ListPipeline(ListPipelineRequest) returns (PipelineInfos) {}
  rpc DeletePipeline(DeletePipelineRequest) returns (google.protobuf.Empty) {}
  rpc StartPipeline(StartPipelineRequest) returns (google.protobuf.Empty) {}
  rpc StopPipeline(StopPipelineRequest) returns (google.protobuf.Empty) {}
  rpc RunPipeline(RunPipelineRequest) returns (google.protobuf.Empty) {}
  rpc RunCron(RunCronRequest) returns (google.protobuf.Empty) {}

  // DeleteAll deletes everything
  rpc DeleteAll(google.protobuf.Empty) returns (google.protobuf.Empty) {}
  rpc GetLogs(GetLogsRequest) returns (stream LogMessage) {}

  // Garbage collection
  rpc GarbageCollect(GarbageCollectRequest) returns (GarbageCollectResponse) {}

  // An internal call that causes PPS to put itself into an auth-enabled state
  // (all pipeline have tokens, correct permissions, etcd)
  rpc ActivateAuth(ActivateAuthRequest) returns (ActivateAuthResponse) {}

  // An internal call used to move a job from one state to another
  rpc UpdateJobState(UpdateJobStateRequest) returns(google.protobuf.Empty) {}
}