github.com/apache/beam/sdks/v2@v2.48.2/python/apache_beam/examples/ml-orchestration/kfp/pipeline.json (about)

     1  {
     2    "pipelineSpec": {
     3      "components": {
     4        "comp-ingestion": {
     5          "executorLabel": "exec-ingestion",
     6          "inputDefinitions": {
     7            "parameters": {
     8              "base_artifact_path": {
     9                "type": "STRING"
    10              }
    11            }
    12          },
    13          "outputDefinitions": {
    14            "parameters": {
    15              "ingested_dataset_path": {
    16                "type": "STRING"
    17              }
    18            }
    19          }
    20        },
    21        "comp-model-training": {
    22          "executorLabel": "exec-model-training",
    23          "inputDefinitions": {
    24            "parameters": {
    25              "base_artifact_path": {
    26                "type": "STRING"
    27              },
    28              "preprocessed_dataset_path": {
    29                "type": "STRING"
    30              }
    31            }
    32          },
    33          "outputDefinitions": {
    34            "parameters": {
    35              "trained_model_path": {
    36                "type": "STRING"
    37              }
    38            }
    39          }
    40        },
    41        "comp-preprocessing": {
    42          "executorLabel": "exec-preprocessing",
    43          "inputDefinitions": {
    44            "parameters": {
    45              "base_artifact_path": {
    46                "type": "STRING"
    47              },
    48              "beam_runner": {
    49                "type": "STRING"
    50              },
    51              "dataflow_staging_root": {
    52                "type": "STRING"
    53              },
    54              "gcp_project_id": {
    55                "type": "STRING"
    56              },
    57              "ingested_dataset_path": {
    58                "type": "STRING"
    59              },
    60              "region": {
    61                "type": "STRING"
    62              }
    63            }
    64          },
    65          "outputDefinitions": {
    66            "parameters": {
    67              "preprocessed_dataset_path": {
    68                "type": "STRING"
    69              }
    70            }
    71          }
    72        }
    73      },
    74      "deploymentSpec": {
    75        "executors": {
    76          "exec-ingestion": {
    77            "container": {
    78              "command": [
    79                "python3",
    80                "ingest.py",
    81                "--base-artifact-path",
    82                "{{$.inputs.parameters['base_artifact_path']}}",
    83                "--ingested-dataset-path",
    84                "{{$.outputs.parameters['ingested_dataset_path'].output_file}}"
    85              ],
    86              "image": "<your-docker-registry/ingestion-image-name:latest>"
    87            }
    88          },
    89          "exec-model-training": {
    90            "container": {
    91              "command": [
    92                "python3",
    93                "train.py",
    94                "--preprocessed-dataset-path",
    95                "{{$.inputs.parameters['preprocessed_dataset_path']}}",
    96                "--base-artifact-path",
    97                "{{$.inputs.parameters['base_artifact_path']}}",
    98                "--trained-model-path",
    99                "{{$.outputs.parameters['trained_model_path'].output_file}}"
   100              ],
   101              "image": "<your-docker-registry/train-image-name:latest>"
   102            }
   103          },
   104          "exec-preprocessing": {
   105            "container": {
   106              "command": [
   107                "python3",
   108                "preprocess.py",
   109                "--ingested-dataset-path",
   110                "{{$.inputs.parameters['ingested_dataset_path']}}",
   111                "--base-artifact-path",
   112                "{{$.inputs.parameters['base_artifact_path']}}",
   113                "--preprocessed-dataset-path",
   114                "{{$.outputs.parameters['preprocessed_dataset_path'].output_file}}",
   115                "--gcp-project-id",
   116                "{{$.inputs.parameters['gcp_project_id']}}",
   117                "--region",
   118                "{{$.inputs.parameters['region']}}",
   119                "--dataflow-staging-root",
   120                "{{$.inputs.parameters['dataflow_staging_root']}}",
   121                "--beam-runner",
   122                "{{$.inputs.parameters['beam_runner']}}"
   123              ],
   124              "image": "<your-docker-registry/preprocessing-image-name:latest>"
   125            }
   126          }
   127        }
   128      },
   129      "pipelineInfo": {
   130        "name": "beam-preprocessing-kfp-example"
   131      },
   132      "root": {
   133        "dag": {
   134          "tasks": {
   135            "ingestion": {
   136              "cachingOptions": {
   137                "enableCache": true
   138              },
   139              "componentRef": {
   140                "name": "comp-ingestion"
   141              },
   142              "inputs": {
   143                "parameters": {
   144                  "base_artifact_path": {
   145                    "componentInputParameter": "component_artifact_root"
   146                  }
   147                }
   148              },
   149              "taskInfo": {
   150                "name": "ingestion"
   151              }
   152            },
   153            "model-training": {
   154              "cachingOptions": {
   155                "enableCache": true
   156              },
   157              "componentRef": {
   158                "name": "comp-model-training"
   159              },
   160              "dependentTasks": [
   161                "preprocessing"
   162              ],
   163              "inputs": {
   164                "parameters": {
   165                  "base_artifact_path": {
   166                    "componentInputParameter": "component_artifact_root"
   167                  },
   168                  "preprocessed_dataset_path": {
   169                    "taskOutputParameter": {
   170                      "outputParameterKey": "preprocessed_dataset_path",
   171                      "producerTask": "preprocessing"
   172                    }
   173                  }
   174                }
   175              },
   176              "taskInfo": {
   177                "name": "model-training"
   178              }
   179            },
   180            "preprocessing": {
   181              "cachingOptions": {
   182                "enableCache": true
   183              },
   184              "componentRef": {
   185                "name": "comp-preprocessing"
   186              },
   187              "dependentTasks": [
   188                "ingestion"
   189              ],
   190              "inputs": {
   191                "parameters": {
   192                  "base_artifact_path": {
   193                    "componentInputParameter": "component_artifact_root"
   194                  },
   195                  "beam_runner": {
   196                    "componentInputParameter": "beam_runner"
   197                  },
   198                  "dataflow_staging_root": {
   199                    "componentInputParameter": "dataflow_staging_root"
   200                  },
   201                  "gcp_project_id": {
   202                    "componentInputParameter": "gcp_project_id"
   203                  },
   204                  "ingested_dataset_path": {
   205                    "taskOutputParameter": {
   206                      "outputParameterKey": "ingested_dataset_path",
   207                      "producerTask": "ingestion"
   208                    }
   209                  },
   210                  "region": {
   211                    "componentInputParameter": "region"
   212                  }
   213                }
   214              },
   215              "taskInfo": {
   216                "name": "preprocessing"
   217              }
   218            }
   219          }
   220        },
   221        "inputDefinitions": {
   222          "parameters": {
   223            "beam_runner": {
   224              "type": "STRING"
   225            },
   226            "component_artifact_root": {
   227              "type": "STRING"
   228            },
   229            "dataflow_staging_root": {
   230              "type": "STRING"
   231            },
   232            "gcp_project_id": {
   233              "type": "STRING"
   234            },
   235            "region": {
   236              "type": "STRING"
   237            }
   238          }
   239        }
   240      },
   241      "schemaVersion": "2.0.0",
   242      "sdkVersion": "kfp-1.8.14"
   243    },
   244    "runtimeConfig": {
   245      "gcsOutputDirectory": "gs://test/test"
   246    }
   247  }