github.com/pachyderm/pachyderm@v1.13.4/examples/shuffle/shuffle.json (about)

     1  {
     2    "input": {
     3      "union": [
     4        {
     5          "pfs": {
     6            "glob": "/*.jpeg",
     7            "repo": "fruits",
     8            "empty_files": true
     9          }
    10        },
    11        {
    12          "pfs": {
    13            "glob": "/*.json",
    14            "repo": "pricing",
    15            "empty_files": true
    16          }
    17        }
    18      ]
    19    },
    20    "pipeline": {
    21      "name": "shuffle"
    22    },
    23    "description": "A pipeline that places files with a matching name pattern and stored in separate repositories to a corresponding folder in the output repository.",
    24    "transform": {
    25      "image": "ubuntu",
    26      "cmd": ["/bin/bash"],
    27      "stdin": [
    28        "echo 'process fruits if any'",
    29        "fn=$(find  -L /pfs -not -path \"*/\\.*\"  -type f \\( -path '*/fruits/*' \\))",
    30        "for f in $fn; do fruit_name=$(basename $f .jpeg); mkdir -p /pfs/out/$fruit_name/; ln -s $f /pfs/out/$fruit_name/img.jpeg; done",
    31        "echo 'process pricing if any'",
    32        "fn=$(find  -L /pfs -not -path \"*/\\.*\"  -type f \\( -path '*/pricing/*' \\))",
    33        "for f in $fn; do fruit_name=$(basename $f .json); mkdir -p /pfs/out/$fruit_name/; ln -s $f /pfs/out/$fruit_name/cost.json; done"
    34      ]
    35    }
    36  }