github.com/pachyderm/pachyderm@v1.13.4/etc/testing/migration/v1_7/diagonal.sh (about)

     1  #!/bin/bash
     2  # diagonal.sh creates a contrived collection of pachyderm pipelines that look
     3  # like:
     4  #    left─┬─filter-left─┐
     5  #         │             ├─join
     6  #    right┴─filter-right┘
     7  #
     8  # inputs:
     9  # left:  0,1,2,... -> 0/00,0/01,0/02,... -> 01, 02, 03, 04, 05, ...
    10  # right: 0,1,2,...    0/00,0/10,0/20,...
    11  #
    12  # Due to https://github.com/pachyderm/pachyderm/issues/3337, restoring this
    13  # cluster doesn't yet work (the cross input causes the restored pipeline to
    14  # process commits with no matching datums)
    15  
    16  HERE="$(dirname "${0}")"
    17  # shellcheck source=./etc/testing/migration/v1_7/deploy.sh
    18  source "${HERE}/deploy.sh"
    19  
    20  set -x
    21  
    22  pachctl_1_7 create-repo left
    23  pachctl_1_7 create-repo right
    24  
    25  pachctl_1_7 create-pipeline -f - <<EOF
    26  {
    27    "pipeline": {
    28      "name": "filter-left"
    29    },
    30    "transform": {
    31      "cmd": [ "/bin/bash" ],
    32      "stdin": [
    33        "left=\$(ls /pfs/left)",
    34        "right=\$(ls /pfs/right)",
    35        "mkdir /pfs/out/\${left}",
    36        "touch /pfs/out/\${left}/\${left}\${right}"
    37      ]
    38    },
    39    "parallelism_spec": {
    40      "constant": 1
    41    },
    42    "input": {
    43      "cross": [
    44        {
    45          "atom": {
    46            "repo": "left",
    47            "glob": "/*"
    48          }
    49        },
    50        {
    51          "atom": {
    52            "repo": "right",
    53            "glob": "/*"
    54          }
    55        }
    56      ]
    57    },
    58    "enable_stats": true
    59  }
    60  {
    61    "pipeline": {
    62      "name": "filter-right"
    63    },
    64    "transform": {
    65      "cmd": [ "/bin/bash" ],
    66      "stdin": [
    67        "left=\$(ls /pfs/left)",
    68        "right=\$(ls /pfs/right)",
    69        "mkdir /pfs/out/\${right}",
    70        "touch /pfs/out/\${right}/\${left}\${right}"
    71      ]
    72    },
    73    "parallelism_spec": {
    74      "constant": 1
    75    },
    76    "input": {
    77      "cross": [
    78        {
    79          "atom": {
    80            "repo": "left",
    81            "glob": "/*"
    82          }
    83        },
    84        {
    85          "atom": {
    86            "repo": "right",
    87            "glob": "/*"
    88          }
    89        }
    90      ]
    91    },
    92    "enable_stats": true
    93  }
    94  {
    95    "pipeline": {
    96      "name": "join"
    97    },
    98    "transform": {
    99      "cmd": [ "/bin/bash" ],
   100      "stdin": [
   101        "find -L /pfs/left /pfs/right -type f -printf '%f\\\\n' \\\\",
   102        "| sort \\\\",
   103        "| uniq --repeated \\\\",
   104        "| xargs -I{} touch /pfs/out/{}"
   105      ]
   106    },
   107    "parallelism_spec": {
   108      "constant": 1
   109    },
   110    "input": {
   111      "cross": [
   112        {
   113          "atom": {
   114            "name": "left",
   115            "repo": "filter-left",
   116            "glob": "/*"
   117          }
   118        },
   119        {
   120          "atom": {
   121            "name": "right",
   122            "repo": "filter-right",
   123            "glob": "/*"
   124          }
   125        }
   126      ]
   127    },
   128    "enable_stats": true
   129  }
   130  EOF
   131  
   132  for repo in left right; do
   133    for c in $(seq 0 9); do
   134      pachctl_1_7 put-file "${repo}" master "/${c}" </dev/null
   135    done
   136  done
   137  
   138  pachctl_1_7 flush-commit left/master
   139  
   140  # Delete a few commits, as that has caused migration bugs in the past
   141  # TODO(msteffen): Split this test up into tests of distinct bugs (stats,
   142  # delete-commit, multiple pipelines)
   143  pachctl_1_7 delete-commit left master~9
   144  pachctl_1_7 delete-commit left master~8
   145  pachctl_1_7 delete-commit left master~7
   146  pachctl_1_7 delete-commit right master~9
   147  pachctl_1_7 delete-commit right master~8
   148  pachctl_1_7 delete-commit right master~7
   149  
   150  pachctl_1_7 extract >"${HERE}/diagonal.dump"