github.com/pachyderm/pachyderm@v1.13.4/etc/testing/migration/v1_11/auth.sh (about)

     1  #!/bin/bash -e
     2  #
     3  # auth.sh enables authorization and creates a collection of pachyderm pipelines that look like:
     4  #    left─┐
     5  #         ├─copy─sort
     6  #    right┘
     7  #
     8  # inputs:
     9  # each input file is named for a digit and contains 2-digit numbers ending in
    10  # that digit. E.g. '0' contains '00\n10\n20...', '1' contains '01\n11\n21\n...'
    11  # left:  0,...,4 -> copy -> sort -> 01, 02, 03, 04, 05, ...
    12  # right: 5,...,9
    13  
    14  HERE="$(dirname "${0}")"
    15  
    16  set -x
    17  
    18  pachctl_1_11 enterprise activate "$ENT_ACT_CODE"
    19  pachctl_1_11 auth activate --initial-admin=robot:test
    20  
    21  pachctl_1_11 create repo left
    22  pachctl_1_11 create repo right
    23  
    24  pachctl_1_11 create pipeline -f - <<EOF
    25  {
    26    "pipeline": {
    27      "name": "copy"
    28    },
    29    "transform": {
    30      "cmd": [ "/bin/bash" ],
    31      "stdin": [
    32        "cp /pfs/*/* /pfs/out"
    33      ]
    34    },
    35    "parallelism_spec": {
    36      "constant": 1
    37    },
    38    "input": {
    39      "union": [
    40        { "pfs": { "repo": "left",  "glob": "/*" } },
    41        { "pfs": { "repo": "right", "glob": "/*" } }
    42      ]
    43    },
    44    "enable_stats": true
    45  }
    46  {
    47    "pipeline": {
    48      "name": "sort"
    49    },
    50    "transform": {
    51      "cmd": [ "/bin/bash" ],
    52      "stdin": [
    53        "sort -n /pfs/copy/* >/pfs/out/nums"
    54      ]
    55    },
    56    "parallelism_spec": {
    57      "constant": 1
    58    },
    59    "input": { "pfs": { "repo": "copy", "glob": "/" } },
    60    "enable_stats": true
    61  }
    62  EOF
    63  
    64  tmpfile=$(mktemp -p. || mktemp -t ./)
    65  for _i in $(seq 0 9); do
    66    # roughly alternate between 'left' and 'right' by committing in a funny order
    67    # (this writes the files left/0, right/7, left/4, left/1, right/8, etc...)
    68    i=$(( _i*7 % 10 ))
    69    [[ "${i}" -ge 5 ]] && repo=right || repo=left
    70  
    71    # Clear tmpfile, write all two-digit numbers with ones place=$i to tmpfile
    72    echo -n "" >"${tmpfile}"
    73    for j in $(seq 0 9); do
    74      echo "${j}${i}" >>"${tmpfile}"
    75    done
    76  
    77    # Write to pachd
    78    pachctl_1_11 put file "${repo}"@master:"/${i}" <"${tmpfile}"
    79  done
    80  
    81  # Wait for pipelines to process all commits
    82  pachctl_1_11 flush commit left@master
    83  
    84  # Delete a few commits, as that has caused migration bugs in the past
    85  # TODO(msteffen): Split this test up into tests of distinct bugs (stats,
    86  # delete commit, multiple pipelines)
    87  pachctl_1_11 delete commit left@master~4
    88  pachctl_1_11 delete commit left@master~3
    89  pachctl_1_11 delete commit right@master~4
    90  pachctl_1_11 delete commit right@master~3
    91  
    92  echo "Extracting metadata from Pachyderm. Note that this step occasionally"
    93  echo "fails due to transient encoding issues, and you may need to re-run it"
    94  
    95  set -x
    96  
    97  pachctl_1_11 extract >"${HERE}/auth.metadata"