github.com/treeverse/lakefs@v1.24.1-0.20240520134607-95648127bfb0/pkg/actions/lua/lakefs/catalogexport/internal.lua (about)

     1  local url = require("net/url")
     2  local pathlib = require("path")
     3  local DEFAULT_SHORT_DIGEST_LEN=6
     4  
     5  local function deepcopy(orig)
     6      local orig_type = type(orig)
     7      local copy
     8      if orig_type == 'table' then
     9          copy = {}
    10          for orig_key, orig_value in next, orig, nil do
    11              copy[deepcopy(orig_key)] = deepcopy(orig_value)
    12          end
    13          setmetatable(copy, deepcopy(getmetatable(orig)))
    14      else -- number, string, boolean, etc
    15          copy = orig
    16      end
    17      return copy
    18  end
    19  
    20  local function short_digest(digest, len)
    21      return digest:sub(1, len or DEFAULT_SHORT_DIGEST_LEN)
    22  end
    23  
    24  -- paginate lakefs api
    25  local function lakefs_paginiated_api(api_call, after)
    26      local next_offset = after
    27      local has_more = true
    28      return function()
    29          if not has_more then
    30              return nil
    31          end
    32          local code, resp = api_call(next_offset)
    33          if code < 200 or code >= 300 then
    34              error("lakeFS: api return non-2xx" .. tostring(code))
    35          end
    36          has_more = resp.pagination.has_more
    37          next_offset = resp.pagination.next_offset
    38          return resp.results
    39      end
    40  end
    41  
    42  -- paginage over lakefs objects
    43  local function lakefs_object_pager(lakefs_client, repo_id, commit_id, after, prefix, delimiter, page_size)
    44      return lakefs_paginiated_api(function(next_offset)
    45          return lakefs_client.list_objects(repo_id, commit_id, next_offset, prefix, delimiter, page_size or 30)
    46      end, after)
    47  end
    48  
    49  -- resolve ref value from action global, used as part of setting default table name
    50  local function ref_from_branch_or_tag(action_info)
    51      local event = action_info.event_type
    52      if event == "pre-create-tag" or event == "post-create-tag" then
    53          return action_info.tag_id
    54      elseif event == "pre-create-branch" or event == "post-create-branch" or "post-commit" or "post-merge" then
    55          return action_info.branch_id
    56      else
    57          error("unsupported event type: " .. action_info.event_type)
    58      end
    59  end
    60  
    61  local function parse_storage_uri(uri)
    62      local u = url.parse(uri)
    63      return {
    64          protocol = u.scheme,
    65          bucket = u.host,
    66          key = (u.path:sub(0, 1) == "/") and u.path:sub(2) or u.path,
    67      }
    68  end
    69  
    70  local function get_storage_uri_prefix(storage_ns, commit_id, action_info)
    71      local branch_or_tag = ref_from_branch_or_tag(action_info)
    72      local sha = short_digest(commit_id)
    73      return pathlib.join("/", storage_ns, string.format("_lakefs/exported/%s/%s/", branch_or_tag, sha))
    74  end
    75  
    76  local function sortedKeys(query, sortFunction)
    77      local keys, len = {}, 0
    78      for k,_ in pairs(query) do
    79          len = len + 1
    80          keys[len] = k
    81      end
    82      table.sort(keys, sortFunction)
    83  
    84      return keys
    85  end
    86  
    87  return {
    88      deepcopy=deepcopy,
    89      parse_storage_uri=parse_storage_uri,
    90      short_digest=short_digest,
    91      ref_from_branch_or_tag=ref_from_branch_or_tag,
    92      lakefs_object_pager=lakefs_object_pager,
    93      lakefs_paginiated_api=lakefs_paginiated_api,
    94      sortedKeys = sortedKeys,
    95      get_storage_uri_prefix = get_storage_uri_prefix,
    96  }