github.com/treeverse/lakefs@v1.24.1-0.20240520134607-95648127bfb0/pkg/actions/testdata/lua/catalogexport_hive_partition_pager.lua (about)

     1  local hive = require("lakefs/catalogexport/hive")
     2  
     3  -- helper function to slice table array
     4  local function slice_dense_array(tbl, first, last, step)
     5      local sliced = {}
     6  
     7      for i = first or 1, last or #tbl, step or 1 do
     8          sliced[#sliced + 1] = tbl[i]
     9      end
    10  
    11      return sliced
    12  end
    13  
    14  -- lakefs mock package
    15  
    16  local lakefs = {
    17      list_objects = function(repo_id, commit_id, next_offset, prefix, delimiter, page_size)
    18          local fs = {
    19              [action.repository_id] = {
    20                  [action.commit_id] = {{
    21                      physical_address = "s3://bucket/a1/b1/b",
    22                      path = "letters/a=1/b=1/b.csv"
    23                  }, {
    24                      physical_address = "s3://bucket/a2/b2/a",
    25                      path = "letters/a=2/b=2/a.csv"
    26                  }, {
    27                      physical_address = "s3://bucket/a2/b2/b",
    28                      path = "letters/a=2/b=2/b.csv"
    29                  }, {
    30                      physical_address = "",
    31                      path = "letters/a=2/b=3/_SUCCESS"
    32                  }, {
    33                      physical_address = "s3://bucket/a2/b3/a",
    34                      path = "letters/a=2/b=3/a.csv"
    35                  }, {
    36                      physical_address = "s3://bucket/a3/b4/a",
    37                      path = "letters/a=3/b=4/a.csv"
    38                  }, {
    39                      physical_address = "s3://bucket/a3/b4/b",
    40                      path = "letters/a=3/b=4/b.csv"
    41                  }}
    42              }
    43          }
    44          local all_entries = fs[repo_id][commit_id]
    45          if next_offset == "" then
    46              next_offset = 1
    47          end
    48          local end_idx = next_offset + page_size
    49          return 200, {
    50              results = slice_dense_array(all_entries, next_offset, end_idx),
    51              pagination = {
    52                  has_more = end_idx < #all_entries,
    53                  next_offset = end_idx + 1
    54              }
    55          }
    56      end
    57  }
    58  
    59  local partitions = {"a", "b"}
    60  local prefix = "letters/"
    61  
    62  for page_size = 1, 10 do
    63      local pager = hive.extract_partition_pager(lakefs, action.repository_id, action.commit_id, prefix, partitions,
    64          page_size)
    65      print("result for page_size " .. tostring(page_size))
    66      for part_key, entries in pager do
    67          print("# partition: " .. part_key)
    68          for _, entry in ipairs(entries) do
    69              print("path: " .. entry.path .. " physical: " .. entry.physical_address)
    70          end
    71      end
    72  end