github.com/treeverse/lakefs@v1.24.1-0.20240520134607-95648127bfb0/examples/hooks/delta_lake_S3_export.lua (about)

     1  --[[
     2      args:
     3          - table_defs (e.g. ["table1.yaml", "table2", ...])
     4          - lakefs.access_key_id
     5          - lakefs.secret_access_key
     6          - aws.access_key_id
     7          - aws.secret_access_key
     8          - aws.region
     9          - region
    10  ]]
    11  local aws = require("aws")
    12  local formats = require("formats")
    13  local delta_export = require("lakefs/catalogexport/delta_exporter")
    14  local json = require("encoding/json")
    15  
    16  local table_descriptors_path = "_lakefs_tables"
    17  local sc = aws.s3_client(args.aws.access_key_id, args.aws.secret_access_key, args.aws.region)
    18  
    19  local delta_client = formats.delta_client(args.lakefs.access_key_id, args.lakefs.secret_access_key, args.aws.region)
    20  local delta_table_details = delta_export.export_delta_log(action, args.table_defs, sc.put_object, delta_client, table_descriptors_path)
    21  for t, details in pairs(delta_table_details) do
    22      print("Delta Lake exported table \"" .. t .. "\"'s location: " .. details["path"] .. "\n")
    23      print("Delta Lake exported table \"" .. t .. "\"'s metadata:\n")
    24      for k, v in pairs(details["metadata"]) do
    25          if type(v) == "table" then
    26              print("\t" .. k .. " = " .. json.marshal(v) .. "\n")
    27          else
    28              print("\t" .. k .. " = " .. v .. "\n")
    29          end
    30      end
    31  end