github.com/treeverse/lakefs@v1.24.1-0.20240520134607-95648127bfb0/examples/hooks/unity_table_export.lua (about)

     1  --[[
     2   As an exhaustive example, it will first start off with a Delta Lake tables export, then continue to register the table
     3   with Unity Catalog
     4  ]]
     5  
     6  local aws = require("aws")
     7  local formats = require("formats")
     8  local databricks = require("databricks")
     9  local delta_export = require("lakefs/catalogexport/delta_exporter")
    10  local unity_export = require("lakefs/catalogexport/unity_exporter")
    11  
    12  local sc = aws.s3_client(args.aws.access_key_id, args.aws.secret_access_key, args.aws.region)
    13  
    14  -- Export Delta Lake tables export:
    15  local delta_client = formats.delta_client(args.lakefs.access_key_id, args.lakefs.secret_access_key, args.aws.region)
    16  local delta_table_locations = delta_export.export_delta_log(action, args.table_defs, sc.put_object, delta_client, "_lakefs_tables")
    17  
    18  -- Register the exported table in Unity Catalog:
    19  local databricks_client = databricks.client(args.databricks_host, args.databricks_token)
    20  local registration_statuses = unity_export.register_tables(action, "_lakefs_tables", delta_table_locations, databricks_client, args.warehouse_id)
    21  
    22  for t, status in pairs(registration_statuses) do
    23      print("Unity catalog registration for table \"" .. t .. "\" completed with status: " .. status .. "\n")
    24  end