github.com/treeverse/lakefs@v1.24.1-0.20240520134607-95648127bfb0/examples/hooks/unity_table_export_azure.lua (about)

     1  --[[
     2   As an exhaustive example, it will first start off with a Delta Lake tables export, then continue to register the table
     3   with Unity Catalog
     4  ]]
     5  
     6  local azure = require("azure")
     7  local formats = require("formats")
     8  local databricks = require("databricks")
     9  local delta_exporter = require("lakefs/catalogexport/delta_exporter")
    10  local unity_exporter = require("lakefs/catalogexport/unity_exporter")
    11  
    12  local table_descriptors_path = "_lakefs_tables"
    13  local sc = azure.blob_client(args.azure.storage_account, args.azure.access_key)
    14  local function write_object(_, key, buf)
    15      return sc.put_object(key,buf)
    16  end
    17  local delta_client = formats.delta_client(args.lakefs.access_key_id, args.lakefs.secret_access_key)
    18  local delta_table_details = delta_exporter.export_delta_log(action, args.table_defs, write_object, delta_client, table_descriptors_path, azure.abfss_transform_path)
    19  
    20  -- Register the exported table in Unity Catalog:
    21  local databricks_client = databricks.client(args.databricks_host, args.databricks_token)
    22  local registration_statuses = unity_exporter.register_tables(action, "_lakefs_tables", delta_table_details, databricks_client, args.warehouse_id)
    23  for t, status in pairs(registration_statuses) do
    24      print("Unity catalog registration for table \"" .. t .. "\" completed with status: " .. status .. "\n")
    25  end