github.com/treeverse/lakefs@v1.24.1-0.20240520134607-95648127bfb0/examples/hooks/unity_table_export_azure.lua (about) 1 --[[ 2 As an exhaustive example, it will first start off with a Delta Lake tables export, then continue to register the table 3 with Unity Catalog 4 ]] 5 6 local azure = require("azure") 7 local formats = require("formats") 8 local databricks = require("databricks") 9 local delta_exporter = require("lakefs/catalogexport/delta_exporter") 10 local unity_exporter = require("lakefs/catalogexport/unity_exporter") 11 12 local table_descriptors_path = "_lakefs_tables" 13 local sc = azure.blob_client(args.azure.storage_account, args.azure.access_key) 14 local function write_object(_, key, buf) 15 return sc.put_object(key,buf) 16 end 17 local delta_client = formats.delta_client(args.lakefs.access_key_id, args.lakefs.secret_access_key) 18 local delta_table_details = delta_exporter.export_delta_log(action, args.table_defs, write_object, delta_client, table_descriptors_path, azure.abfss_transform_path) 19 20 -- Register the exported table in Unity Catalog: 21 local databricks_client = databricks.client(args.databricks_host, args.databricks_token) 22 local registration_statuses = unity_exporter.register_tables(action, "_lakefs_tables", delta_table_details, databricks_client, args.warehouse_id) 23 for t, status in pairs(registration_statuses) do 24 print("Unity catalog registration for table \"" .. t .. "\" completed with status: " .. status .. "\n") 25 end