github.com/treeverse/lakefs@v1.24.1-0.20240520134607-95648127bfb0/examples/hooks/format_validator.lua (about)

     1  --[[
     2  Format Validator
     3  
     4  Args:
     5   - ignore_hidden (boolean): whether or not to disregard objects that are hidden (begin with a "_" or part of a directory that begins with a "_")
     6   - allow_list (list of strings): allowed suffixes. For example, ["parquet", "orc"]
     7  
     8  Example hook declaration: (_lakefs_actions/pre-merge-format-validation.yaml):
     9  
    10  name: pre merge format check on main
    11  on:
    12  pre-merge:
    13    branches:
    14      - main
    15  hooks:
    16    - id: check_formats
    17      type: lua
    18      properties:
    19      script_path: scripts/format_validator.lua # location of this script in the repository!
    20      args:
    21          allow_list: ["parquet", "orc", "log"]
    22          ignore_hidden:  true
    23  ]]
    24  
    25  lakefs = require("lakefs")
    26  strings = require("strings")
    27  
    28  
    29  forbidden_paths = {}
    30  has_more = true
    31  after = ""
    32  while has_more do
    33    local code, resp = lakefs.diff_refs(action.repository_id, action.branch_id, action.source_ref, after)
    34    if code ~= 200 then
    35      error("could not diff: " .. resp.message)
    36    end
    37  
    38    for _, result in pairs(resp.results) do
    39      if result.path_type == "object" and result.type == "added" then
    40          should_check = true
    41          valid = false
    42          path_parts = strings.split(result.path, "/")
    43          base_name = path_parts[#path_parts]
    44  
    45          -- hidden in this case is any file that begins with "_"
    46          -- or that belongs to a directory that begins with foo
    47          if args.ignore_hidden then
    48              for _, path_part in ipairs(path_parts) do
    49                  if strings.has_prefix(path_part, "_") then
    50                      should_check = false
    51                      break
    52                  end
    53              end
    54          end
    55  
    56          -- check if this file is allowed
    57          if should_check then
    58              for _, ext in ipairs(args.allow_list) do
    59                  if strings.has_suffix(base_name, ext) then
    60                      valid = true
    61                      break
    62                  end
    63              end
    64              if not valid then
    65                  table.insert(forbidden_paths, result.path)
    66              end
    67          end
    68      end
    69    end
    70  
    71    -- pagination
    72    has_more = resp.pagination.has_more
    73    after = resp.pagination.next_offset
    74  end
    75  
    76  if #forbidden_paths > 0 then
    77      print("Found forbidden paths:")
    78      for _, p in ipairs(forbidden_paths) do
    79          print(p)
    80      end
    81      error("forbidden paths found")
    82  end