github.com/chainreactors/fingers@v1.2.1/resources/fingerprinthub_v4.py (about)

     1  #!/usr/bin/env python3
     2  """
     3  合并 FingerprintHub 的 web 和 service 指纹到 JSON 文件并压缩
     4  """
     5  import os
     6  import yaml
     7  import json
     8  import gzip
     9  from pathlib import Path
    10  
    11  def merge_fingerprints(source_dir, output_file, fingerprint_type="web"):
    12      """
    13      将所有 yaml 文件合并到一个 JSON 文件
    14  
    15      Args:
    16          source_dir: 指纹目录路径
    17          output_file: 输出的 JSON 文件路径
    18          fingerprint_type: 指纹类型 (web 或 service)
    19      """
    20      fingerprints = []
    21      loaded_count = 0
    22      failed_count = 0
    23      errors = []
    24  
    25      print(f"\n{'='*60}")
    26      print(f"Processing {fingerprint_type.upper()} fingerprints")
    27      print(f"{'='*60}")
    28      print(f"Scanning directory: {source_dir}")
    29  
    30      # 遍历所有 yaml 文件
    31      yaml_files = list(Path(source_dir).rglob("*.yaml")) + list(Path(source_dir).rglob("*.yml"))
    32      total = len(yaml_files)
    33  
    34      print(f"Found {total} yaml files")
    35      print("Loading...")
    36  
    37      for i, yaml_file in enumerate(yaml_files):
    38          try:
    39              with open(yaml_file, 'r', encoding='utf-8') as f:
    40                  data = yaml.safe_load(f)
    41                  if data:
    42                      # 添加文件路径信息用于调试
    43                      data['_source_file'] = str(yaml_file.relative_to(source_dir))
    44                      fingerprints.append(data)
    45                      loaded_count += 1
    46          except Exception as e:
    47              failed_count += 1
    48              if len(errors) < 10:
    49                  errors.append(f"{yaml_file.name}: {str(e)[:50]}")
    50  
    51          # 显示进度
    52          if (i + 1) % 100 == 0 or (i + 1) == total:
    53              print(f"Progress: {i+1}/{total} ({(i+1)/total*100:.1f}%)")
    54  
    55      print(f"\nLoaded: {loaded_count}")
    56      print(f"Failed: {failed_count}")
    57  
    58      if errors:
    59          print(f"\nFirst {len(errors)} errors:")
    60          for err in errors:
    61              print(f"  - {err}")
    62  
    63      # 保存为 JSON
    64      json_file = output_file.replace('.gz', '')
    65      print(f"\nSaving to {json_file}...")
    66      with open(json_file, 'w', encoding='utf-8') as f:
    67          json.dump(fingerprints, f, ensure_ascii=False)
    68  
    69      # 压缩为 gzip
    70      print(f"Compressing to {output_file}...")
    71      with open(json_file, 'rb') as f_in:
    72          with gzip.open(output_file, 'wb', compresslevel=9) as f_out:
    73              f_out.writelines(f_in)
    74  
    75      # 删除未压缩的 JSON 文件
    76      os.remove(json_file)
    77  
    78      # 统计信息
    79      file_size = os.path.getsize(output_file)
    80      print(f"Done! Saved {loaded_count} fingerprints")
    81      print(f"Compressed file size: {file_size / 1024:.2f} KB ({file_size / 1024 / 1024:.2f} MB)")
    82  
    83      return loaded_count, failed_count
    84  
    85  def process_fingerprinthub(base_dir, output_dir):
    86      """
    87      处理 FingerprintHub 的 web 和 service 指纹
    88  
    89      Args:
    90          base_dir: FingerprintHub 根目录
    91          output_dir: 输出目录
    92      """
    93      base_path = Path(base_dir)
    94      output_path = Path(output_dir)
    95  
    96      # 确保输出目录存在
    97      output_path.mkdir(parents=True, exist_ok=True)
    98  
    99      results = {}
   100  
   101      # 处理 web-fingerprint
   102      web_dir = base_path / "web-fingerprint"
   103      if web_dir.exists():
   104          web_output = output_path / "fingerprinthub_web.json.gz"
   105          web_count, web_failed = merge_fingerprints(web_dir, str(web_output), "web")
   106          results['web'] = {'count': web_count, 'failed': web_failed}
   107      else:
   108          print(f"\n⚠️  Web fingerprint directory not found: {web_dir}")
   109  
   110      # 处理 service-fingerprint
   111      service_dir = base_path / "service-fingerprint"
   112      if service_dir.exists():
   113          service_output = output_path / "fingerprinthub_service.json.gz"
   114          service_count, service_failed = merge_fingerprints(service_dir, str(service_output), "service")
   115          results['service'] = {'count': service_count, 'failed': service_failed}
   116      else:
   117          print(f"\n⚠️  Service fingerprint directory not found: {service_dir}")
   118  
   119      # 打印总结
   120      print(f"\n{'='*60}")
   121      print("SUMMARY")
   122      print(f"{'='*60}")
   123      if 'web' in results:
   124          print(f"Web fingerprints:     {results['web']['count']} loaded, {results['web']['failed']} failed")
   125      if 'service' in results:
   126          print(f"Service fingerprints: {results['service']['count']} loaded, {results['service']['failed']} failed")
   127      print(f"{'='*60}\n")
   128  
   129      return results
   130  
   131  if __name__ == "__main__":
   132      import sys
   133  
   134      if len(sys.argv) < 2:
   135          print("Usage: python fingerprinthub_v4.py <FingerprintHub-dir> [output-dir]")
   136          print()
   137          print("Example:")
   138          print("  python fingerprinthub_v4.py ../refer/FingerprintHub .")
   139          print()
   140          print("This will process both web-fingerprint and service-fingerprint directories")
   141          print("and generate compressed JSON files.")
   142          sys.exit(1)
   143  
   144      base_dir = sys.argv[1]
   145      output_dir = sys.argv[2] if len(sys.argv) > 2 else "."
   146  
   147      if not os.path.exists(base_dir):
   148          print(f"Error: Directory not found: {base_dir}")
   149          sys.exit(1)
   150  
   151      process_fingerprinthub(base_dir, output_dir)