feat(etl): stable pipeline, exception registry in SQLite, multi-pass aggregation and db_cli

2026-08-27 19:26:28 +03:00
parent 66087d5806
commit a9680db0aa
77 changed files with 12548 additions and 5625 deletions
@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+"""
+===============================================================================
+FILE: scripts/diagnostics/make_etl_snapshot.py
+ROLE: Генерация компактного слепка ETL-конвейера, БД и сервисов СКУД.
+===============================================================================
+"""
+
+import os
+
+ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+OUTPUT_FILE = os.path.join(ROOT_DIR, "etl_code_snapshot.md")
+
+# Строгий целевой список файлов для ETL-слепка
+TARGET_FILES = [
+    "config.py",
+    "exceptions.json",
+    "main_etl.py",
+    "run_cron_etl.sh",
+    "scripts/db_cli.py",  # ⭐️ Гарантированно включен
+    "core/connection.py",
+    "core/database.py",
+    "core/schema.py",
+    "core/repositories/scud_repo.py",
+    "core/repositories/zup_repo.py",
+    "services/data_loader.py",
+    "services/scud_export.py",
+    "services/share_copier.py",
+    "services/excel_exporter.py",
+    "services/text_reporter.py",
+    "services/exceptions_repo.py",
+    "services/zup_extractor.py",
+    "services/ai_verifier.py",
+    "services/knowledge_base.py",
+    "services/knowledge/service.py",
+    "services/scud_etl/pipeline.py",
+    "services/scud_etl/merger.py",
+    "services/scud_etl/anomaly_detector.py",
+    "services/snapshots/service.py",
+    "services/tasks/repository.py",
+    "services/tasks/service.py"
+]
+
+
+def create_etl_snapshot():
+    content = ["# 📦 ETL-СЛЕПОК ИСХОДНОГО КОДА (СКУД ⟷ 1С & DB CORE)\n"]
+    included_count = 0
+
+    for rel_path in TARGET_FILES:
+        full_path = os.path.join(ROOT_DIR, rel_path)
+        if os.path.exists(full_path):
+            ext = os.path.splitext(rel_path)[1].replace(".", "")
+            lang = "py" if ext == "py" else ("json" if ext == "json" else "bash")
+            try:
+                with open(full_path, "r", encoding="utf-8") as f:
+                    file_text = f.read()
+                content.append(f"## File: `./{rel_path}`\n```{lang}\n{file_text}\n```\n")
+                included_count += 1
+            except Exception as e:
+                print(f"[⚠️] Ошибка чтения {rel_path}: {e}")
+        else:
+            print(f"[ℹ️] Пропущен отсутствующий файл: {rel_path}")
+
+    with open(OUTPUT_FILE, "w", encoding="utf-8") as f:
+        f.write("\n".join(content))
+
+    size_kb = os.path.getsize(OUTPUT_FILE) / 1024
+    print(f"\n[✓] ETL-слепок создан: {OUTPUT_FILE}")
+    print(f"    Включено файлов: {included_count} | Размер: {size_kb:.1f} KB\n")
+
+
+if __name__ == "__main__":
+    create_etl_snapshot()