feat(etl): stable pipeline, exception registry in SQLite, multi-pass aggregation and db_cli
This commit is contained in:
@@ -1,46 +0,0 @@
|
||||
import os
|
||||
|
||||
OUTPUT_SNAPSHOT = "project_code_snapshot.md"
|
||||
|
||||
# Расширения файлов для включения в снимок (только исходный код)
|
||||
ALLOWED_EXTENSIONS = {'.py', '.json', '.sh', '.ini', '.js', '.html', '.css', '.sql'}
|
||||
|
||||
# Исключаемые каталоги (убираем docs, кэши, архивы и окружения)
|
||||
EXCLUDE_DIRS = {
|
||||
'.git', '__pycache__', 'venv', '.venv', 'output', 'logs',
|
||||
'extracted_project', 'docs', 'data'
|
||||
}
|
||||
|
||||
# Исключаемые файлы
|
||||
EXCLUDE_FILES = {
|
||||
OUTPUT_SNAPSHOT,
|
||||
'api_code_snapshot.md',
|
||||
'project_code_snapshot.md',
|
||||
'scud_orion_ai_v2.tar.gz',
|
||||
'scud_context_api.tar.gz',
|
||||
'context_memory.db'
|
||||
}
|
||||
|
||||
print(f"🔄 Сборка оптимизированного слепка кода в {OUTPUT_SNAPSHOT}...")
|
||||
|
||||
with open(OUTPUT_SNAPSHOT, 'w', encoding='utf-8') as out:
|
||||
out.write("# 📦 ОПТИМИЗИРОВАННЫЙ СЛЕПОК ИСХОДНОГО КОДА scud_orion_ai\n\n")
|
||||
|
||||
for root, dirs, files in os.walk('.'):
|
||||
dirs[:] = [d for d in dirs if d not in EXCLUDE_DIRS]
|
||||
|
||||
for file in sorted(files):
|
||||
ext = os.path.splitext(file)[1].lower()
|
||||
if ext in ALLOWED_EXTENSIONS and file not in EXCLUDE_FILES:
|
||||
filepath = os.path.join(root, file)
|
||||
out.write(f"## File: `{filepath}`\n")
|
||||
out.write("```" + ext.replace('.', '') + "\n")
|
||||
try:
|
||||
with open(filepath, 'r', encoding='utf-8', errors='replace') as f:
|
||||
out.write(f.read())
|
||||
except Exception as e:
|
||||
out.write(f"// Ошибка чтения файла: {e}\n")
|
||||
out.write("\n```\n\n")
|
||||
|
||||
size_kb = os.path.getsize(OUTPUT_SNAPSHOT) / 1024
|
||||
print(f"✓ Слепок успешно создан: {OUTPUT_SNAPSHOT} ({size_kb:.1f} KB)")
|
||||
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
===============================================================================
|
||||
FILE: scripts/diagnostics/make_etl_snapshot.py
|
||||
ROLE: Генерация компактного слепка ETL-конвейера, БД и сервисов СКУД.
|
||||
===============================================================================
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
||||
OUTPUT_FILE = os.path.join(ROOT_DIR, "etl_code_snapshot.md")
|
||||
|
||||
# Строгий целевой список файлов для ETL-слепка
|
||||
TARGET_FILES = [
|
||||
"config.py",
|
||||
"exceptions.json",
|
||||
"main_etl.py",
|
||||
"run_cron_etl.sh",
|
||||
"scripts/db_cli.py", # ⭐️ Гарантированно включен
|
||||
"core/connection.py",
|
||||
"core/database.py",
|
||||
"core/schema.py",
|
||||
"core/repositories/scud_repo.py",
|
||||
"core/repositories/zup_repo.py",
|
||||
"services/data_loader.py",
|
||||
"services/scud_export.py",
|
||||
"services/share_copier.py",
|
||||
"services/excel_exporter.py",
|
||||
"services/text_reporter.py",
|
||||
"services/exceptions_repo.py",
|
||||
"services/zup_extractor.py",
|
||||
"services/ai_verifier.py",
|
||||
"services/knowledge_base.py",
|
||||
"services/knowledge/service.py",
|
||||
"services/scud_etl/pipeline.py",
|
||||
"services/scud_etl/merger.py",
|
||||
"services/scud_etl/anomaly_detector.py",
|
||||
"services/snapshots/service.py",
|
||||
"services/tasks/repository.py",
|
||||
"services/tasks/service.py"
|
||||
]
|
||||
|
||||
|
||||
def create_etl_snapshot():
|
||||
content = ["# 📦 ETL-СЛЕПОК ИСХОДНОГО КОДА (СКУД ⟷ 1С & DB CORE)\n"]
|
||||
included_count = 0
|
||||
|
||||
for rel_path in TARGET_FILES:
|
||||
full_path = os.path.join(ROOT_DIR, rel_path)
|
||||
if os.path.exists(full_path):
|
||||
ext = os.path.splitext(rel_path)[1].replace(".", "")
|
||||
lang = "py" if ext == "py" else ("json" if ext == "json" else "bash")
|
||||
try:
|
||||
with open(full_path, "r", encoding="utf-8") as f:
|
||||
file_text = f.read()
|
||||
content.append(f"## File: `./{rel_path}`\n```{lang}\n{file_text}\n```\n")
|
||||
included_count += 1
|
||||
except Exception as e:
|
||||
print(f"[⚠️] Ошибка чтения {rel_path}: {e}")
|
||||
else:
|
||||
print(f"[ℹ️] Пропущен отсутствующий файл: {rel_path}")
|
||||
|
||||
with open(OUTPUT_FILE, "w", encoding="utf-8") as f:
|
||||
f.write("\n".join(content))
|
||||
|
||||
size_kb = os.path.getsize(OUTPUT_FILE) / 1024
|
||||
print(f"\n[✓] ETL-слепок создан: {OUTPUT_FILE}")
|
||||
print(f" Включено файлов: {included_count} | Размер: {size_kb:.1f} KB\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
create_etl_snapshot()
|
||||
@@ -0,0 +1,63 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
===============================================================================
|
||||
FILE: scripts/diagnostics/make_web_snapshot.py
|
||||
ROLE: Генерация слепка Web API, LLM-движка и клиентских скриптов.
|
||||
===============================================================================
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
||||
OUTPUT_FILE = os.path.join(ROOT_DIR, "web_api_code_snapshot.md")
|
||||
|
||||
WEB_TARGET_FILES = [
|
||||
"modules/web_api/main.py",
|
||||
"modules/web_api/routers/chat.py",
|
||||
"modules/web_api/routers/auth.py",
|
||||
"modules/web_api/routers/tasks.py",
|
||||
"modules/web_api/routers/exceptions.py",
|
||||
"modules/web_api/routers/admin.py",
|
||||
"modules/web_api/routers/files.py",
|
||||
"modules/web_api/llm/agent.py",
|
||||
"modules/web_api/llm/db_tools.py",
|
||||
"modules/web_api/llm/schemas.py",
|
||||
"modules/web_api/llm/core/context_manager.py",
|
||||
"modules/web_api/llm/core/tool_injector.py",
|
||||
"modules/web_api/llm/core/ollama_client.py",
|
||||
"modules/web_api/llm/core/fast_path.py",
|
||||
"modules/web_api/static/js/app.js",
|
||||
"modules/web_api/static/js/auth.js",
|
||||
"modules/web_api/static/js/tasks.js",
|
||||
"modules/web_api/static/js/chat/core.js",
|
||||
"modules/web_api/static/js/chat/task_widget.js"
|
||||
]
|
||||
|
||||
|
||||
def create_web_snapshot():
|
||||
content = ["# 🌐 WEB API & LLM AGENT CODE SNAPSHOT\n"]
|
||||
included_count = 0
|
||||
|
||||
for rel_path in WEB_TARGET_FILES:
|
||||
full_path = os.path.join(ROOT_DIR, rel_path)
|
||||
if os.path.exists(full_path):
|
||||
ext = os.path.splitext(rel_path)[1].replace(".", "")
|
||||
lang = "js" if ext == "js" else ("py" if ext == "py" else "text")
|
||||
try:
|
||||
with open(full_path, "r", encoding="utf-8") as f:
|
||||
file_text = f.read()
|
||||
content.append(f"## File: `./{rel_path}`\n```{lang}\n{file_text}\n```\n")
|
||||
included_count += 1
|
||||
except Exception as e:
|
||||
print(f"[⚠️] Ошибка чтения {rel_path}: {e}")
|
||||
|
||||
with open(OUTPUT_FILE, "w", encoding="utf-8") as f:
|
||||
f.write("\n".join(content))
|
||||
|
||||
size_kb = os.path.getsize(OUTPUT_FILE) / 1024
|
||||
print(f"\n[✓] Web API слепок создан: {OUTPUT_FILE}")
|
||||
print(f" Включено файлов: {included_count} | Размер: {size_kb:.1f} KB\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
create_web_snapshot()
|
||||
@@ -0,0 +1,22 @@
|
||||
import os
|
||||
|
||||
OUTPUT_SNAPSHOT = "web_api_code_snapshot.md"
|
||||
ALLOWED_EXTENSIONS = {'.py', '.html', '.css', '.js'}
|
||||
EXCLUDE_DIRS = {'.git', '__pycache__', 'venv', '.venv', 'output', 'logs', 'extracted_project', 'docs', 'data'}
|
||||
|
||||
print(f"🔄 Сборка Web API слепка кода в {OUTPUT_SNAPSHOT}...")
|
||||
with open(OUTPUT_SNAPSHOT, 'w', encoding='utf-8') as out:
|
||||
out.write("# 📦 WEB API & AI СЛЕПОК ИСХОДНОГО КОДА\n\n")
|
||||
if os.path.exists('modules'):
|
||||
for r, d, files in os.walk('modules'):
|
||||
d[:] = [sub for sub in d if sub not in EXCLUDE_DIRS]
|
||||
for file in sorted(files):
|
||||
ext = os.path.splitext(file)[1].lower()
|
||||
if ext in ALLOWED_EXTENSIONS:
|
||||
filepath = os.path.join(r, file)
|
||||
out.write(f"## File: `./{filepath}`\n```" + ext.replace('.', '') + "\n")
|
||||
with open(filepath, 'r', encoding='utf-8', errors='replace') as f:
|
||||
out.write(f.read())
|
||||
out.write("\n```\n\n")
|
||||
|
||||
print(f"✓ Web API слепок готов: {OUTPUT_SNAPSHOT} ({os.path.getsize(OUTPUT_SNAPSHOT) / 1024:.1f} KB)")
|
||||
Reference in New Issue
Block a user