66 lines
2.5 KiB
Python
66 lines
2.5 KiB
Python
"""
|
|
===============================================================================
|
|
FILE: services/scud_etl/pipeline.py
|
|
PROJECT: SCUD Orion AI (Unified Architecture)
|
|
MODULE: services / scud_etl
|
|
ROLE: Оркестрация выборки снапшотов из SQLite и загрузки кадровых файлов 1С.
|
|
===============================================================================
|
|
"""
|
|
|
|
import os
|
|
import json
|
|
import logging
|
|
from typing import Optional
|
|
import pandas as pd
|
|
|
|
from core.connection import get_connection
|
|
|
|
logger = logging.getLogger("SCUD_PIPELINE")
|
|
|
|
|
|
def load_best_snapshot_for_date(date_str: str, prefer_final_y: bool = True) -> pd.DataFrame:
|
|
"""
|
|
Извлекает срез за дату из SQLite. Для отчета за вчера строго ищет
|
|
финальный вечерний срез Y (с зафиксированными выходами за 22:00:00).
|
|
"""
|
|
with get_connection(row_factory=True) as conn:
|
|
cursor = conn.cursor()
|
|
|
|
if prefer_final_y:
|
|
cursor.execute("""
|
|
SELECT raw_data_json
|
|
FROM scud_snapshots
|
|
WHERE (snapshot_date = ? OR date_str = ?) AND (snapshot_id LIKE 'Y%' OR snapshot_time LIKE '22:00%')
|
|
ORDER BY id DESC LIMIT 1
|
|
""", (date_str, date_str))
|
|
row = cursor.fetchone()
|
|
if row and row["raw_data_json"]:
|
|
data = json.loads(row["raw_data_json"])
|
|
return pd.DataFrame(data)
|
|
|
|
cursor.execute("""
|
|
SELECT raw_data_json
|
|
FROM scud_snapshots
|
|
WHERE snapshot_date = ? OR date_str = ?
|
|
ORDER BY id DESC LIMIT 1
|
|
""", (date_str, date_str))
|
|
row = cursor.fetchone()
|
|
if row and row["raw_data_json"]:
|
|
data = json.loads(row["raw_data_json"])
|
|
return pd.DataFrame(data)
|
|
|
|
return pd.DataFrame()
|
|
|
|
|
|
def load_1c_files_for_date(date_str: str) -> tuple[pd.DataFrame, pd.DataFrame]:
|
|
"""
|
|
Загружает файлы Штат_*.xlsx и Отсутствия_*.xlsx за указанную дату из data/1c/.
|
|
"""
|
|
formatted_date = date_str.replace(".", "_")
|
|
staff_file = f"data/1c/Штат_{formatted_date}.xlsx"
|
|
absences_file = f"data/1c/Отсутствия_{formatted_date}.xlsx"
|
|
|
|
df_staff = pd.read_excel(staff_file) if os.path.exists(staff_file) else pd.DataFrame()
|
|
df_absences = pd.read_excel(absences_file) if os.path.exists(absences_file) else pd.DataFrame()
|
|
|
|
return df_staff, df_absences |