feat: implement svodka/otchet generators, Y-23:59:59 and snap-to-grid time finder

services/scud_etl/pipeline.py

@@ -1,59 +1,64 @@
 """
 ===============================================================================
 FILE: services/scud_etl/pipeline.py
-ROLE: Выборка данных СКУД из SQLite (scud_logs) с жестким приоритетом Y-снапшотов.
 ===============================================================================
 """
 
 import os
 import logging
+from typing import Optional, Dict, Any, Tuple
 import pandas as pd
+
 from core.connection import get_connection
-from services.data_loader import load_1c_data_smart
+from core.database import load_scud_from_db_by_snapshot
+from config import DATA_DIR
+from services.data_loader import load_staff_data, load_absent_data
 
 logger = logging.getLogger("SCUD_PIPELINE")
 
 
-def load_best_snapshot_for_date(date_str: str, prefer_final_y: bool = True) -> pd.DataFrame:
-    """Извлекает срез СКУД. Для вчерашнего дня строго берет финальный вечерний срез Y."""
+def load_best_snapshot_for_date(date_str: str, prefer_final_y: bool = False) -> Optional[pd.DataFrame]:
+    """
+    Загружает наилучший срез СКУД за дату.
+    Если prefer_final_y=True — отдает предпочтение финишному Y (23:59:59).
+    """
     with get_connection() as conn:
-        df = pd.DataFrame()
         cursor = conn.cursor()
+        target_snap_id = None
 
         if prefer_final_y:
-            cursor.execute(
-                "SELECT snapshot_id FROM scud_logs WHERE log_date = ? AND snapshot_id LIKE 'Y%' ORDER BY id DESC LIMIT 1",
-                (date_str,)
-            )
+            cursor.execute("""
+                SELECT snapshot_id 
+                FROM scud_logs 
+                WHERE log_date = ? 
+                  AND (snapshot_id LIKE 'Y%' OR snapshot_time LIKE '%23:59:59' OR snapshot_time LIKE '%22:00:00')
+                ORDER BY id DESC LIMIT 1
+            """, (date_str,))
             row = cursor.fetchone()
-            if row and row[0]:
-                df = pd.read_sql_query("SELECT * FROM scud_logs WHERE snapshot_id = ?", conn, params=(row[0],))
+            if row:
+                target_snap_id = row[0]
 
-        if df.empty:
-            cursor.execute(
-                "SELECT snapshot_id FROM scud_logs WHERE log_date = ? ORDER BY id DESC LIMIT 1",
-                (date_str,)
-            )
+        if not target_snap_id:
+            cursor.execute("""
+                SELECT snapshot_id 
+                FROM scud_logs 
+                WHERE log_date = ? 
+                ORDER BY id DESC LIMIT 1
+            """, (date_str,))
             row = cursor.fetchone()
-            if row and row[0]:
-                df = pd.read_sql_query("SELECT * FROM scud_logs WHERE snapshot_id = ?", conn, params=(row[0],))
+            if row:
+                target_snap_id = row[0]
 
-        if not df.empty:
-            rename_map = {
-                'department': 'Подразделение', 'position': 'Должность', 'fio': 'Сотрудник',
-                'time_in': 'Начало_дня', 'first_activity': 'Первая_активность', 'time_out': 'Конец_дня',
-                'time_in_building': 'Находился_в_здании', 'is_present': 'Пришел'
-            }
-            df = df.rename(columns={k: v for k, v in rename_map.items() if k in df.columns})
-            if 'fio_clean' not in df.columns and 'Сотрудник' in df.columns:
-                df['fio_clean'] = df['Сотрудник'].astype(str).str.strip()
-            if 'Пришел' in df.columns:
-                df['Пришел'] = df['Пришел'].astype(bool)
+    if not target_snap_id:
+        return None
 
-        return df
+    return load_scud_from_db_by_snapshot(date_str, snapshot_param=target_snap_id)
 
 
-def load_1c_files_for_date(date_str: str) -> tuple[pd.DataFrame, pd.DataFrame]:
-    df_staff, df_absences = load_1c_data_smart(date_str, use_db=False)
-    return (df_staff if df_staff is not None else pd.DataFrame(),
-            df_absences if df_absences is not None else pd.DataFrame())
+def load_1c_files_for_date(date_str: str) -> Tuple[Optional[pd.DataFrame], Optional[pd.DataFrame]]:
+    """
+    Загружает реестры штата и отсутствий 1С на указанную дату через data_loader.
+    """
+    df_staff = load_staff_data(date_str)
+    df_abs = load_absent_data(date_str)
+    return df_staff, df_abs