feat(scud): добавление таблицы scud_events_raw и сохранение детальной ленты проходов

2026-09-07 18:25:06 +03:00
parent 17c960fa34
commit 6c9b131cf2
4 changed files with 114 additions and 15 deletions
@@ -237,4 +237,40 @@ def delete_snapshots_by_date(date_str: str) -> int:
        cursor.execute("DELETE FROM scud_logs WHERE log_date = ? OR snapshot_id LIKE ?", (date_str, f"%{date_str.replace('.', '')}%"))
        cnt = cursor.rowcount
        conn.commit()
-    return cnt
+    return cnt
+
+def save_raw_events_to_db(df_raw_events: pd.DataFrame, date_str: str) -> int:
+    """
+    Сохраняет ленту сырых физических проходов турникетов в таблицу scud_events_raw.
+    Перезаписывает сырые события за указанную дату для исключения дубликатов.
+    """
+    if df_raw_events is None or df_raw_events.empty:
+        return 0
+
+    records = [
+        (
+            date_str,
+            str(r.get('TimeVal', '')),
+            int(r.get('HozOrgan', 0)),
+            str(r.get('Сотрудник', '')),
+            str(r.get('fio_clean', '')),
+            str(r.get('Подразделение', '')),
+            int(r.get('Event', 0)),
+            int(r.get('Mode', 0)),
+            str(r.get('Direction', 'OTHER'))
+        )
+        for _, r in df_raw_events.iterrows()
+    ]
+
+    with get_connection() as conn:
+        cursor = conn.cursor()
+        cursor.execute("DELETE FROM scud_events_raw WHERE log_date = ?", (date_str,))
+        cursor.executemany("""
+            INSERT INTO scud_events_raw (
+                log_date, time_val, hoz_organ, fio, fio_clean,
+                department, event_code, mode, direction
+            )
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+        """, records)
+        conn.commit()
+        return len(records)
@@ -38,6 +38,23 @@ def init_all_tables() -> None:
        );
        """)

+        # 1.1 Сырые физические события турникетов СКУД (для перекуров и оперативного статуса)
+        cursor.execute("""
+        CREATE TABLE IF NOT EXISTS scud_events_raw (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            log_date TEXT NOT NULL,
+            time_val TEXT NOT NULL,
+            hoz_organ INTEGER NOT NULL,
+            fio TEXT NOT NULL,
+            fio_clean TEXT NOT NULL,
+            department TEXT,
+            event_code INTEGER NOT NULL,
+            mode INTEGER NOT NULL,
+            direction TEXT NOT NULL, -- 'IN' или 'OUT'
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        );
+        """)
+
        # 2. Кадровые реестры 1С
        cursor.execute("""
        CREATE TABLE IF NOT EXISTS zup_staff (
@@ -172,5 +189,7 @@ def init_all_tables() -> None:
        cursor.execute("CREATE INDEX IF NOT EXISTS idx_scud_date ON scud_logs(log_date);")
        cursor.execute("CREATE INDEX IF NOT EXISTS idx_scud_fio ON scud_logs(fio_clean);")
        cursor.execute("CREATE INDEX IF NOT EXISTS idx_mapping_scud ON person_identity_mapping(scud_fio);")
+        cursor.execute("CREATE INDEX IF NOT EXISTS idx_raw_events_date_hoz ON scud_events_raw(log_date, hoz_organ);")
+        cursor.execute("CREATE INDEX IF NOT EXISTS idx_raw_events_date_time ON scud_events_raw(log_date, time_val);")

        conn.commit()
@@ -101,7 +101,7 @@ def print_stats():
    with get_connection() as conn:
        cursor = conn.cursor()
        tables = [
-            'scud_logs', 'zup_staff', 'zup_absences', 'anomalies_history', 
+            'scud_logs', 'scud_events_raw', 'zup_staff', 'zup_absences', 'anomalies_history', 
            'ai_knowledge_base', 'chat_messages', 'session_states', 
            'system_prompt_nodes', 'tasks', 'exceptions_registry'
        ]
@@ -24,6 +24,7 @@ import xlsxwriter

 from config import SCUD_DIR, clean_scud_fio_light, load_exceptions
 from core.database import save_scud_to_db, has_scud_logs_for_date, has_yesterday_final_snapshot
+from core.repositories.scud_repo import save_raw_events_to_db

 warnings.filterwarnings("ignore", message="pandas only supports SQLAlchemy connectable")

@@ -192,6 +193,41 @@ WHERE
 ORDER BY p.Name ASC;
 """

+SQL_RAW_EVENTS_QUERY = r"""
+DECLARE @InputDate DATE = '{target_date}';
+DECLARE @StartDate DATETIME = CAST(@InputDate AS DATETIME);
+DECLARE @EndDate DATETIME = DATEADD(SECOND, -1, DATEADD(DAY, 1, @StartDate));
+
+SELECT 
+    log.TimeVal,
+    log.HozOrgan,
+    LTRIM(RTRIM(
+        ISNULL(CAST(p.Name AS NVARCHAR(255)), N'') +
+        CASE WHEN p.FirstName IS NOT NULL AND CAST(p.FirstName AS NVARCHAR(255)) <> ''
+             THEN N' ' + CAST(p.FirstName AS NVARCHAR(255)) ELSE N'' END +
+        CASE WHEN p.MidName IS NOT NULL AND CAST(p.MidName AS NVARCHAR(255)) <> ''
+             THEN N' ' + CAST(p.MidName AS NVARCHAR(255)) ELSE N'' END
+    )) AS [Сотрудник],
+    ISNULL(CAST(div.Name AS NVARCHAR(255)), N'Без подразделения') AS [Подразделение],
+    log.Event,
+    log.Mode,
+    CASE 
+        WHEN log.Mode = 2 THEN 'OUT'
+        WHEN log.Mode = 1 THEN 'IN'
+        WHEN log.Event IN (2, 27, 29, 33, 55, 65) THEN 'OUT'
+        WHEN log.Event IN (1, 21, 26, 54, 64) THEN 'IN'
+        ELSE 'OTHER' 
+    END AS Direction
+FROM pLogData log WITH (NOLOCK)
+INNER JOIN pList p WITH (NOLOCK) ON log.HozOrgan = p.ID
+LEFT JOIN PDivision div WITH (NOLOCK) ON p.Section = div.ID
+WHERE log.TimeVal BETWEEN @StartDate AND @EndDate
+  AND log.HozOrgan IS NOT NULL
+  AND log.HozOrgan > 0
+  AND log.Event IN (28, 32)
+  AND ISNULL(p.StatusRecord, 0) = 0
+ORDER BY log.TimeVal ASC;
+"""

 def save_df_to_clean_excel(df: pd.DataFrame, file_path: str, sheet_name: str = "Отчет"):
    workbook = xlsxwriter.Workbook(file_path, {'constant_memory': False})
@@ -318,22 +354,30 @@ def run_export(input_date: str | None = None, save_xlsx: bool = True, debug: boo
                df['Пришел'] = df['Статус'].str.contains('Присутствовал', case=False, na=False) & (~mask_anomaly)

                save_scud_to_db(df, processing_date_str, snapshot_time=snapshot_time, is_yesterday=is_yesterday)
-                log(f"[✓] Записи за {processing_date_str} успешно сохранены в SQLite!", "SUCCESS")

-                if save_xlsx:
-                    file_name = f"Сотрудники_{processing_date_str}.xlsx"
-                    file_path = os.path.join(SCUD_DIR, file_name)
+                raw_sql = SQL_RAW_EVENTS_QUERY.format(target_date=processing_date.strftime("%Y-%m-%d"))
+                df_raw = pd.read_sql(raw_sql, connection)
+                if len(df_raw) > 0:
+                    df_raw['fio_clean'] = df_raw['Сотрудник'].apply(clean_scud_fio_light)
+                    inserted_count = save_raw_events_to_db(df_raw, processing_date_str)
+                    log(f"[✓] В scud_events_raw сохранено {inserted_count} сырых событий проходов за {processing_date_str}!", "SUCCESS")

-                    if os.path.exists(file_path):
-                        try:
-                            os.remove(file_path)
-                        except OSError as e:
-                            log(f"ОШИБКА при удалении старого файла {file_name}: {e}", "ERROR")
+                    log(f"[✓] Записи за {processing_date_str} успешно сохранены в SQLite!", "SUCCESS")

-                    save_df_to_clean_excel(df, file_path, sheet_name="Отчет")
-                    log(f"[✓] Успешно экспортирован файл: data/scud/{file_name}", "SUCCESS")
-            else:
-                log(f"Запрос за {processing_date_str} вернул 0 строк.", "WARNING")
+                    if save_xlsx:
+                        file_name = f"Сотрудники_{processing_date_str}.xlsx"
+                        file_path = os.path.join(SCUD_DIR, file_name)
+
+                        if os.path.exists(file_path):
+                            try:
+                                os.remove(file_path)
+                            except OSError as e:
+                                log(f"ОШИБКА при удалении старого файла {file_name}: {e}", "ERROR")
+
+                        save_df_to_clean_excel(df, file_path, sheet_name="Отчет")
+                        log(f"[✓] Успешно экспортирован файл: data/scud/{file_name}", "SUCCESS")
+                else:
+                    log(f"Запрос за {processing_date_str} вернул 0 строк.", "WARNING")

        except Exception as e:
            log(f"🛑 ОШИБКА выгрузки СКУД за {processing_date_str}: {e}", "ERROR")