diff --git a/core/repositories/scud_repo.py b/core/repositories/scud_repo.py index e3d7541..d6c525e 100644 --- a/core/repositories/scud_repo.py +++ b/core/repositories/scud_repo.py @@ -237,4 +237,40 @@ def delete_snapshots_by_date(date_str: str) -> int: cursor.execute("DELETE FROM scud_logs WHERE log_date = ? OR snapshot_id LIKE ?", (date_str, f"%{date_str.replace('.', '')}%")) cnt = cursor.rowcount conn.commit() - return cnt \ No newline at end of file + return cnt + +def save_raw_events_to_db(df_raw_events: pd.DataFrame, date_str: str) -> int: + """ + Сохраняет ленту сырых физических проходов турникетов в таблицу scud_events_raw. + Перезаписывает сырые события за указанную дату для исключения дубликатов. + """ + if df_raw_events is None or df_raw_events.empty: + return 0 + + records = [ + ( + date_str, + str(r.get('TimeVal', '')), + int(r.get('HozOrgan', 0)), + str(r.get('Сотрудник', '')), + str(r.get('fio_clean', '')), + str(r.get('Подразделение', '')), + int(r.get('Event', 0)), + int(r.get('Mode', 0)), + str(r.get('Direction', 'OTHER')) + ) + for _, r in df_raw_events.iterrows() + ] + + with get_connection() as conn: + cursor = conn.cursor() + cursor.execute("DELETE FROM scud_events_raw WHERE log_date = ?", (date_str,)) + cursor.executemany(""" + INSERT INTO scud_events_raw ( + log_date, time_val, hoz_organ, fio, fio_clean, + department, event_code, mode, direction + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, records) + conn.commit() + return len(records) \ No newline at end of file diff --git a/core/schema.py b/core/schema.py index 93e4093..e48b5fd 100644 --- a/core/schema.py +++ b/core/schema.py @@ -38,6 +38,23 @@ def init_all_tables() -> None: ); """) + # 1.1 Сырые физические события турникетов СКУД (для перекуров и оперативного статуса) + cursor.execute(""" + CREATE TABLE IF NOT EXISTS scud_events_raw ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + log_date TEXT NOT NULL, + time_val TEXT NOT NULL, + hoz_organ INTEGER NOT NULL, + fio TEXT NOT NULL, + fio_clean TEXT NOT NULL, + department TEXT, + event_code INTEGER NOT NULL, + mode INTEGER NOT NULL, + direction TEXT NOT NULL, -- 'IN' или 'OUT' + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + """) + # 2. Кадровые реестры 1С cursor.execute(""" CREATE TABLE IF NOT EXISTS zup_staff ( @@ -172,5 +189,7 @@ def init_all_tables() -> None: cursor.execute("CREATE INDEX IF NOT EXISTS idx_scud_date ON scud_logs(log_date);") cursor.execute("CREATE INDEX IF NOT EXISTS idx_scud_fio ON scud_logs(fio_clean);") cursor.execute("CREATE INDEX IF NOT EXISTS idx_mapping_scud ON person_identity_mapping(scud_fio);") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_raw_events_date_hoz ON scud_events_raw(log_date, hoz_organ);") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_raw_events_date_time ON scud_events_raw(log_date, time_val);") conn.commit() \ No newline at end of file diff --git a/scripts/db_cli.py b/scripts/db_cli.py index 82a8444..8e2c8ac 100644 --- a/scripts/db_cli.py +++ b/scripts/db_cli.py @@ -101,7 +101,7 @@ def print_stats(): with get_connection() as conn: cursor = conn.cursor() tables = [ - 'scud_logs', 'zup_staff', 'zup_absences', 'anomalies_history', + 'scud_logs', 'scud_events_raw', 'zup_staff', 'zup_absences', 'anomalies_history', 'ai_knowledge_base', 'chat_messages', 'session_states', 'system_prompt_nodes', 'tasks', 'exceptions_registry' ] diff --git a/services/scud_export.py b/services/scud_export.py index 227a429..cce4e9b 100644 --- a/services/scud_export.py +++ b/services/scud_export.py @@ -24,6 +24,7 @@ import xlsxwriter from config import SCUD_DIR, clean_scud_fio_light, load_exceptions from core.database import save_scud_to_db, has_scud_logs_for_date, has_yesterday_final_snapshot +from core.repositories.scud_repo import save_raw_events_to_db warnings.filterwarnings("ignore", message="pandas only supports SQLAlchemy connectable") @@ -192,6 +193,41 @@ WHERE ORDER BY p.Name ASC; """ +SQL_RAW_EVENTS_QUERY = r""" +DECLARE @InputDate DATE = '{target_date}'; +DECLARE @StartDate DATETIME = CAST(@InputDate AS DATETIME); +DECLARE @EndDate DATETIME = DATEADD(SECOND, -1, DATEADD(DAY, 1, @StartDate)); + +SELECT + log.TimeVal, + log.HozOrgan, + LTRIM(RTRIM( + ISNULL(CAST(p.Name AS NVARCHAR(255)), N'') + + CASE WHEN p.FirstName IS NOT NULL AND CAST(p.FirstName AS NVARCHAR(255)) <> '' + THEN N' ' + CAST(p.FirstName AS NVARCHAR(255)) ELSE N'' END + + CASE WHEN p.MidName IS NOT NULL AND CAST(p.MidName AS NVARCHAR(255)) <> '' + THEN N' ' + CAST(p.MidName AS NVARCHAR(255)) ELSE N'' END + )) AS [Сотрудник], + ISNULL(CAST(div.Name AS NVARCHAR(255)), N'Без подразделения') AS [Подразделение], + log.Event, + log.Mode, + CASE + WHEN log.Mode = 2 THEN 'OUT' + WHEN log.Mode = 1 THEN 'IN' + WHEN log.Event IN (2, 27, 29, 33, 55, 65) THEN 'OUT' + WHEN log.Event IN (1, 21, 26, 54, 64) THEN 'IN' + ELSE 'OTHER' + END AS Direction +FROM pLogData log WITH (NOLOCK) +INNER JOIN pList p WITH (NOLOCK) ON log.HozOrgan = p.ID +LEFT JOIN PDivision div WITH (NOLOCK) ON p.Section = div.ID +WHERE log.TimeVal BETWEEN @StartDate AND @EndDate + AND log.HozOrgan IS NOT NULL + AND log.HozOrgan > 0 + AND log.Event IN (28, 32) + AND ISNULL(p.StatusRecord, 0) = 0 +ORDER BY log.TimeVal ASC; +""" def save_df_to_clean_excel(df: pd.DataFrame, file_path: str, sheet_name: str = "Отчет"): workbook = xlsxwriter.Workbook(file_path, {'constant_memory': False}) @@ -318,22 +354,30 @@ def run_export(input_date: str | None = None, save_xlsx: bool = True, debug: boo df['Пришел'] = df['Статус'].str.contains('Присутствовал', case=False, na=False) & (~mask_anomaly) save_scud_to_db(df, processing_date_str, snapshot_time=snapshot_time, is_yesterday=is_yesterday) - log(f"[✓] Записи за {processing_date_str} успешно сохранены в SQLite!", "SUCCESS") - if save_xlsx: - file_name = f"Сотрудники_{processing_date_str}.xlsx" - file_path = os.path.join(SCUD_DIR, file_name) + raw_sql = SQL_RAW_EVENTS_QUERY.format(target_date=processing_date.strftime("%Y-%m-%d")) + df_raw = pd.read_sql(raw_sql, connection) + if len(df_raw) > 0: + df_raw['fio_clean'] = df_raw['Сотрудник'].apply(clean_scud_fio_light) + inserted_count = save_raw_events_to_db(df_raw, processing_date_str) + log(f"[✓] В scud_events_raw сохранено {inserted_count} сырых событий проходов за {processing_date_str}!", "SUCCESS") - if os.path.exists(file_path): - try: - os.remove(file_path) - except OSError as e: - log(f"ОШИБКА при удалении старого файла {file_name}: {e}", "ERROR") + log(f"[✓] Записи за {processing_date_str} успешно сохранены в SQLite!", "SUCCESS") - save_df_to_clean_excel(df, file_path, sheet_name="Отчет") - log(f"[✓] Успешно экспортирован файл: data/scud/{file_name}", "SUCCESS") - else: - log(f"Запрос за {processing_date_str} вернул 0 строк.", "WARNING") + if save_xlsx: + file_name = f"Сотрудники_{processing_date_str}.xlsx" + file_path = os.path.join(SCUD_DIR, file_name) + + if os.path.exists(file_path): + try: + os.remove(file_path) + except OSError as e: + log(f"ОШИБКА при удалении старого файла {file_name}: {e}", "ERROR") + + save_df_to_clean_excel(df, file_path, sheet_name="Отчет") + log(f"[✓] Успешно экспортирован файл: data/scud/{file_name}", "SUCCESS") + else: + log(f"Запрос за {processing_date_str} вернул 0 строк.", "WARNING") except Exception as e: log(f"🛑 ОШИБКА выгрузки СКУД за {processing_date_str}: {e}", "ERROR")