""" =============================================================================== FILE: services/scud_export.py ROLE: Прямой экспорт данных СКУД Орион (MS SQL) в SQLite и чистый Excel (XlsxWriter). Корректная фильтрация транзитных проходов турникетов парковки и двора. =============================================================================== """ import argparse import logging import os import sys import warnings from datetime import datetime, timedelta CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) ROOT_DIR = os.path.abspath(os.path.join(CURRENT_DIR, "..")) if ROOT_DIR not in sys.path: sys.path.insert(0, ROOT_DIR) import pandas as pd import pyodbc import xlsxwriter from config import SCUD_DIR, clean_scud_fio_light, load_exceptions from core.database import save_scud_to_db, has_scud_logs_for_date, has_yesterday_final_snapshot warnings.filterwarnings("ignore", message="pandas only supports SQLAlchemy connectable") SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) LOG_DIR = os.path.join(SCRIPT_DIR, "..", "logs") os.makedirs(LOG_DIR, exist_ok=True) TODAY_DATE_STR = datetime.now().strftime("%d.%m.%Y") LOG_FILE = os.path.join(LOG_DIR, f"export_{TODAY_DATE_STR}.log") logger = logging.getLogger("scud_export") logger.setLevel(logging.INFO) logger.propagate = False if not logger.handlers: try: _file_handler = logging.FileHandler(LOG_FILE, encoding="utf-8") _formatter = logging.Formatter("[%(asctime)s] [%(levelname)s] %(message)s", datefmt="%Y-%m-%d %H:%M:%S") _file_handler.setFormatter(_formatter) logger.addHandler(_file_handler) except (PermissionError, OSError) as e: sys.stderr.write(f"Предупреждение: невозможно создать лог-файл {LOG_FILE}: {e}\n") _console_handler = logging.StreamHandler(sys.stdout) _formatter = logging.Formatter("[%(asctime)s] [%(levelname)s] %(message)s", datefmt="%Y-%m-%d %H:%M:%S") _console_handler.setFormatter(_formatter) logger.addHandler(_console_handler) def log(message: str, level: str = "INFO"): level_map = { "INFO": logging.INFO, "ERROR": logging.ERROR, "WARNING": logging.WARNING, "SUCCESS": logging.INFO, } if level == "SUCCESS": logger.info(f"[SUCCESS] {message}") else: logger.log(level_map.get(level, logging.INFO), message) SERVER_NAME = r"172.16.200.147\SQL" DATABASE_NAME = "Orion-14.01.21-1" SQL_USER = "sa" SQL_PASSWORD = "123456" ODBC_DRIVER = "ODBC Driver 18 for SQL Server" SQL_QUERY_TEMPLATE = r""" DECLARE @InputDate DATE = '{target_date}'; DECLARE @TargetDate DATE = @InputDate; DECLARE @StartDate DATETIME = CAST(@TargetDate AS DATETIME); DECLARE @EndDate DATETIME = DATEADD(SECOND, -1, DATEADD(DAY, 1, @StartDate)); WITH DailyLogs AS ( SELECT log.HozOrgan AS EmployeeID, log.TimeVal, log.Event, log.Mode, -- Приоритет отдается физическому направлению контроллера (Mode): CASE WHEN log.Mode = 2 THEN 'OUT' WHEN log.Mode = 1 THEN 'IN' WHEN log.Event IN (2, 27, 29, 33, 55, 65) THEN 'OUT' WHEN log.Event IN (1, 21, 26, 54, 64) THEN 'IN' ELSE 'OTHER' END AS Direction, -- Вычисляем самое последнее событие сотрудника за день: ROW_NUMBER() OVER (PARTITION BY log.HozOrgan ORDER BY log.TimeVal DESC, log.Event DESC) AS RnLast FROM pLogData log WITH (NOLOCK) WHERE log.TimeVal BETWEEN @StartDate AND @EndDate AND log.HozOrgan IS NOT NULL AND log.HozOrgan > 0 AND log.Event IN (1, 2, 21, 26, 27, 28, 29, 32, 33, 54, 55, 64, 65) ), Passages AS ( SELECT EmployeeID, MIN(TimeVal) AS FirstRawEvent, MAX(TimeVal) AS LastRawEvent, -- Первый вход за день: MIN(CASE WHEN Direction = 'IN' THEN TimeVal END) AS FirstIn, -- Фиксируем выход ТОЛЬКО если самое последнее событие за день было именно выходом (OUT): MAX(CASE WHEN RnLast = 1 AND Direction = 'OUT' THEN TimeVal END) AS FinalOut FROM DailyLogs GROUP BY EmployeeID ), EvaluatedPassages AS ( SELECT p.*, -- Время выхода проставляется только при окончательном уходе сотрудника из здания: CASE WHEN p.FinalOut IS NOT NULL AND p.FirstIn IS NOT NULL AND p.FinalOut > DATEADD(MINUTE, 5, p.FirstIn) THEN p.FinalOut ELSE NULL END AS FilteredLastOut FROM Passages p ) SELECT N'ЛЕНМОРНИИПРОЕКТ' AS [Фирма], ISNULL(CAST(div.Name AS NVARCHAR(255)), N'Без подразделения') AS [Подразделение], LTRIM(RTRIM( ISNULL(CAST(p.Name AS NVARCHAR(255)), N'') + CASE WHEN p.FirstName IS NOT NULL AND CAST(p.FirstName AS NVARCHAR(255)) <> '' THEN N' ' + CAST(p.FirstName AS NVARCHAR(255)) ELSE N'' END + CASE WHEN p.MidName IS NOT NULL AND CAST(p.MidName AS NVARCHAR(255)) <> '' THEN N' ' + CAST(p.MidName AS NVARCHAR(255)) ELSE N'' END )) AS [Сотрудник], ISNULL(CAST(post.Name AS NVARCHAR(255)), N'—') AS [Должность], ISNULL(CAST(p.TabNumber AS NVARCHAR(50)), N'—') AS [Таб_№], CONVERT(VARCHAR(10), @TargetDate, 104) AS [Дата], ISNULL(CAST(CONVERT(VARCHAR(8), pass.FirstIn, 108) AS NVARCHAR(20)), N'Нет входа') AS [Начало_дня], CASE WHEN pass.FirstIn IS NULL AND pass.FirstRawEvent IS NOT NULL THEN CAST(CONVERT(VARCHAR(8), pass.FirstRawEvent, 108) AS NVARCHAR(20)) ELSE N'—' END AS [Первая_активность], CASE WHEN pass.FilteredLastOut IS NOT NULL THEN CAST(CONVERT(VARCHAR(8), pass.FilteredLastOut, 108) AS NVARCHAR(20)) ELSE N'Нет выхода' END AS [Конец_дня], CASE WHEN pass.EmployeeID IS NOT NULL AND (pass.FirstIn IS NOT NULL OR pass.FirstRawEvent IS NOT NULL) THEN RIGHT('0' + CAST(DATEDIFF(MINUTE, ISNULL(pass.FirstIn, pass.FirstRawEvent), CASE WHEN pass.FilteredLastOut IS NOT NULL THEN pass.FilteredLastOut WHEN @TargetDate = CAST(GETDATE() AS DATE) THEN GETDATE() ELSE ISNULL(pass.FirstIn, pass.FirstRawEvent) END) / 60 AS VARCHAR), 2) + ':' + RIGHT('0' + CAST(DATEDIFF(MINUTE, ISNULL(pass.FirstIn, pass.FirstRawEvent), CASE WHEN pass.FilteredLastOut IS NOT NULL THEN pass.FilteredLastOut WHEN @TargetDate = CAST(GETDATE() AS DATE) THEN GETDATE() ELSE ISNULL(pass.FirstIn, pass.FirstRawEvent) END) % 60 AS VARCHAR), 2) ELSE N'00:00' END AS [Находился_в_здании], CASE WHEN pass.EmployeeID IS NOT NULL THEN N'Присутствовал' ELSE N'Отсутствовал (Нет событий)' END AS [Статус] FROM pList p WITH (NOLOCK) LEFT JOIN PDivision div WITH (NOLOCK) ON p.Section = div.ID LEFT JOIN PPost post WITH (NOLOCK) ON p.Post = post.ID LEFT JOIN EvaluatedPassages pass ON p.ID = pass.EmployeeID WHERE ISNULL(p.StatusRecord, 0) = 0 AND p.DateTimeInArchive IS NULL AND ISNULL(CAST(div.Name AS NVARCHAR(255)), N'') NOT LIKE N'Аренд%' AND ISNULL(CAST(div.Name AS NVARCHAR(255)), N'') NOT IN (N'Без подразделения', N'') AND p.Name NOT LIKE N'бр.%' AND p.Name NOT LIKE N'Гость%' AND ISNULL(CAST(div.Name AS NVARCHAR(255)), N'') NOT IN (N'БГИ', N'КНР') AND ISNULL(CAST(div.Name AS NVARCHAR(255)), N'') NOT LIKE N'Рабоч%' AND ISNULL(CAST(div.Name AS NVARCHAR(255)), N'') NOT LIKE N'Врем%' AND ISNULL(CAST(div.Name AS NVARCHAR(255)), N'') NOT LIKE N'Практика%' AND ISNULL(CAST(div.Name AS NVARCHAR(255)), N'') NOT LIKE N'тест%' AND ISNULL(CAST(post.Name AS NVARCHAR(255)), N'') NOT LIKE N'Практикант%' ORDER BY p.Name ASC; """ def save_df_to_clean_excel(df: pd.DataFrame, file_path: str, sheet_name: str = "Отчет"): workbook = xlsxwriter.Workbook(file_path, {'constant_memory': False}) worksheet = workbook.add_worksheet(sheet_name) fmt_header = workbook.add_format({ 'bold': True, 'bg_color': '#D9E1F2', 'border': 1, 'border_color': '#D3D3D3', 'align': 'center', 'valign': 'vcenter', 'font_name': 'Calibri', 'font_size': 11 }) fmt_cell = workbook.add_format({ 'border': 1, 'border_color': '#D3D3D3', 'valign': 'vcenter', 'align': 'left', 'font_name': 'Calibri', 'font_size': 11 }) headers = list(df.columns) col_widths = [len(str(h)) for h in headers] for col_idx, header in enumerate(headers): worksheet.write(0, col_idx, str(header), fmt_header) for row_idx, row_values in enumerate(df.values, start=1): for col_idx, val in enumerate(row_values): if pd.isna(val) or val is None: val_str = "" elif isinstance(val, bool): val_str = "Да" if val else "Нет" else: val_str = str(val) worksheet.write(row_idx, col_idx, val_str, fmt_cell) if len(val_str) > col_widths[col_idx]: col_widths[col_idx] = len(val_str) for col_idx, width in enumerate(col_widths): worksheet.set_column(col_idx, col_idx, min(max(width + 3, 10), 45)) workbook.close() def get_targets(input_date: str | None): targets = [] if input_date: try: parsed = datetime.strptime(input_date, "%d.%m.%Y").date() targets.append({"name": "Указанная дата", "date": parsed}) except ValueError: log(f"ОШИБКА: Неверный формат даты '{input_date}'. Используйте ДД.ММ.ГГГГ", "ERROR") sys.exit(1) else: now = datetime.now() yesterday = (now - timedelta(days=3 if now.weekday() == 0 else 1)).date() today = now.date() targets.append({"name": "Вчера", "date": yesterday}) targets.append({"name": "Сегодня", "date": today}) return targets def run_export(input_date: str | None = None, save_xlsx: bool = True, debug: bool = False): if debug: logger.setLevel(logging.DEBUG) log("=== ВКЛЮЧЕН РЕЖИМ ОТЛАДКИ (DEBUG MODE) ===", "WARNING") log("=== [ЭТАП 0] Выгрузка свежих данных СКУД напрямую из БД Орион ===") os.makedirs(SCUD_DIR, exist_ok=True) targets = get_targets(input_date) conn_str = ( f"DRIVER={{{ODBC_DRIVER}}};" f"SERVER={SERVER_NAME};" f"DATABASE={DATABASE_NAME};" f"UID={SQL_USER};" f"PWD={SQL_PASSWORD};" f"TrustServerCertificate=yes;" f"Encrypt=no;" ) success = True for target in targets: processing_date = target["date"] processing_date_str = processing_date.strftime("%d.%m.%Y") period_label = target["name"] is_yesterday = (period_label == "Вчера") if is_yesterday and has_yesterday_final_snapshot(processing_date_str): log(f"[ℹ️] Вчерашний день ({processing_date_str}) уже зафиксирован финишным снапшотом Y. Пропускаем запрос к MS SQL.") continue if is_yesterday: snapshot_time = f"{processing_date.strftime('%Y-%m-%d')} 23:59:59" else: snapshot_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") log(f"--- Обработка периода: {period_label} ({processing_date_str}) --- [Снапшот: {snapshot_time}]") sql_query = SQL_QUERY_TEMPLATE.format(target_date=processing_date.strftime("%Y-%m-%d")) connection = None try: connection = pyodbc.connect(conn_str, timeout=120) df = pd.read_sql(sql_query, connection) log(f"База вернула {len(df)} строк за {processing_date_str}") if len(df) > 0: df['fio_clean'] = df['Сотрудник'].apply(clean_scud_fio_light) df['anomaly_flag'] = 'NONE' mask_anomaly = (df['Начало_дня'] == 'Нет входа') & (df['Первая_активность'] != '—') df.loc[mask_anomaly, 'anomaly_flag'] = 'ANOMALY_NO_IN_HAS_ACTIVITY' df['Пришел'] = df['Статус'].str.contains('Присутствовал', case=False, na=False) & (~mask_anomaly) save_scud_to_db(df, processing_date_str, snapshot_time=snapshot_time, is_yesterday=is_yesterday) log(f"[✓] Записи за {processing_date_str} успешно сохранены в SQLite!", "SUCCESS") if save_xlsx: file_name = f"Сотрудники_{processing_date_str}.xlsx" file_path = os.path.join(SCUD_DIR, file_name) if os.path.exists(file_path): try: os.remove(file_path) except OSError as e: log(f"ОШИБКА при удалении старого файла {file_name}: {e}", "ERROR") save_df_to_clean_excel(df, file_path, sheet_name="Отчет") log(f"[✓] Успешно экспортирован файл: data/scud/{file_name}", "SUCCESS") else: log(f"Запрос за {processing_date_str} вернул 0 строк.", "WARNING") except Exception as e: log(f"🛑 ОШИБКА выгрузки СКУД за {processing_date_str}: {e}", "ERROR") success = False finally: if connection is not None: connection.close() log("=== Выгрузка СКУД завершена ===\n") return success if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--date", dest="input_date", default=None) parser.add_argument("-d", "--debug", action="store_true") parser.add_argument("--no-xlsx", dest="save_xlsx", action="store_false", default=True) args = parser.parse_args() run_export(args.input_date, save_xlsx=args.save_xlsx, debug=args.debug)