manoraga/scud_ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(turnstile): двухконтурный учет СКУД, реестры исключений с автокомплитом 1С и калибровка таймзон

Browse Source
This commit is contained in:
manoraga
2026-09-10 15:36:32 +03:00
parent 6c9b131cf2
commit 74332aa38a
31 changed files with 4509 additions and 10769 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/scud_export.py
+109 -53
View File
@@ -3,6 +3,7 @@
FILE: services/scud_export.py
ROLE: Прямой экспорт данных СКУД Орион (MS SQL) в SQLite и чистый Excel (XlsxWriter).
Корректная фильтрация транзитных проходов турникетов парковки и двора.
Учет только левого PERCo (DoorIndex = 1) и факта физического прохода (Event = 32).
===============================================================================
"""
@@ -78,46 +79,50 @@ DECLARE @InputDate DATE = '{target_date}';
DECLARE @TargetDate DATE = @InputDate;
DECLARE @StartDate DATETIME = CAST(@TargetDate AS DATETIME);
DECLARE @EndDate DATETIME = DATEADD(SECOND, -1, DATEADD(DAY, 1, @StartDate));
DECLARE @EndDate DATETIME = {end_datetime_sql};
WITH DailyLogs AS (
WITH PercoPassages AS (
-- Физические факты прохода (Event = 32)
SELECT
log.HozOrgan AS EmployeeID,
log.TimeVal,
log.Event,
log.Mode,
-- Приоритет отдается физическому направлению контроллера (Mode):
CASE
WHEN log.Mode = 2 THEN 'OUT'
WHEN log.Mode = 1 THEN 'IN'
WHEN log.Event IN (2, 27, 29, 33, 55, 65) THEN 'OUT'
WHEN log.Event IN (1, 21, 26, 54, 64) THEN 'IN'
WHEN log.Mode = 2 THEN 'OUT'
ELSE 'OTHER'
END AS Direction,
-- Вычисляем самое последнее событие сотрудника за день:
ROW_NUMBER() OVER (PARTITION BY log.HozOrgan ORDER BY log.TimeVal DESC, log.Event DESC) AS RnLast
END AS Direction
FROM pLogData log WITH (NOLOCK)
INNER JOIN pList p WITH (NOLOCK) ON log.HozOrgan = p.ID
LEFT JOIN PDivision div WITH (NOLOCK) ON p.Section = div.ID
WHERE log.TimeVal BETWEEN @StartDate AND @EndDate
AND log.HozOrgan IS NOT NULL
AND log.HozOrgan > 0
AND log.Event IN (1, 2, 21, 26, 27, 28, 29, 32, 33, 54, 55, 64, 65)
AND log.Event = 32
AND log.Mode IN (1, 2)
AND (
-- Контур 1: Левый турникет открыт для всех
log.DoorIndex = 1
OR
-- Контур 2: Правый турникет разрешен только для реестра двора
(
log.DoorIndex = 2
AND ({turnstile_filter_sql})
)
)
),
Passages AS (
SELECT
EmployeeID,
MIN(TimeVal) AS FirstRawEvent,
MAX(TimeVal) AS LastRawEvent,
-- Первый вход за день:
MIN(CASE WHEN Direction = 'IN' THEN TimeVal END) AS FirstIn,
-- Фиксируем выход ТОЛЬКО если самое последнее событие за день было именно выходом (OUT):
MAX(CASE WHEN RnLast = 1 AND Direction = 'OUT' THEN TimeVal END) AS FinalOut
FROM DailyLogs
MAX(CASE WHEN Direction = 'OUT' THEN TimeVal END) AS FinalOut
FROM PercoPassages
GROUP BY EmployeeID
),
EvaluatedPassages AS (
SELECT
p.*,
-- Время выхода проставляется только при окончательном уходе сотрудника из здания:
CASE
WHEN p.FinalOut IS NOT NULL
AND p.FirstIn IS NOT NULL
@@ -157,15 +162,13 @@ SELECT
ISNULL(pass.FirstIn, pass.FirstRawEvent),
CASE
WHEN pass.FilteredLastOut IS NOT NULL THEN pass.FilteredLastOut
WHEN @TargetDate = CAST(GETDATE() AS DATE) THEN GETDATE()
ELSE ISNULL(pass.FirstIn, pass.FirstRawEvent)
ELSE @EndDate
END) / 60 AS VARCHAR), 2) + ':' +
RIGHT('0' + CAST(DATEDIFF(MINUTE,
ISNULL(pass.FirstIn, pass.FirstRawEvent),
CASE
WHEN pass.FilteredLastOut IS NOT NULL THEN pass.FilteredLastOut
WHEN @TargetDate = CAST(GETDATE() AS DATE) THEN GETDATE()
ELSE ISNULL(pass.FirstIn, pass.FirstRawEvent)
ELSE @EndDate
END) % 60 AS VARCHAR), 2)
ELSE N'00:00'
END AS [Находился_в_здании],
@@ -196,7 +199,7 @@ ORDER BY p.Name ASC;
SQL_RAW_EVENTS_QUERY = r"""
DECLARE @InputDate DATE = '{target_date}';
DECLARE @StartDate DATETIME = CAST(@InputDate AS DATETIME);
DECLARE @EndDate DATETIME = DATEADD(SECOND, -1, DATEADD(DAY, 1, @StartDate));
DECLARE @EndDate DATETIME = {end_datetime_sql};
SELECT
log.TimeVal,
@@ -211,6 +214,7 @@ SELECT
ISNULL(CAST(div.Name AS NVARCHAR(255)), N'Без подразделения') AS [Подразделение],
log.Event,
log.Mode,
log.DoorIndex,
CASE
WHEN log.Mode = 2 THEN 'OUT'
WHEN log.Mode = 1 THEN 'IN'
@@ -229,6 +233,7 @@ WHERE log.TimeVal BETWEEN @StartDate AND @EndDate
ORDER BY log.TimeVal ASC;
"""
def save_df_to_clean_excel(df: pd.DataFrame, file_path: str, sheet_name: str = "Отчет"):
workbook = xlsxwriter.Workbook(file_path, {'constant_memory': False})
worksheet = workbook.add_worksheet(sheet_name)
@@ -279,12 +284,12 @@ def save_df_to_clean_excel(df: pd.DataFrame, file_path: str, sheet_name: str = "
workbook.close()
def get_targets(input_date: str | None):
def get_targets(input_date: str | None, input_time: str | None = None):
targets = []
if input_date:
try:
parsed = datetime.strptime(input_date, "%d.%m.%Y").date()
targets.append({"name": "Указанная дата", "date": parsed})
parsed = datetime.strptime(input_date.replace('_', '.'), "%d.%m.%Y").date()
targets.append({"name": "Указанная дата", "date": parsed, "target_time": input_time})
except ValueError:
log(f"ОШИБКА: Неверный формат даты '{input_date}'. Используйте ДД.ММ.ГГГГ", "ERROR")
sys.exit(1)
@@ -293,13 +298,13 @@ def get_targets(input_date: str | None):
yesterday = (now - timedelta(days=3 if now.weekday() == 0 else 1)).date()
today = now.date()
targets.append({"name": "Вчера", "date": yesterday})
targets.append({"name": "Сегодня", "date": today})
targets.append({"name": "Вчера", "date": yesterday, "target_time": None})
targets.append({"name": "Сегодня", "date": today, "target_time": None})
return targets
def run_export(input_date: str | None = None, save_xlsx: bool = True, debug: bool = False):
def run_export(input_date: str | None = None, input_time: str | None = None, save_xlsx: bool = True, debug: bool = False):
if debug:
logger.setLevel(logging.DEBUG)
log("=== ВКЛЮЧЕН РЕЖИМ ОТЛАДКИ (DEBUG MODE) ===", "WARNING")
@@ -307,7 +312,7 @@ def run_export(input_date: str | None = None, save_xlsx: bool = True, debug: boo
log("=== [ЭТАП 0] Выгрузка свежих данных СКУД напрямую из БД Орион ===")
os.makedirs(SCUD_DIR, exist_ok=True)
targets = get_targets(input_date)
targets = get_targets(input_date, input_time)
conn_str = (
f"DRIVER={{{ODBC_DRIVER}}};"
f"SERVER={SERVER_NAME};"
@@ -323,19 +328,66 @@ def run_export(input_date: str | None = None, save_xlsx: bool = True, debug: boo
processing_date = target["date"]
processing_date_str = processing_date.strftime("%d.%m.%Y")
period_label = target["name"]
is_yesterday = (period_label == "Вчера")
target_time = target["target_time"]
today_date = datetime.now().date()
is_past_day = (processing_date < today_date) or (period_label == "Вчера")
if is_yesterday and has_yesterday_final_snapshot(processing_date_str):
log(f"[ℹ️] Вчерашний день ({processing_date_str}) уже зафиксирован финишным снапшотом Y. Пропускаем запрос к MS SQL.")
if is_past_day and not target_time and has_yesterday_final_snapshot(processing_date_str):
log(f"[ℹ️] День ({processing_date_str}) уже зафиксирован финишным снапшотом _FINAL. Пропускаем.")
continue
if is_yesterday:
# 1. Формируем фильтр правого турникета из exceptions_registry
exc_data = load_exceptions()
t_fios = [f.replace("'", "''") for f in exc_data.get('turnstile_fio', []) if f]
t_depts = [d.replace("'", "''") for d in exc_data.get('turnstile_departments', []) if d]
conditions = []
if t_fios:
fio_in = ", ".join([f"N'{f}'" for f in t_fios])
# Склеиваем Фамилию + Имя + Отчество для точного сравнения с реестром ФИО
full_fio_sql = (
"LTRIM(RTRIM("
"ISNULL(CAST(p.Name AS NVARCHAR(255)), N'') + "
"CASE WHEN p.FirstName IS NOT NULL AND CAST(p.FirstName AS NVARCHAR(255)) <> '' THEN N' ' + CAST(p.FirstName AS NVARCHAR(255)) ELSE N'' END + "
"CASE WHEN p.MidName IS NOT NULL AND CAST(p.MidName AS NVARCHAR(255)) <> '' THEN N' ' + CAST(p.MidName AS NVARCHAR(255)) ELSE N'' END"
"))"
)
conditions.append(f"{full_fio_sql} IN ({fio_in})")
if t_depts:
dept_in = ", ".join([f"N'{d}'" for d in t_depts])
conditions.append(f"ISNULL(CAST(div.Name AS NVARCHAR(255)), N'') IN ({dept_in})")
turnstile_filter_sql = " OR ".join(conditions) if conditions else "1 = 0"
# 2. Безопасное математическое определение @EndDate через DATEADD (независимо от локали сервера)
if target_time:
t_clean = target_time.strip()
t_parts = t_clean.split(":")
h = int(t_parts[0])
m = int(t_parts[1]) if len(t_parts) > 1 else 0
s = int(t_parts[2]) if len(t_parts) > 2 else 0
snapshot_time = f"{processing_date.strftime('%Y-%m-%d')} {h:02d}:{m:02d}:{s:02d}"
end_datetime_sql = f"DATEADD(SECOND, {s}, DATEADD(MINUTE, {m}, DATEADD(HOUR, {h}, @StartDate)))"
is_final = False
elif is_past_day:
snapshot_time = f"{processing_date.strftime('%Y-%m-%d')} 23:59:59"
end_datetime_sql = "DATEADD(SECOND, -1, DATEADD(DAY, 1, @StartDate))"
is_final = True
else:
snapshot_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
end_datetime_sql = "GETDATE()"
is_final = False
log(f"--- Обработка периода: {period_label} ({processing_date_str}) --- [Снапшот: {snapshot_time}]")
sql_query = SQL_QUERY_TEMPLATE.format(target_date=processing_date.strftime("%Y-%m-%d"))
log(f"--- Обработка периода: {period_label} ({processing_date_str}) --- [Срез: {snapshot_time}]")
sql_query = SQL_QUERY_TEMPLATE.format(
target_date=processing_date.strftime("%Y-%m-%d"),
end_datetime_sql=end_datetime_sql,
turnstile_filter_sql=turnstile_filter_sql
)
connection = None
try:
@@ -353,31 +405,34 @@ def run_export(input_date: str | None = None, save_xlsx: bool = True, debug: boo
df['Пришел'] = df['Статус'].str.contains('Присутствовал', case=False, na=False) & (~mask_anomaly)
save_scud_to_db(df, processing_date_str, snapshot_time=snapshot_time, is_yesterday=is_yesterday)
save_scud_to_db(df, processing_date_str, snapshot_time=snapshot_time, is_yesterday=is_final)
raw_sql = SQL_RAW_EVENTS_QUERY.format(target_date=processing_date.strftime("%Y-%m-%d"))
raw_sql = SQL_RAW_EVENTS_QUERY.format(
target_date=processing_date.strftime("%Y-%m-%d"),
end_datetime_sql=end_datetime_sql
)
df_raw = pd.read_sql(raw_sql, connection)
if len(df_raw) > 0:
df_raw['fio_clean'] = df_raw['Сотрудник'].apply(clean_scud_fio_light)
inserted_count = save_raw_events_to_db(df_raw, processing_date_str)
log(f"[✓] В scud_events_raw сохранено {inserted_count} сырых событий проходов за {processing_date_str}!", "SUCCESS")
log(f"[✓] В scud_events_raw сохранено {inserted_count} сырых событий за {processing_date_str}!", "SUCCESS")
log(f"[✓] Записи за {processing_date_str} успешно сохранены в SQLite!", "SUCCESS")
log(f"[✓] Записи за {processing_date_str} успешно сохранены в SQLite!", "SUCCESS")
if save_xlsx:
file_name = f"Сотрудники_{processing_date_str}.xlsx"
file_path = os.path.join(SCUD_DIR, file_name)
if save_xlsx:
file_name = f"Сотрудники_{processing_date_str}.xlsx"
file_path = os.path.join(SCUD_DIR, file_name)
if os.path.exists(file_path):
try:
os.remove(file_path)
except OSError as e:
log(f"ОШИБКА при удалении старого файла {file_name}: {e}", "ERROR")
if os.path.exists(file_path):
try:
os.remove(file_path)
except OSError as e:
log(f"ОШИБКА при удалении старого файла {file_name}: {e}", "ERROR")
save_df_to_clean_excel(df, file_path, sheet_name="Отчет")
log(f"[✓] Успешно экспортирован файл: data/scud/{file_name}", "SUCCESS")
else:
log(f"Запрос за {processing_date_str} вернул 0 строк.", "WARNING")
save_df_to_clean_excel(df, file_path, sheet_name="Отчет")
log(f"[✓] Успешно экспортирован файл: data/scud/{file_name}", "SUCCESS")
else:
log(f"Запрос за {processing_date_str} вернул 0 строк.", "WARNING")
except Exception as e:
log(f"🛑 ОШИБКА выгрузки СКУД за {processing_date_str}: {e}", "ERROR")
@@ -392,9 +447,10 @@ def run_export(input_date: str | None = None, save_xlsx: bool = True, debug: boo
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--date", dest="input_date", default=None)
parser.add_argument("--date", dest="input_date", default=None, help="Дата среза (ДД.ММ.ГГГГ)")
parser.add_argument("--time", dest="input_time", default=None, help="Время среза (ЧЧ:ММ)")
parser.add_argument("-d", "--debug", action="store_true")
parser.add_argument("--no-xlsx", dest="save_xlsx", action="store_false", default=True)
args = parser.parse_args()
run_export(args.input_date, save_xlsx=args.save_xlsx, debug=args.debug)
run_export(args.input_date, input_time=args.input_time, save_xlsx=args.save_xlsx, debug=args.debug)