feat(etl): stable pipeline, exception registry in SQLite, multi-pass aggregation and db_cli
This commit is contained in:
@@ -0,0 +1,60 @@
|
||||
"""
|
||||
===============================================================================
|
||||
FILE: services/scud_etl/anomaly_detector.py
|
||||
PROJECT: SCUD Orion AI (Unified Architecture)
|
||||
MODULE: services / scud_etl
|
||||
ROLE: Детектирование истинных аномалий и конфликтов реестров.
|
||||
(Сотрудники из exceptions.json, удаленка и командировки исключены).
|
||||
===============================================================================
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import List, Dict, Any
|
||||
import pandas as pd
|
||||
|
||||
logger = logging.getLogger("SCUD_ANOMALY")
|
||||
|
||||
|
||||
def detect_registry_anomalies(df_merged: pd.DataFrame) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Выявляет реальные аномалии:
|
||||
- Приход в офис во время отпуска или больничного листа.
|
||||
- Ошибки считывателей СКУД (наличие выхода при отсутствии отметки входа).
|
||||
"""
|
||||
anomalies = []
|
||||
if df_merged is None or df_merged.empty:
|
||||
return anomalies
|
||||
|
||||
for _, row in df_merged.iterrows():
|
||||
# Пропускаем системные исключения
|
||||
if row.get("is_excluded", False):
|
||||
continue
|
||||
|
||||
fio = row.get("fio_clean") or row.get("Сотрудник", "")
|
||||
start_day = str(row.get("Начало_дня", "")).strip()
|
||||
end_day = str(row.get("Конец_дня", "")).strip()
|
||||
reason = str(row.get("причина отсутствия", row.get("Вид_отсутствия", ""))).strip()
|
||||
reason_lower = reason.lower()
|
||||
|
||||
# Пропускаем технические метки
|
||||
if "исключен" in reason_lower or "овк" in reason_lower:
|
||||
continue
|
||||
|
||||
# ⭐️ Физический приход в офис: удаленка и командировки разрешены и НЕ являются аномалией
|
||||
if start_day not in ["Нет входа", "—", "", "nan", "None"] and reason and reason != "nan":
|
||||
if not ("удален" in reason_lower or "дистанцион" in reason_lower or "командировк" in reason_lower or "поездк" in reason_lower):
|
||||
anomalies.append({
|
||||
"fio": fio,
|
||||
"type": "PHYSICAL_PRESENCE_DURING_ABSENCE",
|
||||
"description": f"Сотрудник пришел по СКУД ({start_day}), но в 1С оформлен документ: '{reason}'."
|
||||
})
|
||||
|
||||
# Аномалия оборудования (есть выход без утреннего входа)
|
||||
if start_day in ["Нет входа", "—", ""] and end_day not in ["Нет выхода", "—", "", "nan", "None"]:
|
||||
anomalies.append({
|
||||
"fio": fio,
|
||||
"type": "SCUD_EQUIPMENT_ANOMALY",
|
||||
"description": f"Отсутствует отметка утреннего входа при наличии выхода ({end_day})."
|
||||
})
|
||||
|
||||
return anomalies
|
||||
@@ -0,0 +1,147 @@
|
||||
"""
|
||||
===============================================================================
|
||||
FILE: services/scud_etl/merger.py
|
||||
===============================================================================
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Dict, Any
|
||||
import pandas as pd
|
||||
|
||||
from services.knowledge.service import get_department_synonyms_dict
|
||||
from config import normalize_fio, load_exceptions
|
||||
|
||||
logger = logging.getLogger("SCUD_MERGER")
|
||||
|
||||
|
||||
def merge_scud_and_1c(
|
||||
df_scud: pd.DataFrame,
|
||||
df_staff_1c: pd.DataFrame,
|
||||
df_absences_1c: pd.DataFrame
|
||||
) -> pd.DataFrame:
|
||||
if (df_scud is None or df_scud.empty) and (df_staff_1c is None or df_staff_1c.empty):
|
||||
return pd.DataFrame(columns=[
|
||||
'Сотрудник', 'fio_clean', 'Подразделение', 'Должность',
|
||||
'Начало_дня', 'Первая_активность', 'Конец_дня', 'Находился_в_здании',
|
||||
'Пришел', 'anomaly_flag', 'причина отсутствия', 'Вид_отсутствия', 'is_excluded'
|
||||
])
|
||||
|
||||
synonyms = get_department_synonyms_dict()
|
||||
exceptions_cfg = load_exceptions()
|
||||
|
||||
df_res = df_scud.copy() if df_scud is not None and not df_scud.empty else df_staff_1c.copy()
|
||||
|
||||
if "Сотрудник" in df_res.columns:
|
||||
df_res["fio_clean"] = df_res["Сотрудник"].apply(normalize_fio)
|
||||
elif "ФИО" in df_res.columns:
|
||||
df_res["Сотрудник"] = df_res["ФИО"]
|
||||
df_res["fio_clean"] = df_res["ФИО"].apply(normalize_fio)
|
||||
elif "fio_clean" not in df_res.columns:
|
||||
df_res["fio_clean"] = ""
|
||||
|
||||
for col, default_val in [
|
||||
('Начало_дня', 'Нет входа'),
|
||||
('Первая_активность', '—'),
|
||||
('Конец_дня', 'Нет выхода'),
|
||||
('Находился_в_здании', '00:00'),
|
||||
('Пришел', False),
|
||||
('anomaly_flag', 'NONE')
|
||||
]:
|
||||
if col not in df_res.columns:
|
||||
df_res[col] = default_val
|
||||
|
||||
# 1. Приведение отделов к аббревиатурам
|
||||
reverse_synonyms = {v.lower(): k.upper() for k, v in synonyms.items()}
|
||||
direct_synonyms = {k.lower(): k.upper() for k in synonyms.keys()}
|
||||
all_dept_map = {**reverse_synonyms, **direct_synonyms, "отдел внутреннего контроля": "ОВК", "отдел вневедомственного контроля": "ОВК"}
|
||||
|
||||
if "Подразделение" in df_res.columns:
|
||||
df_res["Подразделение"] = df_res["Подразделение"].apply(
|
||||
lambda d: all_dept_map.get(str(d).strip().lower(), str(d).strip())
|
||||
)
|
||||
|
||||
# 2. Привязка документов отсутствий из 1С ПЕРЕД разметкой исключений
|
||||
absences_map = {}
|
||||
if not df_absences_1c.empty:
|
||||
fio_col = next((c for c in ["fio_clean", "ФИО", "Сотрудник"] if c in df_absences_1c.columns), None)
|
||||
reason_col = next((c for c in ["Вид_отсутствия", "Причина", "причина отсутствия"] if c in df_absences_1c.columns), None)
|
||||
|
||||
if fio_col and reason_col:
|
||||
for _, row in df_absences_1c.iterrows():
|
||||
fio = normalize_fio(str(row[fio_col]))
|
||||
reason = str(row[reason_col]).strip()
|
||||
if reason and reason.lower() != "nan":
|
||||
absences_map[fio] = reason
|
||||
|
||||
df_res["причина отсутствия"] = df_res["fio_clean"].map(absences_map)
|
||||
df_res["Вид_отсутствия"] = df_res["причина отсутствия"]
|
||||
|
||||
# 3. Разметка исключений
|
||||
exc_fios = [normalize_fio(f) for f in exceptions_cfg.get("fio", []) if f]
|
||||
exc_depts = [d.strip().upper() for d in exceptions_cfg.get("departments", []) if d]
|
||||
exc_pos = [p.strip().lower() for p in exceptions_cfg.get("positions", []) if p]
|
||||
pos_kw = [k.strip().lower() for k in exceptions_cfg.get("position_keywords", []) if k]
|
||||
whitelist_fios = [normalize_fio(f) for f in exceptions_cfg.get("include_fio", []) if f]
|
||||
|
||||
df_res["is_excluded"] = False
|
||||
for idx, row in df_res.iterrows():
|
||||
fio = row.get("fio_clean", "")
|
||||
has_official_absence = pd.notna(row.get("Вид_отсутствия")) and str(row.get("Вид_отсутствия")).strip() != ""
|
||||
|
||||
# ⭐️ Если сотрудник в белом списке ИЛИ у него есть официальное отсутствие — он НЕ исключается
|
||||
if fio in whitelist_fios or has_official_absence:
|
||||
df_res.at[idx, "is_excluded"] = False
|
||||
continue
|
||||
|
||||
dep = str(row.get("Подразделение", "")).upper()
|
||||
pos = str(row.get("Должность", "")).lower()
|
||||
|
||||
if fio in exc_fios or dep in exc_depts or any(d in dep for d in exc_depts) or pos in exc_pos or any(k in pos for k in pos_kw):
|
||||
df_res.at[idx, "is_excluded"] = True
|
||||
|
||||
# Чистое "Исключение" только для истинно исключенных без документов 1С
|
||||
mask_exc = (df_res["is_excluded"] == True) & (df_res["Вид_отсутствия"].isna() | (df_res["Вид_отсутствия"] == ""))
|
||||
df_res.loc[mask_exc, "Вид_отсутствия"] = "Исключение"
|
||||
df_res.loc[mask_exc, "причина отсутствия"] = "Исключение"
|
||||
|
||||
return df_res
|
||||
|
||||
|
||||
def calculate_summary_metrics(df_merged: pd.DataFrame) -> Dict[str, Any]:
|
||||
active_df = df_merged[df_merged.get("is_excluded", False) == False].copy()
|
||||
total_staff = len(active_df)
|
||||
|
||||
# 1. Строго физически пришедшие в офис
|
||||
came_to_office_mask = active_df["Начало_дня"].astype(str).str.strip().ne("Нет входа")
|
||||
working_in_office = active_df[came_to_office_mask]
|
||||
working_in_office_count = len(working_in_office)
|
||||
|
||||
# 2. Не пришедшие в офис
|
||||
df_not_came = active_df[~came_to_office_mask]
|
||||
|
||||
# 3. Из не пришедших — удаленщики (работают из дома)
|
||||
reason_series = df_not_came["причина отсутствия"].astype(str).str.lower()
|
||||
is_remote_mask = reason_series.str.contains("удален|дистанцион", regex=True, na=False)
|
||||
remote_home = df_not_came[is_remote_mask]
|
||||
remote_home_count = len(remote_home)
|
||||
|
||||
# 4. Из оставшихся — официальные отсутствия (отпуск, больничный, командировка)
|
||||
df_remaining_absent = df_not_came[~is_remote_mask]
|
||||
has_doc_mask = df_remaining_absent["причина отсутствия"].notna() & \
|
||||
df_remaining_absent["причина отсутствия"].ne("") & \
|
||||
df_remaining_absent["причина отсутствия"].ne("nan") & \
|
||||
(~df_remaining_absent["причина отсутствия"].astype(str).str.startswith("Исключение"))
|
||||
official_absent_count = len(df_remaining_absent[has_doc_mask])
|
||||
|
||||
# 5. Неизвестные
|
||||
unknown = df_remaining_absent[~has_doc_mask]
|
||||
unknown_count = len(unknown)
|
||||
|
||||
return {
|
||||
"total_staff": total_staff,
|
||||
"working_in_office_count": working_in_office_count,
|
||||
"remote_home_count": remote_home_count,
|
||||
"official_absent_count": official_absent_count,
|
||||
"unknown_count": unknown_count,
|
||||
"unknown_list": unknown[["fio_clean", "Подразделение", "Должность"]].to_dict(orient="records") if not unknown.empty else []
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
"""
|
||||
===============================================================================
|
||||
FILE: services/scud_etl/pipeline.py
|
||||
ROLE: Выборка данных СКУД из SQLite (scud_logs) с жестким приоритетом Y-снапшотов.
|
||||
===============================================================================
|
||||
"""
|
||||
|
||||
import os
|
||||
import logging
|
||||
import pandas as pd
|
||||
from core.connection import get_connection
|
||||
from services.data_loader import load_1c_data_smart
|
||||
|
||||
logger = logging.getLogger("SCUD_PIPELINE")
|
||||
|
||||
|
||||
def load_best_snapshot_for_date(date_str: str, prefer_final_y: bool = True) -> pd.DataFrame:
|
||||
"""Извлекает срез СКУД. Для вчерашнего дня строго берет финальный вечерний срез Y."""
|
||||
with get_connection() as conn:
|
||||
df = pd.DataFrame()
|
||||
cursor = conn.cursor()
|
||||
|
||||
if prefer_final_y:
|
||||
cursor.execute(
|
||||
"SELECT snapshot_id FROM scud_logs WHERE log_date = ? AND snapshot_id LIKE 'Y%' ORDER BY id DESC LIMIT 1",
|
||||
(date_str,)
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if row and row[0]:
|
||||
df = pd.read_sql_query("SELECT * FROM scud_logs WHERE snapshot_id = ?", conn, params=(row[0],))
|
||||
|
||||
if df.empty:
|
||||
cursor.execute(
|
||||
"SELECT snapshot_id FROM scud_logs WHERE log_date = ? ORDER BY id DESC LIMIT 1",
|
||||
(date_str,)
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if row and row[0]:
|
||||
df = pd.read_sql_query("SELECT * FROM scud_logs WHERE snapshot_id = ?", conn, params=(row[0],))
|
||||
|
||||
if not df.empty:
|
||||
rename_map = {
|
||||
'department': 'Подразделение', 'position': 'Должность', 'fio': 'Сотрудник',
|
||||
'time_in': 'Начало_дня', 'first_activity': 'Первая_активность', 'time_out': 'Конец_дня',
|
||||
'time_in_building': 'Находился_в_здании', 'is_present': 'Пришел'
|
||||
}
|
||||
df = df.rename(columns={k: v for k, v in rename_map.items() if k in df.columns})
|
||||
if 'fio_clean' not in df.columns and 'Сотрудник' in df.columns:
|
||||
df['fio_clean'] = df['Сотрудник'].astype(str).str.strip()
|
||||
if 'Пришел' in df.columns:
|
||||
df['Пришел'] = df['Пришел'].astype(bool)
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def load_1c_files_for_date(date_str: str) -> tuple[pd.DataFrame, pd.DataFrame]:
|
||||
df_staff, df_absences = load_1c_data_smart(date_str, use_db=False)
|
||||
return (df_staff if df_staff is not None else pd.DataFrame(),
|
||||
df_absences if df_absences is not None else pd.DataFrame())
|
||||
@@ -0,0 +1,116 @@
|
||||
"""
|
||||
===============================================================================
|
||||
FILE: services/scud_etl/sql_queries.py
|
||||
PROJECT: SCUD Orion AI (Unified Architecture)
|
||||
MODULE: services / scud_etl
|
||||
ROLE: Хранилище сырых SQL-шаблонов для выгрузки из MS SQL Server (СКУД Орион Pro).
|
||||
|
||||
AI-CONTEXT-ANCHORS:
|
||||
- ANCHOR[SQL_SCUD_EXPORT_TEMPLATE]: T-SQL запрос с расчетом первой активности и длительности.
|
||||
===============================================================================
|
||||
"""
|
||||
|
||||
# ANCHOR[SQL_SCUD_EXPORT_TEMPLATE]
|
||||
SCUD_EXPORT_QUERY_TEMPLATE = r"""
|
||||
DECLARE @InputDate DATE = '{target_date}';
|
||||
DECLARE @TargetDate DATE = @InputDate;
|
||||
|
||||
DECLARE @StartDate DATETIME = CAST(@TargetDate AS DATETIME);
|
||||
DECLARE @EndDate DATETIME = DATEADD(SECOND, -1, DATEADD(DAY, 1, @StartDate));
|
||||
|
||||
WITH DailyLogs AS (
|
||||
SELECT
|
||||
log.HozOrgan AS EmployeeID,
|
||||
log.TimeVal,
|
||||
log.Event,
|
||||
log.Mode,
|
||||
CASE
|
||||
WHEN log.Mode = 2 OR log.Event IN (29, 27, 33) THEN 'OUT'
|
||||
WHEN log.Mode = 1 OR log.Event IN (28, 26, 32) THEN 'IN'
|
||||
ELSE 'OTHER'
|
||||
END AS Direction,
|
||||
ROW_NUMBER() OVER (PARTITION BY log.HozOrgan ORDER BY log.TimeVal DESC) AS RowNumDesc
|
||||
FROM pLogData log WITH (NOLOCK)
|
||||
WHERE log.TimeVal BETWEEN @StartDate AND @EndDate
|
||||
AND log.HozOrgan IS NOT NULL
|
||||
AND log.HozOrgan > 0
|
||||
AND log.Event IN (26, 27, 28, 29, 32, 33, 54, 55, 64, 65)
|
||||
),
|
||||
Passages AS (
|
||||
SELECT
|
||||
EmployeeID,
|
||||
MIN(TimeVal) AS FirstRawEvent,
|
||||
MAX(TimeVal) AS LastRawEvent,
|
||||
MIN(CASE WHEN Direction = 'IN' THEN TimeVal END) AS FirstIn,
|
||||
MAX(CASE WHEN Direction = 'OUT' THEN TimeVal END) AS LastOut,
|
||||
MAX(CASE WHEN RowNumDesc = 1 THEN Direction END) AS LastEventType
|
||||
FROM DailyLogs
|
||||
GROUP BY EmployeeID
|
||||
)
|
||||
SELECT
|
||||
N'ЛЕНМОРНИИПРОЕКТ' AS [Фирма],
|
||||
ISNULL(CAST(div.Name AS NVARCHAR(255)), N'Без подразделения') AS [Подразделение],
|
||||
LTRIM(RTRIM(
|
||||
ISNULL(CAST(p.Name AS NVARCHAR(255)), N'') +
|
||||
CASE WHEN p.FirstName IS NOT NULL AND CAST(p.FirstName AS NVARCHAR(255)) <> ''
|
||||
THEN N' ' + CAST(p.FirstName AS NVARCHAR(255)) ELSE N'' END +
|
||||
CASE WHEN p.MidName IS NOT NULL AND CAST(p.MidName AS NVARCHAR(255)) <> ''
|
||||
THEN N' ' + CAST(p.MidName AS NVARCHAR(255)) ELSE N'' END
|
||||
)) AS [Сотрудник],
|
||||
ISNULL(CAST(post.Name AS NVARCHAR(255)), N'—') AS [Должность],
|
||||
ISNULL(CAST(p.TabNumber AS NVARCHAR(50)), N'—') AS [Таб_№],
|
||||
CONVERT(VARCHAR(10), @TargetDate, 104) AS [Дата],
|
||||
ISNULL(CAST(CONVERT(VARCHAR(8), pass.FirstIn, 108) AS NVARCHAR(20)), N'Нет входа') AS [Начало_дня],
|
||||
CASE
|
||||
WHEN pass.FirstIn IS NULL AND pass.FirstRawEvent IS NOT NULL
|
||||
THEN CAST(CONVERT(VARCHAR(8), pass.FirstRawEvent, 108) AS NVARCHAR(20))
|
||||
ELSE N'—'
|
||||
END AS [Первая_активность],
|
||||
CASE
|
||||
WHEN @TargetDate = CAST(GETDATE() AS DATE) AND (pass.LastEventType = 'IN' OR pass.LastOut IS NULL OR pass.LastOut <= pass.FirstIn)
|
||||
THEN N'Нет выхода'
|
||||
WHEN pass.LastOut IS NOT NULL AND pass.LastOut > pass.FirstIn
|
||||
THEN CAST(CONVERT(VARCHAR(8), pass.LastOut, 108) AS NVARCHAR(20))
|
||||
WHEN @TargetDate < CAST(GETDATE() AS DATE) AND pass.LastRawEvent IS NOT NULL AND pass.LastRawEvent > ISNULL(pass.FirstIn, pass.FirstRawEvent)
|
||||
THEN CAST(CONVERT(VARCHAR(8), pass.LastRawEvent, 108) AS NVARCHAR(20))
|
||||
ELSE N'Нет выхода'
|
||||
END AS [Конец_дня],
|
||||
CASE
|
||||
WHEN pass.EmployeeID IS NOT NULL AND (pass.FirstIn IS NOT NULL OR pass.FirstRawEvent IS NOT NULL) THEN
|
||||
RIGHT('0' + CAST(DATEDIFF(MINUTE,
|
||||
ISNULL(pass.FirstIn, pass.FirstRawEvent),
|
||||
CASE
|
||||
WHEN @TargetDate = CAST(GETDATE() AS DATE) AND (pass.LastEventType = 'IN' OR pass.LastOut IS NULL OR pass.LastOut <= pass.FirstIn) THEN GETDATE()
|
||||
ELSE ISNULL(pass.LastOut, pass.LastRawEvent)
|
||||
END) / 60 AS VARCHAR), 2) + ':' +
|
||||
RIGHT('0' + CAST(DATEDIFF(MINUTE,
|
||||
ISNULL(pass.FirstIn, pass.FirstRawEvent),
|
||||
CASE
|
||||
WHEN @TargetDate = CAST(GETDATE() AS DATE) AND (pass.LastEventType = 'IN' OR pass.LastOut IS NULL OR pass.LastOut <= pass.FirstIn) THEN GETDATE()
|
||||
ELSE ISNULL(pass.LastOut, pass.LastRawEvent)
|
||||
END) % 60 AS VARCHAR), 2)
|
||||
ELSE N'00:00'
|
||||
END AS [Находился_в_здании],
|
||||
CASE
|
||||
WHEN pass.EmployeeID IS NOT NULL THEN N'Присутствовал'
|
||||
ELSE N'Отсутствовал (Нет событий)'
|
||||
END AS [Статус]
|
||||
FROM pList p WITH (NOLOCK)
|
||||
LEFT JOIN PDivision div WITH (NOLOCK) ON p.Section = div.ID
|
||||
LEFT JOIN PPost post WITH (NOLOCK) ON p.Post = post.ID
|
||||
LEFT JOIN Passages pass ON p.ID = pass.EmployeeID
|
||||
WHERE
|
||||
ISNULL(p.StatusRecord, 0) = 0
|
||||
AND p.DateTimeInArchive IS NULL
|
||||
AND ISNULL(CAST(div.Name AS NVARCHAR(255)), N'') NOT LIKE N'Аренд%'
|
||||
AND ISNULL(CAST(div.Name AS NVARCHAR(255)), N'') NOT IN (N'Без подразделения', N'')
|
||||
AND p.Name NOT LIKE N'бр.%'
|
||||
AND p.Name NOT LIKE N'Гость%'
|
||||
AND ISNULL(CAST(div.Name AS NVARCHAR(255)), N'') NOT IN (N'БГИ', N'КНР')
|
||||
AND ISNULL(CAST(div.Name AS NVARCHAR(255)), N'') NOT LIKE N'Рабоч%'
|
||||
AND ISNULL(CAST(div.Name AS NVARCHAR(255)), N'') NOT LIKE N'Врем%'
|
||||
AND ISNULL(CAST(div.Name AS NVARCHAR(255)), N'') NOT LIKE N'Практика%'
|
||||
AND ISNULL(CAST(div.Name AS NVARCHAR(255)), N'') NOT LIKE N'тест%'
|
||||
AND ISNULL(CAST(post.Name AS NVARCHAR(255)), N'') NOT LIKE N'Практикант%'
|
||||
ORDER BY p.Name ASC;
|
||||
"""
|
||||
Reference in New Issue
Block a user