refactor: step 3 - decompose ETL pipeline into modular services/scud_etl package

This commit is contained in:
2026-08-21 10:12:26 +03:00
parent 16928de5bd
commit e8a542eded
7 changed files with 486 additions and 523 deletions
View File
+72
View File
@@ -0,0 +1,72 @@
"""
===============================================================================
FILE: services/scud_etl/anomaly_detector.py
PROJECT: SCUD Orion AI (Unified Architecture)
MODULE: services / scud_etl
ROLE: Автоматическое выявление аномалий и конфликтов между 1С:ЗУП и СКУД.
AI-CONTEXT-ANCHORS:
- ANCHOR[ANOMALY_DETECTOR_CORE]: Проверка физического присутствия в отпуске и перемещений без входа.
===============================================================================
"""
import pandas as pd
from typing import List, Dict, Any, Optional
ALLOWED_WORK_TRIP_KEYWORDS = ['командировк', 'разъездн', 'поездк']
# ANCHOR[ANOMALY_DETECTOR_CORE]
def detect_all_anomalies(
merged_df: pd.DataFrame,
static_reasons_dict: dict,
kb_rules: List[str],
scud_fios_set: Optional[set] = None
) -> List[Dict[str, Any]]:
"""Выявляет конфликты и аномалии между источниками СКУД и 1С."""
anomalies = []
kb_rules_text = " ".join(kb_rules).lower() if kb_rules else ""
for idx, row in merged_df.iterrows():
fio = row.get('Сотрудник', row.get('fio_clean', ''))
fio_clean = row.get('fio_clean', '')
is_present = row.get('Пришел', False)
is_exc = row.get('is_excluded', False)
reason_1c = str(row.get('Вид_отсутствия', '')).strip()
has_1c_reason = pd.notna(row.get('Вид_отсутствия')) and reason_1c != '' and not reason_1c.startswith('Исключение')
anom_flag = row.get('anomaly_flag', 'NONE')
is_fio_whitelisted = fio_clean.lower() in kb_rules_text
# 1. Присутствие при официальном отсутствии
if is_present and has_1c_reason:
is_allowed_trip = any(kw in reason_1c.lower() for kw in ALLOWED_WORK_TRIP_KEYWORDS)
if not is_allowed_trip and not is_fio_whitelisted:
anomalies.append({
"type": "ФИЗИЧЕСКОЕ ПРИСУТСТВИЕ ПРИ ОФИЦИАЛЬНОМ ОТСУТСТВИИ",
"fio": fio,
"details": f"Сотрудник пришел по СКУД, но в 1С оформлен документ: '{reason_1c}'"
})
if is_exc and not has_1c_reason:
continue
# 2. Перемещение внутри здания без отметки входа на КПП
if anom_flag == 'ANOMALY_NO_IN_HAS_ACTIVITY':
first_act = row.get('Первая_активность', '—')
anomalies.append({
"type": "АНОМАЛИЯ СКУД: ПЕРЕМЕЩЕНИЕ БЕЗ ВХОДА",
"fio": fio,
"details": f"Отсутствует регистрация входа на КПП при зафиксированной первой активности в {first_act}"
})
# 3. Сотрудник в штате 1С, но карты/профиля в СКУД нет
if scud_fios_set is not None:
if fio_clean not in scud_fios_set and not has_1c_reason:
anomalies.append({
"type": "АНОМАЛИЯ УЧЕТА: СОТРУДНИК ОТСУТСТВУЕТ В СКУД ОРИОН PRO",
"fio": fio,
"details": f"Сотрудник числится в Штатном расписании 1С ({row.get('Подразделение', '—')}), но отсутствует в СКУД"
})
return anomalies
+114
View File
@@ -0,0 +1,114 @@
"""
===============================================================================
FILE: services/scud_etl/merger.py
PROJECT: SCUD Orion AI (Unified Architecture)
MODULE: services / scud_etl
ROLE: Агрегация проходов, сопоставление исключений и мердж таблиц 1С:ЗУП и СКУД.
AI-CONTEXT-ANCHORS:
- ANCHOR[MERGER_EXCEPTIONS]: Наложение флага исключений из exceptions.json.
- ANCHOR[MERGER_AGGREGATION]: Агрегация множественных проходов до уникального ФИО.
- ANCHOR[MERGER_BUILD_DATASET]: Сборка итогового датасета для сводок и отчетов.
===============================================================================
"""
import os
import json
import pandas as pd
from config import normalize_fio, DATA_DIR
# ANCHOR[MERGER_EXCEPTIONS]
def load_exceptions_config() -> dict:
"""Загружает exceptions.json из корня проекта."""
root_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../"))
json_path = os.path.join(root_dir, "exceptions.json")
if not os.path.exists(json_path):
return {}
try:
with open(json_path, 'r', encoding='utf-8') as f:
return json.load(f)
except Exception:
return {}
def apply_exceptions_from_json(df: pd.DataFrame, exceptions_cfg: dict) -> pd.DataFrame:
"""Быстрая разметка флага is_excluded на основе exceptions.json."""
if df is None or df.empty or not exceptions_cfg:
if df is not None:
df['is_excluded'] = False
return df
deps = [d.strip().lower() for d in exceptions_cfg.get("departments", []) if d]
exact_pos = [p.strip().lower() for p in exceptions_cfg.get("positions", []) if p]
pos_kw = [k.strip().lower() for k in exceptions_cfg.get("position_keywords", []) if k]
exc_fios = [normalize_fio(f) for f in exceptions_cfg.get("fio", []) if f]
df['is_excluded'] = False
for idx, row in df.iterrows():
fio_clean = row.get('fio_clean', '')
dep_1c = str(row.get('Подразделение', '')).lower()
dep_scud = str(row.get('department_scud', row.get('department', ''))).lower()
pos = str(row.get('Должность', '')).lower()
is_fio_exc = fio_clean in exc_fios
is_pos_exc = (pos in exact_pos) or any(k in pos for k in pos_kw if k) if pos else False
is_dep_exc = any(d in dep_1c or d in dep_scud for d in deps) if deps else False
if is_fio_exc or is_dep_exc or is_pos_exc:
df.at[idx, 'is_excluded'] = True
return df
# ANCHOR[MERGER_AGGREGATION]
def aggregate_scud_by_employee(df: pd.DataFrame) -> pd.DataFrame:
"""Агрегирует проходы СКУД по уникальным сотрудникам."""
if df is None or df.empty or 'fio_clean' not in df.columns:
return df
aggregated = []
for fio_clean, group in df.groupby('fio_clean', sort=False):
is_present = group['Пришел'].any() if 'Пришел' in group.columns else False
if is_present and 'Пришел' in group.columns:
present_rows = group[group['Пришел'] == True]
best_row = present_rows.iloc[0].to_dict() if not present_rows.empty else group.iloc[0].to_dict()
else:
best_row = group.iloc[0].to_dict()
best_row['Пришел'] = is_present
aggregated.append(best_row)
return pd.DataFrame(aggregated)
def filter_report_dataframe(merged_df: pd.DataFrame) -> pd.DataFrame:
"""Исключает подрядчиков и сотрудников без пропуска из детального отчета."""
if merged_df is None or merged_df.empty:
return merged_df
has_1c_reason = (
merged_df['Вид_отсутствия'].notna() &
(merged_df['Вид_отсутствия'].astype(str).str.strip() != '') &
(~merged_df['Вид_отсутствия'].astype(str).str.startswith('Исключение'))
)
is_not_excluded = merged_df.get('is_excluded', False) == False
is_not_no_pass = merged_df.get('no_scud_pass', False) == False
return merged_df[(is_not_excluded & is_not_no_pass) | has_1c_reason].copy()
def load_static_reason_workers() -> dict:
"""Загружает реестр удаленщиков из CSV."""
static_path = os.path.join(DATA_DIR, "static_reason_workers.csv")
if not os.path.exists(static_path):
return {}
try:
df_static = pd.read_csv(static_path, encoding='utf-8')
if 'fio' in df_static.columns and 'reason' in df_static.columns:
df_static['fio_clean'] = df_static['fio'].apply(normalize_fio)
return dict(zip(df_static['fio_clean'], df_static['reason']))
except Exception:
pass
return {}
+139
View File
@@ -0,0 +1,139 @@
"""
===============================================================================
FILE: services/scud_etl/pipeline.py
PROJECT: SCUD Orion AI (Unified Architecture)
MODULE: services / scud_etl
ROLE: Оркестратор этапов контроллинга (Загрузка -> Сверка -> Отчеты -> SQLite).
AI-CONTEXT-ANCHORS:
- ANCHOR[PIPELINE_RUN_CONTROLLING]: Главная функция выполнения ETL-конвейера.
===============================================================================
"""
import os
from datetime import datetime, timedelta
import pandas as pd
from config import DATE_TODAY, DATE_YESTERDAY, OUTPUT_DIR
from core.database import (
save_scud_to_db, save_staff_to_db, save_absences_to_db,
save_anomalies_to_db, load_scud_from_db_by_snapshot, get_latest_snapshot_time
)
from services.data_loader import load_1c_data_smart
from services.excel_exporter import generate_summary_excel, generate_detailed_excel
from services.ai_verifier import ai_verify_scud_against_staff, analyze_scud_mass_failure_ai
from services.text_reporter import generate_markdown_report
from services.feedback_loop import review_ai_decisions
from services.knowledge_base import load_knowledge_base
from .merger import (
load_exceptions_config, apply_exceptions_from_json,
aggregate_scud_by_employee, filter_report_dataframe, load_static_reason_workers
)
from .anomaly_detector import detect_all_anomalies
# ANCHOR[PIPELINE_RUN_CONTROLLING]
def run_controlling_pipeline(snapshot_param: str = None, skip_export: bool = False, debug: bool = False, has_today_1c: bool = True) -> None:
"""Выполняет полный цикл сверки СКУД ⟷ 1С и сохранение результатов."""
kb_rules = load_knowledge_base().get("rules", [])
exceptions_cfg = load_exceptions_config()
static_reasons = load_static_reason_workers()
print("[2/5] Загрузка данных из СКУД, 1С:ЗУП, реестра причин и исключений...")
# 1. Определение дат целевого снапшота и предыдущей смены
if snapshot_param or skip_export:
snap_to_use = snapshot_param or get_latest_snapshot_time()
raw_scud_today_df = load_scud_from_db_by_snapshot(None, snapshot_param=snap_to_use)
if not raw_scud_today_df.empty and 'log_date' in raw_scud_today_df.columns:
target_date_str = str(raw_scud_today_df['log_date'].iloc[0])
else:
target_date_str = DATE_TODAY
else:
target_date_str = DATE_TODAY
raw_scud_today_df = load_scud_from_db_by_snapshot(target_date_str, snapshot_param=snapshot_param)
dt_target = datetime.strptime(target_date_str, "%d.%m.%Y")
dt_yesterday = dt_target - timedelta(days=3 if dt_target.weekday() == 0 else 1)
yesterday_date_str = dt_yesterday.strftime("%d.%m.%Y")
print(f"[📸] СНАПШОТ ОПРЕДЕЛЕН: Целевая дата = {target_date_str}, Накануне = {yesterday_date_str}\n")
raw_scud_yesterday_df = load_scud_from_db_by_snapshot(yesterday_date_str, snapshot_param=None)
df_staff_yesterday, df_absent_yesterday = load_1c_data_smart(yesterday_date_str, use_db=True)
df_staff_today, df_absent_today = load_1c_data_smart(target_date_str, use_db=True)
# 2. Этап 3: Обработка ВЧЕРА (Детальный отчет)
print(f"[3/5] Обработка и построение детального отчета за ВЧЕРА ({yesterday_date_str})...")
if df_staff_yesterday is not None:
save_staff_to_db(df_staff_yesterday, yesterday_date_str)
if df_absent_yesterday is not None:
save_absences_to_db(df_absent_yesterday, yesterday_date_str)
# Проверка опечаток ФИО через AI-аудитор
staff_fios_y_clean = df_staff_yesterday['fio_clean'].dropna().tolist() if df_staff_yesterday is not None else []
if not raw_scud_yesterday_df.empty:
raw_scud_yesterday_df['Пришел'] = raw_scud_yesterday_df['is_present'].astype(int) == 1 if 'is_present' in raw_scud_yesterday_df.columns else False
unrecog = raw_scud_yesterday_df[~raw_scud_yesterday_df['fio_clean'].isin(staff_fios_y_clean)]['fio_clean'].tolist()
fio_map = ai_verify_scud_against_staff(unrecog, staff_fios_y_clean)
if fio_map:
raw_scud_yesterday_df['fio_clean'] = raw_scud_yesterday_df['fio_clean'].apply(lambda x: fio_map[x]['staff_fio'] if x in fio_map else x)
raw_scud_yesterday_df = aggregate_scud_by_employee(raw_scud_yesterday_df)
merged_y = (df_staff_yesterday.copy() if df_staff_yesterday is not None else pd.DataFrame())
if not merged_y.empty:
if not raw_scud_yesterday_df.empty:
merged_y = merged_y.merge(
raw_scud_yesterday_df[['fio_clean', 'Пришел', 'Начало_дня', 'Первая_активность', 'Конец_дня', 'Находился_в_здании', 'anomaly_flag']],
on='fio_clean', how='left'
)
if df_absent_yesterday is not None and not df_absent_yesterday.empty:
merged_y = merged_y.merge(df_absent_yesterday[['fio_clean', 'Вид_отсутствия']], on='fio_clean', how='left')
merged_y['Пришел'] = merged_y['Пришел'].fillna(False) if 'Пришел' in merged_y.columns else False
if 'Сотрудник' not in merged_y.columns:
merged_y['Сотрудник'] = merged_y.get('ФИО', merged_y['fio_clean'])
merged_y = apply_exceptions_from_json(merged_y, exceptions_cfg)
scud_fios_y = set(raw_scud_yesterday_df['fio_clean'].dropna().tolist()) if not raw_scud_yesterday_df.empty else set()
anomalies_y = detect_all_anomalies(merged_y, static_reasons, kb_rules, scud_fios_set=scud_fios_y)
save_anomalies_to_db(anomalies_y, yesterday_date_str)
filtered_y = filter_report_dataframe(merged_y)
generate_detailed_excel(merged_df=filtered_y, date_str=yesterday_date_str)
save_scud_to_db(merged_y, yesterday_date_str, snapshot_time=f"{dt_yesterday.strftime('%Y-%m-%d')} 22:00:00", is_yesterday=True)
print(f"[✓] Детальный отчет за вчера сформирован и зафиксирован в SQLite за {yesterday_date_str}")
# 3. Этап 4 & 5: Обработка СЕГОДНЯ (Ежедневная сводка)
if has_today_1c and df_staff_today is not None and df_absent_today is not None:
print(f"\n[4/5] Обработка и построение Ежедневной сводки за {target_date_str}...")
save_staff_to_db(df_staff_today, target_date_str)
save_absences_to_db(df_absent_today, target_date_str)
raw_scud_today_df = aggregate_scud_by_employee(raw_scud_today_df)
merged_t = df_staff_today.copy()
if not raw_scud_today_df.empty:
merged_t = merged_t.merge(
raw_scud_today_df[['fio_clean', 'Пришел', 'Начало_дня', 'Первая_активность', 'Конец_дня', 'Находился_в_здании', 'anomaly_flag']],
on='fio_clean', how='left'
)
merged_t = merged_t.merge(df_absent_today[['fio_clean', 'Вид_отсутствия']], on='fio_clean', how='left')
merged_t['Пришел'] = merged_t['Пришел'].fillna(False) if 'Пришел' in merged_t.columns else False
if 'Сотрудник' not in merged_t.columns:
merged_t['Сотрудник'] = merged_t.get('ФИО', merged_t['fio_clean'])
merged_t = apply_exceptions_from_json(merged_t, exceptions_cfg)
scud_fios_t = set(raw_scud_today_df['fio_clean'].dropna().tolist()) if not raw_scud_today_df.empty else set()
anomalies_t = detect_all_anomalies(merged_t, static_reasons, kb_rules, scud_fios_set=scud_fios_t)
save_anomalies_to_db(anomalies_t, target_date_str)
print("[5/5] Сохранение Ежедневной сводки...")
generate_summary_excel(merged_df=merged_t, date_str=target_date_str)
print(f"[✓] Ежедневная сводка сохранена за {target_date_str}")
else:
print(f"\n[ℹ️] Формирование Ежедневной сводки за {target_date_str} ПРОПУЩЕНО.")
+116
View File
@@ -0,0 +1,116 @@
"""
===============================================================================
FILE: services/scud_etl/sql_queries.py
PROJECT: SCUD Orion AI (Unified Architecture)
MODULE: services / scud_etl
ROLE: Хранилище сырых SQL-шаблонов для выгрузки из MS SQL Server (СКУД Орион Pro).
AI-CONTEXT-ANCHORS:
- ANCHOR[SQL_SCUD_EXPORT_TEMPLATE]: T-SQL запрос с расчетом первой активности и длительности.
===============================================================================
"""
# ANCHOR[SQL_SCUD_EXPORT_TEMPLATE]
SCUD_EXPORT_QUERY_TEMPLATE = r"""
DECLARE @InputDate DATE = '{target_date}';
DECLARE @TargetDate DATE = @InputDate;
DECLARE @StartDate DATETIME = CAST(@TargetDate AS DATETIME);
DECLARE @EndDate DATETIME = DATEADD(SECOND, -1, DATEADD(DAY, 1, @StartDate));
WITH DailyLogs AS (
SELECT
log.HozOrgan AS EmployeeID,
log.TimeVal,
log.Event,
log.Mode,
CASE
WHEN log.Mode = 2 OR log.Event IN (29, 27, 33) THEN 'OUT'
WHEN log.Mode = 1 OR log.Event IN (28, 26, 32) THEN 'IN'
ELSE 'OTHER'
END AS Direction,
ROW_NUMBER() OVER (PARTITION BY log.HozOrgan ORDER BY log.TimeVal DESC) AS RowNumDesc
FROM pLogData log WITH (NOLOCK)
WHERE log.TimeVal BETWEEN @StartDate AND @EndDate
AND log.HozOrgan IS NOT NULL
AND log.HozOrgan > 0
AND log.Event IN (26, 27, 28, 29, 32, 33, 54, 55, 64, 65)
),
Passages AS (
SELECT
EmployeeID,
MIN(TimeVal) AS FirstRawEvent,
MAX(TimeVal) AS LastRawEvent,
MIN(CASE WHEN Direction = 'IN' THEN TimeVal END) AS FirstIn,
MAX(CASE WHEN Direction = 'OUT' THEN TimeVal END) AS LastOut,
MAX(CASE WHEN RowNumDesc = 1 THEN Direction END) AS LastEventType
FROM DailyLogs
GROUP BY EmployeeID
)
SELECT
N'ЛЕНМОРНИИПРОЕКТ' AS [Фирма],
ISNULL(CAST(div.Name AS NVARCHAR(255)), N'Без подразделения') AS [Подразделение],
LTRIM(RTRIM(
ISNULL(CAST(p.Name AS NVARCHAR(255)), N'') +
CASE WHEN p.FirstName IS NOT NULL AND CAST(p.FirstName AS NVARCHAR(255)) <> ''
THEN N' ' + CAST(p.FirstName AS NVARCHAR(255)) ELSE N'' END +
CASE WHEN p.MidName IS NOT NULL AND CAST(p.MidName AS NVARCHAR(255)) <> ''
THEN N' ' + CAST(p.MidName AS NVARCHAR(255)) ELSE N'' END
)) AS [Сотрудник],
ISNULL(CAST(post.Name AS NVARCHAR(255)), N'—') AS [Должность],
ISNULL(CAST(p.TabNumber AS NVARCHAR(50)), N'—') AS [Таб_№],
CONVERT(VARCHAR(10), @TargetDate, 104) AS [Дата],
ISNULL(CAST(CONVERT(VARCHAR(8), pass.FirstIn, 108) AS NVARCHAR(20)), N'Нет входа') AS [Начало_дня],
CASE
WHEN pass.FirstIn IS NULL AND pass.FirstRawEvent IS NOT NULL
THEN CAST(CONVERT(VARCHAR(8), pass.FirstRawEvent, 108) AS NVARCHAR(20))
ELSE N'—'
END AS [Первая_активность],
CASE
WHEN @TargetDate = CAST(GETDATE() AS DATE) AND (pass.LastEventType = 'IN' OR pass.LastOut IS NULL OR pass.LastOut <= pass.FirstIn)
THEN N'Нет выхода'
WHEN pass.LastOut IS NOT NULL AND pass.LastOut > pass.FirstIn
THEN CAST(CONVERT(VARCHAR(8), pass.LastOut, 108) AS NVARCHAR(20))
WHEN @TargetDate < CAST(GETDATE() AS DATE) AND pass.LastRawEvent IS NOT NULL AND pass.LastRawEvent > ISNULL(pass.FirstIn, pass.FirstRawEvent)
THEN CAST(CONVERT(VARCHAR(8), pass.LastRawEvent, 108) AS NVARCHAR(20))
ELSE N'Нет выхода'
END AS [Конец_дня],
CASE
WHEN pass.EmployeeID IS NOT NULL AND (pass.FirstIn IS NOT NULL OR pass.FirstRawEvent IS NOT NULL) THEN
RIGHT('0' + CAST(DATEDIFF(MINUTE,
ISNULL(pass.FirstIn, pass.FirstRawEvent),
CASE
WHEN @TargetDate = CAST(GETDATE() AS DATE) AND (pass.LastEventType = 'IN' OR pass.LastOut IS NULL OR pass.LastOut <= pass.FirstIn) THEN GETDATE()
ELSE ISNULL(pass.LastOut, pass.LastRawEvent)
END) / 60 AS VARCHAR), 2) + ':' +
RIGHT('0' + CAST(DATEDIFF(MINUTE,
ISNULL(pass.FirstIn, pass.FirstRawEvent),
CASE
WHEN @TargetDate = CAST(GETDATE() AS DATE) AND (pass.LastEventType = 'IN' OR pass.LastOut IS NULL OR pass.LastOut <= pass.FirstIn) THEN GETDATE()
ELSE ISNULL(pass.LastOut, pass.LastRawEvent)
END) % 60 AS VARCHAR), 2)
ELSE N'00:00'
END AS [Находился_в_здании],
CASE
WHEN pass.EmployeeID IS NOT NULL THEN N'Присутствовал'
ELSE N'Отсутствовал (Нет событий)'
END AS [Статус]
FROM pList p WITH (NOLOCK)
LEFT JOIN PDivision div WITH (NOLOCK) ON p.Section = div.ID
LEFT JOIN PPost post WITH (NOLOCK) ON p.Post = post.ID
LEFT JOIN Passages pass ON p.ID = pass.EmployeeID
WHERE
ISNULL(p.StatusRecord, 0) = 0
AND p.DateTimeInArchive IS NULL
AND ISNULL(CAST(div.Name AS NVARCHAR(255)), N'') NOT LIKE N'Аренд%'
AND ISNULL(CAST(div.Name AS NVARCHAR(255)), N'') NOT IN (N'Без подразделения', N'')
AND p.Name NOT LIKE N'бр.%'
AND p.Name NOT LIKE N'Гость%'
AND ISNULL(CAST(div.Name AS NVARCHAR(255)), N'') NOT IN (N'БГИ', N'КНР')
AND ISNULL(CAST(div.Name AS NVARCHAR(255)), N'') NOT LIKE N'Рабоч%'
AND ISNULL(CAST(div.Name AS NVARCHAR(255)), N'') NOT LIKE N'Врем%'
AND ISNULL(CAST(div.Name AS NVARCHAR(255)), N'') NOT LIKE N'Практика%'
AND ISNULL(CAST(div.Name AS NVARCHAR(255)), N'') NOT LIKE N'тест%'
AND ISNULL(CAST(post.Name AS NVARCHAR(255)), N'') NOT LIKE N'Практикант%'
ORDER BY p.Name ASC;
"""