114 lines
4.8 KiB
Python
114 lines
4.8 KiB
Python
"""
|
|
===============================================================================
|
|
FILE: services/scud_etl/merger.py
|
|
PROJECT: SCUD Orion AI (Unified Architecture)
|
|
MODULE: services / scud_etl
|
|
ROLE: Агрегация проходов, сопоставление исключений и мердж таблиц 1С:ЗУП и СКУД.
|
|
|
|
AI-CONTEXT-ANCHORS:
|
|
- ANCHOR[MERGER_EXCEPTIONS]: Наложение флага исключений из exceptions.json.
|
|
- ANCHOR[MERGER_AGGREGATION]: Агрегация множественных проходов до уникального ФИО.
|
|
- ANCHOR[MERGER_BUILD_DATASET]: Сборка итогового датасета для сводок и отчетов.
|
|
===============================================================================
|
|
"""
|
|
|
|
import os
|
|
import json
|
|
import pandas as pd
|
|
from config import normalize_fio, DATA_DIR
|
|
|
|
|
|
# ANCHOR[MERGER_EXCEPTIONS]
|
|
def load_exceptions_config() -> dict:
|
|
"""Загружает exceptions.json из корня проекта."""
|
|
root_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../"))
|
|
json_path = os.path.join(root_dir, "exceptions.json")
|
|
if not os.path.exists(json_path):
|
|
return {}
|
|
try:
|
|
with open(json_path, 'r', encoding='utf-8') as f:
|
|
return json.load(f)
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def apply_exceptions_from_json(df: pd.DataFrame, exceptions_cfg: dict) -> pd.DataFrame:
|
|
"""Быстрая разметка флага is_excluded на основе exceptions.json."""
|
|
if df is None or df.empty or not exceptions_cfg:
|
|
if df is not None:
|
|
df['is_excluded'] = False
|
|
return df
|
|
|
|
deps = [d.strip().lower() for d in exceptions_cfg.get("departments", []) if d]
|
|
exact_pos = [p.strip().lower() for p in exceptions_cfg.get("positions", []) if p]
|
|
pos_kw = [k.strip().lower() for k in exceptions_cfg.get("position_keywords", []) if k]
|
|
exc_fios = [normalize_fio(f) for f in exceptions_cfg.get("fio", []) if f]
|
|
|
|
df['is_excluded'] = False
|
|
|
|
for idx, row in df.iterrows():
|
|
fio_clean = row.get('fio_clean', '')
|
|
dep_1c = str(row.get('Подразделение', '')).lower()
|
|
dep_scud = str(row.get('department_scud', row.get('department', ''))).lower()
|
|
pos = str(row.get('Должность', '')).lower()
|
|
|
|
is_fio_exc = fio_clean in exc_fios
|
|
is_pos_exc = (pos in exact_pos) or any(k in pos for k in pos_kw if k) if pos else False
|
|
is_dep_exc = any(d in dep_1c or d in dep_scud for d in deps) if deps else False
|
|
|
|
if is_fio_exc or is_dep_exc or is_pos_exc:
|
|
df.at[idx, 'is_excluded'] = True
|
|
|
|
return df
|
|
|
|
|
|
# ANCHOR[MERGER_AGGREGATION]
|
|
def aggregate_scud_by_employee(df: pd.DataFrame) -> pd.DataFrame:
|
|
"""Агрегирует проходы СКУД по уникальным сотрудникам."""
|
|
if df is None or df.empty or 'fio_clean' not in df.columns:
|
|
return df
|
|
|
|
aggregated = []
|
|
for fio_clean, group in df.groupby('fio_clean', sort=False):
|
|
is_present = group['Пришел'].any() if 'Пришел' in group.columns else False
|
|
if is_present and 'Пришел' in group.columns:
|
|
present_rows = group[group['Пришел'] == True]
|
|
best_row = present_rows.iloc[0].to_dict() if not present_rows.empty else group.iloc[0].to_dict()
|
|
else:
|
|
best_row = group.iloc[0].to_dict()
|
|
|
|
best_row['Пришел'] = is_present
|
|
aggregated.append(best_row)
|
|
|
|
return pd.DataFrame(aggregated)
|
|
|
|
|
|
def filter_report_dataframe(merged_df: pd.DataFrame) -> pd.DataFrame:
|
|
"""Исключает подрядчиков и сотрудников без пропуска из детального отчета."""
|
|
if merged_df is None or merged_df.empty:
|
|
return merged_df
|
|
|
|
has_1c_reason = (
|
|
merged_df['Вид_отсутствия'].notna() &
|
|
(merged_df['Вид_отсутствия'].astype(str).str.strip() != '') &
|
|
(~merged_df['Вид_отсутствия'].astype(str).str.startswith('Исключение'))
|
|
)
|
|
is_not_excluded = merged_df.get('is_excluded', False) == False
|
|
is_not_no_pass = merged_df.get('no_scud_pass', False) == False
|
|
|
|
return merged_df[(is_not_excluded & is_not_no_pass) | has_1c_reason].copy()
|
|
|
|
|
|
def load_static_reason_workers() -> dict:
|
|
"""Загружает реестр удаленщиков из CSV."""
|
|
static_path = os.path.join(DATA_DIR, "static_reason_workers.csv")
|
|
if not os.path.exists(static_path):
|
|
return {}
|
|
try:
|
|
df_static = pd.read_csv(static_path, encoding='utf-8')
|
|
if 'fio' in df_static.columns and 'reason' in df_static.columns:
|
|
df_static['fio_clean'] = df_static['fio'].apply(normalize_fio)
|
|
return dict(zip(df_static['fio_clean'], df_static['reason']))
|
|
except Exception:
|
|
pass
|
|
return {} |