""" =============================================================================== FILE: services/scud_etl/merger.py PROJECT: SCUD Orion AI (Unified Architecture) MODULE: services / scud_etl ROLE: Агрегация проходов, сопоставление исключений и мердж таблиц 1С:ЗУП и СКУД. AI-CONTEXT-ANCHORS: - ANCHOR[MERGER_EXCEPTIONS]: Наложение флага исключений из exceptions.json. - ANCHOR[MERGER_AGGREGATION]: Агрегация множественных проходов до уникального ФИО. - ANCHOR[MERGER_BUILD_DATASET]: Сборка итогового датасета для сводок и отчетов. =============================================================================== """ import os import json import pandas as pd from config import normalize_fio, DATA_DIR # ANCHOR[MERGER_EXCEPTIONS] def load_exceptions_config() -> dict: """Загружает exceptions.json из корня проекта.""" root_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../")) json_path = os.path.join(root_dir, "exceptions.json") if not os.path.exists(json_path): return {} try: with open(json_path, 'r', encoding='utf-8') as f: return json.load(f) except Exception: return {} def apply_exceptions_from_json(df: pd.DataFrame, exceptions_cfg: dict) -> pd.DataFrame: """Быстрая разметка флага is_excluded на основе exceptions.json.""" if df is None or df.empty or not exceptions_cfg: if df is not None: df['is_excluded'] = False return df deps = [d.strip().lower() for d in exceptions_cfg.get("departments", []) if d] exact_pos = [p.strip().lower() for p in exceptions_cfg.get("positions", []) if p] pos_kw = [k.strip().lower() for k in exceptions_cfg.get("position_keywords", []) if k] exc_fios = [normalize_fio(f) for f in exceptions_cfg.get("fio", []) if f] df['is_excluded'] = False for idx, row in df.iterrows(): fio_clean = row.get('fio_clean', '') dep_1c = str(row.get('Подразделение', '')).lower() dep_scud = str(row.get('department_scud', row.get('department', ''))).lower() pos = str(row.get('Должность', '')).lower() is_fio_exc = fio_clean in exc_fios is_pos_exc = (pos in exact_pos) or any(k in pos for k in pos_kw if k) if pos else False is_dep_exc = any(d in dep_1c or d in dep_scud for d in deps) if deps else False if is_fio_exc or is_dep_exc or is_pos_exc: df.at[idx, 'is_excluded'] = True return df # ANCHOR[MERGER_AGGREGATION] def aggregate_scud_by_employee(df: pd.DataFrame) -> pd.DataFrame: """Агрегирует проходы СКУД по уникальным сотрудникам.""" if df is None or df.empty or 'fio_clean' not in df.columns: return df aggregated = [] for fio_clean, group in df.groupby('fio_clean', sort=False): is_present = group['Пришел'].any() if 'Пришел' in group.columns else False if is_present and 'Пришел' in group.columns: present_rows = group[group['Пришел'] == True] best_row = present_rows.iloc[0].to_dict() if not present_rows.empty else group.iloc[0].to_dict() else: best_row = group.iloc[0].to_dict() best_row['Пришел'] = is_present aggregated.append(best_row) return pd.DataFrame(aggregated) def filter_report_dataframe(merged_df: pd.DataFrame) -> pd.DataFrame: """Исключает подрядчиков и сотрудников без пропуска из детального отчета.""" if merged_df is None or merged_df.empty: return merged_df has_1c_reason = ( merged_df['Вид_отсутствия'].notna() & (merged_df['Вид_отсутствия'].astype(str).str.strip() != '') & (~merged_df['Вид_отсутствия'].astype(str).str.startswith('Исключение')) ) is_not_excluded = merged_df.get('is_excluded', False) == False is_not_no_pass = merged_df.get('no_scud_pass', False) == False return merged_df[(is_not_excluded & is_not_no_pass) | has_1c_reason].copy() def load_static_reason_workers() -> dict: """Загружает реестр удаленщиков из CSV.""" static_path = os.path.join(DATA_DIR, "static_reason_workers.csv") if not os.path.exists(static_path): return {} try: df_static = pd.read_csv(static_path, encoding='utf-8') if 'fio' in df_static.columns and 'reason' in df_static.columns: df_static['fio_clean'] = df_static['fio'].apply(normalize_fio) return dict(zip(df_static['fio_clean'], df_static['reason'])) except Exception: pass return {}