feat(etl): stable pipeline, exception registry in SQLite, multi-pass aggregation and db_cli
This commit is contained in:
@@ -0,0 +1,227 @@
|
||||
"""
|
||||
===============================================================================
|
||||
FILE: core/repositories/scud_repo.py
|
||||
ROLE: Репозиторий логов СКУД, сохранение и загрузка снапшотов.
|
||||
===============================================================================
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
import pandas as pd
|
||||
from core.connection import get_connection
|
||||
|
||||
|
||||
def has_scud_logs_for_date(date_str: str) -> bool:
|
||||
with get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT 1 FROM scud_logs WHERE log_date = ? LIMIT 1", (date_str,))
|
||||
return cursor.fetchone() is not None
|
||||
|
||||
|
||||
def has_yesterday_final_snapshot(date_str: str) -> bool:
|
||||
with get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"SELECT 1 FROM scud_logs WHERE log_date = ? AND (snapshot_id LIKE 'Y%' OR snapshot_time LIKE '%22:00:00') LIMIT 1",
|
||||
(date_str,)
|
||||
)
|
||||
return cursor.fetchone() is not None
|
||||
|
||||
|
||||
def get_or_create_snapshot_id(snapshot_time: str, date_str: str = None, is_yesterday: bool = False) -> str:
|
||||
try:
|
||||
dt_snap = datetime.strptime(snapshot_time, "%Y-%m-%d %H:%M:%S").date()
|
||||
date_prefix = dt_snap.strftime("%Y%m%d")
|
||||
except (ValueError, TypeError):
|
||||
dt_snap = datetime.now().date()
|
||||
date_prefix = dt_snap.strftime("%Y%m%d")
|
||||
|
||||
if date_str:
|
||||
try:
|
||||
dt_log = datetime.strptime(date_str, "%d.%m.%Y").date()
|
||||
if dt_log < dt_snap:
|
||||
is_yesterday = True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
prefix = "Y" if is_yesterday else ""
|
||||
|
||||
with get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
if date_str:
|
||||
cursor.execute(
|
||||
"SELECT snapshot_id FROM scud_logs WHERE log_date = ? AND snapshot_time = ? AND snapshot_id IS NOT NULL LIMIT 1",
|
||||
(date_str, snapshot_time)
|
||||
)
|
||||
else:
|
||||
cursor.execute(
|
||||
"SELECT snapshot_id FROM scud_logs WHERE snapshot_time = ? AND snapshot_id IS NOT NULL LIMIT 1",
|
||||
(snapshot_time,)
|
||||
)
|
||||
|
||||
row = cursor.fetchone()
|
||||
if row and row[0]:
|
||||
return row[0]
|
||||
|
||||
cursor.execute("""
|
||||
SELECT snapshot_id FROM scud_logs
|
||||
WHERE snapshot_id LIKE ? OR snapshot_id LIKE ?
|
||||
ORDER BY snapshot_id DESC LIMIT 1
|
||||
""", (f"{date_prefix}-%", f"Y{date_prefix}-%"))
|
||||
|
||||
last_row = cursor.fetchone()
|
||||
next_seq = 1
|
||||
if last_row and last_row[0]:
|
||||
parts = last_row[0].replace("Y", "").split("-")
|
||||
if len(parts) > 1 and parts[1].isdigit():
|
||||
next_seq = int(parts[1]) + 1
|
||||
|
||||
return f"{prefix}{date_prefix}-{next_seq:03d}"
|
||||
|
||||
|
||||
def save_scud_to_db(df_scud: pd.DataFrame, date_str: str, snapshot_time: str = None, is_yesterday: bool = False) -> None:
|
||||
if df_scud is None or df_scud.empty:
|
||||
return
|
||||
|
||||
if not snapshot_time:
|
||||
snapshot_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
snapshot_id = get_or_create_snapshot_id(snapshot_time, date_str=date_str, is_yesterday=is_yesterday)
|
||||
|
||||
data_to_insert = [
|
||||
(
|
||||
date_str,
|
||||
r.get('Сотрудник', r.get('fio_raw', '')),
|
||||
r.get('fio_clean', ''),
|
||||
r.get('Подразделение', ''),
|
||||
r.get('Должность', ''),
|
||||
r.get('Начало_дня', 'Нет входа'),
|
||||
r.get('Первая_активность', '—'),
|
||||
r.get('Конец_дня', 'Нет выхода'),
|
||||
r.get('Находился_в_здании', '00:00'),
|
||||
1 if r.get('Пришел', False) else 0,
|
||||
r.get('anomaly_flag', 'NONE'),
|
||||
snapshot_time,
|
||||
snapshot_id
|
||||
)
|
||||
for _, r in df_scud.iterrows()
|
||||
]
|
||||
|
||||
with get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("DELETE FROM scud_logs WHERE log_date = ? AND snapshot_time = ?", (date_str, snapshot_time))
|
||||
cursor.executemany("""
|
||||
INSERT INTO scud_logs (
|
||||
log_date, fio, fio_clean, department, position,
|
||||
time_in, first_activity, time_out, time_in_building,
|
||||
is_present, anomaly_flag, snapshot_time, snapshot_id
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", data_to_insert)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def get_latest_snapshot_time(date_str: str = None):
|
||||
with get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
if date_str:
|
||||
cursor.execute("SELECT snapshot_time FROM scud_logs WHERE log_date = ? AND snapshot_time IS NOT NULL ORDER BY snapshot_time DESC LIMIT 1", (date_str,))
|
||||
else:
|
||||
cursor.execute("SELECT snapshot_time FROM scud_logs WHERE snapshot_time IS NOT NULL ORDER BY snapshot_time DESC LIMIT 1")
|
||||
row = cursor.fetchone()
|
||||
return row[0] if row else None
|
||||
|
||||
|
||||
def load_scud_from_db_by_snapshot(date_str: str, snapshot_param: str = None) -> pd.DataFrame:
|
||||
with get_connection() as conn:
|
||||
df = pd.DataFrame()
|
||||
|
||||
# 1. Если передан конкретный ID снапшота (например 'Y20260820-004')
|
||||
if snapshot_param:
|
||||
df = pd.read_sql_query(
|
||||
"SELECT * FROM scud_logs WHERE snapshot_id = ?",
|
||||
conn, params=(str(snapshot_param),)
|
||||
)
|
||||
|
||||
# 2. Если ищем за дату (для вчерашнего дня строго ищем Y-снапшот)
|
||||
if df.empty and date_str:
|
||||
cursor = conn.cursor()
|
||||
|
||||
# ⭐️ Жесткий приоритет 1: Ищем снапшот с префиксом 'Y'
|
||||
cursor.execute(
|
||||
"SELECT snapshot_id FROM scud_logs WHERE log_date = ? AND snapshot_id LIKE 'Y%' ORDER BY snapshot_time DESC, id DESC LIMIT 1",
|
||||
(date_str,)
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
|
||||
# Приоритет 2: Если Y нет (например, за сегодня), берем самый свежий по времени
|
||||
if not row:
|
||||
cursor.execute(
|
||||
"SELECT snapshot_id FROM scud_logs WHERE log_date = ? ORDER BY snapshot_time DESC, id DESC LIMIT 1",
|
||||
(date_str,)
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
|
||||
if row and row[0]:
|
||||
target_id = row[0]
|
||||
df = pd.read_sql_query(
|
||||
"SELECT * FROM scud_logs WHERE snapshot_id = ?",
|
||||
conn, params=(target_id,)
|
||||
)
|
||||
|
||||
if not df.empty:
|
||||
rename_map = {
|
||||
'department': 'department_scud',
|
||||
'position': 'Должность',
|
||||
'fio': 'Сотрудник',
|
||||
'time_in': 'Начало_дня',
|
||||
'first_activity': 'Первая_активность',
|
||||
'time_out': 'Конец_дня',
|
||||
'time_in_building': 'Находился_в_здании',
|
||||
'is_present': 'Пришел'
|
||||
}
|
||||
df = df.rename(columns={k: v for k, v in rename_map.items() if k in df.columns})
|
||||
if 'department_scud' in df.columns and 'Подразделение' not in df.columns:
|
||||
df['Подразделение'] = df['department_scud']
|
||||
|
||||
for col in ['Пришел', 'Начало_дня', 'Первая_активность', 'Конец_дня', 'Находился_в_здании', 'anomaly_flag']:
|
||||
if col not in df.columns:
|
||||
df[col] = False if col == 'Пришел' else '—'
|
||||
if 'Пришел' in df.columns:
|
||||
df['Пришел'] = df['Пришел'].astype(bool)
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def get_available_snapshots(date_str: str = None):
|
||||
with get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
query = """
|
||||
SELECT snapshot_id, log_date, snapshot_time, COUNT(*) as cnt
|
||||
FROM scud_logs
|
||||
WHERE snapshot_time IS NOT NULL
|
||||
"""
|
||||
params = []
|
||||
if date_str:
|
||||
query += " AND log_date = ?"
|
||||
params.append(date_str)
|
||||
query += " GROUP BY snapshot_id, log_date, snapshot_time ORDER BY snapshot_time DESC"
|
||||
cursor.execute(query, params)
|
||||
return cursor.fetchall()
|
||||
|
||||
|
||||
def delete_snapshot_by_id(snapshot_id: str) -> int:
|
||||
with get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("DELETE FROM scud_logs WHERE snapshot_id = ?", (snapshot_id,))
|
||||
cnt = cursor.rowcount
|
||||
conn.commit()
|
||||
return cnt
|
||||
|
||||
|
||||
def delete_snapshots_by_date(date_str: str) -> int:
|
||||
with get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("DELETE FROM scud_logs WHERE log_date = ? OR snapshot_id LIKE ?", (date_str, f"%{date_str.replace('.', '')}%"))
|
||||
cnt = cursor.rowcount
|
||||
conn.commit()
|
||||
return cnt
|
||||
@@ -0,0 +1,97 @@
|
||||
"""
|
||||
===============================================================================
|
||||
FILE: core/repositories/zup_repo.py
|
||||
ROLE: Репозиторий кадровых данных 1С:ЗУП, аномалий и базы знаний.
|
||||
===============================================================================
|
||||
"""
|
||||
|
||||
import pandas as pd
|
||||
from typing import List, Dict, Any
|
||||
from core.connection import get_connection
|
||||
|
||||
|
||||
def save_staff_to_db(df_staff: pd.DataFrame, date_str: str) -> None:
|
||||
if df_staff is None or df_staff.empty:
|
||||
return
|
||||
data = [
|
||||
(date_str, r.get('ФИО', ''), r.get('fio_clean', ''), r.get('Подразделение', ''), r.get('Должность', ''))
|
||||
for _, r in df_staff.iterrows()
|
||||
]
|
||||
with get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("DELETE FROM zup_staff WHERE snapshot_date = ?", (date_str,))
|
||||
cursor.executemany("INSERT INTO zup_staff (snapshot_date, fio, fio_clean, department, position) VALUES (?, ?, ?, ?, ?)", data)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def load_staff_from_db(date_str: str) -> pd.DataFrame:
|
||||
with get_connection() as conn:
|
||||
df = pd.read_sql_query(
|
||||
"SELECT fio as 'ФИО', fio_clean, department as 'Подразделение', position as 'Должность' FROM zup_staff WHERE snapshot_date = ?",
|
||||
conn, params=(date_str,)
|
||||
)
|
||||
return df if not df.empty else None
|
||||
|
||||
|
||||
def save_absences_to_db(df_absent: pd.DataFrame, date_str: str) -> None:
|
||||
if df_absent is None or df_absent.empty:
|
||||
return
|
||||
data = [
|
||||
(date_str, r.get('ФИО', r.get('fio_clean', '')), r.get('fio_clean', ''), r.get('Вид_отсутствия', ''))
|
||||
for _, r in df_absent.iterrows()
|
||||
]
|
||||
with get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("DELETE FROM zup_absences WHERE absence_date = ?", (date_str,))
|
||||
cursor.executemany("INSERT INTO zup_absences (absence_date, fio, fio_clean, absence_type) VALUES (?, ?, ?, ?)", data)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def load_absences_from_db(date_str: str) -> pd.DataFrame:
|
||||
with get_connection() as conn:
|
||||
df = pd.read_sql_query(
|
||||
"SELECT fio as 'ФИО', fio_clean, absence_type as 'Вид_отсутствия' FROM zup_absences WHERE absence_date = ?",
|
||||
conn, params=(date_str,)
|
||||
)
|
||||
return df if not df.empty else None
|
||||
|
||||
|
||||
def save_anomalies_to_db(anomalies_list: List[Dict[str, Any]], date_str: str) -> None:
|
||||
if not anomalies_list:
|
||||
return
|
||||
data = [(date_str, a.get('fio', ''), a.get('type', ''), a.get('details', '')) for a in anomalies_list]
|
||||
with get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("DELETE FROM anomalies_history WHERE anomaly_date = ?", (date_str,))
|
||||
cursor.executemany("INSERT INTO anomalies_history (anomaly_date, fio, anomaly_type, details) VALUES (?, ?, ?, ?)", data)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def get_all_rules_from_db() -> List[str]:
|
||||
with get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT rule_text FROM ai_knowledge_base")
|
||||
return [r[0] for r in cursor.fetchall()]
|
||||
|
||||
|
||||
def add_rule_to_db(rule_text: str, added_by: str = "Human") -> None:
|
||||
if not rule_text or not rule_text.strip():
|
||||
return
|
||||
with get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("INSERT OR IGNORE INTO ai_knowledge_base (rule_text, added_by) VALUES (?, ?)", (rule_text.strip(), added_by))
|
||||
conn.commit()
|
||||
|
||||
|
||||
def get_department_synonyms_dict() -> Dict[str, str]:
|
||||
with get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT LOWER(short_name), LOWER(full_name) FROM department_synonyms")
|
||||
return {row[0]: row[1] for row in cursor.fetchall()}
|
||||
|
||||
|
||||
def add_department_synonym_to_db(short_name: str, full_name: str) -> None:
|
||||
with get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("INSERT OR REPLACE INTO department_synonyms (short_name, full_name) VALUES (?, ?)", (short_name.strip().lower(), full_name.strip().lower()))
|
||||
conn.commit()
|
||||
Reference in New Issue
Block a user