#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""공식 공공정책 API/RSS를 영업지역별 정책·생활 콘텐츠로 갱신한다."""

import argparse
import difflib
import hashlib
import html
import json
import os
import re
import sys
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
from datetime import datetime, timezone, timedelta
from email.utils import parsedate_to_datetime
from pathlib import Path

BASE_DIR = Path(__file__).resolve().parents[1]
if str(BASE_DIR) not in sys.path:
    sys.path.insert(0, str(BASE_DIR))

DEFAULT_KEYWORDS = (
    "부동산", "주택", "공동주택", "아파트", "오피스텔", "전세", "월세", "임대",
    "매매", "분양", "청약", "대출", "주택담보", "재개발", "재건축", "정비사업",
    "도시계획", "토지", "공시가격", "취득세", "재산세", "교통", "철도", "학교",
)


def clean(value):
    text = html.unescape(re.sub(r"<[^>]+>", " ", str(value or "")))
    return re.sub(r"\s+", " ", text).strip()


def load_config(path=None):
    target = Path(path) if path else BASE_DIR / "config" / "public_policy_sources.json"
    try:
        payload = json.loads(target.read_text(encoding="utf-8-sig"))
    except FileNotFoundError:
        return {"sources": []}, target
    if not isinstance(payload, dict) or not isinstance(payload.get("sources"), list):
        raise RuntimeError("public_policy_sources.json의 sources 배열을 확인해 주세요.")
    return payload, target


def secret_values():
    values = dict(os.environ)
    try:
        payload = json.loads((BASE_DIR / "config" / "public_api_keys.json").read_text(encoding="utf-8-sig"))
        if isinstance(payload, dict):
            values.update({str(key): str(value) for key, value in payload.items() if value not in (None, "")})
    except (OSError, ValueError, TypeError):
        pass
    return values


def expand_secrets(value, secrets):
    if isinstance(value, dict):
        return {key: expand_secrets(item, secrets) for key, item in value.items()}
    if isinstance(value, list):
        return [expand_secrets(item, secrets) for item in value]
    if isinstance(value, str):
        return re.sub(r"\$\{([A-Z0-9_]+)\}", lambda match: secrets.get(match.group(1), ""), value)
    return value


def request_bytes(url, headers=None, params=None):
    query = {str(k): str(v) for k, v in (params or {}).items() if v not in (None, "")}
    target = url + (("&" if "?" in url else "?") + urllib.parse.urlencode(query) if query else "")
    request = urllib.request.Request(target, headers={"User-Agent": "MaemulHOME-PolicyFeed/1.0", **(headers or {})})
    with urllib.request.urlopen(request, timeout=30) as response:
        return response.read()


def parse_date(value):
    text = clean(value)
    if not text:
        return None
    try:
        parsed = parsedate_to_datetime(text)
        if parsed.tzinfo:
            parsed = parsed.astimezone(timezone.utc).replace(tzinfo=None)
        return parsed.strftime("%Y-%m-%d %H:%M:%S")
    except (TypeError, ValueError, OverflowError):
        pass
    digits = re.sub(r"\D+", "", text)
    for size, pattern in ((14, "%Y%m%d%H%M%S"), (12, "%Y%m%d%H%M"), (8, "%Y%m%d")):
        if len(digits) >= size:
            try:
                return datetime.strptime(digits[:size], pattern).strftime("%Y-%m-%d %H:%M:%S")
            except ValueError:
                continue
    return None


def child_text(node, names):
    wanted = {name.lower() for name in names}
    for child in list(node):
        if child.tag.split("}")[-1].lower() in wanted:
            return "".join(child.itertext()).strip()
    return ""


def parse_rss(raw):
    root = ET.fromstring(raw)
    nodes = root.findall(".//item")
    if not nodes:
        nodes = [node for node in root.iter() if node.tag.split("}")[-1].lower() in {"item", "entry"}]
    items = []
    for node in nodes:
        link = child_text(node, ("link",))
        if not link:
            for child in list(node):
                if child.tag.split("}")[-1].lower() == "link" and child.attrib.get("href"):
                    link = child.attrib["href"]
                    break
        items.append({
            "title": child_text(node, ("title",)),
            "summary": child_text(node, ("description", "summary", "content")),
            "url": link,
            "published_at": child_text(node, ("pubDate", "published", "updated", "date")),
            "id": child_text(node, ("guid", "id")),
        })
    return items


def dotted_get(node, path, default=None):
    current = node
    for part in str(path or "").split("."):
        if not part:
            continue
        if isinstance(current, dict):
            current = current.get(part)
        elif isinstance(current, list) and part.isdigit() and int(part) < len(current):
            current = current[int(part)]
        else:
            return default
    return current if current is not None else default


def parse_json_items(raw, source):
    payload = json.loads(raw.decode("utf-8-sig"))
    rows = dotted_get(payload, source.get("items_path", "data"), [])
    if isinstance(rows, dict):
        rows = rows.get("item") or rows.get("items") or [rows]
    if not isinstance(rows, list):
        return []
    fields = source.get("fields") or {}
    return [{key: dotted_get(row, fields.get(key, key), "") for key in ("title", "summary", "url", "published_at", "id")} for row in rows if isinstance(row, dict)]


def fetch_items(source):
    raw = request_bytes(source["url"], source.get("headers"), source.get("params"))
    return parse_json_items(raw, source) if source.get("type") == "json" else parse_rss(raw)


def naver_items(source, query):
    params = {
        "query": query,
        "display": max(1, min(100, int(source.get("display", 20)))),
        "start": 1,
        "sort": "date",
    }
    raw = request_bytes(source["url"], source.get("headers"), params)
    payload = json.loads(raw.decode("utf-8-sig"))
    result = []
    for row in payload.get("items", []) if isinstance(payload, dict) else []:
        if not isinstance(row, dict):
            continue
        result.append({
            "title": row.get("title"),
            "summary": row.get("description"),
            "url": row.get("originallink") or row.get("link"),
            "published_at": row.get("pubDate") or row.get("postdate"),
            "id": row.get("originallink") or row.get("link"),
        })
    return result


def site_targets(conn, realtor_id=None):
    sql = "SELECT id AS site_id,realtor_id FROM multi_sites WHERE status='active'"
    params = []
    if realtor_id:
        sql += " AND realtor_id=%s"
        params.append(realtor_id)
    with conn.cursor() as cur:
        cur.execute(sql, params)
        sites = cur.fetchall() or []
        for site in sites:
            cur.execute("""SELECT region_key,region_name,province_name,district_name,neighborhood_name
                           FROM multi_site_regions WHERE site_id=%s AND status='active' AND is_excluded=0""", (site["site_id"],))
            site["regions"] = cur.fetchall() or []
    return sites


def region_aliases(region):
    values = [region.get("region_name"), region.get("province_name"), region.get("district_name"), region.get("neighborhood_name")]
    aliases = set()
    for value in values:
        value = clean(value)
        if not value:
            continue
        aliases.add(value)
        aliases.add(re.sub(r"(특별자치시|특별자치도|특별시|광역시|시|군|구)$", "", value))
    return {value for value in aliases if len(value) >= 2}


def choose_regions(item, source, regions):
    scope = source.get("scope", "auto")
    if scope == "national":
        return [("national", "전국")]
    text = clean(f"{item.get('title')} {item.get('summary')}")
    matched = []
    for region in regions:
        if any(alias in text for alias in region_aliases(region)):
            matched.append((region["region_key"], region["region_name"]))
    if matched:
        return matched
    if scope == "region" and len(regions) == 1:
        return [(regions[0]["region_key"], regions[0]["region_name"])]
    return [("national", "전국")] if source.get("national_fallback", True) else []


def keyword_hits(text, keywords):
    lowered = clean(text).lower()
    return {clean(keyword).lower() for keyword in keywords if clean(keyword) and clean(keyword).lower() in lowered}


def relevant(item, source):
    keywords = source.get("keywords") or DEFAULT_KEYWORDS
    title = clean(item.get("title"))
    text = clean(f"{title} {item.get('summary')}")
    excluded = source.get("exclude_keywords") or []
    if keyword_hits(text, excluded):
        return False
    hits = keyword_hits(text, keywords)
    if len(hits) < max(1, int(source.get("min_keyword_hits", 1))):
        return False
    for group in source.get("required_keyword_groups") or []:
        if not keyword_hits(text, group):
            return False
    title_groups = source.get("title_keyword_groups") or []
    for group in title_groups:
        if not keyword_hits(title, group):
            return False
    if source.get("title_keyword_required") and not keyword_hits(title, keywords):
        return False
    return True


def recent_enough(item, source):
    max_days = int(source.get("max_item_age_days", 0) or 0)
    if max_days <= 0:
        return True
    parsed = parse_date(item.get("published_at"))
    if not parsed:
        return False
    try:
        published = datetime.strptime(parsed, "%Y-%m-%d %H:%M:%S")
    except ValueError:
        return False
    return published >= datetime.now() - timedelta(days=max_days)


def region_search_name(region):
    district = clean(region.get("district_name"))
    neighborhood = clean(region.get("neighborhood_name"))
    if district or neighborhood:
        return clean(f"{district} {neighborhood}")
    return clean(region.get("region_name"))


def item_matches_region(item, region):
    text = clean(f"{item.get('title')} {item.get('summary')}")
    neighborhood = clean(region.get("neighborhood_name"))
    district = clean(region.get("district_name"))
    if neighborhood:
        short = re.sub(r"(읍|면|동|가|리)$", "", neighborhood)
        return neighborhood in text or (len(short) >= 2 and short in text)
    if district:
        short = re.sub(r"(시|군|구)$", "", district)
        return district in text or (len(short) >= 2 and short in text)
    region_name = clean(region.get("region_name"))
    return bool(region_name and region_name in text)


def title_matches_region(item, region):
    return item_matches_region({"title": item.get("title"), "summary": ""}, region)


def normalized_topic_title(value):
    text = clean(value).lower()
    text = re.sub(r"\[[^]]+\]|\([^)]*\)", " ", text)
    text = re.sub(r"[^0-9a-z가-힣]+", "", text)
    return text


def duplicate_topic(title, accepted_titles, threshold=0.76):
    normalized = normalized_topic_title(title)
    if not normalized:
        return True
    for existing in accepted_titles:
        if difflib.SequenceMatcher(None, normalized, existing).ratio() >= threshold:
            return True
    accepted_titles.append(normalized)
    return False


def archive_retired_sources(conn):
    """이전 단계의 광범위한 통계 RSS 결과를 화면에서 즉시 제외한다."""
    with conn.cursor() as cur:
        cur.execute("""UPDATE multi_region_contents SET status='archived',updated_at=NOW()
                       WHERE status='published' AND source_name='국가데이터처'
                         AND content_type='통계·생활'""")
    conn.commit()


def reset_source_results(conn, source):
    """강제 품질 재검사 전에 해당 수집원의 과거 노출 결과를 정리한다."""
    with conn.cursor() as cur:
        cur.execute("""UPDATE multi_region_contents SET status='archived',updated_at=NOW()
                       WHERE status='published' AND source_name=%s AND content_type=%s""",
                    (clean(source.get("source_name"))[:150], clean(source.get("content_type"))[:40]))
    conn.commit()


def save_shared_run(conn, site, source_key, fetched, saved, skipped):
    with conn.cursor() as cur:
        cur.execute("""INSERT INTO multi_policy_feed_runs
            (site_id,realtor_id,source_key,status,fetched_count,saved_count,skipped_count,
             started_at,finished_at,created_at)
            VALUES (%s,%s,%s,'completed',%s,%s,%s,NOW(),NOW(),NOW())""",
            (site["site_id"], site["realtor_id"], source_key, fetched, saved, skipped))


def refresh_naver_source(conn, source, sites, force=False):
    source_key = clean(source.get("key"))[:100]
    pending_sites = [site for site in sites if force or not recently_completed(conn, site["site_id"], source_key)]
    if not pending_sites:
        return
    headers = source.get("headers") or {}
    if not clean(headers.get("X-Naver-Client-Id")) or not clean(headers.get("X-Naver-Client-Secret")):
        print(f"[MULTI POLICY FEED SKIP] source={source_key} NAVER 검색 API 키가 없습니다.")
        return
    scope = source.get("scope", "region")
    targets = {"national": {"region_key": "national", "region_name": "전국"}}
    if scope != "national":
        targets = {}
        for site in pending_sites:
            for region in site["regions"]:
                targets.setdefault(region["region_key"], region)
    fetched = saved = skipped = 0
    seen = set()
    accepted_titles = {}
    reset_done = not force
    try:
        for region_key, region in targets.items():
            region_name = "전국" if region_key == "national" else region_search_name(region)
            for template in source.get("query_templates") or ["{region} 부동산"]:
                query = clean(str(template).replace("{region}", region_name))
                batch = naver_items(source, query)
                if not reset_done:
                    reset_source_results(conn, source)
                    reset_done = True
                for item in batch:
                    fetched += 1
                    identity = clean(item.get("id") or item.get("url") or item.get("title"))
                    identity_key = (region_key, identity)
                    if identity_key in seen:
                        skipped += 1
                        continue
                    seen.add(identity_key)
                    if not recent_enough(item, source) or not relevant(item, source):
                        skipped += 1
                        continue
                    if region_key != "national" and not item_matches_region(item, region):
                        skipped += 1
                        continue
                    if region_key != "national" and source.get("region_title_required") and not title_matches_region(item, region):
                        skipped += 1
                        continue
                    topic_titles = accepted_titles.setdefault(region_key, [])
                    if duplicate_topic(item.get("title"), topic_titles, float(source.get("duplicate_title_threshold", 0.76))):
                        skipped += 1
                        continue
                    if save_item(conn, source, item, region_key, clean(region.get("region_name")) or "전국",
                                 int(source.get("max_age_days", 120))):
                        saved += 1
        for site in pending_sites:
            save_shared_run(conn, site, source_key, fetched, saved, skipped)
        conn.commit()
        print(f"[MULTI POLICY FEED] source={source_key} regions={len(targets)} fetched={fetched} saved={saved} skipped={skipped}")
    except Exception as exc:
        conn.rollback()
        print(f"[MULTI POLICY FEED WARNING] source={source_key} error={clean(exc)}")


def save_item(conn, source, item, region_key, region_name, max_age_days):
    title = clean(item.get("title"))[:500]
    if not title:
        return False
    summary = clean(item.get("summary"))[:2000]
    source_url = clean(item.get("url"))[:1000]
    identity = clean(item.get("id")) or source_url or title
    content_key = hashlib.sha1(f"{source.get('key')}|{identity}".encode("utf-8")).hexdigest()
    published_at = parse_date(item.get("published_at")) or datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    with conn.cursor() as cur:
        cur.execute("""INSERT INTO multi_region_contents
            (region_key,region_name,content_key,content_type,title,summary,source_name,source_url,
             is_featured,status,published_at,expires_at,created_at,updated_at)
            VALUES (%s,%s,%s,%s,%s,%s,%s,%s,0,'published',%s,DATE_ADD(%s,INTERVAL %s DAY),NOW(),NOW())
            ON DUPLICATE KEY UPDATE region_name=VALUES(region_name),content_type=VALUES(content_type),
             title=VALUES(title),summary=VALUES(summary),source_name=VALUES(source_name),
             source_url=VALUES(source_url),published_at=VALUES(published_at),expires_at=VALUES(expires_at),
             status='published',updated_at=NOW()""",
            (region_key, region_name, content_key, clean(source.get("content_type") or "부동산정책")[:40],
             title, summary, clean(source.get("source_name"))[:150], source_url,
             published_at, published_at, max_age_days))
    return True


def recently_completed(conn, site_id, source_key, hours=20):
    with conn.cursor() as cur:
        cur.execute("""SELECT 1 FROM multi_policy_feed_runs WHERE site_id=%s AND source_key=%s
                       AND status='completed' AND started_at>=DATE_SUB(NOW(),INTERVAL %s HOUR) LIMIT 1""",
                    (site_id, source_key, hours))
        return bool(cur.fetchone())


def refresh_policy_feeds(conn, realtor_id=None, force=False, config_path=None):
    config, path = load_config(config_path)
    config = expand_secrets(config, secret_values())
    archive_retired_sources(conn)
    sources = [source for source in config.get("sources", []) if source.get("enabled") and source.get("url")]
    if not sources:
        print(f"[MULTI POLICY FEED SKIP] 활성 출처가 없습니다: {path}")
        return
    item_limit = int(config.get("default_item_limit", 40))
    max_age_days = int(config.get("default_max_age_days", 120))
    sites = site_targets(conn, realtor_id)
    for source in sources:
        if source.get("type") in {"naver_news", "naver_blog"}:
            refresh_naver_source(conn, source, sites, force)
            continue
        source_key = clean(source.get("key"))[:100]
        pending_sites = [site for site in sites if force or not recently_completed(conn, site["site_id"], source_key)]
        if not pending_sites:
            continue
        try:
            items = fetch_items(source)[:max(1, min(200, int(source.get("limit", item_limit))))]
        except Exception as exc:
            print(f"[MULTI POLICY FEED WARNING] source={source_key} error={clean(exc)}")
            continue
        for site in pending_sites:
            with conn.cursor() as cur:
                cur.execute("""INSERT INTO multi_policy_feed_runs
                    (site_id,realtor_id,source_key,status,started_at,created_at)
                    VALUES (%s,%s,%s,'running',NOW(),NOW())""", (site["site_id"], site["realtor_id"], source_key))
                run_id = cur.lastrowid
            conn.commit()
            fetched = saved = skipped = 0
            try:
                fetched = len(items)
                for item in items:
                    if not relevant(item, source):
                        skipped += 1
                        continue
                    targets = choose_regions(item, source, site["regions"])
                    if not targets:
                        skipped += 1
                        continue
                    for region_key, region_name in targets:
                        if save_item(conn, source, item, region_key, region_name, int(source.get("max_age_days", max_age_days))):
                            saved += 1
                with conn.cursor() as cur:
                    cur.execute("""UPDATE multi_policy_feed_runs SET status='completed',fetched_count=%s,
                        saved_count=%s,skipped_count=%s,finished_at=NOW() WHERE id=%s""",
                                (fetched, saved, skipped, run_id))
                conn.commit()
                print(f"[MULTI POLICY FEED] site_id={site['site_id']} source={source_key} fetched={fetched} saved={saved} skipped={skipped}")
            except Exception as exc:
                conn.rollback()
                with conn.cursor() as cur:
                    cur.execute("""UPDATE multi_policy_feed_runs SET status='failed',fetched_count=%s,
                        saved_count=%s,skipped_count=%s,error_message=%s,finished_at=NOW() WHERE id=%s""",
                                (fetched, saved, skipped, clean(exc)[:2000], run_id))
                conn.commit()
                print(f"[MULTI POLICY FEED WARNING] site_id={site['site_id']} source={source_key} error={clean(exc)}")


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--realtor-id", type=int)
    parser.add_argument("--config")
    parser.add_argument("--force", action="store_true")
    args = parser.parse_args()
    from db import get_conn
    conn = get_conn()
    try:
        refresh_policy_feeds(conn, args.realtor_id, args.force, args.config)
    finally:
        conn.close()


if __name__ == "__main__":
    main()