# -*- coding: utf-8 -*-
"""
STEP117 V1.5 Naver Realtor Response Finder - V2 scroll logic transplanted

직접 API 호출 폐기:
- context.request => 429
- page.evaluate(fetch) => 401

V1.5 전략:
- 네이버 페이지가 실제로 발생시키는 /api/articles response만 감시한다.
- window 스크롤이 아니라 내부 scrollable div를 찾아 강제 스크롤한다.
- 매매/전세/월세 탭을 클릭해 각 거래유형 response를 유도한다.
- 최종 후보는 article_no 내림차순으로 최신순 정렬한다.
"""

import re
import json
import time
import random
from urllib.parse import quote

from playwright.sync_api import sync_playwright


USER_AGENT = (
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
    "AppleWebKit/537.36 (KHTML, like Gecko) "
    "Chrome/136.0.0.0 Safari/537.36"
)


def uniq_keep_order(items):
    seen = set()
    result = []
    for item in items:
        item = str(item or "").strip()
        if not item or item in seen:
            continue
        seen.add(item)
        result.append(item)
    return result


def extract_article_nos(text):
    text = str(text or "")
    patterns = [
        r'"articleNo"\s*:\s*"([0-9]{8,12})"',
        r'"articleNo"\s*:\s*([0-9]{8,12})',
        r'"atclNo"\s*:\s*"([0-9]{8,12})"',
        r'"atclNo"\s*:\s*([0-9]{8,12})',
        r'"articleNumber"\s*:\s*"([0-9]{8,12})"',
        r'"articleNumber"\s*:\s*([0-9]{8,12})',
        r"articleNo=([0-9]{8,12})",
    ]
    found = []
    for pattern in patterns:
        found.extend(re.findall(pattern, text, flags=re.I))
    return uniq_keep_order(found)


def first_value(data, keys):
    for key in keys:
        if isinstance(data, dict) and data.get(key) not in (None, ""):
            return data.get(key)
    return ""


def parse_article_summary_items(text):
    """
    STEP107-17:
    /api/articles 응답 JSON에서 article_no와 요약 매물정보를 함께 추출한다.
    네이버 응답 키가 바뀔 수 있으므로 후보 키를 넓게 본다.
    """
    text = str(text or "")
    try:
        payload = json.loads(text)
    except Exception:
        return []

    containers = []

    def collect_containers(obj):
        if isinstance(obj, list):
            # 리스트 안에 articleNo가 있는 dict가 있으면 후보 컨테이너
            if any(isinstance(x, dict) and (x.get("articleNo") or x.get("atclNo")) for x in obj):
                containers.append(obj)
            for x in obj:
                collect_containers(x)
        elif isinstance(obj, dict):
            for v in obj.values():
                collect_containers(v)

    collect_containers(payload)

    items = []
    for arr in containers:
        for row in arr:
            if not isinstance(row, dict):
                continue

            article_no = str(first_value(row, ["articleNo", "atclNo", "articleNumber"]) or "").strip()
            if not article_no:
                continue

            item = {
                "article_no": article_no,
                "trade_type": first_value(row, ["tradeTypeCode", "tradeType", "tradTpCd", "tradeTypeName"]),
                "trade_type_name": first_value(row, ["tradeTypeName", "tradTpNm", "tradeType"]),
                "real_estate_type": first_value(row, [
                    "articleRealEstateTypeCode", "realEstateTypeCode",
                    "articleTypeCode", "realEstateType", "rletTpCd",
                ]),
                "real_estate_type_name": first_value(row, [
                    "articleRealEstateTypeName", "realEstateTypeName",
                    "articleTypeName", "rletTpNm", "realEstateType",
                ]),
                "complex_name": first_value(row, ["complexName", "aptName", "atclNm", "articleName"]),
                "building_name": first_value(row, ["buildingName", "bildNm", "dongName", "dongNm"]),
                "price_text": first_value(row, ["dealOrWarrantPrc", "price", "prc", "rentPrc", "prcInfo"]),
                "area_text": first_value(row, ["areaName", "spc1", "spc2", "area", "exclusiveArea"]),
                "floor_text": first_value(row, ["floorInfo", "flrInfo", "floor"]),
                "direction": first_value(row, ["direction", "directionName", "drctNm"]),
                "confirm_date": first_value(row, ["articleConfirmYmd", "atclCfmYmd", "confirmDate"]),
                "short_description": first_value(row, ["articleFeatureDesc", "atclFetrDesc", "tagList", "summary", "description"]),
            }

            if isinstance(item.get("short_description"), list):
                item["short_description"] = ", ".join([str(x) for x in item["short_description"] if x])

            items.append(item)

    # unique by article_no, keep first occurrence
    unique = {}
    for item in items:
        if item["article_no"] not in unique:
            unique[item["article_no"]] = item

    return list(unique.values())


def sort_candidates_latest_first(candidates):
    def key_func(item):
        try:
            return int(str(item.get("article_no") or "0").strip())
        except Exception:
            return 0
    return sorted(candidates or [], key=key_func, reverse=True)


def build_realtor_url(naver_realtor_id, base_url=None):
    if base_url:
        return base_url
    return f"https://new.land.naver.com/offices?realtorId={quote(str(naver_realtor_id).strip())}"


class NaverRealtorResponseFinder:
    def __init__(self, headless=True):
        self.headless = headless
        self.pw = None
        self.browser = None
        self.context = None
        self.page = None

    def start(self):
        self.pw = sync_playwright().start()
        self.browser = self.pw.chromium.launch(
            headless=self.headless,
            args=[
                "--disable-blink-features=AutomationControlled",
                "--no-sandbox",
            ],
        )
        self.context = self.browser.new_context(
            locale="ko-KR",
            viewport={"width": 1365, "height": 900},
            user_agent=USER_AGENT,
        )
        self.page = self.context.new_page()
        print("[V15_SCROLL UI RESPONSE FINDER START]")

    def close(self):
        try:
            if self.context:
                self.context.close()
        except Exception:
            pass
        try:
            if self.browser:
                self.browser.close()
        except Exception:
            pass
        try:
            if self.pw:
                self.pw.stop()
        except Exception:
            pass
        print("[V15_SCROLL UI RESPONSE FINDER CLOSED]")

    def _click_latest_sort(self):
        selectors = [
            "button:has-text('최신순')",
            "a:has-text('최신순')",
            "span:has-text('최신순')",
            "button:has-text('최신')",
            "a:has-text('최신')",
        ]
        for selector in selectors:
            try:
                loc = self.page.locator(selector)
                count = loc.count()
                if count <= 0:
                    continue
                target = loc.nth(count - 1)
                try:
                    if not target.is_visible(timeout=600):
                        continue
                except Exception:
                    pass
                target.click(timeout=1800)
                self.page.wait_for_timeout(1200)
                print("[V15_SCROLL LATEST SORT CLICK]", selector, "count=", count)
                return True
            except Exception:
                continue

        print("[V15_SCROLL LATEST SORT CLICK SKIP]")
        return False

    def _click_trade_tab(self, label):
        if not label:
            return True

        selectors = [
            f"button:has-text('{label}')",
            f"a:has-text('{label}')",
            f"span:has-text('{label}')",
            f"label:has-text('{label}')",
            f"div:has-text('{label}')",
        ]

        for selector in selectors:
            try:
                loc = self.page.locator(selector)
                count = loc.count()
                if count <= 0:
                    continue

                # 가장 작은 visible 요소를 우선 클릭
                for i in range(min(count, 8)):
                    target = loc.nth(i)
                    try:
                        if not target.is_visible(timeout=500):
                            continue
                    except Exception:
                        pass

                    text = ""
                    try:
                        text = target.inner_text(timeout=500)
                    except Exception:
                        pass

                    # 너무 큰 컨테이너 클릭 방지
                    if text and len(text) > 80:
                        continue

                    target.click(timeout=1800)
                    self.page.wait_for_timeout(1500)
                    print("[V15_SCROLL TRADE TAB CLICK]", label, selector, "index=", i, "count=", count)
                    return True

            except Exception:
                continue

        print("[V15_SCROLL TRADE TAB CLICK SKIP]", label)
        return False

    def _extract_visible_trade_counts(self):
        """
        화면 좌측 중개사 정보 영역에 보이는 매매/전세/월세 총 건수를 추출한다.
        이 값은 실제 전체 수집 기대치 검증용으로 사용한다.
        """
        try:
            result = self.page.evaluate(
                """
                () => {
                    const text = document.body.innerText || '';
                    const out = {sale: null, lease: null, rent: null, total: null, raw: text.slice(0, 3000)};
                    const patterns = [
                        ['sale', /매매\\s*(\\d+)\\s*개/],
                        ['lease', /전세\\s*(\\d+)\\s*개/],
                        ['rent', /월세\\s*(\\d+)\\s*개/],
                    ];
                    for (const [key, re] of patterns) {
                        const m = text.match(re);
                        if (m) out[key] = parseInt(m[1], 10);
                    }
                    out.total = (out.sale || 0) + (out.lease || 0) + (out.rent || 0);
                    return out;
                }
                """
            )
            print("[V15_SCROLL VISIBLE TRADE COUNTS]", result)
            return result or {}
        except Exception as e:
            print("[V15_SCROLL VISIBLE TRADE COUNTS ERROR]", str(e))
            return {}

    def _find_scroll_targets(self):
        """
        네이버 매물 리스트 내부 scrollable div 후보를 찾는다.
        """
        try:
            targets = self.page.evaluate(
                """
                () => {
                    const els = Array.from(document.querySelectorAll('div, section, article, main, ul'));
                    const out = [];

                    els.forEach((el, idx) => {
                        try {
                            const st = getComputedStyle(el);
                            const rect = el.getBoundingClientRect();
                            const scrollable = el.scrollHeight > el.clientHeight + 80;
                            const visible = rect.width > 120 && rect.height > 120;
                            const overflow = (st.overflowY || '') + ' ' + (st.overflow || '');

                            if (scrollable && visible) {
                                out.push({
                                    idx,
                                    tag: el.tagName,
                                    className: String(el.className || '').slice(0, 160),
                                    id: String(el.id || ''),
                                    clientHeight: el.clientHeight,
                                    scrollHeight: el.scrollHeight,
                                    scrollTop: el.scrollTop,
                                    rect: {x: rect.x, y: rect.y, width: rect.width, height: rect.height},
                                    overflow,
                                    text: String(el.innerText || '').slice(0, 120)
                                });
                            }
                        } catch(e) {}
                    });

                    out.sort((a,b) => {
                        const score = (x) => {
                            let s = 0;
                            if ((x.text || '').includes('매매')) s += 5;
                            if ((x.text || '').includes('전세')) s += 5;
                            if ((x.text || '').includes('월세')) s += 5;
                            if ((x.text || '').includes('억')) s += 5;
                            s += Math.min(20, x.scrollHeight / 500);
                            s += Math.min(10, x.rect.height / 100);
                            return s;
                        };
                        return score(b) - score(a);
                    });

                    return out.slice(0, 8);
                }
                """
            )
            print("[V15_SCROLL SCROLL TARGETS]", targets)
            return targets or []
        except Exception as e:
            print("[V15_SCROLL SCROLL TARGETS ERROR]", str(e))
            return []

    def _scroll_internal_targets(self, rounds=18, delay_ms=1000):
        """
        STEP107-07:
        no_growth 기준이 아니라 실제 item_list--article 스크롤 바닥 도달 기준으로 진행한다.
        """
        reached_bottom = False

        for round_no in range(1, int(rounds or 18) + 1):
            try:
                moved = self.page.evaluate(
                    """
                    ({roundNo}) => {
                        function fire(el, type) {
                            try {
                                el.dispatchEvent(new Event(type, {bubbles: true, cancelable: true}));
                            } catch(e) {}
                        }

                        const preferred = Array.from(document.querySelectorAll(
                            '.item_list--article, #articleListArea, .infinite_scroll, [class*="item_list"]'
                        ));

                        const all = Array.from(document.querySelectorAll('div, section, article, main, ul'));
                        const merged = [];
                        const seen = new Set();

                        [...preferred, ...all].forEach((el) => {
                            if (!el || seen.has(el)) return;
                            seen.add(el);
                            try {
                                const rect = el.getBoundingClientRect();
                                if (
                                    el.scrollHeight > el.clientHeight + 40 &&
                                    rect.width > 120 &&
                                    rect.height > 120
                                ) {
                                    const text = String(el.innerText || '');
                                    let score = 0;
                                    const cls = String(el.className || '');
                                    const id = String(el.id || '');
                                    if (cls.includes('item_list--article')) score += 100;
                                    if (id === 'articleListArea') score += 80;
                                    if (cls.includes('infinite_scroll')) score += 60;
                                    if (text.includes('매매')) score += 5;
                                    if (text.includes('전세')) score += 5;
                                    if (text.includes('월세')) score += 5;
                                    if (text.includes('억')) score += 5;
                                    score += Math.min(20, el.scrollHeight / 500);
                                    merged.push({el, score});
                                }
                            } catch(e) {}
                        });

                        merged.sort((a,b) => b.score - a.score);

                        const moved = [];
                        let anyNotBottom = false;

                        merged.slice(0, 4).forEach(({el, score}, idx) => {
                            try {
                                const before = el.scrollTop;
                                const maxTop = Math.max(0, el.scrollHeight - el.clientHeight);
                                const step = Math.max(220, Math.floor(el.clientHeight * 0.62));

                                // 마지막 20%에서는 더 촘촘히 움직여 infinite observer를 자극한다.
                                const remain = maxTop - before;
                                let next = before + step;
                                if (remain < step * 1.5) {
                                    next = Math.min(maxTop, before + Math.max(120, Math.floor(step / 2)));
                                }

                                el.scrollTop = Math.min(maxTop, next);
                                fire(el, 'scroll');
                                fire(el, 'wheel');

                                const after = el.scrollTop;
                                const atBottom = after >= maxTop - 8;

                                if (!atBottom) anyNotBottom = true;

                                moved.push({
                                    idx,
                                    score,
                                    tag: el.tagName,
                                    className: String(el.className || '').slice(0, 100),
                                    id: String(el.id || ''),
                                    before,
                                    after,
                                    maxTop,
                                    atBottom,
                                    scrollHeight: el.scrollHeight,
                                    clientHeight: el.clientHeight
                                });
                            } catch(e) {}
                        });

                        window.scrollBy(0, 350);
                        try { document.dispatchEvent(new Event('scroll', {bubbles:true})); } catch(e) {}

                        return {
                            moved,
                            reachedBottom: moved.length > 0 && !anyNotBottom
                        };
                    }
                    """,
                    {"roundNo": round_no},
                )

                # 실제 마우스 wheel도 리스트 영역 위에서 발생시키기
                try:
                    self.page.mouse.move(260, 620)
                    self.page.mouse.wheel(0, 720)
                except Exception:
                    pass

                self.page.wait_for_timeout(int(delay_ms or 1000))

                print("[V15_SCROLL INTERNAL SCROLL]", f"{round_no}/{rounds}", moved)

                if moved and moved.get("reachedBottom"):
                    reached_bottom = True
                    print("[V15_SCROLL INTERNAL SCROLL BOTTOM REACHED]", f"round={round_no}")
                    # 바닥 도달 후에도 lazy load 여지를 위해 2번 더 자극한다.
                    for extra in range(2):
                        try:
                            self.page.mouse.move(260, 620)
                            self.page.mouse.wheel(0, 900)
                            self.page.wait_for_timeout(int(delay_ms or 1000))
                            print("[V15_SCROLL INTERNAL SCROLL BOTTOM EXTRA]", extra + 1)
                        except Exception:
                            pass
                    break

            except Exception as e:
                print("[V15_SCROLL INTERNAL SCROLL ERROR]", round_no, str(e))

        return {"reached_bottom": reached_bottom}

    def find_all_articles(
        self,
        naver_realtor_id,
        base_url=None,
        wait_ms=3500,
        scroll_steps=12,
        scroll_delay_ms=900,
        max_pages=30,
    ):
        naver_realtor_id = str(naver_realtor_id or "").strip()

        if not naver_realtor_id:
            return {
                "ok": False,
                "error": "naver_realtor_id 없음",
                "article_nos": [],
                "candidates": [],
                "debug": [],
            }

        url = build_realtor_url(naver_realtor_id, base_url=base_url)

        collected = []
        debug = []

        def add_article(article_no, response_url="", collector="v15_ui_response"):
            article_no = str(article_no or "").strip()
            if not article_no:
                return
            collected.append({
                "article_no": article_no,
                "source_url": response_url,
                "source_keyword": naver_realtor_id,
                "source_status_code": 200,
                "collector": collector,
            })

        def unique_count():
            return len({str(x.get("article_no") or "").strip() for x in collected if str(x.get("article_no") or "").strip()})

        def handle_response(response):
            try:
                response_url = response.url
                status = response.status

                if "/api/articles?" not in response_url or "realtorId=" not in response_url:
                    return

                text = response.text() or ""
                article_nos = extract_article_nos(text)
                summary_items = parse_article_summary_items(text)
                summary_map = {str(x.get("article_no") or "").strip(): x for x in summary_items}

                debug.append({
                    "url": response_url,
                    "status": status,
                    "count": len(article_nos),
                    "article_nos": article_nos[:100],
                    "summary_count": len(summary_items),
                    "source": "browser_response",
                })

                print("[V15_SCROLL ARTICLE RESPONSE]", status, len(article_nos), "summary=", len(summary_items), response_url)

                for article_no in article_nos:
                    add_article(
                        article_no,
                        response_url=response_url,
                        collector="v15_browser_xhr_response",
                    )
                    # 방금 추가한 마지막 item에 summary를 병합한다.
                    if collected and str(collected[-1].get("article_no")) == str(article_no):
                        collected[-1].update(summary_map.get(str(article_no), {}))

            except Exception as e:
                debug.append({
                    "url": getattr(response, "url", ""),
                    "error": str(e),
                })

        self.page.on("response", handle_response)

        try:
            print("[V15_SCROLL OPEN ALL]", url)
            self.page.goto(url, wait_until="domcontentloaded", timeout=60000)
            self.page.wait_for_timeout(int(wait_ms or 3500))

            visible_counts = self._extract_visible_trade_counts()

            self._click_latest_sort()
            self.page.wait_for_timeout(1000)

            self._find_scroll_targets()

            # 전체 탭/기본 상태
            no_growth_rounds = 0
            previous_count = unique_count()

            # STEP107-18: ALL에서 화면 총계만큼 확보되면 매매/전세/월세 탭 재수집 생략.
            visible_total = 0
            try:
                visible_total = int((visible_counts or {}).get("total") or 0)
            except Exception:
                visible_total = 0

            def click_more_once():
                """리스트 하단 더보기/다음 버튼이 보이면 1회 클릭한다."""
                for selector in [
                    "button:has-text('더보기')", "a:has-text('더보기')",
                    "button[class*='more']", "a[class*='more']",
                    "button:has-text('다음')", "a:has-text('다음')",
                    "button[class*='next']", "a[class*='next']",
                ]:
                    try:
                        loc = self.page.locator(selector)
                        count = loc.count()
                        if count <= 0:
                            continue
                        target = loc.nth(count - 1)
                        try:
                            if not target.is_visible(timeout=400):
                                continue
                        except Exception:
                            pass
                        target.click(timeout=1200)
                        self.page.wait_for_timeout(int(scroll_delay_ms or 900))
                        print("[V15_SCROLL UI MORE CLICK]", selector, "count=", count)
                        return True
                    except Exception:
                        continue
                return False

            def collect_bucket(label=""):
                if label:
                    self._click_trade_tab(label)
                    self._click_latest_sort()
                    self.page.wait_for_timeout(800)

                bucket_before = unique_count()
                print("[V15_SCROLL BUCKET COLLECT START]", label or "ALL", "before=", bucket_before)

                # STEP118:
                # 기존 STEP117은 scroll_steps 1회 실행 후 종료되어 96건 중 80건에서 멈출 수 있었다.
                # 이제는 article_no 증가가 멈출 때까지 여러 cycle을 반복한다.
                # 단, V1.5 안정성을 위해 무한루프 방지용 max_cycles를 둔다.
                max_cycles = max(3, min(12, int(max_pages or 30)))
                no_growth_cycles = 0
                last_count = unique_count()
                final_scroll_result = {"reached_bottom": False}

                for cycle in range(1, max_cycles + 1):
                    before = unique_count()

                    final_scroll_result = self._scroll_internal_targets(
                        rounds=max(8 if not label else 6, int(scroll_steps or 18)),
                        delay_ms=scroll_delay_ms,
                    )

                    more_clicked = click_more_once()
                    if more_clicked:
                        # 더보기 클릭 직후 lazy response가 늦게 들어오는 경우 대기
                        self.page.wait_for_timeout(int(scroll_delay_ms or 900) * 2)

                    after = unique_count()
                    growth = after - before

                    print(
                        "[V15_SCROLL BUCKET CYCLE]", label or "ALL",
                        f"cycle={cycle}/{max_cycles}",
                        f"before={before}", f"after={after}", f"growth={growth}",
                        f"visible_total={visible_total}",
                        f"bottom={final_scroll_result.get('reached_bottom')}",
                        f"more_clicked={more_clicked}",
                    )

                    # 화면에 표시된 총량 이상 확보하면 즉시 종료
                    if visible_total > 0 and after >= visible_total:
                        print("[V15_SCROLL BUCKET TARGET REACHED]", label or "ALL", f"after={after}", f"visible_total={visible_total}")
                        break

                    if growth <= 0:
                        no_growth_cycles += 1
                    else:
                        no_growth_cycles = 0

                    # 바닥 도달 + 더보기 없음 + 2회 연속 증가 없음이면 종료
                    if no_growth_cycles >= 2 and (final_scroll_result.get("reached_bottom") or not more_clicked):
                        print("[V15_SCROLL BUCKET NO GROWTH STOP]", label or "ALL", f"cycle={cycle}", f"count={after}")
                        break

                    last_count = after

                print(
                    "[V15_SCROLL BUCKET GROWTH]", label or "ALL",
                    f"before={bucket_before}", f"after={unique_count()}",
                    f"visible_total={visible_total}",
                    f"bottom={final_scroll_result.get('reached_bottom')}",
                )
                print("[V15_SCROLL BUCKET COLLECT DONE]", label or "ALL", "after=", unique_count(), "added=", unique_count() - bucket_before)

            collect_bucket("")

            if visible_total > 0 and unique_count() >= visible_total:
                print("[V15_SCROLL TRADE BUCKET SKIP]", f"all_count={unique_count()}", f"visible_total={visible_total}", "reason=ALL already collected expected total")
            else:
                print("[V15_SCROLL TRADE BUCKET FALLBACK START]", f"all_count={unique_count()}", f"visible_total={visible_total}")
                for label in ["매매", "전세", "월세"]:
                    collect_bucket(label)

        except Exception as e:
            debug.append({
                "url": url,
                "error": str(e),
            })
            print("[V15_SCROLL FIND ALL ERROR]", str(e))

        try:
            self.page.remove_listener("response", handle_response)
        except Exception:
            pass

        unique = {}
        for item in collected:
            article_no = str(item.get("article_no") or "").strip()
            if article_no and article_no not in unique:
                unique[article_no] = item

        candidates = sort_candidates_latest_first(list(unique.values()))
        article_nos = [str(item.get("article_no") or "").strip() for item in candidates if str(item.get("article_no") or "").strip()]

        print("[V15_SCROLL ALL ARTICLE NOS LATEST FIRST]", len(article_nos), article_nos[:120])

        return {
            "ok": True,
            "naver_realtor_id": naver_realtor_id,
            "article_nos": article_nos,
            "count": len(article_nos),
            "candidates": candidates,
            "debug": debug,
            "debug_pages": debug,
            "visible_trade_counts": locals().get("visible_counts", {}),
            "page_url": self.page.url if self.page else url,
            "collector": "v15_ui_response_internal_scroll_step107_07",
        }

    def find_latest_article(self, naver_realtor_id, base_url=None, wait_ms=3500):
        result = self.find_all_articles(
            naver_realtor_id=naver_realtor_id,
            base_url=base_url,
            wait_ms=wait_ms,
            max_pages=5,
        )
        candidates = result.get("candidates") or []
        result["article_no"] = candidates[0]["article_no"] if candidates else ""
        return result