#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
北京展览信息爬虫 v3 - 多源直接抓取版

核心改进（对比 v1/v2）：
1. 放弃 Tavily 搜索 + 正则提取（数据质量极差：单字标题、无场馆、无展期）
2. 改为直接 HTTP 抓取已知数据源页面 + HTML 结构化解析
3. 每条展览必须包含：名称、场馆、展期、门票、简介
4. 报告按原始模板结构输出（详细版）

数据源（全部直接 HTTP 抓取）：
1. ⭐⭐⭐ 北京本地宝 7 月展览汇总（主数据源，HTML 结构化解析）
   - URL: http://bj.bendibao.com/news/202674/384308.shtm（第1页国博）
   - 追加抓取第2-12页（其他博物馆）
2. ⭐⭐ 携程展览汇总（补充票价简介）
   - URL: https://hk.trip.com/events/9245182-2026-beijing-exhibitions-collection

使用方法：
    python3 crawler_v3.py [--date YYYY-MM-DD]

输出：
    - data/beijing_exhibitions_YYYY-MM-DD.md
    - JSON 结果到 stdout
"""

import json
import re
import sys
import time
import requests
from datetime import datetime
from pathlib import Path
from html import unescape

# ============================================================
# 配置
# ============================================================
BASE_DIR = Path(__file__).parent
DATA_DIR = BASE_DIR / "data"
LOG_DIR = BASE_DIR / "logs"
DATA_DIR.mkdir(exist_ok=True)
LOG_DIR.mkdir(exist_ok=True)

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
}

# 本地宝分页 URL（第1页是国博，每页一个博物馆）
BENDIBAO_PAGES = []
for i in range(1, 24):
    if i == 1:
        BENDIBAO_PAGES.append(("http://bj.bendibao.com/news/202674/384308.shtm", f"P{i}"))
    else:
        BENDIBAO_PAGES.append((f"http://bj.bendibao.com/news/202674/384308_{i}.shtm", f"P{i}"))

# 博物馆名称映射（全文 → 标准名）
MUSEUM_MAP = {
    "中国国家博物馆": "国家博物馆",
    "国家博物馆": "国家博物馆",
    "故宫博物院": "故宫博物院",
    "首都博物馆": "首都博物馆",
    "北京大运河博物馆": "大运河博物馆",
    "中国美术馆": "中国美术馆",
    "中国人民革命军事博物馆": "军事博物馆",
    "北京自然博物馆": "国家自然博物馆",
    "国家自然博物馆": "国家自然博物馆",
    "中国科学技术馆": "中国科技馆",
    "中华世纪坛": "中华世纪坛",
    "北京展览馆": "北京展览馆",
    "北京石刻艺术博物馆": "石刻艺术博物馆",
    "北京艺术博物馆": "万寿寺博物馆",
    "北京白塔寺管理处": "白塔寺",
    "北京市白塔寺管理处": "白塔寺",
    "中国电影博物馆": "中国电影博物馆",
    "北京汽车博物馆": "北京汽车博物馆",
    "中国园林博物馆": "中国园林博物馆",
    "民航博物馆": "民航博物馆",
    "北京市古代钱币展览馆": "德胜门箭楼",
    "北京天文馆": "北京天文馆",
    "中国现代文学馆": "中国现代文学馆",
    "北京鲁迅博物馆": "鲁迅博物馆",
    "北京文博交流馆": "智化寺",
    "北京市智化寺管理处": "智化寺",
    "香山革命纪念馆": "香山革命纪念馆",
    "北京古钟博物馆": "大钟寺",
    "北京古代建筑博物馆": "先农坛",
    "老舍纪念馆": "老舍纪念馆",
    "北京考古遗址博物馆": "考古遗址博物馆",
    "北京市大觉寺与团城管理处": "大觉寺",
    "北京市大葆台西汉墓博物馆": "大葆台",
    "北京中华民族博物院": "中华民族园",
}


# ============================================================
# 工具函数
# ============================================================

def log_msg(msg, level="INFO"):
    ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    line = f"[{ts}] [{level}] {msg}"
    print(line, flush=True)
    try:
        with open(LOG_DIR / f"crawler_{datetime.now().strftime('%Y%m%d')}.log", "a", encoding="utf-8") as f:
            f.write(line + "\n")
    except:
        pass


def fetch_url(url, timeout=20):
    try:
        r = requests.get(url, timeout=timeout, headers=HEADERS)
        r.encoding = "utf-8"
        r.raise_for_status()
        return r.text
    except Exception as e:
        log_msg(f"抓取失败 {url}: {e}", "WARNING")
        return None


def strip_tags(html):
    """去除 HTML 标签"""
    text = re.sub(r'<[^>]+>', ' ', html)
    text = unescape(text)
    text = re.sub(r'&nbsp;', ' ', text)
    text = re.sub(r'\s+', ' ', text).strip()
    return text


def normalize_museum(text):
    """从文本中识别博物馆名称"""
    # 先去除多余空格再匹配
    clean = re.sub(r'\s+', '', text)
    for full, short in MUSEUM_MAP.items():
        full_clean = re.sub(r'\s+', '', full)
        if full_clean in clean:
            return short
    return ""


# ============================================================
# 数据源 1：北京本地宝（主数据源）
# ============================================================

def crawl_bendibao_page(url, label=""):
    """解析本地宝单个页面"""
    raw = fetch_url(url)
    if not raw:
        return []

    # 提取所有 <p> 标签内容
    p_tags = re.findall(r'<p[^>]*>(.*?)</p>', raw, re.DOTALL)
    
    exhibitions = []
    current_museum = ""
    current_ex = None
    
    # 检测页面标题中是否含博物馆名
    page_text = strip_tags(raw[:5000])
    page_museum = normalize_museum(page_text)
    if page_museum:
        current_museum = page_museum
    
    def save_current():
        nonlocal current_ex
        if current_ex and current_ex.get("title") and len(current_ex["title"]) >= 6:
            skip = ["展览预告", "正在展出", "新展推荐", "展览时间", "展览地址",
                     "展览介绍", "温馨提示", "微信搜索", "导语",
                     "分页导航", "末页", "余下全文", "更多"]
            title = current_ex["title"]
            if not any(s in title for s in skip):
                exhibitions.append(current_ex)
        current_ex = None
    
    for p in p_tags:
        text = strip_tags(p).strip()
        if len(text) < 3:
            continue
        
        # 【优先】检测展览编号 + 名称：01　看·见——xxx展 或 01 展名
        # 必须在博物馆名检测之前，否则展名中含博物馆名会被误判
        num_match = re.match(r'^(\d{1,2})[\s ]+(.+)', text)
        if num_match:
            save_current()
            title = num_match.group(2).strip()
            title = re.sub(r'\s*(展览时间|展览地址|展览介绍|【导语】).*$', '', title)
            if title and len(title) >= 4:
                current_ex = {
                    "title": title,
                    "venue": current_museum or "待确认",
                    "start_date": "",
                    "end_date": "",
                    "price": "免费",
                    "description": "",
                    "location": "",
                    "source": f"本地宝",
                    "url": url,
                }
            continue
        
        # 检测博物馆名（加粗且长度较短，且不含编号）
        bold_match = re.search(r'<strong[^>]*>(.*?)</strong>', p, re.DOTALL)
        if bold_match:
            bold_text = strip_tags(bold_match.group(1)).strip()
            detected = normalize_museum(bold_text)
            if detected and len(bold_text) < 30 and not re.match(r'^\d{1,2}[\s ]', bold_text):
                save_current()
                current_museum = detected
                continue
        
        # 以下处理不以编号开头的后续段落（追加信息到当前展览）
        if current_ex:
            if "展览时间" in text:
                date_str = text.replace("展览时间：", "").replace("展览时间:", "").strip()
                date_str = re.sub(r'\s*(展览地址|展览介绍|温馨提示).*$', '', date_str).strip()
                current_ex["date_raw"] = date_str
                parsed = parse_date(date_str)
                current_ex["start_date"] = parsed.get("start", "")
                current_ex["end_date"] = parsed.get("end", "")
            elif "展览地址" in text:
                addr = text.replace("展览地址：", "").replace("展览地址:", "").strip()
                addr = re.sub(r'\s*(展览介绍|温馨提示).*$', '', addr).strip()
                current_ex["location"] = addr
                if current_ex["venue"] == "待确认":
                    m = normalize_museum(addr)
                    if m:
                        current_ex["venue"] = m
            elif "展览介绍" in text:
                desc = text.replace("展览介绍：", "").replace("展览介绍:", "").strip()
                desc = re.sub(r'\s*(温馨提示|微信搜索|关注后|对话框).*$', '', desc).strip()
                if len(desc) > 15:
                    current_ex["description"] = desc[:500]
                # 即使为空也不处理（等待下一个可能的介绍段落）
            elif current_ex.get("date_raw") and not current_ex.get("description") and len(text) > 30:
                # 不以"展览介绍"开头但看起来是介绍的段落
                skip_in_text = ["温馨提示", "微信搜索", "关注后", "对话框", "分页导航", "本地宝",
                               "预约开票", "闭馆", "开放时间", "暑假", "暑期", "学校校历",
                               "根据北京市教委", "端午佳节", "提升服务水平", "北京市文物局",
                               "为更好地满足", "为满足广大", "暑假来北京", "一年一度",
                               "今天小编", "七月七日", "七月二十一日", "六月来博物馆",
                               "六月盛夏", "七月特色", "出游提前看"]
                if not any(s in text for s in skip_in_text):
                    desc = re.sub(r'\s*(温馨提示|微信搜索|关注后).*$', '', text).strip()
                    if len(desc) > 20:
                        current_ex["description"] = desc[:500]
    
    save_current()
    
    if label:
        log_msg(f"  本地宝[{label}]：{len(exhibitions)} 个展览")
    
    return exhibitions


def crawl_bendibao():
    """抓取本地宝展览汇总（主数据源）"""
    log_msg("【数据源1】北京本地宝展览汇总")
    
    all_ex = []
    for url, label in BENDIBAO_PAGES:
        exs = crawl_bendibao_page(url, label)
        all_ex.extend(exs)
        time.sleep(1)
    
    log_msg(f"  本地宝合计：{len(all_ex)} 个展览")
    for ex in all_ex[:5]:
        log_msg(f"    -> {ex['title'][:35]} | {ex['venue']}")
    if len(all_ex) > 5:
        log_msg(f"    ... 等共 {len(all_ex)} 个")
    
    return all_ex


# ============================================================
# 数据源 2：携程展览汇总（补充）
# ============================================================

def crawl_tripcom():
    """携程展览汇总页"""
    log_msg("【数据源2】携程展览汇总")
    url = "https://hk.trip.com/events/9245182-2026-beijing-exhibitions-collection"
    raw = fetch_url(url, timeout=30)
    if not raw:
        return []
    
    text = strip_tags(raw)
    exhibitions = []
    seen = set()
    
    # 匹配展名（严格过滤碎片）
    patterns = [
        r'《([^》]{8,40}(?:展|特展|大展|主题展))》',
        r'([A-Za-z0-9\u4e00-\u9fa5·\-——]{6,30}[—–][A-Za-z0-9\u4e00-\u9fa5·]{2,25}(?:展|特展|大展))',
    ]
    
    for pattern in patterns:
        for m in re.finditer(pattern, text):
            title = m.group(1).strip().rstrip('。')
            # 严格过滤：必须是展名格式，不能是简介片段
            if title not in seen and len(title) >= 8 and len(title) <= 40:
                skip = ["展览预告", "正在展出", "新展推荐", "展览时间", "展览活动",
                        "展览安排", "更多展览", "展览推荐", "今日展览", "非常值得",
                        "不僅有", "徹底改變", "傾力呈現", "懸浮凝視", "此次展覽",
                        "這些懸浮", "打破距離", "不論你是", "這次展覽"]
                if not any(s in title for s in skip):
                    seen.add(title)
                    exhibitions.append({
                        "title": title,
                        "venue": "",
                        "start_date": "",
                        "end_date": "",
                        "price": "",
                        "description": "",
                        "source": "携程",
                        "url": url,
                    })
    
    log_msg(f"  携程提取：{len(exhibitions)} 个展览")
    return exhibitions


# ============================================================
# 日期解析
# ============================================================

def parse_date(date_str):
    s = date_str.strip()
    patterns = [
        (r'(\d{4})年(\d{1,2})月(\d{1,2})日?[—\-~至到]\s*(\d{4})年(\d{1,2})月(\d{1,2})日?',
         lambda g: (f"{g[0]}-{g[1].zfill(2)}-{g[2].zfill(2)}", f"{g[3]}-{g[4].zfill(2)}-{g[5].zfill(2)}")),
        (r'(\d{4})年(\d{1,2})月(\d{1,2})日?起',
         lambda g: (f"{g[0]}-{g[1].zfill(2)}-{g[2].zfill(2)}", "")),
        (r'(\d{1,2})月(\d{1,2})日?[—\-~至到]\s*(\d{1,2})月(\d{1,2})日?',
         lambda g: (f"{datetime.now().year}-{g[0].zfill(2)}-{g[1].zfill(2)}",
                    f"{datetime.now().year}-{g[2].zfill(2)}-{g[3].zfill(2)}")),
        (r'(\d{1,2})月(\d{1,2})日?(?:起对公众开放|起开放|起)',
         lambda g: (f"{datetime.now().year}-{g[0].zfill(2)}-{g[1].zfill(2)}", "")),
    ]
    for regex, fmt in patterns:
        m = re.search(regex, s)
        if m:
            start, end = fmt(m.groups())
            return {"start": start, "end": end}
    return {"start": s, "end": ""}


# ============================================================
# 去重与合并
# ============================================================

def deduplicate(exhibitions):
    seen = {}
    for ex in exhibitions:
        key = ex["title"][:30]  # 用前30字去重（减少碎片误匹配）
        if key in seen:
            old = seen[key]
            for field in ["venue", "start_date", "end_date", "price", "description", "location"]:
                if not old.get(field) and ex.get(field):
                    old[field] = ex[field]
            if old.get("source") == "携程" and ex.get("source") != "携程":
                old["source"] = ex["source"]
        else:
            seen[key] = dict(ex)
    return list(seen.values())


def classify_exhibition(ex):
    text = ex.get("title", "") + " " + ex.get("description", "") + " " + ex.get("venue", "")
    
    top_kw = [
        "历史", "人文", "古代", "考古", "文物", "古建筑", "书法", "藏传",
        "佛教", "古蜀", "青铜", "文明", "庞贝", "三星堆", "敦煌",
        "文艺复兴", "乌菲齐", "卡拉瓦乔", "达·芬奇", "达芬奇",
        "丝绸", "玉器", "陶瓷", "书画", "雕刻", "石窟", "壁画",
        "文献", "简牍", "金石", "碑刻", "宗教", "丝绸之路",
        "写意", "八大山人", "齐白石", "黄宾虹",
        "玛雅", "安第斯", "美洲豹", "叙利亚", "古埃及", "科威特",
        "捐献", "百年", "李静训", "月背", "航天",
    ]
    mid_kw = [
        "艺术", "摄影", "油画", "版画", "水彩", "雕塑",
        "国际", "交流", "文化", "非遗", "工艺",
        "卡塔尔", "巴西", "柬埔寨", "吴哥",
        "国博", "故宫", "首博", "美术馆",
    ]
    
    score_top = sum(1 for kw in top_kw if kw in text)
    score_mid = sum(1 for kw in mid_kw if kw in text)
    
    if score_top >= 2 or (score_top >= 1 and score_mid >= 1):
        return "top", "强烈推荐"
    elif score_top >= 1 or score_mid >= 2:
        return "mid", "值得一看"
    else:
        return "low", "供参考"


# ============================================================
# 报告生成
# ============================================================

def generate_report(exhibitions, report_date=None):
    today = report_date or datetime.now().strftime("%Y-%m-%d")
    
    classified = {"top": [], "mid": [], "low": []}
    for ex in exhibitions:
        level, label = classify_exhibition(ex)
        classified[level].append({**ex, "label": label})
    
    total = len(exhibitions)
    top_n = len(classified["top"])
    mid_n = len(classified["mid"])
    
    def fmt_date(ex):
        s, e = ex.get("start_date", ""), ex.get("end_date", "")
        raw = ex.get("date_raw", "")
        if s and e:
            return f"{s} — {e}"
        elif s:
            return f"{s}起"
        elif raw:
            return raw
        return "展期待确认"
    
    def fmt_desc(ex):
        d = ex.get("description", "")
        if not d:
            return None
        return d[:200] + "…" if len(d) > 200 else d
    
    def fmt_price(ex):
        p = ex.get("price", "")
        if not p or p == "免费":
            return "免费（需预约）"
        return p
    
    def fmt_venue(ex):
        v = ex.get("venue", "")
        loc = ex.get("location", "")
        if v and loc and loc != v:
            return f"{v} · {loc}"
        return v or "待确认"
    
    L = []
    L.append(f"# 北京展览推荐 - {today}")
    L.append("")
    L.append("## 📅 今日概览")
    L.append(f"- **在展数量**：{total} 个（精选）")
    L.append(f"- **强烈推荐**：{top_n} 个（历史人文/古代艺术/世界文明）")
    L.append(f"- **值得一看**：{mid_n} 个（文化交流/艺术大展）")
    L.append(f"- **供参考**：{len(classified['low'])} 个（亲子/科普/科技类）")
    L.append(f"- **数据来源**：北京本地宝、携程展览")
    L.append(f"- **数据更新时间**：{today} {datetime.now().strftime('%H:%M')}")
    L.append("")
    L.append("---")
    L.append("")
    
    # 第一梯队
    if classified["top"]:
        L.append("## 🌟 第一梯队：强烈推荐（历史人文 · 古代艺术 · 世界文明）")
        L.append("")
        for i, ex in enumerate(classified["top"], 1):
            L.append(f"### {i}. {ex['title']}")
            L.append(f"- **场馆**：{fmt_venue(ex)}")
            L.append(f"- **展期**：{fmt_date(ex)}")
            L.append(f"- **门票**：{fmt_price(ex)}")
            desc = fmt_desc(ex)
            if desc:
                L.append(f"- **简介**：{desc}")
            L.append("")
    
    # 第二梯队
    if classified["mid"]:
        L.append("## 🌟 第二梯队：值得一看（文化交流 · 艺术大展）")
        L.append("")
        for i, ex in enumerate(classified["mid"], 1):
            idx = top_n + i
            L.append(f"### {idx}. {ex['title']}")
            L.append(f"- **场馆**：{fmt_venue(ex)}")
            L.append(f"- **展期**：{fmt_date(ex)}")
            L.append(f"- **门票**：{fmt_price(ex)}")
            desc = fmt_desc(ex)
            if desc:
                L.append(f"- **简介**：{desc}")
            L.append("")
    
    # 第三梯队
    if classified["low"]:
        L.append("## 📌 第三梯队：亲子/科普/科技类（供参考）")
        L.append("")
        L.append("| 序号 | 展览名称 | 场馆 | 展期 | 门票 |")
        L.append("|------|---------|------|------|------|")
        for i, ex in enumerate(classified["low"], 1):
            idx = top_n + mid_n + i
            L.append(f"| {idx} | {ex['title']} | {fmt_venue(ex)} | {fmt_date(ex)} | {fmt_price(ex)} |")
        L.append("")
    
    # 参观提示
    L.append("---")
    L.append("")
    L.append("## 📌 参观实用信息")
    L.append("")
    L.append("### 🎫 预约方式")
    L.append("- **故宫**：提前7天20:00开始预订（www.dpm.org.cn）")
    L.append("- **国家博物馆**：提前7天预约（www.chnmuseum.cn）")
    L.append("- **中国美术馆**：提前预约（www.namoc.cn）")
    L.append("- **首都博物馆**：关注「首都博物馆」微信公众号预约")
    L.append("- **免费展览**：部分也需预约，请提前确认")
    L.append("")
    L.append("### 🚇 交通建议")
    L.append("- **优先地铁**：北京停车困难，建议地铁出行")
    L.append("- **798 艺术区**：地铁 14 号线望京南站，换乘公交/打车")
    L.append("- **故宫周边**：地铁 1 号线天安门东站")
    L.append("")
    L.append("### 👨‍👩‍👧 亲子观展")
    L.append("- **建议时长**：每展 1.5-2 小时，避免孩子疲劳")
    L.append("- **携带物品**：水杯、小零食（馆外食用）、湿巾")
    L.append("- **休息安排**：选择有休息区的场馆")
    L.append("")
    L.append("---")
    L.append("")
    L.append(f"*由 Travel Agent 自动生成 | {today} {datetime.now().strftime('%H:%M')} | 每日 9:00 更新*")
    
    content = "\n".join(L)
    report_path = DATA_DIR / f"beijing_exhibitions_{today}.md"
    with open(report_path, "w", encoding="utf-8") as f:
        f.write(content)
    
    log_msg(f"报告已生成：{report_path}（{total} 个展览）")
    return content, report_path


# ============================================================
# 主流程
# ============================================================

def main():
    target_date = datetime.now().strftime("%Y-%m-%d")
    output_only = False
    
    args = sys.argv[1:]
    for i, arg in enumerate(args):
        if arg == "--date" and i + 1 < len(args):
            target_date = args[i + 1]
        elif arg == "--output-only":
            output_only = True
    
    log_msg("=" * 60)
    log_msg(f"北京展览爬取 v3（多源直接抓取）- {target_date}")
    log_msg("=" * 60)
    
    if not output_only:
        ex1 = crawl_bendibao()
        time.sleep(2)
        ex2 = crawl_tripcom()
        
        all_ex = ex1 + ex2
        unique = deduplicate(all_ex)
        
        log_msg(f"合计：{len(all_ex)} 条，去重后：{len(unique)} 条")
    else:
        unique = []
        ex1, ex2 = [], []
    
    if len(unique) < 5:
        log_msg(f"⚠️ 数据量偏少（{len(unique)} 条）", "WARNING")
    
    content, report_path = generate_report(unique, target_date)
    
    result = {
        "status": "success" if len(unique) >= 10 else ("warning" if len(unique) >= 5 else "error"),
        "total": len(unique),
        "total_items": len(unique),
        "saved_items": len(unique),
        "report_path": str(report_path),
        "sources": {
            "bendibao": len(ex1),
            "tripcom": len(ex2),
        }
    }
    print(f"\n__JSON_RESULT__\n{json.dumps(result, ensure_ascii=False, indent=2)}")
    
    log_msg(f"完成！共 {len(unique)} 个展览")
    return 0 if len(unique) >= 5 else 1


if __name__ == "__main__":
    exit(main())