gigachad-library/tools/crawl.py

#!/usr/bin/env python3
"""Crawl the Library of Chadnet into a structured JSON tree for faithful replica."""
import json, os, re, sys, time
from html.parser import HTMLParser
import requests

BASE = "https://wiki.chadnet.org"
HDRS = {"User-Agent": "Mozilla/5.0 (compatible; library-mirror/1.0)"}
S = requests.Session(); S.headers.update(HDRS)
OUT = "/home/hermes/chad-library"
DATA = os.path.join(OUT, "site_data.json")

def classify_icon(src):
    s = src.lower()
    if "folder" in s: return "folder"
    if "globe" in s: return "globe"
    if "document" in s: return "document"
    if "painting" in s: return "painting"
    if "youtube" in s: return "youtube"
    if "article" in s: return "article"
    if "chad-square" in s: return "chadsquare"
    return "link"

class PageParser(HTMLParser):
    """Capture title, parent (from nav), and category lists as sections of items."""
    def __init__(self):
        super().__init__(convert_charrefs=True)
        self.title = None
        self.parent = None
        self.sections = []     # list of [h2key, [items]]
        self._intitle = False
        # element state
        self._in_a = 0
        self._h2 = None        # accumulating h2 text or None
        self._active = None    # current items list
        self._ul_depth = 0
        self._li = None        # current item dict being built or None
        # breadcrumbs: capture parent once from nav <p class="nav">
        self._nav_seen = False

    # -- start tags --
    def handle_starttag(self, tag, attrs):
        a = dict(attrs)
        if tag == "title":
            self._intitle = True
        elif tag == "h2":
            self._h2 = ""
        elif tag == "ul":
            self._ul_depth += 1
        elif tag == "li":
            if self._ul_depth >= 1 and self._li is None:
                self._li = {"type": None, "icon": None, "text": "",
                            "href": "", "note": ""}
        elif tag == "img":
            if self._li is not None and self._li["icon"] is None:
                self._li["icon"] = a.get("src") or ""
        elif tag == "a":
            self._in_a += 1
            if self._li is not None and not self._li["href"]:
                self._li["href"] = a.get("href") or ""
            if self._in_a == 1 and self._li is not None and self._li["type"] is None and self._li["icon"]:
                # first link inside item is the title link
                pass

    # -- end tags --
    def handle_endtag(self, tag):
        if tag == "title":
            self._intitle = False
        elif tag == "h2":
            key = (self._h2 or "").strip()
            self._h2 = None
            # new section becomes active
            self._active = []
            self.sections.append([key, self._active])
        elif tag == "ul":
            self._ul_depth = max(0, self._ul_depth - 1)
        elif tag == "li":
            if self._li is not None:
                self._li["text"] = self._li["text"].strip()
                self._li["note"] = self._li["note"].strip()
                self._li["icon"] = self._li["icon"] or ""
                self._li["type"] = classify_icon(self._li["icon"])
                if self._active is None:
                    self._active = []
                    self.sections.append([None, self._active])
                self._active.append(self._li)
                self._li = None
        elif tag == "a":
            self._in_a = max(0, self._in_a - 1)

    # -- data --
    def handle_data(self, data):
        if self._intitle:
            t = data.strip()
            if t:
                self.title = re.sub(r"\s*[–—-]\s*Chadnet\s*$", "", t)
        if self._h2 is not None:
            self._h2 += data
        if self._li is not None:
            if self._in_a > 0:
                self._li["text"] += data
            else:
                self._li["note"] += data

def fetch(url):
    for attempt in range(4):
        try:
            r = S.get(url, timeout=30)
            if r.status_code == 200:
                return r.text
            print(f"  !! {url} -> HTTP {r.status_code}", flush=True)
        except Exception as e:
            print(f"  !! {url} err {e}", flush=True)
        time.sleep(2 * (attempt + 1))
    return None

def main():
    if os.path.exists(DATA):
        with open(DATA) as f:
            return json.load(f)
    home = fetch(BASE + "/")
    if not home:
        print("FAILED homepage"); sys.exit(1)
    maincats = re.findall(r'<a href="([^"]+)"><div class="maincat"', home)
    print("top-level:", maincats, flush=True)
    to_visit = [(s.lstrip("/"), None) for s in maincats]
    pages = {}
    while to_visit:
        slug, parent = to_visit.pop(0)
        if slug in pages:
            continue
        url = BASE + "/" + slug
        html = fetch(url)
        if not html:
            continue
        p = PageParser(); p.feed(html)
        # detect parent from nav back link
        m = re.search(r'Back:\s*<a href="([^"]+)">', html)
        if m:
            parent = m.group(1).strip().lstrip("/")
        page = {"slug": slug, "title": p.title or slug, "parent": parent,
                "sections": p.sections, "url": url}
        pages[slug] = page
        n = sum(len(s[1]) for s in p.sections)
        print(f"crawled {slug} ({n} items, parent={parent})", flush=True)
        for _, items in p.sections:
            for it in items:
                if it["type"] == "folder" and it["href"] and not it["href"].startswith("http"):
                    child = it["href"].rstrip("/").split("/")[-1].split("?")[0]
                    if child:
                        to_visit.append((child, slug))
        time.sleep(0.3)
    with open(DATA, "w") as f:
        json.dump({"pages": pages}, f, ensure_ascii=False, indent=1)
    print(f"\nTotal pages: {len(pages)}")
    return pages

if __name__ == "__main__":
    main()