#!/usr/bin/env python3
"""Build the directory pages: data/, figures/, scripts/ and papers/.

Why these exist at all: the site is served by GitHub Pages, which has no
directory listing. index.html links to `data/`, `figures/`, `scripts/` and
`papers/`, and without an index.html in each of them those four links 404 --
except `papers/`, which resolved only because Jekyll rendered papers/README.md
through its stock theme, in none of this project's typography. A real page in
each directory fixes both problems at once and takes the site's own tokens.

Everything here is derived, not retyped:
  - file sizes come from the files on disk;
  - the bibliography comes from data/citations.csv;
  - a paper's local copy is found by globbing its DOI slug in papers/, so a
    renamed or added copy is picked up without touching this script;
  - the CSS and the shell come from scripts/tokens.css and
    scripts/page_template.html, shared with index.html and the Moat.

Deterministic: no build date, no clock -- rerunning reproduces every file
byte-for-byte. Asserts that every git-tracked file in a directory is described,
so adding a file to the repo and forgetting to describe it fails the build.
"""
import csv, glob, os, re, subprocess

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
TPL = open(os.path.join(ROOT, "scripts", "page_template.html")).read()
TOKENS = open(os.path.join(ROOT, "scripts", "tokens.css")).read()


ENTITY = re.compile(r"&(?:#x[0-9A-Fa-f]+|#[0-9]+|[A-Za-z][A-Za-z0-9]*);")


def ent(s):
    """Non-ASCII to numeric entities, so every generated file stays 7-bit ASCII.

    Deliberately does not escape markup: the descriptions and the citation
    fields carry intentional <i> tags for scientific names and written-out
    entities like &mdash;. A bare ampersand is the one thing that cannot be
    told apart from a broken entity, so it is rejected rather than guessed at.
    """
    assert "&" not in ENTITY.sub("", s), f"bare ampersand needs an entity: {s!r}"
    return "".join(c if ord(c) < 128 else f"&#x{ord(c):04X};" for c in s)


def size(path):
    n = os.path.getsize(path)
    if n >= 1 << 20:
        return f"{n / (1 << 20):.1f} MB"
    if n >= 1 << 10:
        return f"{n / (1 << 10):.0f} KB"
    return f"{n} B"


# --- the file manifests ----------------------------------------------------
# (filename, description), in reading order rather than alphabetical: the file
# you most likely came for goes first. Every one is asserted to exist, and the
# build fails if a tracked file is left undescribed.

DATA = [
    ("nd2_alignment.fasta",
     "The alignment everything else is computed from: 156 sequences "
     "&times; 1,041 bp of mitochondrial ND2."),
    ("nd2_ml.treefile",
     "The maximum-likelihood tree in Newick, carrying SH-aLRT and ultrafast "
     "bootstrap support at every node."),
    ("nd2_ml_iqtree_log.txt",
     "The full IQ-TREE run log: model selection, likelihoods and bootstrap "
     "settings, kept so the tree can be audited rather than trusted."),
    ("divergence_from_sihek.csv",
     "Uncorrected p-distance and Jukes-Cantor distance from the sihek to every "
     "other sequence, with the number of comparable sites for each."),
    ("within_sihek_distances.csv",
     "Distances among the sihek sequences themselves &mdash; the scale bar that "
     "makes the 2.30% nearest neighbour legible."),
    ("sihek_placement_support.csv",
     "Support for the nodes that place the sihek, walked outward from its "
     "sister lineage. Step 1 is the 66% bootstrap the caveats turn on."),
    ("taxa_accessions.csv",
     "Every tip: tree label, species, GenBank accession, subfamily and "
     "ingroup/outgroup role."),
    ("citations.csv",
     "The four papers behind the analysis, with DOI, access route and what "
     "each one is used for. Rendered at <a href=\"../papers/\">papers/</a>."),
]

FIGURES = [
    ("moat.html",
     "The Moat, self-contained and interactive: every sequence by its distance "
     "from the sihek, with the empty intervals inked instead of the data."),
    ("moat_data.json",
     "The plot's input, joined by <code>scripts/prep_data.py</code> from the "
     "tree order, the distances and the taxonomy."),
    ("wiki_links.json",
     "Each taxon resolved against the Wikipedia API, recording whether the "
     "match was exact or fell back to the parent species."),
    ("fig1_todiramphus_tree.png",
     "Figure 1. Maximum-likelihood tree of <i>Todiramphus</i>: the sihek's "
     "closest kin are not the birds it was once classified with."),
    ("fig2_family_tree.png",
     "Figure 2. The family-level tree, with subfamilies marked &mdash; the "
     "sihek is a tree kingfisher."),
    ("fig3_divergence.png",
     "Figure 3. Divergence from the sihek as a histogram and dot plot; the gap "
     "around it is empty, not sparse."),
    ("fig4_conservation_genomics.png",
     "Figure 4. Runs of homozygosity and heterozygosity from the whole-genome "
     "read set, built in <code>genomics/</code>."),
]

SCRIPTS = [
    ("prep_data.py",
     "Joins tree tip order, divergence and taxonomy into "
     "<code>figures/moat_data.json</code>, asserting all 156 tips resolve."),
    ("wiki_lookup.py",
     "Resolves every taxon against the Wikipedia API, recording exact matches "
     "and parent-species fallbacks separately."),
    ("build_moat.py",
     "Renders the Moat's SVG server-side, so the figure needs no build step "
     "and no JavaScript to draw itself."),
    ("build_index.py",
     "Builds <code>index.html</code>, taking its headline numbers from "
     "<code>moat_data.json</code> so the page cannot go stale against the data."),
    ("build_pages.py",
     "Builds the four directory pages, including this one, from the manifests "
     "and <code>data/citations.csv</code>."),
    ("validate_palette.py",
     "Checks the palette's contrast and separation in both light and dark, so "
     "a colour cannot be changed by eye alone."),
    ("tokens.css",
     "The design tokens. Inlined into every page at build time, which is why "
     "the pages cannot drift apart."),
    ("moat_template.html",
     "The Moat's shell: layout, interaction and the tooltip."),
    ("index_template.html",
     "The landing page's shell."),
    ("page_template.html",
     "The shell these four directory pages share."),
    ("fetch_genome.sh",
     "Fetches assembly <code>GCA_033439825.1</code> from NCBI, ~353 MB, "
     "MD5-verified against NCBI's own manifest."),
]


def files_block(dirname, entries):
    """Render one directory listing, and check it against what git tracks."""
    rows = []
    for name, desc in entries:
        path = os.path.join(ROOT, dirname, name)
        assert os.path.exists(path), f"{dirname}/{name} is described but missing"
        rows.append(
            f'    <div class="file">'
            f'<a class="fn" href="{name}">{name}</a>'
            f'<span class="fd">{ent(desc)}</span>'
            f'<span class="fs">{size(path)}</span></div>'
        )
    return '<div class="files">\n' + "\n".join(rows) + "\n  </div>"


def tracked(dirname):
    """Files git tracks directly in a directory, or None if git is unavailable."""
    try:
        out = subprocess.run(["git", "-C", ROOT, "ls-files", dirname],
                             capture_output=True, text=True, check=True).stdout
    except (OSError, subprocess.CalledProcessError):
        return None
    names = {p.split("/", 1)[1] for p in out.split() if p.startswith(dirname + "/")}
    return {n for n in names if "/" not in n}


# --- the bibliography ------------------------------------------------------

def refs_block():
    """Render papers/ from data/citations.csv, in the order the file lists them.

    The local copy is found by globbing the DOI slug, which is how the files in
    papers/ are already named, so this does not hardcode a filename per paper.
    """
    with open(os.path.join(ROOT, "data", "citations.csv"), newline="") as fh:
        rows = list(csv.DictReader(fh))
    assert rows, "citations.csv is empty"

    out = []
    for i, r in enumerate(rows, 1):
        doi = r["doi"]
        slug = doi.replace("/", "_")
        local = sorted(p for p in glob.glob(os.path.join(ROOT, "papers", slug + ".*")))
        is_open = not r["access"].upper().startswith("CLOSED")

        links = [f'<a href="https://doi.org/{doi}">Publisher (DOI)</a>']
        if r.get("full_text_url"):
            links.append(f'<a href="{r["full_text_url"]}">Free full text</a>')
        if r.get("abstract_url"):
            links.append(f'<a href="{r["abstract_url"]}">Abstract</a>')
        for p in local:
            name = os.path.basename(p)
            kind = "PDF" if name.endswith(".pdf") else "full text"
            links.append(f'<a href="{name}">Local copy &middot; {kind} &middot; {size(p)}</a>')
        if not local:
            links.append('<span class="none">No local copy</span>')

        cls = "open" if is_open else "closed"
        out.append(f"""    <article class="ref">
      <p class="ref-k"><span>{i:02d}</span><span class="{cls}">{ent(r["access"])}</span></p>
      <h3>{ent(r["title"])}</h3>
      <p class="who">{ent(r["authors"])}</p>
      <p class="where">{ent(r["journal"])} &middot; {r["year"]} &middot; doi:{doi}</p>
      <p class="why">{ent(r["relevance"])}</p>
      <p class="links">{" ".join(links)}</p>
    </article>""")
    return '<div class="refs">\n' + "\n".join(out) + "\n  </div>"


# --- the pages -------------------------------------------------------------

PAGES = [
    dict(
        dir="data",
        title="data &mdash; sihek phylogenetics",
        eyebrow="156 ND2 sequences &middot; 1,041 bp &middot; GenBank",
        h1="Everything the tree was built from.",
        standfirst="The alignment, the maximum-likelihood tree and its log, the distance "
                   "tables, and the accession list that ties every tip back to GenBank. "
                   "Each file is plain text and readable on its own.",
        entries=DATA,
        foot="<p>Sequences are GenBank records; accessions are listed per tip in "
             "<code>taxa_accessions.csv</code>. Distances are uncorrected p-distances "
             "over comparable sites only, which is why the site count travels with "
             "every number.</p>",
    ),
    dict(
        dir="figures",
        title="figures &mdash; sihek phylogenetics",
        eyebrow="One interactive figure &middot; four static &middot; two JSON inputs",
        h1="The figures, and the data they are drawn from.",
        standfirst="<a href=\"moat.html\">moat.html</a> is the argument; the PNGs are the "
                   "conventional views of the same result. Both JSON files are build "
                   "inputs, kept so the figures can be regenerated rather than trusted.",
        entries=FIGURES,
        foot="<p>The static figures are built by <code>scripts/</code> from "
             "<code>data/</code>; the Moat is rendered server-side into "
             "<code>moat.html</code>, which needs no build step and no network to "
             "view.</p>",
    ),
    dict(
        dir="scripts",
        title="scripts &mdash; sihek phylogenetics",
        eyebrow="Deterministic builds &middot; no clock &middot; no build date",
        h1="How every page and figure here is made.",
        standfirst="Each script rebuilds its output byte-for-byte from the data in "
                   "<a href=\"../data/\">data/</a>. <code>tokens.css</code> is shared by "
                   "every page, which is the mechanism that keeps them from drifting.",
        entries=SCRIPTS,
        foot="<p>Rebuild in order: <code>prep_data.py</code>, <code>wiki_lookup.py</code>, "
             "<code>build_moat.py</code>, then <code>build_index.py</code> and "
             "<code>build_pages.py</code>. Only <code>wiki_lookup.py</code> and "
             "<code>fetch_genome.sh</code> touch the network.</p>",
    ),
    dict(
        dir="papers",
        title="papers &mdash; sihek phylogenetics",
        eyebrow="Four sources &middot; three open access &middot; one paywalled",
        h1="The four papers this analysis stands on.",
        standfirst="Three are CC-BY and are mirrored here in full. The fourth is the "
                   "genome-scale phylogeny that explains why the sister node in this "
                   "tree is weakly supported, and it is behind a paywall with no open "
                   "copy anywhere.",
        body=None,  # filled by refs_block()
        foot="<p>The mirrored copies are redistributed under each paper's CC-BY licence; "
             "the DOI link is the version of record in every case. "
             "<code>10.1093/sysbio/syaf075</code> has no PMC deposit, no repository copy "
             "and no preprint &mdash; checked against Unpaywall, Semantic Scholar, PMC, "
             "CrossRef and OpenAlex. Institutional access or an author reprint request "
             "are the only routes.</p>",
    ),
]


def build(page):
    d = page["dir"]
    if "entries" in page:
        main = "  " + files_block(d, page["entries"])
        described = {n for n, _ in page["entries"]}
    else:
        main = "  " + refs_block()
        described = set()

    have = tracked(d)
    if have is not None:
        # README.md is the GitHub-facing view of a directory and is not listed
        # on the page; index.html is this file.
        ignore = {"README.md", "index.html"}
        missed = sorted(have - described - ignore) if described else []
        assert not missed, f"{d}/: tracked but not described: {missed}"

    out = (TPL.replace("/*TOKENS*/", TOKENS)
              .replace("<!--TITLE-->", page["title"])
              .replace("<!--EYEBROW-->", page["eyebrow"])
              .replace("<!--H1-->", page["h1"])
              .replace("<!--STANDFIRST-->", page["standfirst"])
              .replace("<!--MAIN-->", main)
              .replace("<!--FOOT-->", "    " + page["foot"]))

    for marker in ("/*TOKENS*/", "<!--TITLE-->", "<!--EYEBROW-->", "<!--H1-->",
                   "<!--STANDFIRST-->", "<!--MAIN-->", "<!--FOOT-->"):
        assert marker not in out, f"unfilled placeholder {marker} in {d}/index.html"
    non_ascii = sorted({c for c in out if ord(c) > 127})
    assert not non_ascii, f"non-ASCII in {d}/index.html: {non_ascii}"

    path = os.path.join(ROOT, d, "index.html")
    open(path, "w").write(out)
    print(f"wrote {os.path.relpath(path, ROOT)}  {len(out)} bytes")


if __name__ == "__main__":
    for page in PAGES:
        build(page)