for (data_path in nli_overview_data_paths) {
x <- readRDS(data_path)
cat(sprintf("\n### %s (%s)\n\n", x$assessment, x$granularity %||% "sub_bm"))
if (isTRUE(x$empty)) {
cat("No NLI scores available yet for this assessment/granularity combination.\n\n")
next
}
cat(sprintf(
"- **%s** works scored\n- **%d** background messages\n- **%d** key messages\n- **%s%%** uncertain (confidence < 0.60)\n\n",
format(x$n_total, big.mark = ","),
x$n_bm,
x$n_km,
x$pct_unc
))
fig_this <- nli_overview_figures_paths[
grepl(paste0("_", x$assessment, "\\.png$"), nli_overview_figures_paths)
]
plot_overall <- fig_this[grepl("plot_overall", fig_this)]
plot_km <- fig_this[grepl("plot_km", fig_this)]
plot_bm <- fig_this[grepl("plot_bm", fig_this)]
plot_conf <- fig_this[grepl("plot_conf", fig_this)]
plot_aln <- fig_this[grepl("plot_aln", fig_this)]
cat("#### Label distribution\n\n")
cat(
knitr::kable(
x$label_overall |>
dplyr::mutate(
label = factor(
label,
levels = c("SUPPORTS", "NOT_ENOUGH_INFO", "REFUTES")
)
) |>
dplyr::arrange(label),
col.names = c("Label", "Works", "%")
),
sep = "\n"
)
cat("\n\n")
if (length(plot_overall)) {
cat(sprintf("\n\n", plot_overall))
}
cat("#### Interactive BM Explorer\n\n")
cat(
"Select a background message from the dropdown and a minimum confidence ",
"from the slider. All works are always shown; the slider instead splits ",
"each bar into a **solid** segment (confidence at or above the ",
"threshold) and a **hollow/outlined** segment on top (below the ",
"threshold, stacked) — in the label-distribution panel (each bar also ",
"labelled `n / N`: works at/above the threshold vs. all works with that ",
"label), the confidence panel (with a dashed line marking the ",
"threshold), and the alignment panel. Mean/median confidence and the % ",
"meeting the threshold are reported above the panels. Below the panels, ",
"a second dropdown (default **REFUTES**) selects which label the ",
"drill-down table lists — the actual works for the current BM + ",
"threshold + label, sorted by confidence descending and capped at 50 ",
"rows (the caption reports the true total when capped). Each work links ",
"to its DOI where one exists (OpenAlex otherwise). **Download table ",
"(CSV)** is not limited to the 50 rows shown — it exports up to 5,000 ",
"matching works for the current BM + threshold + label (sorted by ",
"confidence descending, flagged with a trailing comment line if even ",
"that limit truncates the true match count), including the assessment ",
"and BM as explicit columns.\n\n"
)
explorer_this <- nli_bm_explorer_paths[
grepl(paste0("_", x$assessment, "\\.html$"), nli_bm_explorer_paths)
]
if (length(explorer_this)) {
cat(sprintf(
'<iframe src="%s" width="100%%" height="1020px" style="border:none;"></iframe>\n\n',
explorer_this
))
}
cat("#### Per Key Message\n\n")
cat(
knitr::kable(
x$label_km,
col.names = c("KM", "SUPPORTS %", "NEI %", "REFUTES %")
),
sep = "\n"
)
cat("\n\n")
if (length(plot_km)) {
cat(sprintf("\n\n", plot_km))
}
cat("#### Per Background Message\n\n")
if (length(plot_bm)) {
cat(sprintf("\n\n", plot_bm))
}
cat(
knitr::kable(
x$table_bm,
col.names = c("KM", "BM", "SUPPORTS %", "NEI %", "REFUTES %")
),
sep = "\n"
)
cat("\n\n")
cat("#### Confidence by label\n\n")
cat(
knitr::kable(
x$conf_table,
col.names = c("Label", "n", "Mean", "Median", "P25", "P75")
),
sep = "\n"
)
cat("\n\n")
if (length(plot_conf)) {
cat(sprintf("\n\n", plot_conf))
}
cat("#### Alignment score per BM\n\n")
cat(
knitr::kable(
x$align_table,
col.names = c(
"KM",
"BM",
"n",
"Mean aln",
"SD",
"SUPP %",
"REF %",
"NEI %",
"Unc %"
)
),
sep = "\n"
)
cat("\n\n")
if (length(plot_aln)) cat(sprintf("\n\n", plot_aln))
# AI Based Interpretation — last subsection of each assessment. Hand-
# written commentary, deliberately scoped to the two signals of interest
# (SUPPORTS and REFUTES) rather than the dominant NOT_ENOUGH_INFO label.
# Recomputed against the current scored data (below) — will go stale as
# scoring continues or the NLI config changes, so re-derive the numbers
# from x$table_bm / x$label_km before trusting this after a rerun.
cat("#### AI Based Interpretation\n\n")
if (identical(x$assessment, "GA1")) {
cat(r"----(
::: {.callout-note}
## About this section
Focused specifically on the SUPPORTS/REFUTES signals — NOT_ENOUGH_INFO (~71% of GA1 pairs) is set aside here as the least informative label. Written against the current scored data (2026-07-06); re-derive the numbers below from the tables above before trusting this after a rerun.
:::
**Key finding — KM D is the only reliable signal.** GA1 overall SUPPORTS (16.1%) edges out REFUTES (13.0%), but that gap is almost entirely driven by KM D (34.7% SUPPORTS vs 10.9% REFUTES). KMs A–C sit close to parity or worse — A: 15.0% vs 12.5%, B: 10.0% vs 14.2% (REFUTES-inflated), C: 14.5% vs 20.9% (REFUTES-inflated) — so the argmax label there carries little information on its own.
Within KM D, four BMs are unambiguous: **D3** (62.6% SUPPORTS vs 4.4% REFUTES), **D2** (44.0% vs 6.5%), **D1** (40.8% vs 10.3%), and **D12** (40.7% vs 11.1%) — all over 40% SUPPORTS at single-digit-to-low-teens REFUTES. The one exception inside KM D is **D10**, where REFUTES (15.2%) exceeds SUPPORTS (11.6%) — an outlier against its own KM's pattern, worth treating differently from its D-series siblings.
Likely reason: D-series BMs are probably more concrete and empirically testable — biodiversity metrics, species trends, ecosystem services — the kind of claims abstracts engage with directly.
::: {.callout-warning}
## REFUTES inflation — KMs A, B, C
The BMs with the widest REFUTES-over-SUPPORTS gap:
| BM | SUPPORTS | REFUTES | Gap (REFUTES − SUPPORTS) |
|----|---------|---------|---------|
| C6 | 5.2 % | 28.6 % | +23.4 |
| A9 | 13.8 % | 30.7 % | +16.9 |
| C8 | 8.3 % | 24.8 % | +16.5 |
| B11 | 15.4 % | 28.4 % | +13.0 |
| C7 | 7.1 % | 16.1 % | +9.0 |
| B3 | 3.3 % | 12.0 % | +8.7 |
Citing papers of an assessment being scored as majority-REFUTES at these margins is not credible. This is the well-documented zero-shot NLI failure mode: the hypothesis *"This paper refutes the following claim: …"* draws spurious entailment when the BM contains complex, multi-part, or politically nuanced language. The model fires on surface-level topical tension rather than genuine contradiction.
:::
::: {.callout-note}
## Closest to parity — weakest signal
A handful of BMs have SUPPORTS and REFUTES within a couple of points of each other — the argmax label there is close to a coin flip:
- **A1** 12.9 % vs 11.0 %
- **B10** 12.7 % vs 14.7 %
- **A4** 9.7 % vs 11.8 %
For these, only the continuous alignment score (`p_supports − p_refutes`) carries any real information.
:::
##### Notable exception in KM A
**A2** (20.5% SUPPORTS, 11.0% REFUTES) is the clearest positive signal in KMs A–C outside of KM D — a nearly 10-point SUPPORTS lead, well ahead of its KM-A neighbours. Its claim is likely more concrete or empirically specific than the rest of KM A — worth inspecting manually as a calibration reference.
##### Operational implications
1. **Do not act on raw REFUTES labels for KMs A–C** without Phase-2 LLM review. NLI here is a coarse NEI filter, not a judgment.
2. **KM D SUPPORTS can be treated as the most reliable signal in the assessment** — D3, D2, D1, D12 in particular (all >40% SUPPORTS, single-digit-to-low-teens REFUTES). D10 is the one KM D BM that does NOT fit this pattern.
3. **Use `p_supports − p_refutes` for ranking** rather than argmax labels, especially for the near-parity BMs above.
4. **`max_length: 256` is probably hurting KMs A–C.** Truncating abstract tails removes findings — the part that matters most for support/refute judgements on conceptual BMs. The base-model-at-512 comparison (`deberta_zeroshot_v3`) is the highest-priority next experiment.
)----")
cat(sprintf(
"5. **Fine-tuning on BM citations** ([TD_NLI_training](%s)) would most help KMs A–C — the support/refute axis that zero-shot is fumbling. D-series BMs are already working.\n\n",
td_doc_link("TD_NLI_training")
))
} else if (identical(x$assessment, "IAS")) {
cat(r"----(
::: {.callout-note}
## About this section
Focused specifically on the SUPPORTS/REFUTES signals — NOT_ENOUGH_INFO (~81% of IAS pairs) is set aside here as the least informative label. Written against the current scored data (2026-07-06); re-derive the numbers below from the tables above before trusting this after a rerun.
:::
**Key finding — IAS is weaker and more REFUTES-skewed than GA1.** Overall, IAS REFUTES (11.1%) actually *exceeds* SUPPORTS (8.4%) — the opposite of GA1's overall SUPPORTS lean (16.1% vs 13.0%). Unlike GA1, no IAS BM reaches D-series levels of confidence; the strongest SUPPORTS signal is markedly weaker.
::: {.callout-tip}
## Strongest SUPPORTS signal
**D33** (25.9% SUPPORTS vs 9.5% REFUTES, a 16-point lead) is the clearest positive signal in IAS, appearing under both KM-D4 and KM-D7. A handful of others lean the same way but at much lower absolute rates: **B13** (19.4% vs 10.8%), **D30** (8.7% vs 3.0%), **D27** (7.5% vs 1.8%), and **D25** (12.5% vs 7.2%). None of these approach GA1's D3/D2/D1/D12 margins.
:::
::: {.callout-warning}
## REFUTES inflation
The BMs with the widest REFUTES-over-SUPPORTS gap — the same zero-shot failure mode documented for GA1's KMs A–C:
| BM | SUPPORTS | REFUTES | Gap (REFUTES − SUPPORTS) |
|----|---------|---------|---------|
| A1 | 7.5 % | 22.6 % | +15.1 |
| A8 | 3.6 % | 16.7 % | +13.1 |
| D32 | 22.5 % | 33.5 % | +11.0 |
| A4 | 7.6 % | 17.6 % | +10.0 |
| C19 | 5.0 % | 14.3 % | +9.3 |
| D28 | 8.1 % | 14.7 % | +6.6 |
**D32** is a special case: both SUPPORTS and REFUTES are elevated (22.5% and 33.5%) rather than REFUTES alone — it appears under three different KMs (KM-D5, KM-D6, KM-D7), which may mean it's a genuinely contested or multi-faceted claim rather than a pure model artifact, and is worth a manual look before writing it off as noise.
:::
##### Operational implications
1. **Do not act on raw REFUTES labels anywhere in IAS** without Phase-2 LLM review — the overall REFUTES-over-SUPPORTS skew makes this an even higher priority here than for GA1's KMs A–C.
2. **D33 is the one IAS BM currently worth treating as a real signal** — everything else is either near parity or REFUTES-leaning.
3. **Use `p_supports − p_refutes` for ranking** rather than argmax labels; IAS's overall skew means a plain SUPPORTS-label count will systematically undercount real support.
4. **D32 deserves manual review before either accepting or discarding it** — high confidence in both directions, across multiple KMs, is a different failure mode from the single-direction REFUTES inflation seen elsewhere.
)----")
} else {
cat("No AI-based interpretation has been written yet for this assessment.\n\n")
}
}
