first commit

This commit is contained in:
libroot 2026-11-19 04:40:59 +02:00
commit 8c4dcae475
Signed by: libroot
GPG key ID: 2AC68C029EBE9ACE
89 changed files with 24623 additions and 0 deletions

199
tests/corpus_test.go Normal file
View file

@ -0,0 +1,199 @@
package main
import (
"html"
"net/http"
"os"
"reflect"
"regexp"
"slices"
"strings"
"testing"
)
// Checks against a REAL corpus. Point $SNOWCHIVE_BENCH_DATA at one, as for
// the page benchmarks, and run: go test -run TestCorpus -v
//
// The fixtures are a handful of documents, each built to show one thing. A
// count and its link are computed by different code, and whether the two
// agree turns on what the documents happen to hold: a second publication in
// a later year, a byline on only one of them, a value found only in FOIA
// releases. The disagreements these tests walk the archive for were all
// found on the full corpus and on no fixture.
func TestCorpus(t *testing.T) {
root := os.Getenv("SNOWCHIVE_BENCH_DATA")
if root == "" {
t.Skip("set SNOWCHIVE_BENCH_DATA to a corpus directory to check a real corpus")
}
withCorpus(t, root)
t.Run("counts", corpusCounts)
t.Run("FOIA links", corpusFOIALinks)
t.Run("inflections", corpusInflections)
t.Run("snippets", corpusSnippets)
t.Run("index", corpusIndex)
}
// corpusCounts follows the counted links of the loaded corpus, as a reader
// who leaves FOIA releases out and as one who includes them.
func corpusCounts(t *testing.T) {
// The pages whose bars and refine links are followed: the year pages of
// both date categories, the three commonest values of every other
// category, each cell of the outlets table (a publisher's page for one
// publication year) and searches under each kind of date range.
var pages []string
for ci := range categories {
c := &categories[ci]
n := 3
if c.isDate() {
n = -1
}
for _, v := range rankCounts(c.Facet(corpus()), n) {
pages = append(pages, categoryValueURL(c.Slug, v))
}
}
for _, row := range corpus().outletTables[""].Rows {
for _, cell := range row.Cells {
if cell.URL != "" {
pages = append(pages, cell.URL)
}
}
}
pages = append(pages,
"/search?q=nsa",
"/search?q=nsa+date%3A2012-06..2013-06",
"/search?q=nsa+published%3A2016",
"/search?q=published%3A2014",
"/search?do_search=1&fqv=nsa&fqf=TE&fqc=and&date_pub_start=2015-01-01&date_pub_end=2016-06-30",
"/search?do_search=1&fqv=nsa&fqf=TE&fqc=and&fqv=gchq&fqf=TE&fqc=or&date_pub_start=2016&date_pub_end=2016",
)
for name, reader := range map[string][]*http.Cookie{"FOIA left out": nil, "FOIA included": {foiaOn}} {
t.Run(name, func(t *testing.T) {
bars, refines := 0, 0
for _, p := range pages {
bars += followLinks(t, p, reBarLink, "bar", reader...)
refines += followLinks(t, p, reFacetLink, "refine link", reader...)
}
if bars == 0 || refines == 0 {
t.Errorf("nothing exercised: %d bars, %d refine links", bars, refines)
}
boxes := checkCategoryBoxes(t, reader...)
t.Logf("%d pages: %d bars, %d refine links and %d category boxes checked", len(pages), bars, refines, boxes)
})
}
}
// corpusFOIALinks opens the page of every tenth FOIA release as a reader
// who leaves FOIA out, and follows each category value it links: the page
// behind the link has to list the release.
func corpusFOIALinks(t *testing.T) {
reValueLink := regexp.MustCompile(`href="(/browse/[a-z]+/[^"]+)"`)
links := 0
for i, d := range corpus().collectionDocs[collectionFOIA] {
if i%10 != 0 {
continue
}
// The release's own metadata: the cards under it are other documents.
page, _, _ := strings.Cut(get(t, docPath(d.ID)).Body.String(), `class="related-section"`)
for _, m := range reValueLink.FindAllStringSubmatch(page, -1) {
link := html.UnescapeString(m[1])
links++
if !strings.Contains(get(t, withQuery(link, "view=table")).Body.String(), `href="`+docPath(d.ID)+`"`) {
t.Errorf("%s links %s, which does not list it", d.ID, link)
}
}
}
if links == 0 {
t.Error("no FOIA release with a value link to follow")
}
t.Logf("%d value links followed", links)
}
// A word in -ies or -ied stems as the word in -y it is a form of, whatever
// rules that word goes through - checked over every such pair the corpus
// holds, since a suffix rule that reads one of the two forms and not the
// other splits them silently ("capability" from "capabilities").
func corpusInflections(t *testing.T) {
vocab := map[string]bool{}
for i := range corpus().archive.Docs {
for _, w := range splitWords(corpus().archive.Docs[i].SearchText) {
vocab[w] = true
}
}
pairs := 0
for w := range vocab {
if len(w) < 5 || !(strings.HasSuffix(w, "ies") || strings.HasSuffix(w, "ied")) {
continue
}
base := w[:len(w)-3] + "y"
if !vocab[base] {
continue
}
pairs++
if stem(w) != stem(base) {
t.Errorf("stem(%q) = %q, stem(%q) = %q; want equal", w, stem(w), base, stem(base))
}
}
if pairs == 0 {
t.Error("the corpus holds no -ies/-ied word next to its -y form")
}
t.Logf("%d pairs checked in a vocabulary of %d words", pairs, len(vocab))
}
// corpusSnippets holds every document's snippets off the fold kept at load
// to the ones folding its transcript again gives - the versions' lines sit
// behind that fold in the same string - and a page of "all" cards to its
// bound.
func corpusSnippets(t *testing.T) {
for _, q := range []string{"nsa", "cable tapping", `"upstream collection"`, "tap*", "the"} {
for i := range corpus().archive.Docs {
d := &corpus().archive.Docs[i]
if got, want := d.bodySnippets(q, 3), getSnippets(d.FullText, q, 3); !reflect.DeepEqual(got, want) {
t.Fatalf("%s, %q: %q from the loaded fold, %q from the text", d.ID, q, got, want)
}
}
}
all := &http.Cookie{Name: "pref_per_page", Value: "all"}
for _, c := range categories {
for _, v := range rankCounts(c.Facet(corpus()), 1) {
if c.Facet(corpus())[v] <= maxCardsPerPage {
continue
}
body := get(t, categoryValueURL(c.Slug, v), all, foiaOn).Body.String()
if n := strings.Count(body, `class="doc-card"`); n != maxCardsPerPage {
t.Errorf("%s=%s: %d cards on a page of all, want %d", c.Slug, v, n, maxCardsPerPage)
}
}
}
}
// corpusIndex holds the shortcut a one-word all-fields clause takes - it
// asks the document's entry in the index instead of walking its text - to
// the walk, for a spread of the vocabulary over every document.
func corpusIndex(t *testing.T) {
var vocab []string
for w := range corpus().tfidf.DocFreq {
vocab = append(vocab, w)
}
slices.Sort(vocab)
checked := 0
for i := 0; i < len(vocab); i += 131 {
w := vocab[i]
docs := 0
for di := range corpus().archive.Docs {
d := &corpus().archive.Docs[di]
_, indexed := d.terms[w]
if indexed != containsTokens(d.StemmedText, w) {
t.Fatalf("%q in %s: the index says %v, the all-fields stream the opposite", w, d.ID, indexed)
}
if indexed {
docs++
}
}
if docs != corpus().tfidf.DocFreq[w] {
t.Errorf("%q: in %d documents' entries, with a document frequency of %d", w, docs, corpus().tfidf.DocFreq[w])
}
checked++
}
t.Logf("%d terms checked over %d documents", checked, len(corpus().archive.Docs))
}