package main import ( "html" "net/http" "os" "reflect" "regexp" "slices" "strings" "testing" ) // Checks against a REAL corpus. Point $SNOWCHIVE_BENCH_DATA at one, as for // the page benchmarks, and run: go test -run TestCorpus -v // // The fixtures are a handful of documents, each built to show one thing. A // count and its link are computed by different code, and whether the two // agree turns on what the documents happen to hold: a second publication in // a later year, a byline on only one of them, a value found only in FOIA // releases. The disagreements these tests walk the archive for were all // found on the full corpus and on no fixture. func TestCorpus(t *testing.T) { root := os.Getenv("SNOWCHIVE_BENCH_DATA") if root == "" { t.Skip("set SNOWCHIVE_BENCH_DATA to a corpus directory to check a real corpus") } withCorpus(t, root) t.Run("counts", corpusCounts) t.Run("FOIA links", corpusFOIALinks) t.Run("inflections", corpusInflections) t.Run("snippets", corpusSnippets) t.Run("index", corpusIndex) } // corpusCounts follows the counted links of the loaded corpus, as a reader // who leaves FOIA releases out and as one who includes them. func corpusCounts(t *testing.T) { // The pages whose bars and refine links are followed: the year pages of // both date categories, the three commonest values of every other // category, each cell of the outlets table (a publisher's page for one // publication year) and searches under each kind of date range. var pages []string for ci := range categories { c := &categories[ci] n := 3 if c.isDate() { n = -1 } for _, v := range rankCounts(c.Facet(corpus()), n) { pages = append(pages, categoryValueURL(c.Slug, v)) } } for _, row := range corpus().outletTables[""].Rows { for _, cell := range row.Cells { if cell.URL != "" { pages = append(pages, cell.URL) } } } pages = append(pages, "/search?q=nsa", "/search?q=nsa+date%3A2012-06..2013-06", "/search?q=nsa+published%3A2016", "/search?q=published%3A2014", "/search?do_search=1&fqv=nsa&fqf=TE&fqc=and&date_pub_start=2015-01-01&date_pub_end=2016-06-30", "/search?do_search=1&fqv=nsa&fqf=TE&fqc=and&fqv=gchq&fqf=TE&fqc=or&date_pub_start=2016&date_pub_end=2016", ) for name, reader := range map[string][]*http.Cookie{"FOIA left out": nil, "FOIA included": {foiaOn}} { t.Run(name, func(t *testing.T) { bars, refines := 0, 0 for _, p := range pages { bars += followLinks(t, p, reBarLink, "bar", reader...) refines += followLinks(t, p, reFacetLink, "refine link", reader...) } if bars == 0 || refines == 0 { t.Errorf("nothing exercised: %d bars, %d refine links", bars, refines) } boxes := checkCategoryBoxes(t, reader...) t.Logf("%d pages: %d bars, %d refine links and %d category boxes checked", len(pages), bars, refines, boxes) }) } } // corpusFOIALinks opens the page of every tenth FOIA release as a reader // who leaves FOIA out, and follows each category value it links: the page // behind the link has to list the release. func corpusFOIALinks(t *testing.T) { reValueLink := regexp.MustCompile(`href="(/browse/[a-z]+/[^"]+)"`) links := 0 for i, d := range corpus().collectionDocs[collectionFOIA] { if i%10 != 0 { continue } // The release's own metadata: the cards under it are other documents. page, _, _ := strings.Cut(get(t, docPath(d.ID)).Body.String(), `class="related-section"`) for _, m := range reValueLink.FindAllStringSubmatch(page, -1) { link := html.UnescapeString(m[1]) links++ if !strings.Contains(get(t, withQuery(link, "view=table")).Body.String(), `href="`+docPath(d.ID)+`"`) { t.Errorf("%s links %s, which does not list it", d.ID, link) } } } if links == 0 { t.Error("no FOIA release with a value link to follow") } t.Logf("%d value links followed", links) } // A word in -ies or -ied stems as the word in -y it is a form of, whatever // rules that word goes through - checked over every such pair the corpus // holds, since a suffix rule that reads one of the two forms and not the // other splits them silently ("capability" from "capabilities"). func corpusInflections(t *testing.T) { vocab := map[string]bool{} for i := range corpus().archive.Docs { for _, w := range splitWords(corpus().archive.Docs[i].SearchText) { vocab[w] = true } } pairs := 0 for w := range vocab { if len(w) < 5 || !(strings.HasSuffix(w, "ies") || strings.HasSuffix(w, "ied")) { continue } base := w[:len(w)-3] + "y" if !vocab[base] { continue } pairs++ if stem(w) != stem(base) { t.Errorf("stem(%q) = %q, stem(%q) = %q; want equal", w, stem(w), base, stem(base)) } } if pairs == 0 { t.Error("the corpus holds no -ies/-ied word next to its -y form") } t.Logf("%d pairs checked in a vocabulary of %d words", pairs, len(vocab)) } // corpusSnippets holds every document's snippets off the fold kept at load // to the ones folding its transcript again gives - the versions' lines sit // behind that fold in the same string - and a page of "all" cards to its // bound. func corpusSnippets(t *testing.T) { for _, q := range []string{"nsa", "cable tapping", `"upstream collection"`, "tap*", "the"} { for i := range corpus().archive.Docs { d := &corpus().archive.Docs[i] if got, want := d.bodySnippets(q, 3), getSnippets(d.FullText, q, 3); !reflect.DeepEqual(got, want) { t.Fatalf("%s, %q: %q from the loaded fold, %q from the text", d.ID, q, got, want) } } } all := &http.Cookie{Name: "pref_per_page", Value: "all"} for _, c := range categories { for _, v := range rankCounts(c.Facet(corpus()), 1) { if c.Facet(corpus())[v] <= maxCardsPerPage { continue } body := get(t, categoryValueURL(c.Slug, v), all, foiaOn).Body.String() if n := strings.Count(body, `class="doc-card"`); n != maxCardsPerPage { t.Errorf("%s=%s: %d cards on a page of all, want %d", c.Slug, v, n, maxCardsPerPage) } } } } // corpusIndex holds the shortcut a one-word all-fields clause takes - it // asks the document's entry in the index instead of walking its text - to // the walk, for a spread of the vocabulary over every document. func corpusIndex(t *testing.T) { var vocab []string for w := range corpus().tfidf.DocFreq { vocab = append(vocab, w) } slices.Sort(vocab) checked := 0 for i := 0; i < len(vocab); i += 131 { w := vocab[i] docs := 0 for di := range corpus().archive.Docs { d := &corpus().archive.Docs[di] _, indexed := d.terms[w] if indexed != containsTokens(d.StemmedText, w) { t.Fatalf("%q in %s: the index says %v, the all-fields stream the opposite", w, d.ID, indexed) } if indexed { docs++ } } if docs != corpus().tfidf.DocFreq[w] { t.Errorf("%q: in %d documents' entries, with a document frequency of %d", w, docs, corpus().tfidf.DocFreq[w]) } checked++ } t.Logf("%d terms checked over %d documents", checked, len(corpus().archive.Docs)) }