first commit
This commit is contained in:
commit
8c4dcae475
89 changed files with 24623 additions and 0 deletions
199
tests/corpus_test.go
Normal file
199
tests/corpus_test.go
Normal file
|
|
@ -0,0 +1,199 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"html"
|
||||
"net/http"
|
||||
"os"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Checks against a REAL corpus. Point $SNOWCHIVE_BENCH_DATA at one, as for
|
||||
// the page benchmarks, and run: go test -run TestCorpus -v
|
||||
//
|
||||
// The fixtures are a handful of documents, each built to show one thing. A
|
||||
// count and its link are computed by different code, and whether the two
|
||||
// agree turns on what the documents happen to hold: a second publication in
|
||||
// a later year, a byline on only one of them, a value found only in FOIA
|
||||
// releases. The disagreements these tests walk the archive for were all
|
||||
// found on the full corpus and on no fixture.
|
||||
func TestCorpus(t *testing.T) {
|
||||
root := os.Getenv("SNOWCHIVE_BENCH_DATA")
|
||||
if root == "" {
|
||||
t.Skip("set SNOWCHIVE_BENCH_DATA to a corpus directory to check a real corpus")
|
||||
}
|
||||
withCorpus(t, root)
|
||||
t.Run("counts", corpusCounts)
|
||||
t.Run("FOIA links", corpusFOIALinks)
|
||||
t.Run("inflections", corpusInflections)
|
||||
t.Run("snippets", corpusSnippets)
|
||||
t.Run("index", corpusIndex)
|
||||
}
|
||||
|
||||
// corpusCounts follows the counted links of the loaded corpus, as a reader
|
||||
// who leaves FOIA releases out and as one who includes them.
|
||||
func corpusCounts(t *testing.T) {
|
||||
// The pages whose bars and refine links are followed: the year pages of
|
||||
// both date categories, the three commonest values of every other
|
||||
// category, each cell of the outlets table (a publisher's page for one
|
||||
// publication year) and searches under each kind of date range.
|
||||
var pages []string
|
||||
for ci := range categories {
|
||||
c := &categories[ci]
|
||||
n := 3
|
||||
if c.isDate() {
|
||||
n = -1
|
||||
}
|
||||
for _, v := range rankCounts(c.Facet(corpus()), n) {
|
||||
pages = append(pages, categoryValueURL(c.Slug, v))
|
||||
}
|
||||
}
|
||||
for _, row := range corpus().outletTables[""].Rows {
|
||||
for _, cell := range row.Cells {
|
||||
if cell.URL != "" {
|
||||
pages = append(pages, cell.URL)
|
||||
}
|
||||
}
|
||||
}
|
||||
pages = append(pages,
|
||||
"/search?q=nsa",
|
||||
"/search?q=nsa+date%3A2012-06..2013-06",
|
||||
"/search?q=nsa+published%3A2016",
|
||||
"/search?q=published%3A2014",
|
||||
"/search?do_search=1&fqv=nsa&fqf=TE&fqc=and&date_pub_start=2015-01-01&date_pub_end=2016-06-30",
|
||||
"/search?do_search=1&fqv=nsa&fqf=TE&fqc=and&fqv=gchq&fqf=TE&fqc=or&date_pub_start=2016&date_pub_end=2016",
|
||||
)
|
||||
|
||||
for name, reader := range map[string][]*http.Cookie{"FOIA left out": nil, "FOIA included": {foiaOn}} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
bars, refines := 0, 0
|
||||
for _, p := range pages {
|
||||
bars += followLinks(t, p, reBarLink, "bar", reader...)
|
||||
refines += followLinks(t, p, reFacetLink, "refine link", reader...)
|
||||
}
|
||||
if bars == 0 || refines == 0 {
|
||||
t.Errorf("nothing exercised: %d bars, %d refine links", bars, refines)
|
||||
}
|
||||
boxes := checkCategoryBoxes(t, reader...)
|
||||
t.Logf("%d pages: %d bars, %d refine links and %d category boxes checked", len(pages), bars, refines, boxes)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// corpusFOIALinks opens the page of every tenth FOIA release as a reader
|
||||
// who leaves FOIA out, and follows each category value it links: the page
|
||||
// behind the link has to list the release.
|
||||
func corpusFOIALinks(t *testing.T) {
|
||||
reValueLink := regexp.MustCompile(`href="(/browse/[a-z]+/[^"]+)"`)
|
||||
links := 0
|
||||
for i, d := range corpus().collectionDocs[collectionFOIA] {
|
||||
if i%10 != 0 {
|
||||
continue
|
||||
}
|
||||
// The release's own metadata: the cards under it are other documents.
|
||||
page, _, _ := strings.Cut(get(t, docPath(d.ID)).Body.String(), `class="related-section"`)
|
||||
for _, m := range reValueLink.FindAllStringSubmatch(page, -1) {
|
||||
link := html.UnescapeString(m[1])
|
||||
links++
|
||||
if !strings.Contains(get(t, withQuery(link, "view=table")).Body.String(), `href="`+docPath(d.ID)+`"`) {
|
||||
t.Errorf("%s links %s, which does not list it", d.ID, link)
|
||||
}
|
||||
}
|
||||
}
|
||||
if links == 0 {
|
||||
t.Error("no FOIA release with a value link to follow")
|
||||
}
|
||||
t.Logf("%d value links followed", links)
|
||||
}
|
||||
|
||||
// A word in -ies or -ied stems as the word in -y it is a form of, whatever
|
||||
// rules that word goes through - checked over every such pair the corpus
|
||||
// holds, since a suffix rule that reads one of the two forms and not the
|
||||
// other splits them silently ("capability" from "capabilities").
|
||||
func corpusInflections(t *testing.T) {
|
||||
vocab := map[string]bool{}
|
||||
for i := range corpus().archive.Docs {
|
||||
for _, w := range splitWords(corpus().archive.Docs[i].SearchText) {
|
||||
vocab[w] = true
|
||||
}
|
||||
}
|
||||
pairs := 0
|
||||
for w := range vocab {
|
||||
if len(w) < 5 || !(strings.HasSuffix(w, "ies") || strings.HasSuffix(w, "ied")) {
|
||||
continue
|
||||
}
|
||||
base := w[:len(w)-3] + "y"
|
||||
if !vocab[base] {
|
||||
continue
|
||||
}
|
||||
pairs++
|
||||
if stem(w) != stem(base) {
|
||||
t.Errorf("stem(%q) = %q, stem(%q) = %q; want equal", w, stem(w), base, stem(base))
|
||||
}
|
||||
}
|
||||
if pairs == 0 {
|
||||
t.Error("the corpus holds no -ies/-ied word next to its -y form")
|
||||
}
|
||||
t.Logf("%d pairs checked in a vocabulary of %d words", pairs, len(vocab))
|
||||
}
|
||||
|
||||
// corpusSnippets holds every document's snippets off the fold kept at load
|
||||
// to the ones folding its transcript again gives - the versions' lines sit
|
||||
// behind that fold in the same string - and a page of "all" cards to its
|
||||
// bound.
|
||||
func corpusSnippets(t *testing.T) {
|
||||
for _, q := range []string{"nsa", "cable tapping", `"upstream collection"`, "tap*", "the"} {
|
||||
for i := range corpus().archive.Docs {
|
||||
d := &corpus().archive.Docs[i]
|
||||
if got, want := d.bodySnippets(q, 3), getSnippets(d.FullText, q, 3); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("%s, %q: %q from the loaded fold, %q from the text", d.ID, q, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
all := &http.Cookie{Name: "pref_per_page", Value: "all"}
|
||||
for _, c := range categories {
|
||||
for _, v := range rankCounts(c.Facet(corpus()), 1) {
|
||||
if c.Facet(corpus())[v] <= maxCardsPerPage {
|
||||
continue
|
||||
}
|
||||
body := get(t, categoryValueURL(c.Slug, v), all, foiaOn).Body.String()
|
||||
if n := strings.Count(body, `class="doc-card"`); n != maxCardsPerPage {
|
||||
t.Errorf("%s=%s: %d cards on a page of all, want %d", c.Slug, v, n, maxCardsPerPage)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// corpusIndex holds the shortcut a one-word all-fields clause takes - it
|
||||
// asks the document's entry in the index instead of walking its text - to
|
||||
// the walk, for a spread of the vocabulary over every document.
|
||||
func corpusIndex(t *testing.T) {
|
||||
var vocab []string
|
||||
for w := range corpus().tfidf.DocFreq {
|
||||
vocab = append(vocab, w)
|
||||
}
|
||||
slices.Sort(vocab)
|
||||
checked := 0
|
||||
for i := 0; i < len(vocab); i += 131 {
|
||||
w := vocab[i]
|
||||
docs := 0
|
||||
for di := range corpus().archive.Docs {
|
||||
d := &corpus().archive.Docs[di]
|
||||
_, indexed := d.terms[w]
|
||||
if indexed != containsTokens(d.StemmedText, w) {
|
||||
t.Fatalf("%q in %s: the index says %v, the all-fields stream the opposite", w, d.ID, indexed)
|
||||
}
|
||||
if indexed {
|
||||
docs++
|
||||
}
|
||||
}
|
||||
if docs != corpus().tfidf.DocFreq[w] {
|
||||
t.Errorf("%q: in %d documents' entries, with a document frequency of %d", w, docs, corpus().tfidf.DocFreq[w])
|
||||
}
|
||||
checked++
|
||||
}
|
||||
t.Logf("%d terms checked over %d documents", checked, len(corpus().archive.Docs))
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue