538 lines
23 KiB
Go
538 lines
23 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"html"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/url"
|
|
"path/filepath"
|
|
"reflect"
|
|
"regexp"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// A corpus built to exercise the places where one fact used to be computed
|
|
// twice: search rows and their negations, the page and the API, counts and
|
|
// the pages they link to. Each document's words are chosen so the expected
|
|
// result sets can be written down by hand.
|
|
//
|
|
// n1 news alpha gamma ZEBRA 2010 The Intercept 2014 + the intercept 2016
|
|
// n2 news alpha ZEBRA 2011 The Intercept 2014 + Der Spiegel 2014
|
|
// n3 news beta gamma 2012 Der Spiegel 2015
|
|
// n4 news beta 2012 Der Spiegel 2015
|
|
// f1 foia alpha ZEBRA 2010 (court filing: "(c)" list labels)
|
|
// s1 sidtoday ZEBRA 2011
|
|
// a1 news "aka_id": null
|
|
// d1 news date_created "ODNI", date_published "15 May 2015"
|
|
func writeConsistencyCorpus(t *testing.T) string {
|
|
t.Helper()
|
|
root := t.TempDir()
|
|
doc := func(dir, id string, meta map[string]interface{}, text string) {
|
|
b, err := json.Marshal(meta)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
d := filepath.Join(root, dir, id)
|
|
writeFixtureFile(t, filepath.Join(d, "metadata.json"), string(b))
|
|
writeFixtureFile(t, filepath.Join(d, id+".pdf"), "%PDF-1.4\n")
|
|
if text != "" {
|
|
writeFixtureFile(t, filepath.Join(d, id+".md"), text)
|
|
}
|
|
}
|
|
pub := func(publisher, date, link string, reporters ...string) map[string]interface{} {
|
|
return map[string]interface{}{"publisher": publisher, "date_published": date,
|
|
"link": link, "reporters": reporters}
|
|
}
|
|
meta := func(title, typ, created string, programs []string, pubs ...map[string]interface{}) map[string]interface{} {
|
|
return map[string]interface{}{"title": title, "type": typ,
|
|
"document": map[string]interface{}{"creator": "NSA", "date_created": created, "programs": programs},
|
|
"publication": pubs}
|
|
}
|
|
zebra := []string{"ZEBRA"}
|
|
doc("", "n1", meta("Report one", "REPORTED", "2010-03-01", zebra,
|
|
pub("The Intercept", "2014-01-01", "https://example.org/a", "Glenn Greenwald"),
|
|
pub("the intercept", "2016-01-01", "https://example.org/b", "glenn greenwald")),
|
|
"alpha gamma\n\n(S//SI) a secret line\n(S/SI) the same marking, spelled by OCR\n(U) plain\n")
|
|
doc("", "n2", meta("Report two", "REPORTED", "2011-03-01", zebra,
|
|
pub("The Intercept", "2014-02-01", "https://example.org/c"),
|
|
pub("Der Spiegel", "2014-02-02", "https://example.org/f")),
|
|
"alpha\n\n(TS//SI//NF) top line\n")
|
|
doc("", "n3", meta("Report three", "REPORTED", "2012-03-01", nil,
|
|
pub("Der Spiegel", "2015-01-01", "https://example.org/d")), "beta gamma\n")
|
|
doc("", "n4", meta("Report four", "REPORTED", "2012-04-01", nil,
|
|
pub("Der Spiegel", "2015-01-01", "https://example.org/d")), "beta\n")
|
|
doc("FOIA", "f1", meta("Court filing", "FOIA", "2010-05-01", zebra),
|
|
"alpha\n\n(c) The Court shall issue an order.\n(s) another subsection\n")
|
|
doc("SIDtoday-archive", "s1", meta("Issue", "SIDTODAY", "2011-05-01", zebra), "(S) issue line\n")
|
|
a1 := meta("Null aka", "REPORTED", "", nil)
|
|
a1["aka_id"] = nil
|
|
doc("", "a1", a1, "")
|
|
// A date of no shape the archive reads: served, undated.
|
|
doc("", "d1", meta("Odd date", "REPORTED", "ODNI", nil, pub("Der Spiegel", "15 May 2015", "https://example.org/e")), "")
|
|
writeFixtureFile(t, filepath.Join(root, "glossary.json"), `[]`)
|
|
return root
|
|
}
|
|
|
|
// matchingIDs runs the search engine and returns the matching ids, sorted.
|
|
func matchingIDs(queries, fields, conjs []string) []string {
|
|
q, f, c := normalizeClauses(queries, fields, conjs)
|
|
var ids []string
|
|
scored, _ := corpus().executeSearch(searchParams{Queries: q, Fields: f, Conjs: c})
|
|
for _, s := range scored {
|
|
ids = append(ids, s.Doc.ID)
|
|
}
|
|
sort.Strings(ids)
|
|
return ids
|
|
}
|
|
|
|
// A row is one unit: its -terms are exclusions within it, whatever joins
|
|
// it to the rows before. And an empty row starts nothing - a blank first
|
|
// row with "OR x" under it is a search for x.
|
|
func TestSearchRowsAndNegations(t *testing.T) {
|
|
withCorpus(t, writeConsistencyCorpus(t))
|
|
te2 := []string{"TE", "TE"}
|
|
for _, c := range []struct {
|
|
name string
|
|
queries, conjs []string
|
|
want []string
|
|
}{
|
|
{"inline negation", []string{"alpha -gamma"}, []string{"and"}, []string{"f1", "n2"}},
|
|
// alpha AND NOT (beta AND NOT gamma): no alpha document is lost
|
|
// for mentioning gamma.
|
|
{"negation inside NOT", []string{"alpha", "beta -gamma"}, []string{"and", "not"}, []string{"f1", "n1", "n2"}},
|
|
// alpha OR (beta AND NOT gamma): the exclusion stays inside its row.
|
|
{"negation inside OR", []string{"alpha", "beta -gamma"}, []string{"and", "or"}, []string{"f1", "n1", "n2", "n4"}},
|
|
{"blank first row, OR", []string{"", "alpha"}, []string{"and", "or"}, []string{"f1", "n1", "n2"}},
|
|
{"spaces first row, OR", []string{" ", "alpha"}, []string{"and", "or"}, []string{"f1", "n1", "n2"}},
|
|
{"blank first row, OR nothing", []string{"", "zzzzunmatched"}, []string{"and", "or"}, nil},
|
|
{"blank first row, NOT", []string{"", "gamma"}, []string{"and", "not"}, []string{"a1", "d1", "f1", "n2", "n4", "s1"}},
|
|
{"only exclusions", []string{"-gamma -alpha"}, []string{"and"}, []string{"a1", "d1", "n4", "s1"}},
|
|
} {
|
|
if got := matchingIDs(c.queries, te2[:len(c.queries)], c.conjs); !reflect.DeepEqual(got, c.want) {
|
|
t.Errorf("%s: %q %v matched %v, want %v", c.name, c.queries, c.conjs, got, c.want)
|
|
}
|
|
}
|
|
// The scorer and the highlighter read the same rows: an excluded term
|
|
// is neither rewarded nor marked.
|
|
if got := positiveTerms([]string{"alpha -gamma", "", "beta"}, []string{"and", "and", "not"}); !reflect.DeepEqual(got, []string{"alpha"}) {
|
|
t.Errorf("positiveTerms = %q, want [alpha]", got)
|
|
}
|
|
if term, _ := snippetAnchor([]string{"-gamma", "alpha -beta"}, te2, []string{"and", "and"}); term != "alpha" {
|
|
t.Errorf("snippetAnchor = %q, want alpha", term)
|
|
}
|
|
}
|
|
|
|
// "+ Add row" and "remove" re-run the search: on the same rows, meaning the
|
|
// same thing, as the Search button would.
|
|
func TestRowEditsSearchTheSameRows(t *testing.T) {
|
|
withCorpus(t, writeConsistencyCorpus(t))
|
|
rows := "do_search=1&fqv=alpha+-gamma&fqf=TE&fqc=and"
|
|
want := resultTotal(t, "/search?action=search&"+rows, foiaOn)
|
|
if want != 2 {
|
|
t.Fatalf("search for alpha -gamma found %d, want 2", want)
|
|
}
|
|
for _, action := range []string{"add", "remove-1"} {
|
|
path := "/search?action=" + action + "&" + rows + "&fqv=beta&fqf=TE&fqc=or"
|
|
if action == "add" {
|
|
path = "/search?action=add&" + rows
|
|
}
|
|
if got := resultTotal(t, path, foiaOn); got != want {
|
|
t.Errorf("action=%s: %d results, the same rows searched find %d", action, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The type filter resubmits the search the page shows, not the request: a
|
|
// removed row stays removed, and the chosen sort is kept.
|
|
func TestTypeFilterFormCarriesTheEffectiveSearch(t *testing.T) {
|
|
withCorpus(t, writeConsistencyCorpus(t))
|
|
body := get(t, "/search?do_search=1&action=remove-1&sort=title"+
|
|
"&fqv=alpha&fqf=TE&fqc=and&fqv=removedrow&fqf=TE&fqc=or", foiaOn).Body.String()
|
|
if !strings.Contains(body, `class="type-filter"`) {
|
|
t.Fatal("no type filter on a result set holding two types")
|
|
}
|
|
if strings.Contains(body, "removedrow") {
|
|
t.Error("the removed row is still in the page - the type filter would bring it back")
|
|
}
|
|
if !strings.Contains(body, `name="sort" value="title"`) {
|
|
t.Error("the type filter drops the chosen sort")
|
|
}
|
|
}
|
|
|
|
// /api/search runs the search page's pipeline, type filter included. The
|
|
// API has no preference: it matches the page of a reader who includes FOIA
|
|
// releases.
|
|
func TestAPISearchMatchesThePage(t *testing.T) {
|
|
withCorpus(t, writeConsistencyCorpus(t))
|
|
for qs, want := range map[string]int{
|
|
"q=alpha": 3,
|
|
"q=alpha&type=REPORTED": 2,
|
|
"q=alpha&type=FOIA&type=REPORTED": 3,
|
|
"q=alpha&type=reported": 2,
|
|
"q=zebra&coll=news": 2,
|
|
"q=zebra&coll=foia&type=FOIA": 1,
|
|
"do_search=1&fqv=alpha&fqf=TE&fqc=and&fqv=beta+-gamma&fqf=TE&fqc=or": 4,
|
|
} {
|
|
page := resultTotal(t, "/search?"+qs, foiaOn)
|
|
var api struct{ Total int }
|
|
if err := json.Unmarshal(get(t, "/api/search?"+qs).Body.Bytes(), &api); err != nil {
|
|
t.Fatalf("%s: %v", qs, err)
|
|
}
|
|
if page != want || api.Total != want {
|
|
t.Errorf("%s: the page finds %d, the API %d, want %d", qs, page, api.Total, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Every count in this corpus is the number its page or search returns.
|
|
func TestConsistencyCorpusCounts(t *testing.T) {
|
|
withCorpus(t, writeConsistencyCorpus(t))
|
|
|
|
// Facets, with journalist and publisher spellings that differ by case
|
|
// across one document's publications.
|
|
for ci := range categories {
|
|
c := &categories[ci]
|
|
for val, want := range c.Facet(corpus()) {
|
|
if got := len(corpus().categoryMatches(c.Slug, val)); got != want {
|
|
t.Errorf("%s=%q: facet says %d, the page lists %d", c.Slug, val, want, got)
|
|
}
|
|
}
|
|
}
|
|
if js := corpus().archive.Facets["journalist"]; len(js) != 1 || js["Glenn Greenwald"] != 1 {
|
|
t.Errorf("journalists = %v, want Glenn Greenwald once", js)
|
|
}
|
|
if n := corpus().archive.Facets["publisher"]["The Intercept"]; n != 2 {
|
|
t.Errorf("The Intercept: %d documents, want 2 (%v)", n, corpus().archive.Facets["publisher"])
|
|
}
|
|
|
|
// The value-page timelines and refine links, scoped to a collection:
|
|
// f1 (FOIA) shares 2010 with n1 and s1 (SIDtoday) shares 2011 with n2,
|
|
// so an unscoped bar link finds more than its bar counts.
|
|
for _, p := range []string{"/browse/program/ZEBRA?coll=news", "/browse/program/ZEBRA"} {
|
|
if n := followLinks(t, p, reBarLink, "timeline bar"); n < 2 {
|
|
t.Errorf("%s: %d timeline bars checked, want the doc and pub timelines", p, n)
|
|
}
|
|
}
|
|
|
|
// /explore/outlets: rows, cells and the per-year footer.
|
|
for coll, table := range corpus().outletTables {
|
|
for _, row := range table.Rows {
|
|
if got := resultTotal(t, row.URL); got != row.Total {
|
|
t.Errorf("outlets[%q] %s: total %d, its page lists %d", coll, row.Name, row.Total, got)
|
|
}
|
|
for _, cell := range row.Cells {
|
|
if cell.URL != "" && resultTotal(t, cell.URL) != cell.Docs {
|
|
t.Errorf("outlets[%q] %s: a cell says %d, its search finds %d", coll, row.Name, cell.Docs, resultTotal(t, cell.URL))
|
|
}
|
|
}
|
|
}
|
|
// The footer adds up its column: a document several outlets
|
|
// published counts once for each of them.
|
|
for i, y := range table.Years {
|
|
sum := 0
|
|
for _, row := range table.Rows {
|
|
sum += row.Cells[i].Docs
|
|
}
|
|
if table.Totals[i] != sum {
|
|
t.Errorf("outlets[%q] %s: footer %d, its column adds up to %d", coll, y, table.Totals[i], sum)
|
|
}
|
|
}
|
|
}
|
|
cell := func(outlet, year string) (docs, footer int) {
|
|
table := corpus().outletTables[""]
|
|
for i, y := range table.Years {
|
|
if y != year {
|
|
continue
|
|
}
|
|
footer = table.Totals[i]
|
|
for _, row := range table.Rows {
|
|
if row.Name == outlet {
|
|
docs = row.Cells[i].Docs
|
|
}
|
|
}
|
|
}
|
|
return docs, footer
|
|
}
|
|
// n2: The Intercept and Der Spiegel both published it in 2014, so it
|
|
// counts for both outlets and twice in the year's footer (n1 is the
|
|
// third: The Intercept, 2014).
|
|
if spiegel, footer := cell("Der Spiegel", "2014"); spiegel != 1 || footer != 3 {
|
|
t.Errorf("2014: Der Spiegel cell %d, footer %d - want 1 and 3 (n2 once per outlet)", spiegel, footer)
|
|
}
|
|
if intercept, _ := cell("The Intercept", "2014"); intercept != 2 {
|
|
t.Errorf("2014: The Intercept cell %d, want 2", intercept)
|
|
}
|
|
intercept := 0
|
|
for _, row := range corpus().outletTables[""].Rows {
|
|
if row.Name == "The Intercept" {
|
|
intercept = row.Total
|
|
}
|
|
}
|
|
if intercept != 2 {
|
|
t.Errorf("The Intercept's outlet total = %d, want 2 documents (n1 counts once for its two years)", intercept)
|
|
}
|
|
|
|
// /explore/portions: lower-case list labels are not markings, and every
|
|
// row's document count is what its search finds.
|
|
for _, l := range corpus().portionStats.Levels {
|
|
if l.Name == "C" {
|
|
t.Errorf("a CONFIDENTIAL row from the court filing's (c) labels: %+v", l)
|
|
}
|
|
}
|
|
rows := append(append(append([]portionRow(nil), corpus().portionStats.Levels...), corpus().portionStats.Markings...), corpus().portionStats.Controls...)
|
|
if len(rows) < 6 {
|
|
t.Fatalf("only %d portion rows", len(rows))
|
|
}
|
|
for _, r := range rows {
|
|
if got := resultTotal(t, string(r.SearchURL)); got != r.Docs {
|
|
t.Errorf("portion row %s: %d documents, its search finds %d (%s)", r.Name, r.Docs, got, r.SearchURL)
|
|
}
|
|
}
|
|
if got := corpus().docIndex["n1"].PortionMarkings; !reflect.DeepEqual(got, []string{"S//SI", "U"}) {
|
|
t.Errorf("n1 markings = %v, want [S//SI U]", got)
|
|
}
|
|
|
|
// A malformed date is no date: no "ODNI" year that no year page opens.
|
|
if d := corpus().docIndex["d1"]; d == nil || d.DocMeta.DateCreated != "" || d.Publication[0].DatePublished != "" {
|
|
t.Errorf("d1's malformed dates were kept: %+v", d)
|
|
}
|
|
if _, ok := corpus().archive.Facets["docdate"]["ODNI"]; ok {
|
|
t.Error(`"ODNI" counted as a document year`)
|
|
}
|
|
|
|
// aka_id: null is no akas, not a reason to drop the document.
|
|
if corpus().docIndex["a1"] == nil {
|
|
t.Error(`a document with "aka_id": null was not loaded`)
|
|
}
|
|
}
|
|
|
|
// A corpus for the counts that turn on WHICH publication entry carries a
|
|
// date or a byline, and on the reader's collection scope. Every document
|
|
// says "alpha".
|
|
//
|
|
// p1 news 2010-03 Outlet X 2013 (Able), Outlet Y 2014 (Baker)
|
|
// p2 news 2010-06 Outlet X 2014 (Able) people: Clerk
|
|
// p3 news 2011-01 Outlet X 2014 (Baker)
|
|
// p4 news 2012-03 Outlet Y 2014
|
|
// p5 news 2012-09 Outlet Y 2014
|
|
// p6 news 2013-02 Outlet Y 2014, Partner Z 2014 (its own story; reporting only)
|
|
// f1 foia 2009-07 Outlet Y 2014 (a story of its own) people: Docket
|
|
func writeEntryCorpus(t *testing.T) string {
|
|
t.Helper()
|
|
root := t.TempDir()
|
|
pub := func(publisher, date string, reporters ...string) map[string]interface{} {
|
|
return map[string]interface{}{"publisher": publisher, "date_published": date, "reporters": reporters}
|
|
}
|
|
doc := func(dir, id, typ, created string, people []string, pubs ...map[string]interface{}) {
|
|
b, err := json.Marshal(map[string]interface{}{"title": "Document " + id, "type": typ,
|
|
"document": map[string]interface{}{"creator": "NSA", "date_created": created, "people": people},
|
|
"publication": pubs})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
d := filepath.Join(root, dir, id)
|
|
writeFixtureFile(t, filepath.Join(d, "metadata.json"), string(b))
|
|
writeFixtureFile(t, filepath.Join(d, id+".pdf"), "%PDF-1.4\n")
|
|
writeFixtureFile(t, filepath.Join(d, id+".md"), "alpha\n")
|
|
}
|
|
doc("", "p1", "REPORTED", "2010-03-01", nil,
|
|
pub("Outlet X", "2013-05-01", "Able, Ann"), pub("Outlet Y", "2014-02-01", "Baker, Bob"))
|
|
doc("", "p2", "REPORTED", "2010-06-01", []string{"Clerk, Carol"}, pub("Outlet X", "2014-03-01", "Able, Ann"))
|
|
doc("", "p3", "REPORTED", "2011-01-01", nil, pub("Outlet X", "2014-04-01", "Baker, Bob"))
|
|
doc("", "p4", "REPORTED", "2012-03-01", nil, pub("Outlet Y", "2014-05-01"))
|
|
doc("", "p5", "REPORTED", "2012-09-01", nil, pub("Outlet Y", "2014-06-01"))
|
|
story := func(p map[string]interface{}, title, link string) map[string]interface{} {
|
|
p["title"], p["link"] = title, link
|
|
return p
|
|
}
|
|
partner := story(pub("Partner Z", "2014-07-02"), "Partner story", "https://example.org/partner")
|
|
partner["reporting_only"] = true
|
|
doc("", "p6", "REPORTED", "2013-02-01", nil, pub("Outlet Y", "2014-07-01"), partner)
|
|
doc("FOIA", "f1", "FOIA", "2009-07-01", []string{"Docket, Dan"},
|
|
story(pub("Outlet Y", "2014-08-01"), "FOIA story", "https://example.org/foia-story"))
|
|
writeFixtureFile(t, filepath.Join(root, "glossary.json"), `[]`)
|
|
return root
|
|
}
|
|
|
|
// Counts that turn on a publication entry's date or byline, or on the
|
|
// reader's scope, are still the numbers their links return.
|
|
func TestCountsTiedToEntriesAndScope(t *testing.T) {
|
|
withCorpus(t, writeEntryCorpus(t))
|
|
|
|
// A publication-year page files a document under its first publication,
|
|
// which no search asks for: p1, first published in 2013, would join the
|
|
// 2010 bar's search for 2014. So those bars carry no link. The
|
|
// document-year page's publication bars do, and they are exact.
|
|
body := get(t, "/browse/published/2014").Body.String()
|
|
if !strings.Contains(body, "By document date") || strings.Contains(body, `<a class="hist-col"`) {
|
|
t.Error("/browse/published/2014: want the document-date bars, and no link on them")
|
|
}
|
|
if n := followLinks(t, "/browse/docdate/2010", reBarLink, "publication bar"); n != 2 {
|
|
t.Errorf("/browse/docdate/2010: %d publication bars checked, want 2013 and 2014", n)
|
|
}
|
|
|
|
// Under a publication year a journalist counts for the documents that
|
|
// byline them on an entry of that year, because that is what the refine
|
|
// link's search matches: Able's byline on p1 is Outlet X's of 2013, not
|
|
// Outlet Y's of 2014.
|
|
for _, p := range []string{
|
|
"/browse/publisher/Outlet%20Y?year=2014",
|
|
"/search?q=alpha+published%3A2014",
|
|
// An OR row unties the dates, and the bylines with them.
|
|
"/search?do_search=1&fqv=alpha&fqf=TE&fqc=and&fqv=beta&fqf=TE&fqc=or&date_pub_start=2014&date_pub_end=2014",
|
|
} {
|
|
if n := followLinks(t, p, reFacetLink, "refine link"); n == 0 {
|
|
t.Errorf("%s: no refine links to check", p)
|
|
}
|
|
}
|
|
|
|
// A year bar narrows the search it stands under: in a range that starts
|
|
// in June 2012, the 2012 bar counts p5 and not p4 of March.
|
|
if n := followLinks(t, "/search?q=alpha+date%3A2012-06..2013-06", reBarLink, "year bar"); n != 2 {
|
|
t.Errorf("%d year bars checked, want 2012 and 2013", n)
|
|
}
|
|
|
|
// Docket is named only in the FOIA release and 2009 is only its year:
|
|
// neither is on the default reader's lists, nor in the boxes' counts.
|
|
for _, reader := range [][]*http.Cookie{nil, {foiaOn}} {
|
|
if n := checkCategoryBoxes(t, reader...); n < 10 {
|
|
t.Errorf("%d category boxes checked", n)
|
|
}
|
|
}
|
|
|
|
// The home page's publication chart counts first publications, as the
|
|
// year pages its bars link file them: p1 counts for 2013 and not 2014.
|
|
body = get(t, "/").Body.String()
|
|
for year, want := range map[string]string{"2013": "1", "2014": "5"} {
|
|
if !strings.Contains(body, `href="/browse/published/`+year+`?coll=news" title="`+want+` documents first published in `+year+`"`) {
|
|
t.Errorf("home: no bar of %s documents first published in %s linking to that year's page", want, year)
|
|
}
|
|
}
|
|
|
|
// In a search result a publication year links to its year page only
|
|
// when that page lists the document: p1 is filed under 2013, its first.
|
|
body = get(t, "/search?q=title%3Ap1").Body.String()
|
|
if !strings.Contains(body, `href="/browse/published/2013"`) || strings.Contains(body, `href="/browse/published/2014"`) {
|
|
t.Error("p1's result should link 2013, its first publication year, and not 2014")
|
|
}
|
|
|
|
// A partner that only reported has no publisher page for its story's
|
|
// byline to link.
|
|
body = get(t, "/article/"+articleKeyFor("https://example.org/partner")).Body.String()
|
|
if !strings.Contains(body, "Partner Z") || strings.Contains(body, "/browse/publisher/Partner") {
|
|
t.Error("the partner's article should name Partner Z without linking a publisher page")
|
|
}
|
|
|
|
// The outlets table of one collection counts the stories that have a
|
|
// document in it: Outlet Y's 2014 story published only the FOIA release.
|
|
stories := func(coll string) int {
|
|
table := corpus().outletTables[coll]
|
|
for _, row := range table.Rows {
|
|
if row.Name == "Outlet Y" {
|
|
return row.Cells[slices.Index(table.Years, "2014")].Articles
|
|
}
|
|
}
|
|
return -1
|
|
}
|
|
if all, news := stories(""), stories(collectionNews); all != 1 || news != 0 {
|
|
t.Errorf("Outlet Y's 2014 stories: %d over every collection and %d reported, want 1 and 0", all, news)
|
|
}
|
|
}
|
|
|
|
var reCategoryBox = regexp.MustCompile(`<a href="/browse/([a-z]+)" class="cat-box[^"]*"><strong>[^<]*</strong><span class="count">(\d+)</span></a>`)
|
|
|
|
// checkCategoryBoxes holds the number on each category box in the browse
|
|
// nav to the rows of the list its click opens, for the reader with these
|
|
// cookies.
|
|
func checkCategoryBoxes(t *testing.T, cookies ...*http.Cookie) int {
|
|
t.Helper()
|
|
boxes := reCategoryBox.FindAllStringSubmatch(get(t, "/browse", cookies...).Body.String(), -1)
|
|
for _, m := range boxes {
|
|
slug, shown := m[1], mustAtoi(t, m[2])
|
|
list := get(t, "/browse/"+slug, cookies...).Body.String()
|
|
if rows := strings.Count(list, `href="/browse/`+slug+`/`); rows != shown {
|
|
t.Errorf("the %s box says %d, its list has %d rows (%d cookies)", slug, shown, rows, len(cookies))
|
|
}
|
|
}
|
|
return len(boxes)
|
|
}
|
|
|
|
// -check reads the load counters, so the glossary's problems must still be
|
|
// in them when it does.
|
|
func TestGlossaryWarningsReachCheck(t *testing.T) {
|
|
root := writeConsistencyCorpus(t)
|
|
withCorpus(t, root)
|
|
clean := loadWarnings
|
|
writeFixtureFile(t, filepath.Join(root, "glossary.json"), `{not json`)
|
|
loadData()
|
|
if loadWarnings != clean+1 {
|
|
t.Errorf("malformed glossary.json: %d warnings, want %d", loadWarnings, clean+1)
|
|
}
|
|
}
|
|
|
|
// A 404 under /files must not carry the files' public cache policy.
|
|
func TestMissingFileIsNotCached(t *testing.T) {
|
|
w := get(t, "/files/prism-slides/no-such-file.pdf")
|
|
if w.Code != http.StatusNotFound {
|
|
t.Fatalf("status %d, want 404", w.Code)
|
|
}
|
|
if cc := w.Header().Get("Cache-Control"); strings.Contains(cc, "public") {
|
|
t.Errorf("404 carries Cache-Control %q", cc)
|
|
}
|
|
}
|
|
|
|
// -dev serves the on-disk assets so edits show without a restart: no
|
|
// startup ETag may pin the old content.
|
|
func TestDevModeStaticRevalidatesEdits(t *testing.T) {
|
|
buildFixture(t)
|
|
devMode = true
|
|
initAssets()
|
|
t.Cleanup(func() {
|
|
devMode = false
|
|
initAssets()
|
|
})
|
|
if staticHashes != nil {
|
|
t.Errorf("dev mode built %d static hashes", len(staticHashes))
|
|
}
|
|
w := httptest.NewRecorder()
|
|
handleStatic(w, httptest.NewRequest("GET", "/static/style.css", nil))
|
|
if w.Code != http.StatusOK || w.Header().Get("ETag") != "" || w.Header().Get("Last-Modified") == "" {
|
|
t.Errorf("dev static: status %d, ETag %q, Last-Modified %q - want 200, none, the file's",
|
|
w.Code, w.Header().Get("ETag"), w.Header().Get("Last-Modified"))
|
|
}
|
|
}
|
|
|
|
// What the search page offers to pin is what the pinned page can list.
|
|
func TestPinnableSearchesAreListed(t *testing.T) {
|
|
h, _ := buildFixture(t)
|
|
if label, ok := corpus().searchLabel(mustQuery(t, "/search?do_search=1&fqv=&fqf=TE&fqc=and&date_doc_start=2013&date_doc_end=2013")); !ok || label != "all documents · (doc 2013)" {
|
|
t.Errorf("date-only search label = %q, %v", label, ok)
|
|
}
|
|
if strings.Contains(get(t, "/search?do_search=1&fqv=&fqf=TE&fqc=and").Body.String(), "pin-search-btn") {
|
|
t.Error("a search with no terms and no filters offers a pin")
|
|
}
|
|
if !strings.Contains(get(t, "/search?q=date:2013").Body.String(), "pin-search-btn") {
|
|
t.Error("a date-only search offers no pin")
|
|
}
|
|
|
|
pin := func(qs string) *httptest.ResponseRecorder {
|
|
r := httptest.NewRequest("POST", "/pinsearch", strings.NewReader(url.Values{"qs": {qs}, "back": {"/pinned"}}.Encode()))
|
|
r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
|
return record(h, r)
|
|
}
|
|
if w := pin("do_search=1&fqc=and&fqf=TE&fqv="); w.Code != http.StatusBadRequest {
|
|
t.Errorf("pinning an empty search: status %d, want 400", w.Code)
|
|
}
|
|
w := pin("date_doc_end=2013&date_doc_start=2013&do_search=1&fqc=and&fqf=TE&fqv=")
|
|
if w.Code != http.StatusSeeOther {
|
|
t.Fatalf("pinning a date-only search: status %d", w.Code)
|
|
}
|
|
body := get(t, "/pinned", w.Result().Cookies()...).Body.String()
|
|
if !strings.Contains(html.UnescapeString(body), "all documents · (doc 2013)") {
|
|
t.Error("the pinned date-only search is not listed on /pinned")
|
|
}
|
|
}
|