package main import ( "encoding/json" "html" "net/http" "net/http/httptest" "net/url" "path/filepath" "reflect" "regexp" "slices" "sort" "strings" "testing" ) // A corpus built to exercise the places where one fact used to be computed // twice: search rows and their negations, the page and the API, counts and // the pages they link to. Each document's words are chosen so the expected // result sets can be written down by hand. // // n1 news alpha gamma ZEBRA 2010 The Intercept 2014 + the intercept 2016 // n2 news alpha ZEBRA 2011 The Intercept 2014 + Der Spiegel 2014 // n3 news beta gamma 2012 Der Spiegel 2015 // n4 news beta 2012 Der Spiegel 2015 // f1 foia alpha ZEBRA 2010 (court filing: "(c)" list labels) // s1 sidtoday ZEBRA 2011 // a1 news "aka_id": null // d1 news date_created "ODNI", date_published "15 May 2015" func writeConsistencyCorpus(t *testing.T) string { t.Helper() root := t.TempDir() doc := func(dir, id string, meta map[string]interface{}, text string) { b, err := json.Marshal(meta) if err != nil { t.Fatal(err) } d := filepath.Join(root, dir, id) writeFixtureFile(t, filepath.Join(d, "metadata.json"), string(b)) writeFixtureFile(t, filepath.Join(d, id+".pdf"), "%PDF-1.4\n") if text != "" { writeFixtureFile(t, filepath.Join(d, id+".md"), text) } } pub := func(publisher, date, link string, reporters ...string) map[string]interface{} { return map[string]interface{}{"publisher": publisher, "date_published": date, "link": link, "reporters": reporters} } meta := func(title, typ, created string, programs []string, pubs ...map[string]interface{}) map[string]interface{} { return map[string]interface{}{"title": title, "type": typ, "document": map[string]interface{}{"creator": "NSA", "date_created": created, "programs": programs}, "publication": pubs} } zebra := []string{"ZEBRA"} doc("", "n1", meta("Report one", "REPORTED", "2010-03-01", zebra, pub("The Intercept", "2014-01-01", "https://example.org/a", "Glenn Greenwald"), pub("the intercept", "2016-01-01", "https://example.org/b", "glenn greenwald")), "alpha gamma\n\n(S//SI) a secret line\n(S/SI) the same marking, spelled by OCR\n(U) plain\n") doc("", "n2", meta("Report two", "REPORTED", "2011-03-01", zebra, pub("The Intercept", "2014-02-01", "https://example.org/c"), pub("Der Spiegel", "2014-02-02", "https://example.org/f")), "alpha\n\n(TS//SI//NF) top line\n") doc("", "n3", meta("Report three", "REPORTED", "2012-03-01", nil, pub("Der Spiegel", "2015-01-01", "https://example.org/d")), "beta gamma\n") doc("", "n4", meta("Report four", "REPORTED", "2012-04-01", nil, pub("Der Spiegel", "2015-01-01", "https://example.org/d")), "beta\n") doc("FOIA", "f1", meta("Court filing", "FOIA", "2010-05-01", zebra), "alpha\n\n(c) The Court shall issue an order.\n(s) another subsection\n") doc("SIDtoday-archive", "s1", meta("Issue", "SIDTODAY", "2011-05-01", zebra), "(S) issue line\n") a1 := meta("Null aka", "REPORTED", "", nil) a1["aka_id"] = nil doc("", "a1", a1, "") // A date of no shape the archive reads: served, undated. doc("", "d1", meta("Odd date", "REPORTED", "ODNI", nil, pub("Der Spiegel", "15 May 2015", "https://example.org/e")), "") writeFixtureFile(t, filepath.Join(root, "glossary.json"), `[]`) return root } // matchingIDs runs the search engine and returns the matching ids, sorted. func matchingIDs(queries, fields, conjs []string) []string { q, f, c := normalizeClauses(queries, fields, conjs) var ids []string scored, _ := corpus().executeSearch(searchParams{Queries: q, Fields: f, Conjs: c}) for _, s := range scored { ids = append(ids, s.Doc.ID) } sort.Strings(ids) return ids } // A row is one unit: its -terms are exclusions within it, whatever joins // it to the rows before. And an empty row starts nothing - a blank first // row with "OR x" under it is a search for x. func TestSearchRowsAndNegations(t *testing.T) { withCorpus(t, writeConsistencyCorpus(t)) te2 := []string{"TE", "TE"} for _, c := range []struct { name string queries, conjs []string want []string }{ {"inline negation", []string{"alpha -gamma"}, []string{"and"}, []string{"f1", "n2"}}, // alpha AND NOT (beta AND NOT gamma): no alpha document is lost // for mentioning gamma. {"negation inside NOT", []string{"alpha", "beta -gamma"}, []string{"and", "not"}, []string{"f1", "n1", "n2"}}, // alpha OR (beta AND NOT gamma): the exclusion stays inside its row. {"negation inside OR", []string{"alpha", "beta -gamma"}, []string{"and", "or"}, []string{"f1", "n1", "n2", "n4"}}, {"blank first row, OR", []string{"", "alpha"}, []string{"and", "or"}, []string{"f1", "n1", "n2"}}, {"spaces first row, OR", []string{" ", "alpha"}, []string{"and", "or"}, []string{"f1", "n1", "n2"}}, {"blank first row, OR nothing", []string{"", "zzzzunmatched"}, []string{"and", "or"}, nil}, {"blank first row, NOT", []string{"", "gamma"}, []string{"and", "not"}, []string{"a1", "d1", "f1", "n2", "n4", "s1"}}, {"only exclusions", []string{"-gamma -alpha"}, []string{"and"}, []string{"a1", "d1", "n4", "s1"}}, } { if got := matchingIDs(c.queries, te2[:len(c.queries)], c.conjs); !reflect.DeepEqual(got, c.want) { t.Errorf("%s: %q %v matched %v, want %v", c.name, c.queries, c.conjs, got, c.want) } } // The scorer and the highlighter read the same rows: an excluded term // is neither rewarded nor marked. if got := positiveTerms([]string{"alpha -gamma", "", "beta"}, []string{"and", "and", "not"}); !reflect.DeepEqual(got, []string{"alpha"}) { t.Errorf("positiveTerms = %q, want [alpha]", got) } if term, _ := snippetAnchor([]string{"-gamma", "alpha -beta"}, te2, []string{"and", "and"}); term != "alpha" { t.Errorf("snippetAnchor = %q, want alpha", term) } } // "+ Add row" and "remove" re-run the search: on the same rows, meaning the // same thing, as the Search button would. func TestRowEditsSearchTheSameRows(t *testing.T) { withCorpus(t, writeConsistencyCorpus(t)) rows := "do_search=1&fqv=alpha+-gamma&fqf=TE&fqc=and" want := resultTotal(t, "/search?action=search&"+rows, foiaOn) if want != 2 { t.Fatalf("search for alpha -gamma found %d, want 2", want) } for _, action := range []string{"add", "remove-1"} { path := "/search?action=" + action + "&" + rows + "&fqv=beta&fqf=TE&fqc=or" if action == "add" { path = "/search?action=add&" + rows } if got := resultTotal(t, path, foiaOn); got != want { t.Errorf("action=%s: %d results, the same rows searched find %d", action, got, want) } } } // The type filter resubmits the search the page shows, not the request: a // removed row stays removed, and the chosen sort is kept. func TestTypeFilterFormCarriesTheEffectiveSearch(t *testing.T) { withCorpus(t, writeConsistencyCorpus(t)) body := get(t, "/search?do_search=1&action=remove-1&sort=title"+ "&fqv=alpha&fqf=TE&fqc=and&fqv=removedrow&fqf=TE&fqc=or", foiaOn).Body.String() if !strings.Contains(body, `class="type-filter"`) { t.Fatal("no type filter on a result set holding two types") } if strings.Contains(body, "removedrow") { t.Error("the removed row is still in the page - the type filter would bring it back") } if !strings.Contains(body, `name="sort" value="title"`) { t.Error("the type filter drops the chosen sort") } } // /api/search runs the search page's pipeline, type filter included. The // API has no preference: it matches the page of a reader who includes FOIA // releases. func TestAPISearchMatchesThePage(t *testing.T) { withCorpus(t, writeConsistencyCorpus(t)) for qs, want := range map[string]int{ "q=alpha": 3, "q=alpha&type=REPORTED": 2, "q=alpha&type=FOIA&type=REPORTED": 3, "q=alpha&type=reported": 2, "q=zebra&coll=news": 2, "q=zebra&coll=foia&type=FOIA": 1, "do_search=1&fqv=alpha&fqf=TE&fqc=and&fqv=beta+-gamma&fqf=TE&fqc=or": 4, } { page := resultTotal(t, "/search?"+qs, foiaOn) var api struct{ Total int } if err := json.Unmarshal(get(t, "/api/search?"+qs).Body.Bytes(), &api); err != nil { t.Fatalf("%s: %v", qs, err) } if page != want || api.Total != want { t.Errorf("%s: the page finds %d, the API %d, want %d", qs, page, api.Total, want) } } } // Every count in this corpus is the number its page or search returns. func TestConsistencyCorpusCounts(t *testing.T) { withCorpus(t, writeConsistencyCorpus(t)) // Facets, with journalist and publisher spellings that differ by case // across one document's publications. for ci := range categories { c := &categories[ci] for val, want := range c.Facet(corpus()) { if got := len(corpus().categoryMatches(c.Slug, val)); got != want { t.Errorf("%s=%q: facet says %d, the page lists %d", c.Slug, val, want, got) } } } if js := corpus().archive.Facets["journalist"]; len(js) != 1 || js["Glenn Greenwald"] != 1 { t.Errorf("journalists = %v, want Glenn Greenwald once", js) } if n := corpus().archive.Facets["publisher"]["The Intercept"]; n != 2 { t.Errorf("The Intercept: %d documents, want 2 (%v)", n, corpus().archive.Facets["publisher"]) } // The value-page timelines and refine links, scoped to a collection: // f1 (FOIA) shares 2010 with n1 and s1 (SIDtoday) shares 2011 with n2, // so an unscoped bar link finds more than its bar counts. for _, p := range []string{"/browse/program/ZEBRA?coll=news", "/browse/program/ZEBRA"} { if n := followLinks(t, p, reBarLink, "timeline bar"); n < 2 { t.Errorf("%s: %d timeline bars checked, want the doc and pub timelines", p, n) } } // /explore/outlets: rows, cells and the per-year footer. for coll, table := range corpus().outletTables { for _, row := range table.Rows { if got := resultTotal(t, row.URL); got != row.Total { t.Errorf("outlets[%q] %s: total %d, its page lists %d", coll, row.Name, row.Total, got) } for _, cell := range row.Cells { if cell.URL != "" && resultTotal(t, cell.URL) != cell.Docs { t.Errorf("outlets[%q] %s: a cell says %d, its search finds %d", coll, row.Name, cell.Docs, resultTotal(t, cell.URL)) } } } // The footer adds up its column: a document several outlets // published counts once for each of them. for i, y := range table.Years { sum := 0 for _, row := range table.Rows { sum += row.Cells[i].Docs } if table.Totals[i] != sum { t.Errorf("outlets[%q] %s: footer %d, its column adds up to %d", coll, y, table.Totals[i], sum) } } } cell := func(outlet, year string) (docs, footer int) { table := corpus().outletTables[""] for i, y := range table.Years { if y != year { continue } footer = table.Totals[i] for _, row := range table.Rows { if row.Name == outlet { docs = row.Cells[i].Docs } } } return docs, footer } // n2: The Intercept and Der Spiegel both published it in 2014, so it // counts for both outlets and twice in the year's footer (n1 is the // third: The Intercept, 2014). if spiegel, footer := cell("Der Spiegel", "2014"); spiegel != 1 || footer != 3 { t.Errorf("2014: Der Spiegel cell %d, footer %d - want 1 and 3 (n2 once per outlet)", spiegel, footer) } if intercept, _ := cell("The Intercept", "2014"); intercept != 2 { t.Errorf("2014: The Intercept cell %d, want 2", intercept) } intercept := 0 for _, row := range corpus().outletTables[""].Rows { if row.Name == "The Intercept" { intercept = row.Total } } if intercept != 2 { t.Errorf("The Intercept's outlet total = %d, want 2 documents (n1 counts once for its two years)", intercept) } // /explore/portions: lower-case list labels are not markings, and every // row's document count is what its search finds. for _, l := range corpus().portionStats.Levels { if l.Name == "C" { t.Errorf("a CONFIDENTIAL row from the court filing's (c) labels: %+v", l) } } rows := append(append(append([]portionRow(nil), corpus().portionStats.Levels...), corpus().portionStats.Markings...), corpus().portionStats.Controls...) if len(rows) < 6 { t.Fatalf("only %d portion rows", len(rows)) } for _, r := range rows { if got := resultTotal(t, string(r.SearchURL)); got != r.Docs { t.Errorf("portion row %s: %d documents, its search finds %d (%s)", r.Name, r.Docs, got, r.SearchURL) } } if got := corpus().docIndex["n1"].PortionMarkings; !reflect.DeepEqual(got, []string{"S//SI", "U"}) { t.Errorf("n1 markings = %v, want [S//SI U]", got) } // A malformed date is no date: no "ODNI" year that no year page opens. if d := corpus().docIndex["d1"]; d == nil || d.DocMeta.DateCreated != "" || d.Publication[0].DatePublished != "" { t.Errorf("d1's malformed dates were kept: %+v", d) } if _, ok := corpus().archive.Facets["docdate"]["ODNI"]; ok { t.Error(`"ODNI" counted as a document year`) } // aka_id: null is no akas, not a reason to drop the document. if corpus().docIndex["a1"] == nil { t.Error(`a document with "aka_id": null was not loaded`) } } // A corpus for the counts that turn on WHICH publication entry carries a // date or a byline, and on the reader's collection scope. Every document // says "alpha". // // p1 news 2010-03 Outlet X 2013 (Able), Outlet Y 2014 (Baker) // p2 news 2010-06 Outlet X 2014 (Able) people: Clerk // p3 news 2011-01 Outlet X 2014 (Baker) // p4 news 2012-03 Outlet Y 2014 // p5 news 2012-09 Outlet Y 2014 // p6 news 2013-02 Outlet Y 2014, Partner Z 2014 (its own story; reporting only) // f1 foia 2009-07 Outlet Y 2014 (a story of its own) people: Docket func writeEntryCorpus(t *testing.T) string { t.Helper() root := t.TempDir() pub := func(publisher, date string, reporters ...string) map[string]interface{} { return map[string]interface{}{"publisher": publisher, "date_published": date, "reporters": reporters} } doc := func(dir, id, typ, created string, people []string, pubs ...map[string]interface{}) { b, err := json.Marshal(map[string]interface{}{"title": "Document " + id, "type": typ, "document": map[string]interface{}{"creator": "NSA", "date_created": created, "people": people}, "publication": pubs}) if err != nil { t.Fatal(err) } d := filepath.Join(root, dir, id) writeFixtureFile(t, filepath.Join(d, "metadata.json"), string(b)) writeFixtureFile(t, filepath.Join(d, id+".pdf"), "%PDF-1.4\n") writeFixtureFile(t, filepath.Join(d, id+".md"), "alpha\n") } doc("", "p1", "REPORTED", "2010-03-01", nil, pub("Outlet X", "2013-05-01", "Able, Ann"), pub("Outlet Y", "2014-02-01", "Baker, Bob")) doc("", "p2", "REPORTED", "2010-06-01", []string{"Clerk, Carol"}, pub("Outlet X", "2014-03-01", "Able, Ann")) doc("", "p3", "REPORTED", "2011-01-01", nil, pub("Outlet X", "2014-04-01", "Baker, Bob")) doc("", "p4", "REPORTED", "2012-03-01", nil, pub("Outlet Y", "2014-05-01")) doc("", "p5", "REPORTED", "2012-09-01", nil, pub("Outlet Y", "2014-06-01")) story := func(p map[string]interface{}, title, link string) map[string]interface{} { p["title"], p["link"] = title, link return p } partner := story(pub("Partner Z", "2014-07-02"), "Partner story", "https://example.org/partner") partner["reporting_only"] = true doc("", "p6", "REPORTED", "2013-02-01", nil, pub("Outlet Y", "2014-07-01"), partner) doc("FOIA", "f1", "FOIA", "2009-07-01", []string{"Docket, Dan"}, story(pub("Outlet Y", "2014-08-01"), "FOIA story", "https://example.org/foia-story")) writeFixtureFile(t, filepath.Join(root, "glossary.json"), `[]`) return root } // Counts that turn on a publication entry's date or byline, or on the // reader's scope, are still the numbers their links return. func TestCountsTiedToEntriesAndScope(t *testing.T) { withCorpus(t, writeEntryCorpus(t)) // A publication-year page files a document under its first publication, // which no search asks for: p1, first published in 2013, would join the // 2010 bar's search for 2014. So those bars carry no link. The // document-year page's publication bars do, and they are exact. body := get(t, "/browse/published/2014").Body.String() if !strings.Contains(body, "By document date") || strings.Contains(body, `[^<]*(\d+)`) // checkCategoryBoxes holds the number on each category box in the browse // nav to the rows of the list its click opens, for the reader with these // cookies. func checkCategoryBoxes(t *testing.T, cookies ...*http.Cookie) int { t.Helper() boxes := reCategoryBox.FindAllStringSubmatch(get(t, "/browse", cookies...).Body.String(), -1) for _, m := range boxes { slug, shown := m[1], mustAtoi(t, m[2]) list := get(t, "/browse/"+slug, cookies...).Body.String() if rows := strings.Count(list, `href="/browse/`+slug+`/`); rows != shown { t.Errorf("the %s box says %d, its list has %d rows (%d cookies)", slug, shown, rows, len(cookies)) } } return len(boxes) } // -check reads the load counters, so the glossary's problems must still be // in them when it does. func TestGlossaryWarningsReachCheck(t *testing.T) { root := writeConsistencyCorpus(t) withCorpus(t, root) clean := loadWarnings writeFixtureFile(t, filepath.Join(root, "glossary.json"), `{not json`) loadData() if loadWarnings != clean+1 { t.Errorf("malformed glossary.json: %d warnings, want %d", loadWarnings, clean+1) } } // A 404 under /files must not carry the files' public cache policy. func TestMissingFileIsNotCached(t *testing.T) { w := get(t, "/files/prism-slides/no-such-file.pdf") if w.Code != http.StatusNotFound { t.Fatalf("status %d, want 404", w.Code) } if cc := w.Header().Get("Cache-Control"); strings.Contains(cc, "public") { t.Errorf("404 carries Cache-Control %q", cc) } } // -dev serves the on-disk assets so edits show without a restart: no // startup ETag may pin the old content. func TestDevModeStaticRevalidatesEdits(t *testing.T) { buildFixture(t) devMode = true initAssets() t.Cleanup(func() { devMode = false initAssets() }) if staticHashes != nil { t.Errorf("dev mode built %d static hashes", len(staticHashes)) } w := httptest.NewRecorder() handleStatic(w, httptest.NewRequest("GET", "/static/style.css", nil)) if w.Code != http.StatusOK || w.Header().Get("ETag") != "" || w.Header().Get("Last-Modified") == "" { t.Errorf("dev static: status %d, ETag %q, Last-Modified %q - want 200, none, the file's", w.Code, w.Header().Get("ETag"), w.Header().Get("Last-Modified")) } } // What the search page offers to pin is what the pinned page can list. func TestPinnableSearchesAreListed(t *testing.T) { h, _ := buildFixture(t) if label, ok := corpus().searchLabel(mustQuery(t, "/search?do_search=1&fqv=&fqf=TE&fqc=and&date_doc_start=2013&date_doc_end=2013")); !ok || label != "all documents · (doc 2013)" { t.Errorf("date-only search label = %q, %v", label, ok) } if strings.Contains(get(t, "/search?do_search=1&fqv=&fqf=TE&fqc=and").Body.String(), "pin-search-btn") { t.Error("a search with no terms and no filters offers a pin") } if !strings.Contains(get(t, "/search?q=date:2013").Body.String(), "pin-search-btn") { t.Error("a date-only search offers no pin") } pin := func(qs string) *httptest.ResponseRecorder { r := httptest.NewRequest("POST", "/pinsearch", strings.NewReader(url.Values{"qs": {qs}, "back": {"/pinned"}}.Encode())) r.Header.Set("Content-Type", "application/x-www-form-urlencoded") return record(h, r) } if w := pin("do_search=1&fqc=and&fqf=TE&fqv="); w.Code != http.StatusBadRequest { t.Errorf("pinning an empty search: status %d, want 400", w.Code) } w := pin("date_doc_end=2013&date_doc_start=2013&do_search=1&fqc=and&fqf=TE&fqv=") if w.Code != http.StatusSeeOther { t.Fatalf("pinning a date-only search: status %d", w.Code) } body := get(t, "/pinned", w.Result().Cookies()...).Body.String() if !strings.Contains(html.UnescapeString(body), "all documents · (doc 2013)") { t.Error("the pinned date-only search is not listed on /pinned") } }