package main import ( "html" "net/http" "net/url" "regexp" "strconv" "strings" "testing" ) // foiaOn is the cookie of a reader who opted in to FOIA releases. var foiaOn = &http.Cookie{Name: "pref_include_foia", Value: "1"} // In the consistency corpus "alpha" is in n1, n2 and the FOIA release f1. func TestSearchLeavesOutFOIAByDefault(t *testing.T) { withCorpus(t, writeConsistencyCorpus(t)) if got := resultTotal(t, "/search?q=alpha"); got != 2 { t.Errorf("q=alpha, no preference: %d results, want 2 (FOIA left out)", got) } body := get(t, "/search?q=alpha").Body.String() if strings.Contains(body, "left out") { t.Error("the page counts the FOIA releases it left out") } // The page's own links name the collections it searched, so a copied // link shows the same results to a reader with the other preference. if !strings.Contains(body, "coll=news&coll=sidtoday-archive") { t.Error("the page's links do not carry the default collections") } if got := resultTotal(t, "/search?q=alpha&coll=news&coll=sidtoday-archive&coll=foia"); got != 3 { t.Errorf("q=alpha in every collection: %d results, want 3", got) } if got := resultTotal(t, "/search?q=alpha", foiaOn); got != 3 { t.Errorf("q=alpha with FOIA on: %d results, want 3", got) } if strings.Contains(get(t, "/search?q=alpha", foiaOn).Body.String(), "left out") { t.Error("FOIA on, yet the page says something was left out") } // An explicit collection choice wins over the preference. if got := resultTotal(t, "/search?q=alpha&coll=foia"); got != 1 { t.Errorf("q=alpha&coll=foia, no preference: %d results, want 1", got) } if got := resultTotal(t, "/search?q=alpha&coll=news", foiaOn); got != 2 { t.Errorf("q=alpha&coll=news with FOIA on: %d results, want 2", got) } } // ZEBRA is on n1, n2, f1 (FOIA) and s1 (SIDtoday). func TestCategoryPagesLeaveOutFOIAByDefault(t *testing.T) { withCorpus(t, writeConsistencyCorpus(t)) if got := resultTotal(t, "/browse/program/ZEBRA"); got != 3 { t.Errorf("ZEBRA, no preference: %d documents, want 3", got) } if strings.Contains(get(t, "/browse/program/ZEBRA").Body.String(), "left out") { t.Error("the page still carries a FOIA notice") } if got := resultTotal(t, "/browse/program/ZEBRA", foiaOn); got != 4 { t.Errorf("ZEBRA with FOIA on: %d documents, want 4", got) } if strings.Contains(get(t, "/browse/program/ZEBRA", foiaOn).Body.String(), "left out") { t.Error("FOIA on, yet the page says FOIA releases are left out") } } // Every total in a category table is the number its row's link lists, for // either preference. func TestCategoryTotalsMatchTheirPages(t *testing.T) { withCorpus(t, writeConsistencyCorpus(t)) reRow := regexp.MustCompile(`(?s)\s*([^<]+?)\s*.*?\s*(\d+)`) for _, cookies := range [][]*http.Cookie{nil, {foiaOn}} { for _, slug := range []string{"program", "agency", "docdate", "published", "publisher"} { body := get(t, "/browse/"+slug, cookies...).Body.String() _, body, _ = strings.Cut(body, "") body, _, _ = strings.Cut(body, "") rows := reRow.FindAllStringSubmatch(body, -1) if len(rows) == 0 { t.Fatalf("/browse/%s: no rows", slug) } for _, row := range rows { want, _ := strconv.Atoi(row[3]) if strings.Contains(row[1], "coll=") { t.Errorf("/browse/%s %s: row links with a collection scope: %s", slug, row[2], row[1]) } if got := resultTotal(t, html.UnescapeString(row[1]), cookies...); got != want { t.Errorf("/browse/%s %s (FOIA on: %v): total %d, its page lists %d", slug, row[2], cookies != nil, want, got) } } } } off, on := get(t, "/browse/program").Body.String(), get(t, "/browse/program", foiaOn).Body.String() if strings.Contains(off, "Totals leave out") || strings.Contains(off, "sort=foia") { t.Error("FOIA off: the table still has a FOIA notice or column") } if !strings.Contains(on, "sort=foia") { t.Error("FOIA on: the table has no FOIA column") } } func TestRandomAndRelatedLeaveOutFOIA(t *testing.T) { withCorpus(t, writeConsistencyCorpus(t)) sawFOIA := false for i := 0; i < 300; i++ { if d := corpus().randomDoc(false); d == nil || d.Collection == collectionFOIA { t.Fatalf("randomDoc(false) = %v", d) } if corpus().randomDoc(true).Collection == collectionFOIA { sawFOIA = true } } if !sawFOIA { t.Error("randomDoc(true) never picked the FOIA release in 300 tries") } corpus().archive.RelatedDocs["n2"] = []string{"f1", "n1"} corpus().archive.RelatedDocs["f1"] = []string{"n1"} body := get(t, "/doc/n2").Body.String() if strings.Contains(body, `href="/doc/f1"`) || !strings.Contains(body, `href="/doc/n1"`) { t.Error("/doc/n2, no preference: want n1 related and f1 left out") } if !strings.Contains(get(t, "/doc/n2", foiaOn).Body.String(), `href="/doc/f1"`) { t.Error("/doc/n2 with FOIA on: f1 is not related") } if !strings.Contains(get(t, "/doc/f1").Body.String(), `href="/doc/n1"`) { t.Error("/doc/f1: its related documents are gone") } } // The home page counts the Snowden documents apart from the FOIA releases and // says how the preference treats them. The corpus: n1-n4, a1, d1 (news), s1 // (SIDtoday archive), f1 (FOIA). func TestHomeStatesTheFOIAScope(t *testing.T) { withCorpus(t, writeConsistencyCorpus(t)) body := get(t, "/").Body.String() for _, want := range []string{"holds 7 published Snowden documents. 6 documents are", "1 documents are the NSA newsletter", "also holds 1", "are left out by default but can be included via"} { if !strings.Contains(body, want) { t.Errorf("home page lacks %q", want) } } if !strings.Contains(get(t, "/", foiaOn).Body.String(), "you have included via") { t.Error("FOIA on: the home page does not say so") } } func TestFOIAPreferenceIsSaved(t *testing.T) { withCorpus(t, writeConsistencyCorpus(t)) if !strings.Contains(get(t, "/preferences").Body.String(), `name="include_foia"`) { t.Error("/preferences has no FOIA checkbox") } if readPrefs(httptest_NewGet("/")).IncludeFOIA { t.Error("FOIA releases are included by default") } } // The form shows the scope the search ran with: FOIA left out by default // is an unchecked box, so submitting the form keeps it out. func TestSearchFormKeepsTheDefaultScope(t *testing.T) { withCorpus(t, writeConsistencyCorpus(t)) checked := regexp.MustCompile(`value="foia" checked`) if checked.MatchString(get(t, "/search?q=alpha").Body.String()) { t.Error("FOIA left out, yet its box is checked") } if !checked.MatchString(get(t, "/search?q=alpha", foiaOn).Body.String()) { t.Error("FOIA on, yet its box is not checked") } if got := resultTotal(t, "/search?q=alpha&coll=news&coll=sidtoday-archive&do_search=1"); got != 2 { t.Errorf("the form's resubmit finds %d, want 2", got) } } // A FOIA release's metadata links name every collection, so its value // pages list it under the default scope. func TestFOIADocLinksIncludeFOIA(t *testing.T) { buildFixture(t) // Every value link printed for the release - on its own page, on its // card and its table row in the FOIA tab, in a search result - opens a // page that lists the release. const id = "FISC_-_fixture_order" reValueLink := regexp.MustCompile(`href="(/browse/[a-z]+/[^"]+)"`) for _, page := range []string{"/doc/" + id, "/browse?collection=foia", "/browse?collection=foia&view=table", "/search?q=court&coll=foia"} { links := reValueLink.FindAllStringSubmatch(get(t, page).Body.String(), -1) if len(links) < 2 { t.Fatalf("%s: %d value links, want the release's agency and classification at least", page, len(links)) } for _, m := range links { link := html.UnescapeString(m[1]) if !strings.Contains(get(t, withQuery(link, "view=table")).Body.String(), `href="/doc/`+id+`"`) { t.Errorf("%s links %s, which does not list the release", page, link) } } } // /explore/classification counts every collection, and so do its links. for _, m := range regexp.MustCompile(`href="(/browse/[^"]+)"`).FindAllStringSubmatch(get(t, "/explore/classification").Body.String(), -1) { if !strings.Contains(m[1], "coll=foia") { t.Errorf("/explore/classification links %s without its scope", m[1]) } } body := get(t, "/doc/FISC_-_fixture_order").Body.String() m := regexp.MustCompile(`href="(/browse/agency/FISC[^"]*)"`).FindStringSubmatch(body) if m == nil || !strings.Contains(m[1], "coll=foia") { t.Fatalf("agency link %v does not include FOIA", m) } if got := resultTotal(t, html.UnescapeString(m[1])); got != 1 { t.Errorf("the agency link lists %d documents, want 1", got) } if strings.Contains(get(t, "/doc/FISC_-_fixture_order", foiaOn).Body.String(), "/browse/agency/FISC?") { t.Error("FOIA on: the links need no scope") } } // On a publisher page scoped to a publication year, the links built from // its documents carry that year. func TestPublisherYearLinksCarryTheYear(t *testing.T) { buildFixture(t) body := get(t, "/browse/publisher/The%20Intercept?year=2016").Body.String() if !strings.Contains(body, "date_pub_start=2016") { t.Error("no link on the year-scoped page carries the publication year") } } // collScope is the one place a page's collection scope is decided: FOIA // releases are opt-in when the URL names no collection, an explicit coll= // wins, and only the search pins a defaulted scope into its links. func TestCollScope(t *testing.T) { off, on := Prefs{}, Prefs{IncludeFOIA: true} none := url.Values{} for _, c := range []struct { name string v url.Values p Prefs pin bool foia, def bool qs string }{ {"default, browse", none, off, false, false, true, ""}, {"default, search", none, off, true, false, true, "coll=news&coll=sidtoday-archive"}, {"opted in, browse", none, on, false, true, true, ""}, {"opted in, search", none, on, true, true, true, "coll=news&coll=sidtoday-archive&coll=foia"}, {"explicit foia", url.Values{"coll": {"foia"}}, off, false, true, false, "coll=foia"}, {"explicit news over opt-in", url.Values{"coll": {"news"}}, on, false, false, false, "coll=news"}, {"explicit everything", url.Values{"coll": {"news", "foia", "sidtoday-archive"}}, off, false, true, false, "coll=news&coll=sidtoday-archive&coll=foia"}, } { s := pageScope(c.v, c.p, c.pin) foia := s.shows(&Document{Collection: collectionFOIA}) if foia != c.foia || s.defaulted != c.def || s.qs() != c.qs || s.shown[collectionFOIA] != c.foia { t.Errorf("%s: shows FOIA %v, defaulted %v, qs %q; want %v, %v, %q", c.name, foia, s.defaulted, s.qs(), c.foia, c.def, c.qs) } } // A FOIA release's links widen the default to the whole archive and // say so; a reader who opted in needs no widening. if got := pageScope(none, off, false).including(collectionFOIA).suffix(); got != "?coll=news&coll=sidtoday-archive&coll=foia" { t.Errorf("widened default: %q", got) } if got := pageScope(none, on, false).including(collectionFOIA).suffix(); got != "" { t.Errorf("opted in, widened: %q, want no scope", got) } // and adds to the links, not to the documents. s := pageScope(none, off, false).and("date_pub_start=2014") if s.link("/search?q=x") != "/search?q=x&date_pub_start=2014" || s.shows(&Document{Collection: collectionFOIA}) { t.Errorf("and: %q", s.link("/search?q=x")) } }