272 lines
11 KiB
Go
272 lines
11 KiB
Go
package main
|
|
|
|
import (
|
|
"html"
|
|
"net/http"
|
|
"net/url"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// foiaOn is the cookie of a reader who opted in to FOIA releases.
|
|
var foiaOn = &http.Cookie{Name: "pref_include_foia", Value: "1"}
|
|
|
|
// In the consistency corpus "alpha" is in n1, n2 and the FOIA release f1.
|
|
func TestSearchLeavesOutFOIAByDefault(t *testing.T) {
|
|
withCorpus(t, writeConsistencyCorpus(t))
|
|
|
|
if got := resultTotal(t, "/search?q=alpha"); got != 2 {
|
|
t.Errorf("q=alpha, no preference: %d results, want 2 (FOIA left out)", got)
|
|
}
|
|
body := get(t, "/search?q=alpha").Body.String()
|
|
if strings.Contains(body, "left out") {
|
|
t.Error("the page counts the FOIA releases it left out")
|
|
}
|
|
// The page's own links name the collections it searched, so a copied
|
|
// link shows the same results to a reader with the other preference.
|
|
if !strings.Contains(body, "coll=news&coll=sidtoday-archive") {
|
|
t.Error("the page's links do not carry the default collections")
|
|
}
|
|
if got := resultTotal(t, "/search?q=alpha&coll=news&coll=sidtoday-archive&coll=foia"); got != 3 {
|
|
t.Errorf("q=alpha in every collection: %d results, want 3", got)
|
|
}
|
|
|
|
if got := resultTotal(t, "/search?q=alpha", foiaOn); got != 3 {
|
|
t.Errorf("q=alpha with FOIA on: %d results, want 3", got)
|
|
}
|
|
if strings.Contains(get(t, "/search?q=alpha", foiaOn).Body.String(), "left out") {
|
|
t.Error("FOIA on, yet the page says something was left out")
|
|
}
|
|
// An explicit collection choice wins over the preference.
|
|
if got := resultTotal(t, "/search?q=alpha&coll=foia"); got != 1 {
|
|
t.Errorf("q=alpha&coll=foia, no preference: %d results, want 1", got)
|
|
}
|
|
if got := resultTotal(t, "/search?q=alpha&coll=news", foiaOn); got != 2 {
|
|
t.Errorf("q=alpha&coll=news with FOIA on: %d results, want 2", got)
|
|
}
|
|
}
|
|
|
|
// ZEBRA is on n1, n2, f1 (FOIA) and s1 (SIDtoday).
|
|
func TestCategoryPagesLeaveOutFOIAByDefault(t *testing.T) {
|
|
withCorpus(t, writeConsistencyCorpus(t))
|
|
|
|
if got := resultTotal(t, "/browse/program/ZEBRA"); got != 3 {
|
|
t.Errorf("ZEBRA, no preference: %d documents, want 3", got)
|
|
}
|
|
if strings.Contains(get(t, "/browse/program/ZEBRA").Body.String(), "left out") {
|
|
t.Error("the page still carries a FOIA notice")
|
|
}
|
|
if got := resultTotal(t, "/browse/program/ZEBRA", foiaOn); got != 4 {
|
|
t.Errorf("ZEBRA with FOIA on: %d documents, want 4", got)
|
|
}
|
|
if strings.Contains(get(t, "/browse/program/ZEBRA", foiaOn).Body.String(), "left out") {
|
|
t.Error("FOIA on, yet the page says FOIA releases are left out")
|
|
}
|
|
}
|
|
|
|
// Every total in a category table is the number its row's link lists, for
|
|
// either preference.
|
|
func TestCategoryTotalsMatchTheirPages(t *testing.T) {
|
|
withCorpus(t, writeConsistencyCorpus(t))
|
|
reRow := regexp.MustCompile(`(?s)<a href="([^"]+)">\s*([^<]+?)\s*</a>.*?</td>\s*<td class="num-col">(\d+)</td>`)
|
|
for _, cookies := range [][]*http.Cookie{nil, {foiaOn}} {
|
|
for _, slug := range []string{"program", "agency", "docdate", "published", "publisher"} {
|
|
body := get(t, "/browse/"+slug, cookies...).Body.String()
|
|
_, body, _ = strings.Cut(body, "<tbody>")
|
|
body, _, _ = strings.Cut(body, "</tbody>")
|
|
rows := reRow.FindAllStringSubmatch(body, -1)
|
|
if len(rows) == 0 {
|
|
t.Fatalf("/browse/%s: no rows", slug)
|
|
}
|
|
for _, row := range rows {
|
|
want, _ := strconv.Atoi(row[3])
|
|
if strings.Contains(row[1], "coll=") {
|
|
t.Errorf("/browse/%s %s: row links with a collection scope: %s", slug, row[2], row[1])
|
|
}
|
|
if got := resultTotal(t, html.UnescapeString(row[1]), cookies...); got != want {
|
|
t.Errorf("/browse/%s %s (FOIA on: %v): total %d, its page lists %d", slug, row[2], cookies != nil, want, got)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
off, on := get(t, "/browse/program").Body.String(), get(t, "/browse/program", foiaOn).Body.String()
|
|
if strings.Contains(off, "Totals leave out") || strings.Contains(off, "sort=foia") {
|
|
t.Error("FOIA off: the table still has a FOIA notice or column")
|
|
}
|
|
if !strings.Contains(on, "sort=foia") {
|
|
t.Error("FOIA on: the table has no FOIA column")
|
|
}
|
|
}
|
|
|
|
func TestRandomAndRelatedLeaveOutFOIA(t *testing.T) {
|
|
withCorpus(t, writeConsistencyCorpus(t))
|
|
|
|
sawFOIA := false
|
|
for i := 0; i < 300; i++ {
|
|
if d := corpus().randomDoc(false); d == nil || d.Collection == collectionFOIA {
|
|
t.Fatalf("randomDoc(false) = %v", d)
|
|
}
|
|
if corpus().randomDoc(true).Collection == collectionFOIA {
|
|
sawFOIA = true
|
|
}
|
|
}
|
|
if !sawFOIA {
|
|
t.Error("randomDoc(true) never picked the FOIA release in 300 tries")
|
|
}
|
|
|
|
corpus().archive.RelatedDocs["n2"] = []string{"f1", "n1"}
|
|
corpus().archive.RelatedDocs["f1"] = []string{"n1"}
|
|
body := get(t, "/doc/n2").Body.String()
|
|
if strings.Contains(body, `href="/doc/f1"`) || !strings.Contains(body, `href="/doc/n1"`) {
|
|
t.Error("/doc/n2, no preference: want n1 related and f1 left out")
|
|
}
|
|
if !strings.Contains(get(t, "/doc/n2", foiaOn).Body.String(), `href="/doc/f1"`) {
|
|
t.Error("/doc/n2 with FOIA on: f1 is not related")
|
|
}
|
|
if !strings.Contains(get(t, "/doc/f1").Body.String(), `href="/doc/n1"`) {
|
|
t.Error("/doc/f1: its related documents are gone")
|
|
}
|
|
}
|
|
|
|
// The home page counts the Snowden documents apart from the FOIA releases and
|
|
// says how the preference treats them. The corpus: n1-n4, a1, d1 (news), s1
|
|
// (SIDtoday archive), f1 (FOIA).
|
|
func TestHomeStatesTheFOIAScope(t *testing.T) {
|
|
withCorpus(t, writeConsistencyCorpus(t))
|
|
body := get(t, "/").Body.String()
|
|
for _, want := range []string{"holds 7 published Snowden documents. 6 documents are", "1 documents are the NSA newsletter",
|
|
"also holds 1", "are left out by default but can be included via"} {
|
|
if !strings.Contains(body, want) {
|
|
t.Errorf("home page lacks %q", want)
|
|
}
|
|
}
|
|
if !strings.Contains(get(t, "/", foiaOn).Body.String(), "you have included via") {
|
|
t.Error("FOIA on: the home page does not say so")
|
|
}
|
|
}
|
|
|
|
func TestFOIAPreferenceIsSaved(t *testing.T) {
|
|
withCorpus(t, writeConsistencyCorpus(t))
|
|
if !strings.Contains(get(t, "/preferences").Body.String(), `name="include_foia"`) {
|
|
t.Error("/preferences has no FOIA checkbox")
|
|
}
|
|
if readPrefs(httptest_NewGet("/")).IncludeFOIA {
|
|
t.Error("FOIA releases are included by default")
|
|
}
|
|
}
|
|
|
|
// The form shows the scope the search ran with: FOIA left out by default
|
|
// is an unchecked box, so submitting the form keeps it out.
|
|
func TestSearchFormKeepsTheDefaultScope(t *testing.T) {
|
|
withCorpus(t, writeConsistencyCorpus(t))
|
|
|
|
checked := regexp.MustCompile(`value="foia" checked`)
|
|
if checked.MatchString(get(t, "/search?q=alpha").Body.String()) {
|
|
t.Error("FOIA left out, yet its box is checked")
|
|
}
|
|
if !checked.MatchString(get(t, "/search?q=alpha", foiaOn).Body.String()) {
|
|
t.Error("FOIA on, yet its box is not checked")
|
|
}
|
|
if got := resultTotal(t, "/search?q=alpha&coll=news&coll=sidtoday-archive&do_search=1"); got != 2 {
|
|
t.Errorf("the form's resubmit finds %d, want 2", got)
|
|
}
|
|
}
|
|
|
|
// A FOIA release's metadata links name every collection, so its value
|
|
// pages list it under the default scope.
|
|
func TestFOIADocLinksIncludeFOIA(t *testing.T) {
|
|
buildFixture(t)
|
|
// Every value link printed for the release - on its own page, on its
|
|
// card and its table row in the FOIA tab, in a search result - opens a
|
|
// page that lists the release.
|
|
const id = "FISC_-_fixture_order"
|
|
reValueLink := regexp.MustCompile(`href="(/browse/[a-z]+/[^"]+)"`)
|
|
for _, page := range []string{"/doc/" + id, "/browse?collection=foia", "/browse?collection=foia&view=table", "/search?q=court&coll=foia"} {
|
|
links := reValueLink.FindAllStringSubmatch(get(t, page).Body.String(), -1)
|
|
if len(links) < 2 {
|
|
t.Fatalf("%s: %d value links, want the release's agency and classification at least", page, len(links))
|
|
}
|
|
for _, m := range links {
|
|
link := html.UnescapeString(m[1])
|
|
if !strings.Contains(get(t, withQuery(link, "view=table")).Body.String(), `href="/doc/`+id+`"`) {
|
|
t.Errorf("%s links %s, which does not list the release", page, link)
|
|
}
|
|
}
|
|
}
|
|
// /explore/classification counts every collection, and so do its links.
|
|
for _, m := range regexp.MustCompile(`href="(/browse/[^"]+)"`).FindAllStringSubmatch(get(t, "/explore/classification").Body.String(), -1) {
|
|
if !strings.Contains(m[1], "coll=foia") {
|
|
t.Errorf("/explore/classification links %s without its scope", m[1])
|
|
}
|
|
}
|
|
|
|
body := get(t, "/doc/FISC_-_fixture_order").Body.String()
|
|
m := regexp.MustCompile(`href="(/browse/agency/FISC[^"]*)"`).FindStringSubmatch(body)
|
|
if m == nil || !strings.Contains(m[1], "coll=foia") {
|
|
t.Fatalf("agency link %v does not include FOIA", m)
|
|
}
|
|
if got := resultTotal(t, html.UnescapeString(m[1])); got != 1 {
|
|
t.Errorf("the agency link lists %d documents, want 1", got)
|
|
}
|
|
if strings.Contains(get(t, "/doc/FISC_-_fixture_order", foiaOn).Body.String(), "/browse/agency/FISC?") {
|
|
t.Error("FOIA on: the links need no scope")
|
|
}
|
|
}
|
|
|
|
// On a publisher page scoped to a publication year, the links built from
|
|
// its documents carry that year.
|
|
func TestPublisherYearLinksCarryTheYear(t *testing.T) {
|
|
buildFixture(t)
|
|
body := get(t, "/browse/publisher/The%20Intercept?year=2016").Body.String()
|
|
if !strings.Contains(body, "date_pub_start=2016") {
|
|
t.Error("no link on the year-scoped page carries the publication year")
|
|
}
|
|
}
|
|
|
|
// collScope is the one place a page's collection scope is decided: FOIA
|
|
// releases are opt-in when the URL names no collection, an explicit coll=
|
|
// wins, and only the search pins a defaulted scope into its links.
|
|
func TestCollScope(t *testing.T) {
|
|
off, on := Prefs{}, Prefs{IncludeFOIA: true}
|
|
none := url.Values{}
|
|
for _, c := range []struct {
|
|
name string
|
|
v url.Values
|
|
p Prefs
|
|
pin bool
|
|
foia, def bool
|
|
qs string
|
|
}{
|
|
{"default, browse", none, off, false, false, true, ""},
|
|
{"default, search", none, off, true, false, true, "coll=news&coll=sidtoday-archive"},
|
|
{"opted in, browse", none, on, false, true, true, ""},
|
|
{"opted in, search", none, on, true, true, true, "coll=news&coll=sidtoday-archive&coll=foia"},
|
|
{"explicit foia", url.Values{"coll": {"foia"}}, off, false, true, false, "coll=foia"},
|
|
{"explicit news over opt-in", url.Values{"coll": {"news"}}, on, false, false, false, "coll=news"},
|
|
{"explicit everything", url.Values{"coll": {"news", "foia", "sidtoday-archive"}}, off, false, true, false,
|
|
"coll=news&coll=sidtoday-archive&coll=foia"},
|
|
} {
|
|
s := pageScope(c.v, c.p, c.pin)
|
|
foia := s.shows(&Document{Collection: collectionFOIA})
|
|
if foia != c.foia || s.defaulted != c.def || s.qs() != c.qs || s.shown[collectionFOIA] != c.foia {
|
|
t.Errorf("%s: shows FOIA %v, defaulted %v, qs %q; want %v, %v, %q",
|
|
c.name, foia, s.defaulted, s.qs(), c.foia, c.def, c.qs)
|
|
}
|
|
}
|
|
|
|
// A FOIA release's links widen the default to the whole archive and
|
|
// say so; a reader who opted in needs no widening.
|
|
if got := pageScope(none, off, false).including(collectionFOIA).suffix(); got != "?coll=news&coll=sidtoday-archive&coll=foia" {
|
|
t.Errorf("widened default: %q", got)
|
|
}
|
|
if got := pageScope(none, on, false).including(collectionFOIA).suffix(); got != "" {
|
|
t.Errorf("opted in, widened: %q, want no scope", got)
|
|
}
|
|
// and adds to the links, not to the documents.
|
|
s := pageScope(none, off, false).and("date_pub_start=2014")
|
|
if s.link("/search?q=x") != "/search?q=x&date_pub_start=2014" || s.shows(&Document{Collection: collectionFOIA}) {
|
|
t.Errorf("and: %q", s.link("/search?q=x"))
|
|
}
|
|
}
|