first commit

This commit is contained in:
libroot 2026-11-19 04:40:59 +02:00
commit 8c4dcae475
Signed by: libroot
GPG key ID: 2AC68C029EBE9ACE
89 changed files with 24623 additions and 0 deletions

272
tests/foia_test.go Normal file
View file

@ -0,0 +1,272 @@
package main
import (
"html"
"net/http"
"net/url"
"regexp"
"strconv"
"strings"
"testing"
)
// foiaOn is the cookie of a reader who opted in to FOIA releases.
var foiaOn = &http.Cookie{Name: "pref_include_foia", Value: "1"}
// In the consistency corpus "alpha" is in n1, n2 and the FOIA release f1.
func TestSearchLeavesOutFOIAByDefault(t *testing.T) {
withCorpus(t, writeConsistencyCorpus(t))
if got := resultTotal(t, "/search?q=alpha"); got != 2 {
t.Errorf("q=alpha, no preference: %d results, want 2 (FOIA left out)", got)
}
body := get(t, "/search?q=alpha").Body.String()
if strings.Contains(body, "left out") {
t.Error("the page counts the FOIA releases it left out")
}
// The page's own links name the collections it searched, so a copied
// link shows the same results to a reader with the other preference.
if !strings.Contains(body, "coll=news&coll=sidtoday-archive") {
t.Error("the page's links do not carry the default collections")
}
if got := resultTotal(t, "/search?q=alpha&coll=news&coll=sidtoday-archive&coll=foia"); got != 3 {
t.Errorf("q=alpha in every collection: %d results, want 3", got)
}
if got := resultTotal(t, "/search?q=alpha", foiaOn); got != 3 {
t.Errorf("q=alpha with FOIA on: %d results, want 3", got)
}
if strings.Contains(get(t, "/search?q=alpha", foiaOn).Body.String(), "left out") {
t.Error("FOIA on, yet the page says something was left out")
}
// An explicit collection choice wins over the preference.
if got := resultTotal(t, "/search?q=alpha&coll=foia"); got != 1 {
t.Errorf("q=alpha&coll=foia, no preference: %d results, want 1", got)
}
if got := resultTotal(t, "/search?q=alpha&coll=news", foiaOn); got != 2 {
t.Errorf("q=alpha&coll=news with FOIA on: %d results, want 2", got)
}
}
// ZEBRA is on n1, n2, f1 (FOIA) and s1 (SIDtoday).
func TestCategoryPagesLeaveOutFOIAByDefault(t *testing.T) {
withCorpus(t, writeConsistencyCorpus(t))
if got := resultTotal(t, "/browse/program/ZEBRA"); got != 3 {
t.Errorf("ZEBRA, no preference: %d documents, want 3", got)
}
if strings.Contains(get(t, "/browse/program/ZEBRA").Body.String(), "left out") {
t.Error("the page still carries a FOIA notice")
}
if got := resultTotal(t, "/browse/program/ZEBRA", foiaOn); got != 4 {
t.Errorf("ZEBRA with FOIA on: %d documents, want 4", got)
}
if strings.Contains(get(t, "/browse/program/ZEBRA", foiaOn).Body.String(), "left out") {
t.Error("FOIA on, yet the page says FOIA releases are left out")
}
}
// Every total in a category table is the number its row's link lists, for
// either preference.
func TestCategoryTotalsMatchTheirPages(t *testing.T) {
withCorpus(t, writeConsistencyCorpus(t))
reRow := regexp.MustCompile(`(?s)<a href="([^"]+)">\s*([^<]+?)\s*</a>.*?</td>\s*<td class="num-col">(\d+)</td>`)
for _, cookies := range [][]*http.Cookie{nil, {foiaOn}} {
for _, slug := range []string{"program", "agency", "docdate", "published", "publisher"} {
body := get(t, "/browse/"+slug, cookies...).Body.String()
_, body, _ = strings.Cut(body, "<tbody>")
body, _, _ = strings.Cut(body, "</tbody>")
rows := reRow.FindAllStringSubmatch(body, -1)
if len(rows) == 0 {
t.Fatalf("/browse/%s: no rows", slug)
}
for _, row := range rows {
want, _ := strconv.Atoi(row[3])
if strings.Contains(row[1], "coll=") {
t.Errorf("/browse/%s %s: row links with a collection scope: %s", slug, row[2], row[1])
}
if got := resultTotal(t, html.UnescapeString(row[1]), cookies...); got != want {
t.Errorf("/browse/%s %s (FOIA on: %v): total %d, its page lists %d", slug, row[2], cookies != nil, want, got)
}
}
}
}
off, on := get(t, "/browse/program").Body.String(), get(t, "/browse/program", foiaOn).Body.String()
if strings.Contains(off, "Totals leave out") || strings.Contains(off, "sort=foia") {
t.Error("FOIA off: the table still has a FOIA notice or column")
}
if !strings.Contains(on, "sort=foia") {
t.Error("FOIA on: the table has no FOIA column")
}
}
func TestRandomAndRelatedLeaveOutFOIA(t *testing.T) {
withCorpus(t, writeConsistencyCorpus(t))
sawFOIA := false
for i := 0; i < 300; i++ {
if d := corpus().randomDoc(false); d == nil || d.Collection == collectionFOIA {
t.Fatalf("randomDoc(false) = %v", d)
}
if corpus().randomDoc(true).Collection == collectionFOIA {
sawFOIA = true
}
}
if !sawFOIA {
t.Error("randomDoc(true) never picked the FOIA release in 300 tries")
}
corpus().archive.RelatedDocs["n2"] = []string{"f1", "n1"}
corpus().archive.RelatedDocs["f1"] = []string{"n1"}
body := get(t, "/doc/n2").Body.String()
if strings.Contains(body, `href="/doc/f1"`) || !strings.Contains(body, `href="/doc/n1"`) {
t.Error("/doc/n2, no preference: want n1 related and f1 left out")
}
if !strings.Contains(get(t, "/doc/n2", foiaOn).Body.String(), `href="/doc/f1"`) {
t.Error("/doc/n2 with FOIA on: f1 is not related")
}
if !strings.Contains(get(t, "/doc/f1").Body.String(), `href="/doc/n1"`) {
t.Error("/doc/f1: its related documents are gone")
}
}
// The home page counts the Snowden documents apart from the FOIA releases and
// says how the preference treats them. The corpus: n1-n4, a1, d1 (news), s1
// (SIDtoday archive), f1 (FOIA).
func TestHomeStatesTheFOIAScope(t *testing.T) {
withCorpus(t, writeConsistencyCorpus(t))
body := get(t, "/").Body.String()
for _, want := range []string{"holds 7 published Snowden documents. 6 documents are", "1 documents are the NSA newsletter",
"also holds 1", "are left out by default but can be included via"} {
if !strings.Contains(body, want) {
t.Errorf("home page lacks %q", want)
}
}
if !strings.Contains(get(t, "/", foiaOn).Body.String(), "you have included via") {
t.Error("FOIA on: the home page does not say so")
}
}
func TestFOIAPreferenceIsSaved(t *testing.T) {
withCorpus(t, writeConsistencyCorpus(t))
if !strings.Contains(get(t, "/preferences").Body.String(), `name="include_foia"`) {
t.Error("/preferences has no FOIA checkbox")
}
if readPrefs(httptest_NewGet("/")).IncludeFOIA {
t.Error("FOIA releases are included by default")
}
}
// The form shows the scope the search ran with: FOIA left out by default
// is an unchecked box, so submitting the form keeps it out.
func TestSearchFormKeepsTheDefaultScope(t *testing.T) {
withCorpus(t, writeConsistencyCorpus(t))
checked := regexp.MustCompile(`value="foia" checked`)
if checked.MatchString(get(t, "/search?q=alpha").Body.String()) {
t.Error("FOIA left out, yet its box is checked")
}
if !checked.MatchString(get(t, "/search?q=alpha", foiaOn).Body.String()) {
t.Error("FOIA on, yet its box is not checked")
}
if got := resultTotal(t, "/search?q=alpha&coll=news&coll=sidtoday-archive&do_search=1"); got != 2 {
t.Errorf("the form's resubmit finds %d, want 2", got)
}
}
// A FOIA release's metadata links name every collection, so its value
// pages list it under the default scope.
func TestFOIADocLinksIncludeFOIA(t *testing.T) {
buildFixture(t)
// Every value link printed for the release - on its own page, on its
// card and its table row in the FOIA tab, in a search result - opens a
// page that lists the release.
const id = "FISC_-_fixture_order"
reValueLink := regexp.MustCompile(`href="(/browse/[a-z]+/[^"]+)"`)
for _, page := range []string{"/doc/" + id, "/browse?collection=foia", "/browse?collection=foia&view=table", "/search?q=court&coll=foia"} {
links := reValueLink.FindAllStringSubmatch(get(t, page).Body.String(), -1)
if len(links) < 2 {
t.Fatalf("%s: %d value links, want the release's agency and classification at least", page, len(links))
}
for _, m := range links {
link := html.UnescapeString(m[1])
if !strings.Contains(get(t, withQuery(link, "view=table")).Body.String(), `href="/doc/`+id+`"`) {
t.Errorf("%s links %s, which does not list the release", page, link)
}
}
}
// /explore/classification counts every collection, and so do its links.
for _, m := range regexp.MustCompile(`href="(/browse/[^"]+)"`).FindAllStringSubmatch(get(t, "/explore/classification").Body.String(), -1) {
if !strings.Contains(m[1], "coll=foia") {
t.Errorf("/explore/classification links %s without its scope", m[1])
}
}
body := get(t, "/doc/FISC_-_fixture_order").Body.String()
m := regexp.MustCompile(`href="(/browse/agency/FISC[^"]*)"`).FindStringSubmatch(body)
if m == nil || !strings.Contains(m[1], "coll=foia") {
t.Fatalf("agency link %v does not include FOIA", m)
}
if got := resultTotal(t, html.UnescapeString(m[1])); got != 1 {
t.Errorf("the agency link lists %d documents, want 1", got)
}
if strings.Contains(get(t, "/doc/FISC_-_fixture_order", foiaOn).Body.String(), "/browse/agency/FISC?") {
t.Error("FOIA on: the links need no scope")
}
}
// On a publisher page scoped to a publication year, the links built from
// its documents carry that year.
func TestPublisherYearLinksCarryTheYear(t *testing.T) {
buildFixture(t)
body := get(t, "/browse/publisher/The%20Intercept?year=2016").Body.String()
if !strings.Contains(body, "date_pub_start=2016") {
t.Error("no link on the year-scoped page carries the publication year")
}
}
// collScope is the one place a page's collection scope is decided: FOIA
// releases are opt-in when the URL names no collection, an explicit coll=
// wins, and only the search pins a defaulted scope into its links.
func TestCollScope(t *testing.T) {
off, on := Prefs{}, Prefs{IncludeFOIA: true}
none := url.Values{}
for _, c := range []struct {
name string
v url.Values
p Prefs
pin bool
foia, def bool
qs string
}{
{"default, browse", none, off, false, false, true, ""},
{"default, search", none, off, true, false, true, "coll=news&coll=sidtoday-archive"},
{"opted in, browse", none, on, false, true, true, ""},
{"opted in, search", none, on, true, true, true, "coll=news&coll=sidtoday-archive&coll=foia"},
{"explicit foia", url.Values{"coll": {"foia"}}, off, false, true, false, "coll=foia"},
{"explicit news over opt-in", url.Values{"coll": {"news"}}, on, false, false, false, "coll=news"},
{"explicit everything", url.Values{"coll": {"news", "foia", "sidtoday-archive"}}, off, false, true, false,
"coll=news&coll=sidtoday-archive&coll=foia"},
} {
s := pageScope(c.v, c.p, c.pin)
foia := s.shows(&Document{Collection: collectionFOIA})
if foia != c.foia || s.defaulted != c.def || s.qs() != c.qs || s.shown[collectionFOIA] != c.foia {
t.Errorf("%s: shows FOIA %v, defaulted %v, qs %q; want %v, %v, %q",
c.name, foia, s.defaulted, s.qs(), c.foia, c.def, c.qs)
}
}
// A FOIA release's links widen the default to the whole archive and
// say so; a reader who opted in needs no widening.
if got := pageScope(none, off, false).including(collectionFOIA).suffix(); got != "?coll=news&coll=sidtoday-archive&coll=foia" {
t.Errorf("widened default: %q", got)
}
if got := pageScope(none, on, false).including(collectionFOIA).suffix(); got != "" {
t.Errorf("opted in, widened: %q, want no scope", got)
}
// and adds to the links, not to the documents.
s := pageScope(none, off, false).and("date_pub_start=2014")
if s.link("/search?q=x") != "/search?q=x&date_pub_start=2014" || s.shows(&Document{Collection: collectionFOIA}) {
t.Errorf("and: %q", s.link("/search?q=x"))
}
}