705 lines
29 KiB
Go
705 lines
29 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"html/template"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/url"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// Every route, through the real mux: the status it must answer, and a
|
|
// fragment of the body that proves the page is the one it should be. The
|
|
// handlers had no coverage at all before this - a missing template
|
|
// function, a nil map or a renamed data key reached production as a 500.
|
|
func TestRoutes(t *testing.T) {
|
|
cases := []struct {
|
|
path string
|
|
code int
|
|
want string // substring of the body
|
|
}{
|
|
{"/", 200, "Snowden"},
|
|
{"/browse", 200, "Reported"},
|
|
{"/browse?collection=foia", 200, "FOIA"},
|
|
{"/browse?collection=sidtoday-archive", 200, "SIDtoday Archive"},
|
|
{"/browse?collection=sidtoday-archive&view=card", 200, "SIDtoday Archive"},
|
|
{"/browse?collection=sidtoday-archive&view=table", 200, "SIDtoday Archive"},
|
|
{"/browse?collection=bogus", 200, "Reported"}, // unknown collection falls back
|
|
{"/browse/program", 200, "PRISM"},
|
|
{"/browse/program/PRISM", 200, "PRISM Collection Slides"},
|
|
{"/browse/program/NOSUCHTHING", 200, "0 document"},
|
|
{"/browse/tag/PRISM", 200, "PRISM Collection Slides"},
|
|
{"/browse/agency/NSA", 200, "PRISM Collection Slides"},
|
|
{"/browse/publisher/The%20Guardian", 200, "PRISM Collection Slides"},
|
|
{"/browse/publisher/NRK", 200, "0 document"}, // reporting-only: not a publisher
|
|
{"/browse/journalist/Greenwald%2C%20Glenn", 200, "PRISM"},
|
|
{"/browse/classification/TOP%20SECRET", 200, "PRISM"},
|
|
{"/browse/dissemination/NOFORN", 200, "PRISM"},
|
|
{"/browse/docdate/2013", 200, "PRISM"},
|
|
{"/browse/docdate/2013-04", 200, "PRISM"},
|
|
{"/browse/docdate/20", 200, "0 document"}, // not a valid date prefix
|
|
{"/browse/published/2013", 200, "PRISM"},
|
|
{"/browse/series", 200, "A Day in the Life"},
|
|
{"/browse/nosuchcategory", 404, ""},
|
|
{"/browse/nosuchcategory/value", 404, ""},
|
|
{"/doc/prism-slides", 200, "PRISM Collection Slides"},
|
|
{"/doc/prism-slides/pages", 200, "All pages"},
|
|
{"/doc/prism-slides/versions/lemonde", 200, "lemonde"},
|
|
{"/doc/prism-slides/versions/lemonde/pages", 200, "lemonde"},
|
|
{"/doc/prism-slides/versions/nope", 404, ""},
|
|
{"/doc/broken-no-pdf", 404, ""}, // skipped at load: no PDF
|
|
{"/doc/nope", 404, ""},
|
|
{"/search", 200, "Search"},
|
|
{"/search?q=prism&do_search=1", 200, "Collection Slides"}, // the term itself is <mark>ed
|
|
{"/search?q=zzzznomatch&do_search=1", 200, "No matches"},
|
|
{"/search?q=fairview&do_search=1", 200, "Copy:"}, // matched in a version
|
|
{"/articles", 200, "NSA Prism program"},
|
|
{"/explore", 200, "Explore"},
|
|
{"/explore/pages", 200, "Longest"},
|
|
{"/explore/ids", 200, "prism-slides"},
|
|
{"/explore/ids.txt", 200, "prism-slides.pdf"},
|
|
{"/explore/ids?missing=1", 200, "Technical IDs"},
|
|
{"/explore/outlets", 200, "Guardian"},
|
|
{"/explore/terms", 200, "PRISM"},
|
|
{"/explore/classification", 200, "Classification"},
|
|
{"/explore/nosuchmetric", 404, ""},
|
|
{"/glossary", 200, "XKEYSCORE"},
|
|
{"/changelog", 200, ""},
|
|
{"/about", 200, "About"},
|
|
{"/preferences", 200, "Preferences"},
|
|
{"/pinned", 200, "Pinned"},
|
|
{"/pinned/export", 200, ""},
|
|
{"/random", 302, ""},
|
|
{"/robots.txt", 200, "Sitemap"},
|
|
{"/sitemap.xml", 200, "/doc/prism-slides"},
|
|
{"/feed.xml", 404, ""},
|
|
{"/api/docs", 200, `"total"`},
|
|
{"/api/docs?per_page=all", 200, `"docs"`},
|
|
{"/api/doc/prism-slides", 200, `"full_text"`},
|
|
{"/api/doc/nope", 404, `"error"`},
|
|
{"/api/search?q=prism", 200, `"results"`},
|
|
{"/api/search", 400, `"error"`},
|
|
{"/api/facets", 200, `"programs"`},
|
|
{"/api/nosuch", 404, `"error"`},
|
|
{"/static/style.css", 200, "--"},
|
|
{"/static/nosuch.css", 404, ""},
|
|
{"/files/prism-slides/prism-slides.pdf", 200, "%PDF"},
|
|
{"/nosuchpage", 404, ""},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
w := get(t, c.path)
|
|
if w.Code != c.code {
|
|
t.Errorf("%s: status %d, want %d", c.path, w.Code, c.code)
|
|
continue
|
|
}
|
|
if c.want != "" && !strings.Contains(w.Body.String(), c.want) {
|
|
t.Errorf("%s: body does not contain %q", c.path, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The document file handler is the one place a URL reaches the filesystem.
|
|
func TestFileHandlerContainment(t *testing.T) {
|
|
for _, path := range []string{
|
|
"/files/prism-slides/images/page-1.jpg",
|
|
"/files/prism-slides/versions/lemonde/lemonde.pdf",
|
|
"/files/prism-slides/versions/lemonde/images/page-1.jpg",
|
|
} {
|
|
if w := get(t, path); w.Code != 200 {
|
|
t.Fatalf("%s: a file of the document should serve, got %d", path, w.Code)
|
|
}
|
|
}
|
|
// A file the loader never indexed is not served, whatever its type: a
|
|
// document directory gathers backups and second copies.
|
|
_, root := buildFixture(t)
|
|
dir := filepath.Join(root, "prism-slides")
|
|
for _, stray := range []string{"zz-stray.pdf", "images/notes/scan.jpg", "old-versions/draft.pdf"} {
|
|
writeFixtureFile(t, filepath.Join(dir, filepath.FromSlash(stray)), "%PDF-1.4\n")
|
|
}
|
|
t.Cleanup(func() {
|
|
for _, stray := range []string{"zz-stray.pdf", "images/notes", "old-versions"} {
|
|
os.RemoveAll(filepath.Join(dir, stray))
|
|
}
|
|
})
|
|
for _, path := range []string{
|
|
"/files/prism-slides/zz-stray.pdf", // a second PDF
|
|
"/files/prism-slides/images/notes/scan.jpg", // not a page image
|
|
"/files/prism-slides/old-versions/draft.pdf",
|
|
"/files/prism-slides/versions/nosuch/x.pdf",
|
|
"/files/prism-slides/thumb.jpg", // the fixture has no thumbnail
|
|
"/files/prism-slides/../../../etc/passwd",
|
|
"/files/prism-slides/..%2f..%2fetc%2fpasswd",
|
|
"/files/prism-slides/versions/lemonde/metadata.json", // a version has none
|
|
"/files/prism-slides/glossary.json",
|
|
"/files/prism-slides/", // no file
|
|
"/files/prism-slides/nope.pdf", // inside the doc, missing
|
|
"/files/nosuchdoc/x.pdf", // unknown id
|
|
"/files/prism-slides/versions/lemonde/", // directory
|
|
} {
|
|
if w := get(t, path); w.Code == 200 {
|
|
t.Errorf("%s: served 200, want a refusal", path)
|
|
}
|
|
}
|
|
// The curated metadata is a served file like the PDF it describes.
|
|
if m := get(t, "/files/prism-slides/metadata.json"); m.Code != 200 || !strings.HasPrefix(m.Header().Get("Content-Type"), "application/json") {
|
|
t.Errorf("metadata.json: status %d, Content-Type %q, want 200 application/json", m.Code, m.Header().Get("Content-Type"))
|
|
}
|
|
// A .md transcript is served as plain text, never as HTML.
|
|
w := get(t, "/files/prism-slides/prism-slides.md")
|
|
if w.Code != 200 {
|
|
t.Fatalf("transcript status %d", w.Code)
|
|
}
|
|
if ct := w.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") {
|
|
t.Errorf("transcript Content-Type %q, want text/plain", ct)
|
|
}
|
|
}
|
|
|
|
// Cacheability is decided by one rule - a request carrying any cookie is
|
|
// private - and personal pages go further with no-store. The hosting story
|
|
// (a shared microcache in front) rests on this.
|
|
func TestCacheHeaders(t *testing.T) {
|
|
anon := get(t, "/browse")
|
|
if cc := anon.Header().Get("Cache-Control"); cc != "public, max-age=10" {
|
|
t.Errorf("cookieless page: Cache-Control %q", cc)
|
|
}
|
|
if v := anon.Header().Get("Vary"); v != "Cookie" {
|
|
t.Errorf("cookieless page: Vary %q, want Cookie", v)
|
|
}
|
|
|
|
cookied := get(t, "/browse", &http.Cookie{Name: "pref_theme", Value: "dark"})
|
|
if cc := cookied.Header().Get("Cache-Control"); cc != "private, no-cache" {
|
|
t.Errorf("cookied page: Cache-Control %q", cc)
|
|
}
|
|
|
|
for _, p := range []string{"/preferences", "/pinned", "/pinned/export"} {
|
|
if cc := get(t, p).Header().Get("Cache-Control"); !strings.Contains(cc, "no-store") {
|
|
t.Errorf("%s: Cache-Control %q, want no-store", p, cc)
|
|
}
|
|
}
|
|
|
|
if cc := get(t, "/files/prism-slides/prism-slides.pdf").Header().Get("Cache-Control"); cc != "public, max-age=86400" {
|
|
t.Errorf("document file: Cache-Control %q", cc)
|
|
}
|
|
// A 404 must never inherit the immutable static cache.
|
|
if cc := get(t, "/static/nosuch.css?v=abc123").Header().Get("Cache-Control"); strings.Contains(cc, "immutable") {
|
|
t.Errorf("missing static asset marked immutable: %q", cc)
|
|
}
|
|
}
|
|
|
|
// Every response carries the baseline security headers, whatever it is.
|
|
func TestSecurityHeaders(t *testing.T) {
|
|
for _, p := range []string{"/", "/doc/prism-slides", "/api/facets", "/nosuchpage", "/files/prism-slides/prism-slides.pdf"} {
|
|
h := get(t, p).Header()
|
|
for _, want := range []string{"Content-Security-Policy", "X-Frame-Options", "X-Content-Type-Options", "Referrer-Policy"} {
|
|
if h.Get(want) == "" {
|
|
t.Errorf("%s: missing %s", p, want)
|
|
}
|
|
}
|
|
if csp := h.Get("Content-Security-Policy"); !strings.Contains(csp, "default-src 'self'") {
|
|
t.Errorf("%s: CSP %q", p, csp)
|
|
}
|
|
}
|
|
// The method gate's 405 too: scanner traffic is what it answers.
|
|
mux, _ := buildFixture(t)
|
|
w := record(mux, httptest.NewRequest("DELETE", "/", nil))
|
|
if w.Code != http.StatusMethodNotAllowed || w.Header().Get("Content-Security-Policy") == "" {
|
|
t.Errorf("DELETE: status %d, CSP %q; want 405 with the headers", w.Code, w.Header().Get("Content-Security-Policy"))
|
|
}
|
|
}
|
|
|
|
// State-changing POSTs are cookie-backed with no login, so a cross-site
|
|
// submission must be refused rather than silently flipping preferences.
|
|
func TestCrossSitePOSTRejected(t *testing.T) {
|
|
mux, _ := buildFixture(t)
|
|
for _, p := range []string{"/preferences/save", "/pin/prism-slides", "/history/clear", "/pinsearch"} {
|
|
// same-site is a sibling subdomain: another origin, not this site.
|
|
for _, site := range []string{"cross-site", "same-site"} {
|
|
r := newPost(p)
|
|
r.Header.Set("Sec-Fetch-Site", site)
|
|
if w := record(mux, r); w.Code != http.StatusForbidden {
|
|
t.Errorf("%s from %s: status %d, want 403", p, site, w.Code)
|
|
}
|
|
}
|
|
r := newPost(p)
|
|
r.Header.Set("Sec-Fetch-Site", "same-origin")
|
|
if w := record(mux, r); w.Code == http.StatusForbidden {
|
|
t.Errorf("%s from this site: rejected", p)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Loading is idempotent: every build* starts from empty, so a second load
|
|
// (a test with another corpus, a future reload) produces the same archive
|
|
// rather than a doubled one. Several builders used to append to package
|
|
// slices that nothing reset.
|
|
func TestLoadDataIsIdempotent(t *testing.T) {
|
|
buildFixture(t)
|
|
type snapshot struct {
|
|
docs, timeline, codewords, groups, titled, glossary, sitemap, articles int
|
|
}
|
|
take := func() snapshot {
|
|
years := 0
|
|
for _, y := range corpus().archiveTimeline {
|
|
years += len(y.Months)
|
|
}
|
|
return snapshot{
|
|
docs: len(corpus().archive.Docs), timeline: years, codewords: len(corpus().codewords),
|
|
groups: len(corpus().articleGroups), titled: corpus().articlesTitled,
|
|
glossary: len(corpus().glossaryIndex) + len(corpus().glossaryGroups), sitemap: len(corpus().sitemapPaths), articles: len(corpus().articles),
|
|
}
|
|
}
|
|
before := take()
|
|
loadData()
|
|
if after := take(); after != before {
|
|
t.Errorf("a second load changed the archive:\n before %+v\n after %+v", before, after)
|
|
}
|
|
}
|
|
|
|
// robots.txt keeps crawlers off the routes that cost a corpus scan, and the
|
|
// sitemap never advertises a path robots.txt disallows.
|
|
func TestRobotsAndSitemapAgree(t *testing.T) {
|
|
robots := get(t, "/robots.txt").Body.String()
|
|
for _, want := range []string{"Disallow: /search", "Disallow: /api/", "Sitemap: "} {
|
|
if !strings.Contains(robots, want) {
|
|
t.Errorf("robots.txt lacks %q:\n%s", want, robots)
|
|
}
|
|
}
|
|
var disallowed []string
|
|
for _, line := range strings.Split(robots, "\n") {
|
|
if v := strings.TrimPrefix(line, "Disallow:"); v != line {
|
|
disallowed = append(disallowed, strings.TrimSpace(v))
|
|
}
|
|
}
|
|
sitemap := get(t, "/sitemap.xml").Body.String()
|
|
for _, m := range regexp.MustCompile(`<loc>([^<]+)</loc>`).FindAllStringSubmatch(sitemap, -1) {
|
|
path := m[1]
|
|
if i := strings.Index(path[len("https://"):], "/"); i >= 0 {
|
|
path = path[len("https://")+i:]
|
|
}
|
|
for _, d := range disallowed {
|
|
if d != "" && strings.HasPrefix(path, d) {
|
|
t.Errorf("sitemap advertises %s, which robots.txt disallows (%s)", path, d)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// With a canonical base configured, /sitemap.xml is marshaled once at load
|
|
// rather than per request - and still says the same thing.
|
|
func TestCanonicalSitemapPrebuilt(t *testing.T) {
|
|
buildFixture(t)
|
|
prev := canonicalBase
|
|
t.Cleanup(func() {
|
|
canonicalBase = prev
|
|
corpus().canonicalSitemap = nil
|
|
corpus().buildCanonicalSitemap()
|
|
})
|
|
canonicalBase = "https://archive.example.org"
|
|
corpus().buildCanonicalSitemap()
|
|
if corpus().canonicalSitemap == nil {
|
|
t.Fatal("no sitemap was built for the canonical base")
|
|
}
|
|
body := get(t, "/sitemap.xml").Body.String()
|
|
if !strings.Contains(body, "<loc>https://archive.example.org/doc/prism-slides</loc>") {
|
|
t.Errorf("sitemap does not use the canonical base:\n%s", body[:200])
|
|
}
|
|
if body != string(corpus().canonicalSitemap) {
|
|
t.Error("the served sitemap is not the one built at load")
|
|
}
|
|
}
|
|
|
|
// Without a canonical base the sitemap is written from parts cut at load,
|
|
// with the request's base between them: the document marshalSitemap gives
|
|
// for that base, to the byte - a base that needs escaping included.
|
|
func TestSitemapFromParts(t *testing.T) {
|
|
h, _ := buildFixture(t)
|
|
if canonicalBase != "" || corpus().sitemapParts == nil {
|
|
t.Fatalf("the fixture should serve its sitemap from parts (base %q, %d parts)", canonicalBase, len(corpus().sitemapParts))
|
|
}
|
|
for _, host := range []string{"archive.example.org", "odd&host<x>.example"} {
|
|
r := httptest.NewRequest("GET", "/sitemap.xml", nil)
|
|
r.Host = host
|
|
want, err := corpus().marshalSitemap(requestBase(r))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := record(h, r).Body.String(); got != string(want) {
|
|
t.Errorf("host %q: the sitemap written from parts differs from marshalSitemap", host)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The big visitor-independent page bodies are executed once at startup
|
|
// (prerenderPages). The page must look exactly the same whether it used
|
|
// that copy or rendered its rows inline - which is what dev mode does, and
|
|
// what every one of these pages falls back to if a block ever fails to
|
|
// pre-render.
|
|
func TestPrerenderedMatchesLive(t *testing.T) {
|
|
buildFixture(t)
|
|
if len(corpus().prerendered) == 0 {
|
|
t.Fatal("nothing was pre-rendered")
|
|
}
|
|
pages := []string{
|
|
"/explore/ids",
|
|
"/explore/ids?missing=1",
|
|
"/glossary",
|
|
"/browse?collection=sidtoday-archive",
|
|
"/browse/program",
|
|
"/browse/tag",
|
|
// Category orderings are ASSEMBLED from pre-rendered rows rather
|
|
// than pre-rendered whole, so each one has to match too.
|
|
"/browse/tag?sort=count",
|
|
"/browse/program?sort=count&order=asc",
|
|
"/browse/agency?sort=name&order=desc",
|
|
"/browse/people?sort=foia",
|
|
// So are the doc tables, out of pre-rendered document rows.
|
|
"/browse?view=table",
|
|
"/browse?collection=sidtoday-archive&view=table&sort=title&order=desc",
|
|
"/browse/program/PRISM?view=table",
|
|
}
|
|
was, wasRows, wasAZ, wasDocRows := corpus().prerendered, corpus().categoryRowHTML, corpus().azRowHTML, corpus().docRowHTML
|
|
t.Cleanup(func() {
|
|
corpus().prerendered, corpus().categoryRowHTML, corpus().azRowHTML, corpus().docRowHTML = was, wasRows, wasAZ, wasDocRows
|
|
})
|
|
|
|
live := make([]string, len(pages))
|
|
corpus().prerendered = map[string]template.HTML{} // force the inline path
|
|
corpus().categoryRowHTML, corpus().azRowHTML, corpus().docRowHTML = nil, nil, nil
|
|
for i, p := range pages {
|
|
live[i] = get(t, p).Body.String()
|
|
}
|
|
corpus().prerendered, corpus().categoryRowHTML, corpus().azRowHTML, corpus().docRowHTML = was, wasRows, wasAZ, wasDocRows
|
|
for i, p := range pages {
|
|
if got := get(t, p).Body.String(); got != live[i] {
|
|
t.Errorf("%s: pre-rendered output differs from the live one", p)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The table view's headings are its sorters, and the one marked active is
|
|
// the order the rows are actually in - the page's default when the URL
|
|
// names none (the SIDtoday tab sorts by date, not title).
|
|
func TestDocTableActiveHeading(t *testing.T) {
|
|
for path, want := range map[string]string{
|
|
"/browse?collection=sidtoday-archive&view=table": "Doc Date ▲",
|
|
"/browse?view=table": "Title ▲",
|
|
"/browse?view=table&sort=country&order=desc": "Countries ▼",
|
|
"/browse?view=table&sort=bogus": "Title ▲",
|
|
} {
|
|
body := get(t, path).Body.String()
|
|
if !strings.Contains(body, want) {
|
|
t.Errorf("%s: no active heading %q", path, want)
|
|
}
|
|
if n := strings.Count(body, "▲") + strings.Count(body, "▼"); n != 1 {
|
|
t.Errorf("%s: %d active headings, want 1", path, n)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A reporter credited by two co-publishing outlets is one name in the
|
|
// table's Journalists column, as on the cards.
|
|
func TestDocTableJournalistsDeduped(t *testing.T) {
|
|
buildFixture(t)
|
|
d := testDoc(t, `{"id": "x", "publication": [
|
|
{"publisher": "A", "reporters": ["Greenwald, Glenn"]},
|
|
{"publisher": "B", "reporters": ["Greenwald, Glenn", "Poitras, Laura"]}]}`)
|
|
var b strings.Builder
|
|
err := tmplSet["list.html"].ExecuteTemplate(&b, "doc-table", listView{Docs: []*Document{&d}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n := strings.Count(b.String(), "/browse/journalist/Greenwald"); n != 1 {
|
|
t.Errorf("Greenwald linked %d times, want once", n)
|
|
}
|
|
if !strings.Contains(b.String(), "/browse/journalist/Poitras") {
|
|
t.Error("Poitras missing")
|
|
}
|
|
}
|
|
|
|
// Without fetch metadata the Origin decides, and it omits a default port
|
|
// the Host header may carry.
|
|
func TestOriginIsHost(t *testing.T) {
|
|
for _, c := range []struct {
|
|
origin, host string
|
|
want bool
|
|
}{
|
|
{"https://example.org", "example.org", true},
|
|
{"https://example.org", "example.org:443", true},
|
|
{"https://EXAMPLE.org", "example.org", true},
|
|
{"http://127.0.0.1:8080", "127.0.0.1:8080", true},
|
|
{"http://[::1]:8080", "[::1]:8080", true},
|
|
{"https://example.org", "example.org:8443", false},
|
|
{"https://example.org:8443", "example.org", false},
|
|
{"http://example.org", "example.org:443", false},
|
|
{"https://evil.example", "example.org", false},
|
|
} {
|
|
u, err := url.Parse(c.origin)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := originIsHost(u, c.host); got != c.want {
|
|
t.Errorf("originIsHost(%s, %s) = %v, want %v", c.origin, c.host, got, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestEnvBool(t *testing.T) {
|
|
for v, want := range map[string]bool{"": false, "1": true, "true": true, "TRUE": true, "0": false, "false": false} {
|
|
t.Setenv("SNOWCHIVE_TEST_BOOL", v)
|
|
if got := envBool("SNOWCHIVE_TEST_BOOL"); got != want {
|
|
t.Errorf("envBool(%q) = %v, want %v", v, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestBrowseValueSpelling: a value matches ignoring case, so another
|
|
// spelling of it redirects to the one the lists link instead of serving a
|
|
// second page for the same documents; the query string goes along.
|
|
func TestBrowseValueSpelling(t *testing.T) {
|
|
w := get(t, "/browse/program/prism?view=table")
|
|
if w.Code != http.StatusMovedPermanently || w.Header().Get("Location") != "/browse/program/PRISM?view=table" {
|
|
t.Errorf("/browse/program/prism: %d to %q, want 301 to /browse/program/PRISM?view=table", w.Code, w.Header().Get("Location"))
|
|
}
|
|
if w := get(t, "/browse/program/NOSUCHTHING"); w.Code != 200 {
|
|
t.Errorf("an unknown value: %d, want the empty page", w.Code)
|
|
}
|
|
}
|
|
|
|
// TestCardHighlightNeedsTheCategory: the cards on a value's page mark the
|
|
// links to that value - not every link that happens to read the same, such
|
|
// as the document date 2013 on the publication-year page for 2013.
|
|
func TestCardHighlightNeedsTheCategory(t *testing.T) {
|
|
if body := get(t, "/browse/published/2013").Body.String(); strings.Contains(body, "doc-created active") {
|
|
t.Error("/browse/published/2013 marks the document-date links")
|
|
}
|
|
if body := get(t, "/browse/docdate/2013").Body.String(); !strings.Contains(body, "doc-created active") {
|
|
t.Error("/browse/docdate/2013 does not mark the document-date links")
|
|
}
|
|
if body := get(t, "/browse/program/PRISM").Body.String(); !strings.Contains(body, `class="active">PRISM</a>`) {
|
|
t.Error("/browse/program/PRISM does not mark the program links")
|
|
}
|
|
}
|
|
|
|
// TestUnknownSortIsNotEchoed: a sort key no list has is the default sort,
|
|
// and stays out of the page's links and labels.
|
|
func TestUnknownSortIsNotEchoed(t *testing.T) {
|
|
for _, path := range []string{"/browse?sort=bogus", "/browse/program/PRISM?sort=bogus", "/search?q=prism&do_search=1&sort=bogus"} {
|
|
body := get(t, path).Body.String()
|
|
if strings.Contains(body, "sort=bogus") || strings.Contains(body, "Bogus") {
|
|
t.Errorf("%s: the unknown sort is echoed", path)
|
|
}
|
|
}
|
|
if body := get(t, "/search?q=prism&do_search=1").Body.String(); strings.Contains(body, "sort=relevance&order=desc") {
|
|
t.Error("relevance offers a direction it does not have")
|
|
}
|
|
}
|
|
|
|
// TestOutletYearLinksKeepTheScope: on the reported-only outlets table the
|
|
// year headings open the reported-only year, like the rows and cells do.
|
|
func TestOutletYearLinksKeepTheScope(t *testing.T) {
|
|
body := get(t, "/explore/outlets?coll=news").Body.String()
|
|
if !strings.Contains(body, `href="/browse/published/2013?coll=news"`) {
|
|
t.Error("the 2013 heading drops coll=news")
|
|
}
|
|
}
|
|
|
|
// pinSearch posts qs to /pinsearch from this site, with the given pinned
|
|
// searches already in the cookie.
|
|
func pinSearch(t *testing.T, qs string, pinned []string) *httptest.ResponseRecorder {
|
|
t.Helper()
|
|
mux, _ := buildFixture(t)
|
|
r := httptest.NewRequest("POST", "/pinsearch", strings.NewReader(url.Values{"qs": {qs}, "back": {"/search"}}.Encode()))
|
|
r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
|
r.Header.Set("Sec-Fetch-Site", "same-origin")
|
|
if len(pinned) > 0 {
|
|
r.AddCookie(&http.Cookie{Name: pinnedSearchesCookie, Value: strings.Join(pinned, "|")})
|
|
}
|
|
return record(mux, r)
|
|
}
|
|
|
|
// TestPinListFull: a pin the cookie has no room for is refused, and the
|
|
// reader lands on /pinned, which says so, rather than back on a page whose
|
|
// button still reads "Pin" as though nothing happened.
|
|
func TestPinListFull(t *testing.T) {
|
|
qs := url.Values{"do_search": {"1"}, "fqc": {"and"}, "fqf": {"TE"}, "fqv": {"prism"}}.Encode()
|
|
if w := pinSearch(t, qs, nil); w.Code != http.StatusSeeOther || w.Header().Get("Location") != "/search" {
|
|
t.Fatalf("pinning into an empty list: %d to %q", w.Code, w.Header().Get("Location"))
|
|
}
|
|
// Filler searches up to the byte budget: stored entries are not
|
|
// checked on read, so any escaped query string stands in for one.
|
|
var fill []string
|
|
for i := 0; ; i++ {
|
|
e := url.QueryEscape(fmt.Sprintf("fqv=filler%02d", i) + strings.Repeat("x", 180))
|
|
if len(strings.Join(append(fill, e), "|")) > cookieValueMax {
|
|
break
|
|
}
|
|
fill = append(fill, e)
|
|
}
|
|
// The last filler takes the cookie to exactly its budget.
|
|
fill[len(fill)-1] += strings.Repeat("x", cookieValueMax-len(strings.Join(fill, "|")))
|
|
w := pinSearch(t, qs, fill)
|
|
if w.Code != http.StatusSeeOther || w.Header().Get("Location") != "/pinned?full=1" {
|
|
t.Errorf("pinning into a full list: %d to %q, want 303 to /pinned?full=1", w.Code, w.Header().Get("Location"))
|
|
}
|
|
if body := get(t, "/pinned?full=1").Body.String(); !strings.Contains(body, "Your pinned list is full") {
|
|
t.Error("/pinned?full=1 does not say the list is full")
|
|
}
|
|
}
|
|
|
|
// TestSavableSearchMatchesPinSearch: the search page offers the pin button
|
|
// for exactly the searches /pinsearch accepts - one size limit for both.
|
|
func TestSavableSearchMatchesPinSearch(t *testing.T) {
|
|
long := strings.TrimSpace(strings.Repeat("abcdefgh ", 58)) // 58 words, under the word cap
|
|
q := url.Values{"do_search": {"1"}, "fqc": {"and"}, "fqf": {"TE"}, "fqv": {long}}
|
|
if len(q.Encode()) <= searchQSMax {
|
|
t.Fatalf("test query is %d bytes, not over the %d limit", len(q.Encode()), searchQSMax)
|
|
}
|
|
if body := get(t, "/search?"+q.Encode()).Body.String(); strings.Contains(body, `action="/pinsearch"`) {
|
|
t.Error("the search page offers to pin a search /pinsearch refuses")
|
|
}
|
|
if w := pinSearch(t, q.Encode(), nil); w.Code != http.StatusBadRequest {
|
|
t.Errorf("/pinsearch with an over-long search: %d, want 400", w.Code)
|
|
}
|
|
short := url.Values{"do_search": {"1"}, "fqc": {"and"}, "fqf": {"TE"}, "fqv": {"prism"}}
|
|
if body := get(t, "/search?"+short.Encode()).Body.String(); !strings.Contains(body, `action="/pinsearch"`) {
|
|
t.Error("the search page does not offer to pin a short search")
|
|
}
|
|
}
|
|
|
|
// TestAssetStamp: pages link each asset by its content hash; that URL is
|
|
// cached for a year, and any other stamp only revalidates.
|
|
func TestAssetStamp(t *testing.T) {
|
|
body := get(t, "/").Body.String()
|
|
want := "/static/style.css?v=" + staticHashes["style.css"]
|
|
if !strings.Contains(body, `href="`+want+`"`) {
|
|
t.Fatalf("home page does not link %s", want)
|
|
}
|
|
if cc := get(t, want).Header().Get("Cache-Control"); !strings.Contains(cc, "immutable") {
|
|
t.Errorf("%s: Cache-Control %q, want immutable", want, cc)
|
|
}
|
|
w := get(t, "/static/style.css?v=0000000000000000")
|
|
if cc := w.Header().Get("Cache-Control"); strings.Contains(cc, "immutable") || w.Header().Get("ETag") == "" {
|
|
t.Errorf("a stale stamp: Cache-Control %q, ETag %q; want revalidation", cc, w.Header().Get("ETag"))
|
|
}
|
|
}
|
|
|
|
// TestSitemapIgnoresFile: the sitemap is always the one built from the
|
|
// loaded corpus - a sitemap.xml lying in the data or project directory is
|
|
// never served, with -base or without it.
|
|
func TestSitemapIgnoresFile(t *testing.T) {
|
|
root := t.TempDir()
|
|
writeFixtureFile(t, filepath.Join(root, "sitemap.xml"), xmlHeaderForTest+"<urlset><url><loc>https://stale.example/</loc></url></urlset>\n")
|
|
withCorpus(t, root)
|
|
prev := canonicalBase
|
|
t.Cleanup(func() { canonicalBase = prev })
|
|
for _, base := range []string{"", "https://archive.example.org"} {
|
|
canonicalBase = base
|
|
loadData()
|
|
if body := get(t, "/sitemap.xml").Body.String(); strings.Contains(body, "stale.example") {
|
|
t.Errorf("base %q: a sitemap.xml file was served", base)
|
|
}
|
|
}
|
|
}
|
|
|
|
const xmlHeaderForTest = `<?xml version="1.0" encoding="UTF-8"?>` + "\n"
|
|
|
|
// TestPinnedSearchListed: /pinned lists a pinned search with its label,
|
|
// its link and the unpin form carrying the canonical query string.
|
|
func TestPinnedSearchListed(t *testing.T) {
|
|
qs := url.Values{"do_search": {"1"}, "fqc": {"and"}, "fqf": {"TE"}, "fqv": {"prism"}}.Encode()
|
|
body := get(t, "/pinned", &http.Cookie{Name: pinnedSearchesCookie, Value: url.QueryEscape(qs)}).Body.String()
|
|
for _, want := range []string{`href="/search?` + template.HTMLEscapeString(qs) + `"`, `name="qs" value="` + template.HTMLEscapeString(qs) + `"`, ">prism<"} {
|
|
if !strings.Contains(body, want) {
|
|
t.Errorf("/pinned lacks %s", want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestPageTitles: every page names itself in the tab, not just the site.
|
|
func TestPageTitles(t *testing.T) {
|
|
buildFixture(t) // the article key comes from the loaded corpus
|
|
for path, want := range map[string]string{
|
|
"/browse/program/PRISM": "<title>Program: PRISM - Snowden Archive</title>",
|
|
"/browse/program": "<title>Browse by Programs - Snowden Archive</title>",
|
|
"/search?q=prism&do_search=1": "<title>prism - Search - Snowden Archive</title>",
|
|
"/search": "<title>Search - Snowden Archive</title>",
|
|
"/browse?collection=foia": "<title>FOIA documents - Snowden Archive</title>",
|
|
"/pinned": "<title>Pinned - Snowden Archive</title>",
|
|
"/nosuchpage": "<title>Page not found - Snowden Archive</title>",
|
|
"/explore/pages": "- Snowden Archive</title>",
|
|
"/article/" + corpus().articleByLink["https://example.org/prism"].Key: "<title>NSA Prism program - Snowden Archive</title>",
|
|
} {
|
|
if body := get(t, path).Body.String(); !strings.Contains(body, want) {
|
|
t.Errorf("%s: no %q", path, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestNoBlockContentInHeadings: a heading holds text and inline controls
|
|
// only - a form or a disclosure inside one is invalid HTML and lands in
|
|
// the heading's accessible name. Forms tie their buttons from outside.
|
|
func TestNoBlockContentInHeadings(t *testing.T) {
|
|
heading := regexp.MustCompile(`(?s)<h([1-6])\b[^>]*>(.*?)</h[1-6]>`)
|
|
block := regexp.MustCompile(`<(form|details|div|p|ul|ol|table)\b`)
|
|
cookie := &http.Cookie{Name: pinnedCookie, Value: "prism-slides"}
|
|
recent := []*http.Cookie{{Name: "pref_track_recent", Value: "1"}, {Name: recentDocsCookie, Value: "prism-slides"}}
|
|
pages := map[string][]*http.Cookie{
|
|
"/pinned": {cookie},
|
|
"/": recent,
|
|
"/search?q=prism&do_search=1": nil,
|
|
"/doc/prism-slides": nil,
|
|
}
|
|
for path, cookies := range pages {
|
|
body := get(t, path, cookies...).Body.String()
|
|
for _, m := range heading.FindAllStringSubmatch(body, -1) {
|
|
if b := block.FindString(m[2]); b != "" {
|
|
t.Errorf("%s: %s inside <h%s>", path, b, m[1])
|
|
}
|
|
}
|
|
}
|
|
if body := get(t, "/pinned", cookie).Body.String(); !strings.Contains(body, `form="unpin-prism-slides"`) || !strings.Contains(body, `id="unpin-prism-slides"`) {
|
|
t.Error("/pinned: the unpin button is not tied to its form")
|
|
}
|
|
}
|
|
|
|
// TestGlossarySeeAlsoResolves: a see-also name links to the entry it
|
|
// names - through an alias or another case too - and a name no entry has
|
|
// is plain text rather than a link to nowhere.
|
|
func TestGlossarySeeAlsoResolves(t *testing.T) {
|
|
root := t.TempDir()
|
|
writeFixtureFile(t, filepath.Join(root, "glossary.json"), `[
|
|
{"term": "XKEYSCORE", "definition": "A search system.", "aka": ["XKS"], "see": ["rt-rg", "TURMOIL", "NOSUCHTERM"]},
|
|
{"term": "RT-RG", "definition": "Real time regional gateway."},
|
|
{"term": "TURMOIL", "definition": "Passive collection.", "see": ["xks"]}
|
|
]`)
|
|
withCorpus(t, root)
|
|
body := get(t, "/glossary").Body.String()
|
|
for _, want := range []string{`<a href="#term-RT-RG">rt-rg</a>`, `<a href="#term-TURMOIL">TURMOIL</a>`, `<a href="#term-XKEYSCORE">xks</a>`} {
|
|
if !strings.Contains(body, want) {
|
|
t.Errorf("/glossary lacks %s", want)
|
|
}
|
|
}
|
|
if strings.Contains(body, `href="#term-NOSUCHTERM"`) {
|
|
t.Error("/glossary links a see-also name no entry has")
|
|
}
|
|
}
|
|
|
|
// A pager or sort link appends its parameter to queryBase, with nothing
|
|
// left over when the page has no query of its own.
|
|
func TestQueryBase(t *testing.T) {
|
|
for want, q := range map[string]url.Values{
|
|
"/browse?": {},
|
|
"/browse?view=table&": {"view": {"table"}},
|
|
"/browse?coll=foia&view=card&": {"view": {"card"}, "coll": {"foia"}},
|
|
} {
|
|
if got := queryBase("/browse", q); got != want {
|
|
t.Errorf("queryBase(%v) = %q, want %q", q, got, want)
|
|
}
|
|
}
|
|
}
|