package main import ( "fmt" "html/template" "net/http" "net/http/httptest" "net/url" "os" "path/filepath" "regexp" "strings" "testing" ) // Every route, through the real mux: the status it must answer, and a // fragment of the body that proves the page is the one it should be. The // handlers had no coverage at all before this - a missing template // function, a nil map or a renamed data key reached production as a 500. func TestRoutes(t *testing.T) { cases := []struct { path string code int want string // substring of the body }{ {"/", 200, "Snowden"}, {"/browse", 200, "Reported"}, {"/browse?collection=foia", 200, "FOIA"}, {"/browse?collection=sidtoday-archive", 200, "SIDtoday Archive"}, {"/browse?collection=sidtoday-archive&view=card", 200, "SIDtoday Archive"}, {"/browse?collection=sidtoday-archive&view=table", 200, "SIDtoday Archive"}, {"/browse?collection=bogus", 200, "Reported"}, // unknown collection falls back {"/browse/program", 200, "PRISM"}, {"/browse/program/PRISM", 200, "PRISM Collection Slides"}, {"/browse/program/NOSUCHTHING", 200, "0 document"}, {"/browse/tag/PRISM", 200, "PRISM Collection Slides"}, {"/browse/agency/NSA", 200, "PRISM Collection Slides"}, {"/browse/publisher/The%20Guardian", 200, "PRISM Collection Slides"}, {"/browse/publisher/NRK", 200, "0 document"}, // reporting-only: not a publisher {"/browse/journalist/Greenwald%2C%20Glenn", 200, "PRISM"}, {"/browse/classification/TOP%20SECRET", 200, "PRISM"}, {"/browse/dissemination/NOFORN", 200, "PRISM"}, {"/browse/docdate/2013", 200, "PRISM"}, {"/browse/docdate/2013-04", 200, "PRISM"}, {"/browse/docdate/20", 200, "0 document"}, // not a valid date prefix {"/browse/published/2013", 200, "PRISM"}, {"/browse/series", 200, "A Day in the Life"}, {"/browse/nosuchcategory", 404, ""}, {"/browse/nosuchcategory/value", 404, ""}, {"/doc/prism-slides", 200, "PRISM Collection Slides"}, {"/doc/prism-slides/pages", 200, "All pages"}, {"/doc/prism-slides/versions/lemonde", 200, "lemonde"}, {"/doc/prism-slides/versions/lemonde/pages", 200, "lemonde"}, {"/doc/prism-slides/versions/nope", 404, ""}, {"/doc/broken-no-pdf", 404, ""}, // skipped at load: no PDF {"/doc/nope", 404, ""}, {"/search", 200, "Search"}, {"/search?q=prism&do_search=1", 200, "Collection Slides"}, // the term itself is ed {"/search?q=zzzznomatch&do_search=1", 200, "No matches"}, {"/search?q=fairview&do_search=1", 200, "Copy:"}, // matched in a version {"/articles", 200, "NSA Prism program"}, {"/explore", 200, "Explore"}, {"/explore/pages", 200, "Longest"}, {"/explore/ids", 200, "prism-slides"}, {"/explore/ids.txt", 200, "prism-slides.pdf"}, {"/explore/ids?missing=1", 200, "Technical IDs"}, {"/explore/outlets", 200, "Guardian"}, {"/explore/terms", 200, "PRISM"}, {"/explore/classification", 200, "Classification"}, {"/explore/nosuchmetric", 404, ""}, {"/glossary", 200, "XKEYSCORE"}, {"/changelog", 200, ""}, {"/about", 200, "About"}, {"/preferences", 200, "Preferences"}, {"/pinned", 200, "Pinned"}, {"/pinned/export", 200, ""}, {"/random", 302, ""}, {"/robots.txt", 200, "Sitemap"}, {"/sitemap.xml", 200, "/doc/prism-slides"}, {"/feed.xml", 404, ""}, {"/api/docs", 200, `"total"`}, {"/api/docs?per_page=all", 200, `"docs"`}, {"/api/doc/prism-slides", 200, `"full_text"`}, {"/api/doc/nope", 404, `"error"`}, {"/api/search?q=prism", 200, `"results"`}, {"/api/search", 400, `"error"`}, {"/api/facets", 200, `"programs"`}, {"/api/nosuch", 404, `"error"`}, {"/static/style.css", 200, "--"}, {"/static/nosuch.css", 404, ""}, {"/files/prism-slides/prism-slides.pdf", 200, "%PDF"}, {"/nosuchpage", 404, ""}, } for _, c := range cases { w := get(t, c.path) if w.Code != c.code { t.Errorf("%s: status %d, want %d", c.path, w.Code, c.code) continue } if c.want != "" && !strings.Contains(w.Body.String(), c.want) { t.Errorf("%s: body does not contain %q", c.path, c.want) } } } // The document file handler is the one place a URL reaches the filesystem. func TestFileHandlerContainment(t *testing.T) { for _, path := range []string{ "/files/prism-slides/images/page-1.jpg", "/files/prism-slides/versions/lemonde/lemonde.pdf", "/files/prism-slides/versions/lemonde/images/page-1.jpg", } { if w := get(t, path); w.Code != 200 { t.Fatalf("%s: a file of the document should serve, got %d", path, w.Code) } } // A file the loader never indexed is not served, whatever its type: a // document directory gathers backups and second copies. _, root := buildFixture(t) dir := filepath.Join(root, "prism-slides") for _, stray := range []string{"zz-stray.pdf", "images/notes/scan.jpg", "old-versions/draft.pdf"} { writeFixtureFile(t, filepath.Join(dir, filepath.FromSlash(stray)), "%PDF-1.4\n") } t.Cleanup(func() { for _, stray := range []string{"zz-stray.pdf", "images/notes", "old-versions"} { os.RemoveAll(filepath.Join(dir, stray)) } }) for _, path := range []string{ "/files/prism-slides/zz-stray.pdf", // a second PDF "/files/prism-slides/images/notes/scan.jpg", // not a page image "/files/prism-slides/old-versions/draft.pdf", "/files/prism-slides/versions/nosuch/x.pdf", "/files/prism-slides/thumb.jpg", // the fixture has no thumbnail "/files/prism-slides/../../../etc/passwd", "/files/prism-slides/..%2f..%2fetc%2fpasswd", "/files/prism-slides/versions/lemonde/metadata.json", // a version has none "/files/prism-slides/glossary.json", "/files/prism-slides/", // no file "/files/prism-slides/nope.pdf", // inside the doc, missing "/files/nosuchdoc/x.pdf", // unknown id "/files/prism-slides/versions/lemonde/", // directory } { if w := get(t, path); w.Code == 200 { t.Errorf("%s: served 200, want a refusal", path) } } // The curated metadata is a served file like the PDF it describes. if m := get(t, "/files/prism-slides/metadata.json"); m.Code != 200 || !strings.HasPrefix(m.Header().Get("Content-Type"), "application/json") { t.Errorf("metadata.json: status %d, Content-Type %q, want 200 application/json", m.Code, m.Header().Get("Content-Type")) } // A .md transcript is served as plain text, never as HTML. w := get(t, "/files/prism-slides/prism-slides.md") if w.Code != 200 { t.Fatalf("transcript status %d", w.Code) } if ct := w.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") { t.Errorf("transcript Content-Type %q, want text/plain", ct) } } // Cacheability is decided by one rule - a request carrying any cookie is // private - and personal pages go further with no-store. The hosting story // (a shared microcache in front) rests on this. func TestCacheHeaders(t *testing.T) { anon := get(t, "/browse") if cc := anon.Header().Get("Cache-Control"); cc != "public, max-age=10" { t.Errorf("cookieless page: Cache-Control %q", cc) } if v := anon.Header().Get("Vary"); v != "Cookie" { t.Errorf("cookieless page: Vary %q, want Cookie", v) } cookied := get(t, "/browse", &http.Cookie{Name: "pref_theme", Value: "dark"}) if cc := cookied.Header().Get("Cache-Control"); cc != "private, no-cache" { t.Errorf("cookied page: Cache-Control %q", cc) } for _, p := range []string{"/preferences", "/pinned", "/pinned/export"} { if cc := get(t, p).Header().Get("Cache-Control"); !strings.Contains(cc, "no-store") { t.Errorf("%s: Cache-Control %q, want no-store", p, cc) } } if cc := get(t, "/files/prism-slides/prism-slides.pdf").Header().Get("Cache-Control"); cc != "public, max-age=86400" { t.Errorf("document file: Cache-Control %q", cc) } // A 404 must never inherit the immutable static cache. if cc := get(t, "/static/nosuch.css?v=abc123").Header().Get("Cache-Control"); strings.Contains(cc, "immutable") { t.Errorf("missing static asset marked immutable: %q", cc) } } // Every response carries the baseline security headers, whatever it is. func TestSecurityHeaders(t *testing.T) { for _, p := range []string{"/", "/doc/prism-slides", "/api/facets", "/nosuchpage", "/files/prism-slides/prism-slides.pdf"} { h := get(t, p).Header() for _, want := range []string{"Content-Security-Policy", "X-Frame-Options", "X-Content-Type-Options", "Referrer-Policy"} { if h.Get(want) == "" { t.Errorf("%s: missing %s", p, want) } } if csp := h.Get("Content-Security-Policy"); !strings.Contains(csp, "default-src 'self'") { t.Errorf("%s: CSP %q", p, csp) } } // The method gate's 405 too: scanner traffic is what it answers. mux, _ := buildFixture(t) w := record(mux, httptest.NewRequest("DELETE", "/", nil)) if w.Code != http.StatusMethodNotAllowed || w.Header().Get("Content-Security-Policy") == "" { t.Errorf("DELETE: status %d, CSP %q; want 405 with the headers", w.Code, w.Header().Get("Content-Security-Policy")) } } // State-changing POSTs are cookie-backed with no login, so a cross-site // submission must be refused rather than silently flipping preferences. func TestCrossSitePOSTRejected(t *testing.T) { mux, _ := buildFixture(t) for _, p := range []string{"/preferences/save", "/pin/prism-slides", "/history/clear", "/pinsearch"} { // same-site is a sibling subdomain: another origin, not this site. for _, site := range []string{"cross-site", "same-site"} { r := newPost(p) r.Header.Set("Sec-Fetch-Site", site) if w := record(mux, r); w.Code != http.StatusForbidden { t.Errorf("%s from %s: status %d, want 403", p, site, w.Code) } } r := newPost(p) r.Header.Set("Sec-Fetch-Site", "same-origin") if w := record(mux, r); w.Code == http.StatusForbidden { t.Errorf("%s from this site: rejected", p) } } } // Loading is idempotent: every build* starts from empty, so a second load // (a test with another corpus, a future reload) produces the same archive // rather than a doubled one. Several builders used to append to package // slices that nothing reset. func TestLoadDataIsIdempotent(t *testing.T) { buildFixture(t) type snapshot struct { docs, timeline, codewords, groups, titled, glossary, sitemap, articles int } take := func() snapshot { years := 0 for _, y := range corpus().archiveTimeline { years += len(y.Months) } return snapshot{ docs: len(corpus().archive.Docs), timeline: years, codewords: len(corpus().codewords), groups: len(corpus().articleGroups), titled: corpus().articlesTitled, glossary: len(corpus().glossaryIndex) + len(corpus().glossaryGroups), sitemap: len(corpus().sitemapPaths), articles: len(corpus().articles), } } before := take() loadData() if after := take(); after != before { t.Errorf("a second load changed the archive:\n before %+v\n after %+v", before, after) } } // robots.txt keeps crawlers off the routes that cost a corpus scan, and the // sitemap never advertises a path robots.txt disallows. func TestRobotsAndSitemapAgree(t *testing.T) { robots := get(t, "/robots.txt").Body.String() for _, want := range []string{"Disallow: /search", "Disallow: /api/", "Sitemap: "} { if !strings.Contains(robots, want) { t.Errorf("robots.txt lacks %q:\n%s", want, robots) } } var disallowed []string for _, line := range strings.Split(robots, "\n") { if v := strings.TrimPrefix(line, "Disallow:"); v != line { disallowed = append(disallowed, strings.TrimSpace(v)) } } sitemap := get(t, "/sitemap.xml").Body.String() for _, m := range regexp.MustCompile(`([^<]+)`).FindAllStringSubmatch(sitemap, -1) { path := m[1] if i := strings.Index(path[len("https://"):], "/"); i >= 0 { path = path[len("https://")+i:] } for _, d := range disallowed { if d != "" && strings.HasPrefix(path, d) { t.Errorf("sitemap advertises %s, which robots.txt disallows (%s)", path, d) } } } } // With a canonical base configured, /sitemap.xml is marshaled once at load // rather than per request - and still says the same thing. func TestCanonicalSitemapPrebuilt(t *testing.T) { buildFixture(t) prev := canonicalBase t.Cleanup(func() { canonicalBase = prev corpus().canonicalSitemap = nil corpus().buildCanonicalSitemap() }) canonicalBase = "https://archive.example.org" corpus().buildCanonicalSitemap() if corpus().canonicalSitemap == nil { t.Fatal("no sitemap was built for the canonical base") } body := get(t, "/sitemap.xml").Body.String() if !strings.Contains(body, "https://archive.example.org/doc/prism-slides") { t.Errorf("sitemap does not use the canonical base:\n%s", body[:200]) } if body != string(corpus().canonicalSitemap) { t.Error("the served sitemap is not the one built at load") } } // Without a canonical base the sitemap is written from parts cut at load, // with the request's base between them: the document marshalSitemap gives // for that base, to the byte - a base that needs escaping included. func TestSitemapFromParts(t *testing.T) { h, _ := buildFixture(t) if canonicalBase != "" || corpus().sitemapParts == nil { t.Fatalf("the fixture should serve its sitemap from parts (base %q, %d parts)", canonicalBase, len(corpus().sitemapParts)) } for _, host := range []string{"archive.example.org", "odd&host.example"} { r := httptest.NewRequest("GET", "/sitemap.xml", nil) r.Host = host want, err := corpus().marshalSitemap(requestBase(r)) if err != nil { t.Fatal(err) } if got := record(h, r).Body.String(); got != string(want) { t.Errorf("host %q: the sitemap written from parts differs from marshalSitemap", host) } } } // The big visitor-independent page bodies are executed once at startup // (prerenderPages). The page must look exactly the same whether it used // that copy or rendered its rows inline - which is what dev mode does, and // what every one of these pages falls back to if a block ever fails to // pre-render. func TestPrerenderedMatchesLive(t *testing.T) { buildFixture(t) if len(corpus().prerendered) == 0 { t.Fatal("nothing was pre-rendered") } pages := []string{ "/explore/ids", "/explore/ids?missing=1", "/glossary", "/browse?collection=sidtoday-archive", "/browse/program", "/browse/tag", // Category orderings are ASSEMBLED from pre-rendered rows rather // than pre-rendered whole, so each one has to match too. "/browse/tag?sort=count", "/browse/program?sort=count&order=asc", "/browse/agency?sort=name&order=desc", "/browse/people?sort=foia", // So are the doc tables, out of pre-rendered document rows. "/browse?view=table", "/browse?collection=sidtoday-archive&view=table&sort=title&order=desc", "/browse/program/PRISM?view=table", } was, wasRows, wasAZ, wasDocRows := corpus().prerendered, corpus().categoryRowHTML, corpus().azRowHTML, corpus().docRowHTML t.Cleanup(func() { corpus().prerendered, corpus().categoryRowHTML, corpus().azRowHTML, corpus().docRowHTML = was, wasRows, wasAZ, wasDocRows }) live := make([]string, len(pages)) corpus().prerendered = map[string]template.HTML{} // force the inline path corpus().categoryRowHTML, corpus().azRowHTML, corpus().docRowHTML = nil, nil, nil for i, p := range pages { live[i] = get(t, p).Body.String() } corpus().prerendered, corpus().categoryRowHTML, corpus().azRowHTML, corpus().docRowHTML = was, wasRows, wasAZ, wasDocRows for i, p := range pages { if got := get(t, p).Body.String(); got != live[i] { t.Errorf("%s: pre-rendered output differs from the live one", p) } } } // The table view's headings are its sorters, and the one marked active is // the order the rows are actually in - the page's default when the URL // names none (the SIDtoday tab sorts by date, not title). func TestDocTableActiveHeading(t *testing.T) { for path, want := range map[string]string{ "/browse?collection=sidtoday-archive&view=table": "Doc Date ▲", "/browse?view=table": "Title ▲", "/browse?view=table&sort=country&order=desc": "Countries ▼", "/browse?view=table&sort=bogus": "Title ▲", } { body := get(t, path).Body.String() if !strings.Contains(body, want) { t.Errorf("%s: no active heading %q", path, want) } if n := strings.Count(body, "▲") + strings.Count(body, "▼"); n != 1 { t.Errorf("%s: %d active headings, want 1", path, n) } } } // A reporter credited by two co-publishing outlets is one name in the // table's Journalists column, as on the cards. func TestDocTableJournalistsDeduped(t *testing.T) { buildFixture(t) d := testDoc(t, `{"id": "x", "publication": [ {"publisher": "A", "reporters": ["Greenwald, Glenn"]}, {"publisher": "B", "reporters": ["Greenwald, Glenn", "Poitras, Laura"]}]}`) var b strings.Builder err := tmplSet["list.html"].ExecuteTemplate(&b, "doc-table", listView{Docs: []*Document{&d}}) if err != nil { t.Fatal(err) } if n := strings.Count(b.String(), "/browse/journalist/Greenwald"); n != 1 { t.Errorf("Greenwald linked %d times, want once", n) } if !strings.Contains(b.String(), "/browse/journalist/Poitras") { t.Error("Poitras missing") } } // Without fetch metadata the Origin decides, and it omits a default port // the Host header may carry. func TestOriginIsHost(t *testing.T) { for _, c := range []struct { origin, host string want bool }{ {"https://example.org", "example.org", true}, {"https://example.org", "example.org:443", true}, {"https://EXAMPLE.org", "example.org", true}, {"http://127.0.0.1:8080", "127.0.0.1:8080", true}, {"http://[::1]:8080", "[::1]:8080", true}, {"https://example.org", "example.org:8443", false}, {"https://example.org:8443", "example.org", false}, {"http://example.org", "example.org:443", false}, {"https://evil.example", "example.org", false}, } { u, err := url.Parse(c.origin) if err != nil { t.Fatal(err) } if got := originIsHost(u, c.host); got != c.want { t.Errorf("originIsHost(%s, %s) = %v, want %v", c.origin, c.host, got, c.want) } } } func TestEnvBool(t *testing.T) { for v, want := range map[string]bool{"": false, "1": true, "true": true, "TRUE": true, "0": false, "false": false} { t.Setenv("SNOWCHIVE_TEST_BOOL", v) if got := envBool("SNOWCHIVE_TEST_BOOL"); got != want { t.Errorf("envBool(%q) = %v, want %v", v, got, want) } } } // TestBrowseValueSpelling: a value matches ignoring case, so another // spelling of it redirects to the one the lists link instead of serving a // second page for the same documents; the query string goes along. func TestBrowseValueSpelling(t *testing.T) { w := get(t, "/browse/program/prism?view=table") if w.Code != http.StatusMovedPermanently || w.Header().Get("Location") != "/browse/program/PRISM?view=table" { t.Errorf("/browse/program/prism: %d to %q, want 301 to /browse/program/PRISM?view=table", w.Code, w.Header().Get("Location")) } if w := get(t, "/browse/program/NOSUCHTHING"); w.Code != 200 { t.Errorf("an unknown value: %d, want the empty page", w.Code) } } // TestCardHighlightNeedsTheCategory: the cards on a value's page mark the // links to that value - not every link that happens to read the same, such // as the document date 2013 on the publication-year page for 2013. func TestCardHighlightNeedsTheCategory(t *testing.T) { if body := get(t, "/browse/published/2013").Body.String(); strings.Contains(body, "doc-created active") { t.Error("/browse/published/2013 marks the document-date links") } if body := get(t, "/browse/docdate/2013").Body.String(); !strings.Contains(body, "doc-created active") { t.Error("/browse/docdate/2013 does not mark the document-date links") } if body := get(t, "/browse/program/PRISM").Body.String(); !strings.Contains(body, `class="active">PRISM`) { t.Error("/browse/program/PRISM does not mark the program links") } } // TestUnknownSortIsNotEchoed: a sort key no list has is the default sort, // and stays out of the page's links and labels. func TestUnknownSortIsNotEchoed(t *testing.T) { for _, path := range []string{"/browse?sort=bogus", "/browse/program/PRISM?sort=bogus", "/search?q=prism&do_search=1&sort=bogus"} { body := get(t, path).Body.String() if strings.Contains(body, "sort=bogus") || strings.Contains(body, "Bogus") { t.Errorf("%s: the unknown sort is echoed", path) } } if body := get(t, "/search?q=prism&do_search=1").Body.String(); strings.Contains(body, "sort=relevance&order=desc") { t.Error("relevance offers a direction it does not have") } } // TestOutletYearLinksKeepTheScope: on the reported-only outlets table the // year headings open the reported-only year, like the rows and cells do. func TestOutletYearLinksKeepTheScope(t *testing.T) { body := get(t, "/explore/outlets?coll=news").Body.String() if !strings.Contains(body, `href="/browse/published/2013?coll=news"`) { t.Error("the 2013 heading drops coll=news") } } // pinSearch posts qs to /pinsearch from this site, with the given pinned // searches already in the cookie. func pinSearch(t *testing.T, qs string, pinned []string) *httptest.ResponseRecorder { t.Helper() mux, _ := buildFixture(t) r := httptest.NewRequest("POST", "/pinsearch", strings.NewReader(url.Values{"qs": {qs}, "back": {"/search"}}.Encode())) r.Header.Set("Content-Type", "application/x-www-form-urlencoded") r.Header.Set("Sec-Fetch-Site", "same-origin") if len(pinned) > 0 { r.AddCookie(&http.Cookie{Name: pinnedSearchesCookie, Value: strings.Join(pinned, "|")}) } return record(mux, r) } // TestPinListFull: a pin the cookie has no room for is refused, and the // reader lands on /pinned, which says so, rather than back on a page whose // button still reads "Pin" as though nothing happened. func TestPinListFull(t *testing.T) { qs := url.Values{"do_search": {"1"}, "fqc": {"and"}, "fqf": {"TE"}, "fqv": {"prism"}}.Encode() if w := pinSearch(t, qs, nil); w.Code != http.StatusSeeOther || w.Header().Get("Location") != "/search" { t.Fatalf("pinning into an empty list: %d to %q", w.Code, w.Header().Get("Location")) } // Filler searches up to the byte budget: stored entries are not // checked on read, so any escaped query string stands in for one. var fill []string for i := 0; ; i++ { e := url.QueryEscape(fmt.Sprintf("fqv=filler%02d", i) + strings.Repeat("x", 180)) if len(strings.Join(append(fill, e), "|")) > cookieValueMax { break } fill = append(fill, e) } // The last filler takes the cookie to exactly its budget. fill[len(fill)-1] += strings.Repeat("x", cookieValueMax-len(strings.Join(fill, "|"))) w := pinSearch(t, qs, fill) if w.Code != http.StatusSeeOther || w.Header().Get("Location") != "/pinned?full=1" { t.Errorf("pinning into a full list: %d to %q, want 303 to /pinned?full=1", w.Code, w.Header().Get("Location")) } if body := get(t, "/pinned?full=1").Body.String(); !strings.Contains(body, "Your pinned list is full") { t.Error("/pinned?full=1 does not say the list is full") } } // TestSavableSearchMatchesPinSearch: the search page offers the pin button // for exactly the searches /pinsearch accepts - one size limit for both. func TestSavableSearchMatchesPinSearch(t *testing.T) { long := strings.TrimSpace(strings.Repeat("abcdefgh ", 58)) // 58 words, under the word cap q := url.Values{"do_search": {"1"}, "fqc": {"and"}, "fqf": {"TE"}, "fqv": {long}} if len(q.Encode()) <= searchQSMax { t.Fatalf("test query is %d bytes, not over the %d limit", len(q.Encode()), searchQSMax) } if body := get(t, "/search?"+q.Encode()).Body.String(); strings.Contains(body, `action="/pinsearch"`) { t.Error("the search page offers to pin a search /pinsearch refuses") } if w := pinSearch(t, q.Encode(), nil); w.Code != http.StatusBadRequest { t.Errorf("/pinsearch with an over-long search: %d, want 400", w.Code) } short := url.Values{"do_search": {"1"}, "fqc": {"and"}, "fqf": {"TE"}, "fqv": {"prism"}} if body := get(t, "/search?"+short.Encode()).Body.String(); !strings.Contains(body, `action="/pinsearch"`) { t.Error("the search page does not offer to pin a short search") } } // TestAssetStamp: pages link each asset by its content hash; that URL is // cached for a year, and any other stamp only revalidates. func TestAssetStamp(t *testing.T) { body := get(t, "/").Body.String() want := "/static/style.css?v=" + staticHashes["style.css"] if !strings.Contains(body, `href="`+want+`"`) { t.Fatalf("home page does not link %s", want) } if cc := get(t, want).Header().Get("Cache-Control"); !strings.Contains(cc, "immutable") { t.Errorf("%s: Cache-Control %q, want immutable", want, cc) } w := get(t, "/static/style.css?v=0000000000000000") if cc := w.Header().Get("Cache-Control"); strings.Contains(cc, "immutable") || w.Header().Get("ETag") == "" { t.Errorf("a stale stamp: Cache-Control %q, ETag %q; want revalidation", cc, w.Header().Get("ETag")) } } // TestSitemapIgnoresFile: the sitemap is always the one built from the // loaded corpus - a sitemap.xml lying in the data or project directory is // never served, with -base or without it. func TestSitemapIgnoresFile(t *testing.T) { root := t.TempDir() writeFixtureFile(t, filepath.Join(root, "sitemap.xml"), xmlHeaderForTest+"https://stale.example/\n") withCorpus(t, root) prev := canonicalBase t.Cleanup(func() { canonicalBase = prev }) for _, base := range []string{"", "https://archive.example.org"} { canonicalBase = base loadData() if body := get(t, "/sitemap.xml").Body.String(); strings.Contains(body, "stale.example") { t.Errorf("base %q: a sitemap.xml file was served", base) } } } const xmlHeaderForTest = `` + "\n" // TestPinnedSearchListed: /pinned lists a pinned search with its label, // its link and the unpin form carrying the canonical query string. func TestPinnedSearchListed(t *testing.T) { qs := url.Values{"do_search": {"1"}, "fqc": {"and"}, "fqf": {"TE"}, "fqv": {"prism"}}.Encode() body := get(t, "/pinned", &http.Cookie{Name: pinnedSearchesCookie, Value: url.QueryEscape(qs)}).Body.String() for _, want := range []string{`href="/search?` + template.HTMLEscapeString(qs) + `"`, `name="qs" value="` + template.HTMLEscapeString(qs) + `"`, ">prism<"} { if !strings.Contains(body, want) { t.Errorf("/pinned lacks %s", want) } } } // TestPageTitles: every page names itself in the tab, not just the site. func TestPageTitles(t *testing.T) { buildFixture(t) // the article key comes from the loaded corpus for path, want := range map[string]string{ "/browse/program/PRISM": "Program: PRISM - Snowden Archive", "/browse/program": "Browse by Programs - Snowden Archive", "/search?q=prism&do_search=1": "prism - Search - Snowden Archive", "/search": "Search - Snowden Archive", "/browse?collection=foia": "FOIA documents - Snowden Archive", "/pinned": "Pinned - Snowden Archive", "/nosuchpage": "Page not found - Snowden Archive", "/explore/pages": "- Snowden Archive", "/article/" + corpus().articleByLink["https://example.org/prism"].Key: "NSA Prism program - Snowden Archive", } { if body := get(t, path).Body.String(); !strings.Contains(body, want) { t.Errorf("%s: no %q", path, want) } } } // TestNoBlockContentInHeadings: a heading holds text and inline controls // only - a form or a disclosure inside one is invalid HTML and lands in // the heading's accessible name. Forms tie their buttons from outside. func TestNoBlockContentInHeadings(t *testing.T) { heading := regexp.MustCompile(`(?s)]*>(.*?)`) block := regexp.MustCompile(`<(form|details|div|p|ul|ol|table)\b`) cookie := &http.Cookie{Name: pinnedCookie, Value: "prism-slides"} recent := []*http.Cookie{{Name: "pref_track_recent", Value: "1"}, {Name: recentDocsCookie, Value: "prism-slides"}} pages := map[string][]*http.Cookie{ "/pinned": {cookie}, "/": recent, "/search?q=prism&do_search=1": nil, "/doc/prism-slides": nil, } for path, cookies := range pages { body := get(t, path, cookies...).Body.String() for _, m := range heading.FindAllStringSubmatch(body, -1) { if b := block.FindString(m[2]); b != "" { t.Errorf("%s: %s inside ", path, b, m[1]) } } } if body := get(t, "/pinned", cookie).Body.String(); !strings.Contains(body, `form="unpin-prism-slides"`) || !strings.Contains(body, `id="unpin-prism-slides"`) { t.Error("/pinned: the unpin button is not tied to its form") } } // TestGlossarySeeAlsoResolves: a see-also name links to the entry it // names - through an alias or another case too - and a name no entry has // is plain text rather than a link to nowhere. func TestGlossarySeeAlsoResolves(t *testing.T) { root := t.TempDir() writeFixtureFile(t, filepath.Join(root, "glossary.json"), `[ {"term": "XKEYSCORE", "definition": "A search system.", "aka": ["XKS"], "see": ["rt-rg", "TURMOIL", "NOSUCHTERM"]}, {"term": "RT-RG", "definition": "Real time regional gateway."}, {"term": "TURMOIL", "definition": "Passive collection.", "see": ["xks"]} ]`) withCorpus(t, root) body := get(t, "/glossary").Body.String() for _, want := range []string{`rt-rg`, `TURMOIL`, `xks`} { if !strings.Contains(body, want) { t.Errorf("/glossary lacks %s", want) } } if strings.Contains(body, `href="#term-NOSUCHTERM"`) { t.Error("/glossary links a see-also name no entry has") } } // A pager or sort link appends its parameter to queryBase, with nothing // left over when the page has no query of its own. func TestQueryBase(t *testing.T) { for want, q := range map[string]url.Values{ "/browse?": {}, "/browse?view=table&": {"view": {"table"}}, "/browse?coll=foia&view=card&": {"view": {"card"}, "coll": {"foia"}}, } { if got := queryBase("/browse", q); got != want { t.Errorf("queryBase(%v) = %q, want %q", q, got, want) } } }