first commit
This commit is contained in:
commit
8c4dcae475
89 changed files with 24623 additions and 0 deletions
122
tests/api_test.go
Normal file
122
tests/api_test.go
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAPIPageSize(t *testing.T) {
|
||||
for raw, want := range map[string]int{
|
||||
"": apiPerPageDefault,
|
||||
"25": 25,
|
||||
"500": 500,
|
||||
"1000": apiMaxPerPage,
|
||||
"99999999": apiMaxPerPage,
|
||||
"0": apiPerPageDefault, // not a page size; fall back
|
||||
"-5": apiPerPageDefault,
|
||||
"lots": apiPerPageDefault,
|
||||
"all": 0,
|
||||
} {
|
||||
v := url.Values{}
|
||||
if raw != "" {
|
||||
v.Set("per_page", raw)
|
||||
}
|
||||
if got := apiPageSize(v); got != want {
|
||||
t.Errorf("apiPageSize(per_page=%q) = %d, want %d", raw, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPIRespPerPage(t *testing.T) {
|
||||
for _, c := range []struct{ perPage, total, want int }{
|
||||
{50, 3106, 50},
|
||||
{0, 3106, 3106},
|
||||
{0, 0, 1},
|
||||
} {
|
||||
if got := apiRespPerPage(c.perPage, c.total); got != c.want {
|
||||
t.Errorf("apiRespPerPage(%d, %d) = %d, want %d", c.perPage, c.total, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPIVersionsShape(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeTestFile(t, filepath.Join(root, "doc-a", "metadata.json"), `{
|
||||
"title": "A", "type": "REPORTED",
|
||||
"publication": [{"publisher": "Le Monde", "version": "entier"}],
|
||||
"versions": {"entier": {"changed_pages": [1]}}
|
||||
}`)
|
||||
writeTestFile(t, filepath.Join(root, "doc-a", "a.pdf"), "%PDF-1.4")
|
||||
writeTestFile(t, filepath.Join(root, "doc-a", "versions", "entier", "entier.pdf"), "%PDF-1.4")
|
||||
withCorpus(t, root)
|
||||
|
||||
type versionsObj struct {
|
||||
Versions map[string]VersionInfo `json:"versions"`
|
||||
}
|
||||
var detail struct {
|
||||
versionsObj
|
||||
VersionCopies []apiVersion `json:"version_copies"`
|
||||
}
|
||||
decode := func(path string, v interface{}) {
|
||||
t.Helper()
|
||||
w := get(t, path)
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("%s: %d", path, w.Code)
|
||||
}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), v); err != nil {
|
||||
t.Fatalf("%s: %v", path, err)
|
||||
}
|
||||
}
|
||||
decode("/api/doc/doc-a", &detail)
|
||||
if len(detail.Versions["entier"].ChangedPages) != 1 {
|
||||
t.Errorf("/api/doc versions = %+v, want the metadata object", detail.Versions)
|
||||
}
|
||||
if len(detail.VersionCopies) != 1 || detail.VersionCopies[0].Slug != "entier" {
|
||||
t.Errorf("/api/doc version_copies = %+v, want the entier copy", detail.VersionCopies)
|
||||
}
|
||||
var list struct{ Docs []versionsObj }
|
||||
decode("/api/docs", &list)
|
||||
var search struct{ Results []versionsObj }
|
||||
decode("/api/search?q=A", &search)
|
||||
for name, docs := range map[string][]versionsObj{"/api/docs": list.Docs, "/api/search": search.Results} {
|
||||
if len(docs) != 1 || len(docs[0].Versions["entier"].ChangedPages) != 1 {
|
||||
t.Errorf("%s versions = %+v, want the metadata object", name, docs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPIFacetsRevalidates(t *testing.T) {
|
||||
w := get(t, "/api/facets")
|
||||
tag := w.Header().Get("ETag")
|
||||
if w.Code != 200 || tag == "" || w.Header().Get("Access-Control-Allow-Origin") != "*" {
|
||||
t.Fatalf("/api/facets: %d, ETag %q, CORS %q", w.Code, tag, w.Header().Get("Access-Control-Allow-Origin"))
|
||||
}
|
||||
mux, _ := buildFixture(t)
|
||||
r := httptest.NewRequest("GET", "/api/facets", nil)
|
||||
r.Header.Set("If-None-Match", tag)
|
||||
if w := record(mux, r); w.Code != http.StatusNotModified || w.Body.Len() != 0 {
|
||||
t.Errorf("revalidation: %d with %d bytes, want 304 and none", w.Code, w.Body.Len())
|
||||
}
|
||||
}
|
||||
|
||||
// /api/docs takes coll= as /api/search and the pages do.
|
||||
func TestAPIDocsCollections(t *testing.T) {
|
||||
total := func(query string) int {
|
||||
var body struct{ Total int }
|
||||
if err := json.Unmarshal(get(t, "/api/docs?"+query).Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("%s: %v", query, err)
|
||||
}
|
||||
return body.Total
|
||||
}
|
||||
all, news, foia := total(""), total("coll=news"), total("coll=foia")
|
||||
if news == 0 || foia == 0 || news+foia >= all {
|
||||
t.Fatalf("fixture totals: %d documents, %d news, %d foia", all, news, foia)
|
||||
}
|
||||
if got := total("coll=news&coll=foia"); got != news+foia {
|
||||
t.Errorf("coll=news&coll=foia lists %d, want %d", got, news+foia)
|
||||
}
|
||||
}
|
||||
211
tests/assemblies_test.go
Normal file
211
tests/assemblies_test.go
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func writeAssemblyDoc(t *testing.T, root, id string, meta map[string]interface{}, pages []string, versions map[string][]string) {
|
||||
t.Helper()
|
||||
dir := filepath.Join(root, id)
|
||||
b, err := json.Marshal(meta)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeFixtureFile(t, filepath.Join(dir, "metadata.json"), string(b))
|
||||
writeFixtureFile(t, filepath.Join(dir, id+".pdf"), "%PDF-1.4\n")
|
||||
for _, p := range pages {
|
||||
writeFixtureFile(t, filepath.Join(dir, "images", p), fixtureJPEG)
|
||||
}
|
||||
for slug, imgs := range versions {
|
||||
v := filepath.Join(dir, "versions", slug)
|
||||
writeFixtureFile(t, filepath.Join(v, slug+".pdf"), "%PDF-1.4\n")
|
||||
for _, p := range imgs {
|
||||
writeFixtureFile(t, filepath.Join(v, "images", p), fixtureJPEG)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assemblyCorpus(t *testing.T, manifest string) string {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
writeAssemblyDoc(t, root, "deck", map[string]interface{}{
|
||||
"title": "The Deck",
|
||||
"publication": []map[string]interface{}{
|
||||
{"publisher": "Outlet A"},
|
||||
{"publisher": "Partner", "reporting_only": true},
|
||||
{"publisher": "Outlet B", "version": "fuller"},
|
||||
},
|
||||
}, []string{"page-1.jpg", "page-2.jpg", "page-3.jpg"}, map[string][]string{"fuller": {"page-1.jpg", "page-2.jpg"}})
|
||||
writeAssemblyDoc(t, root, "single", map[string]interface{}{
|
||||
"title": "One Slide",
|
||||
"publication": []map[string]interface{}{{"publisher": "Outlet C"}},
|
||||
}, []string{"page-1.jpg"}, nil)
|
||||
writeAssemblyDoc(t, root, "bystander", map[string]interface{}{"title": "Bystander"}, []string{"page-1.jpg"}, nil)
|
||||
if manifest != "" {
|
||||
writeFixtureFile(t, filepath.Join(root, "assemblies.json"), manifest)
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
const assemblyManifest = `[{
|
||||
"id": "set", "title": "A Set", "note": "Order is a guess.",
|
||||
"slides": [
|
||||
{"doc": "deck", "pages": "1-2", "alt": [{"doc": "deck", "version": "fuller", "pages": "1-2"}]},
|
||||
{"doc": "deck", "version": "fuller", "pages": "2", "note": "Only here."},
|
||||
{"doc": "single", "pages": "1"}
|
||||
]
|
||||
}]`
|
||||
|
||||
func TestParsePageList(t *testing.T) {
|
||||
for in, want := range map[string][]int{
|
||||
"8": {8}, "1-3": {1, 2, 3}, "1,3,5-6": {1, 3, 5, 6}, " 2 , 4 ": {2, 4}, "9,1": {9, 1},
|
||||
} {
|
||||
got, err := parsePageList(in)
|
||||
if err != nil || !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("parsePageList(%q) = %v, %v; want %v", in, got, err, want)
|
||||
}
|
||||
}
|
||||
for _, in := range []string{"", "a", "0", "3-1", "1-", "1,,2"} {
|
||||
if got, err := parsePageList(in); err == nil {
|
||||
t.Errorf("parsePageList(%q) = %v, want an error", in, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssemblyResolves(t *testing.T) {
|
||||
withCorpus(t, assemblyCorpus(t, assemblyManifest))
|
||||
a := corpus().assemblyIndex["set"]
|
||||
if a == nil {
|
||||
t.Fatal("assembly not loaded")
|
||||
}
|
||||
if len(a.Slides) != 4 {
|
||||
t.Fatalf("%d slides, want 4", len(a.Slides))
|
||||
}
|
||||
for i, s := range a.Slides {
|
||||
if s.Pos != i+1 {
|
||||
t.Errorf("slide %d has Pos %d", i+1, s.Pos)
|
||||
}
|
||||
}
|
||||
first := a.Slides[0]
|
||||
if first.Image != "/files/deck/images/page-1.jpg" || first.Source != "Outlet A" {
|
||||
t.Errorf("first slide %+v", first)
|
||||
}
|
||||
if len(first.Alts) != 1 || first.Alts[0].Image != "/files/deck/versions/fuller/images/page-1.jpg" || first.Alts[0].Source != "Outlet B" {
|
||||
t.Errorf("first slide alts %+v", first.Alts)
|
||||
}
|
||||
if a.Slides[2].Note != "Only here." || a.Slides[2].Path != "/doc/deck/versions/fuller" {
|
||||
t.Errorf("third slide %+v", a.Slides[2])
|
||||
}
|
||||
if fmt.Sprint(a.DocList) != "[deck single]" {
|
||||
t.Errorf("DocList %v", a.DocList)
|
||||
}
|
||||
if a.Copies != 3 {
|
||||
t.Errorf("Copies %d, want 3", a.Copies)
|
||||
}
|
||||
if len(corpus().assembliesByDoc["single"]) != 1 || len(corpus().assembliesByDoc["bystander"]) != 0 {
|
||||
t.Errorf("assembliesByDoc %v", corpus().assembliesByDoc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssemblyPages(t *testing.T) {
|
||||
withCorpus(t, assemblyCorpus(t, assemblyManifest))
|
||||
h := withBaseHeaders(newMux())
|
||||
body := func(path string, code int) string {
|
||||
t.Helper()
|
||||
w := record(h, httptest_NewGet(path))
|
||||
if w.Code != code {
|
||||
t.Fatalf("%s: %d, want %d", path, w.Code, code)
|
||||
}
|
||||
return w.Body.String()
|
||||
}
|
||||
list := body("/explore/assemblies", 200)
|
||||
if !strings.Contains(list, `href="/explore/assemblies/set"`) || !strings.Contains(list, "4 slides from 3 copies") {
|
||||
t.Errorf("/explore/assemblies:\n%s", list)
|
||||
}
|
||||
page := body("/explore/assemblies/set", 200)
|
||||
for _, want := range []string{
|
||||
"<title>A Set - Snowden Archive</title>", "from 3 separately published copies.", "Order is a guess.", "not a document in its own right",
|
||||
`id="lb-4"`, `href="#lb-a1-1"`, `id="lb-a1-1"`, "Other copy: Outlet B", "Back to slide 1", "(copy: fuller)", "Outlet C", "· Only here.",
|
||||
} {
|
||||
if !strings.Contains(page, want) {
|
||||
t.Errorf("/explore/assemblies/set lacks %q", want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(page, `id="lb-5"`) || strings.Contains(page, "Partner") {
|
||||
t.Errorf("/explore/assemblies/set has a fifth overlay or a reporting-only credit")
|
||||
}
|
||||
if !strings.Contains(body("/doc/single", 200), `href="/explore/assemblies/set"`) {
|
||||
t.Errorf("/doc/single has no assembly link")
|
||||
}
|
||||
if strings.Contains(body("/doc/bystander", 200), "/explore/assemblies/") {
|
||||
t.Errorf("/doc/bystander links an assembly")
|
||||
}
|
||||
// Assemblies live under /explore only: listed there, not on /browse.
|
||||
if !strings.Contains(body("/explore", 200), `href="/explore/assemblies"`) {
|
||||
t.Errorf("/explore has no assemblies link")
|
||||
}
|
||||
if strings.Contains(body("/browse", 200), "assemblies") {
|
||||
t.Errorf("/browse still links assemblies")
|
||||
}
|
||||
body("/explore/assemblies/nope", 404)
|
||||
if !slicesContains(corpus().sitemapPaths, "/explore/assemblies/set") {
|
||||
t.Errorf("sitemap lacks the assembly page")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssemblyProblemsWarn(t *testing.T) {
|
||||
cases := map[string]struct {
|
||||
manifest string
|
||||
slides int
|
||||
loaded bool
|
||||
}{
|
||||
"missing doc": {`[{"id":"x","title":"X","slides":[{"doc":"ghost","pages":"1"},{"doc":"single","pages":"1"}]}]`, 1, true},
|
||||
"missing page": {`[{"id":"x","title":"X","slides":[{"doc":"deck","pages":"3-4"}]}]`, 1, true},
|
||||
"bad version": {`[{"id":"x","title":"X","slides":[{"doc":"deck","version":"no","pages":"1"},{"doc":"single","pages":"1"}]}]`, 1, true},
|
||||
"bad pages": {`[{"id":"x","title":"X","slides":[{"doc":"deck","pages":"two"},{"doc":"single","pages":"1"}]}]`, 1, true},
|
||||
"alt mismatch": {`[{"id":"x","title":"X","slides":[{"doc":"deck","pages":"1-2","alt":[{"doc":"single","pages":"1"}]}]}]`, 2, true},
|
||||
"nothing left": {`[{"id":"x","title":"X","slides":[{"doc":"ghost","pages":"1"}]}]`, 0, false},
|
||||
"unsafe id": {`[{"id":"a/b","title":"X","slides":[{"doc":"single","pages":"1"}]}]`, 0, false},
|
||||
"malformed": {`[{`, 0, false},
|
||||
}
|
||||
for name, c := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
withCorpus(t, assemblyCorpus(t, c.manifest))
|
||||
if loadWarnings == 0 {
|
||||
t.Errorf("no warning")
|
||||
}
|
||||
a := corpus().assemblyIndex["x"]
|
||||
if (a != nil) != c.loaded {
|
||||
t.Fatalf("loaded = %v, want %v", a != nil, c.loaded)
|
||||
}
|
||||
if a != nil && len(a.Slides) != c.slides {
|
||||
t.Errorf("%d slides, want %d", len(a.Slides), c.slides)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoAssembliesFile(t *testing.T) {
|
||||
withCorpus(t, assemblyCorpus(t, ""))
|
||||
if len(corpus().assemblies) != 0 {
|
||||
t.Fatalf("assemblies %v", corpus().assemblies)
|
||||
}
|
||||
h := withBaseHeaders(newMux())
|
||||
if w := record(h, httptest_NewGet("/explore/assemblies")); w.Code != 200 {
|
||||
t.Errorf("/explore/assemblies: %d", w.Code)
|
||||
}
|
||||
if w := record(h, httptest_NewGet("/explore")); strings.Contains(w.Body.String(), `href="/explore/assemblies"`) {
|
||||
t.Errorf("/explore links assemblies with none loaded")
|
||||
}
|
||||
if slicesContains(corpus().sitemapPaths, "/explore/assemblies") {
|
||||
t.Errorf("sitemap lists assemblies with none loaded")
|
||||
}
|
||||
}
|
||||
|
||||
func slicesContains(s []string, v string) bool { return slices.Contains(s, v) }
|
||||
312
tests/bench_test.go
Normal file
312
tests/bench_test.go
Normal file
|
|
@ -0,0 +1,312 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Page benchmarks against a REAL corpus - the fixture is four documents, so
|
||||
// it cannot show what 3,000 do. Point $SNOWCHIVE_BENCH_ROOT at an archive
|
||||
// root (the directory holding data/ and project/) and run:
|
||||
// tests/run.sh -run XXX -bench . -benchtime 40x
|
||||
//
|
||||
// What they are for: every page but /search is built from data indexed at
|
||||
// load, so a request should cost what its OUTPUT costs (template execution)
|
||||
// and nothing more. A page whose time or allocations grow with the CORPUS
|
||||
// rather than with the page has picked up a scan.
|
||||
var (
|
||||
benchOnce sync.Once
|
||||
benchMux http.Handler
|
||||
)
|
||||
|
||||
func benchCorpus(tb testing.TB) http.Handler {
|
||||
root := os.Getenv("SNOWCHIVE_BENCH_ROOT")
|
||||
if root == "" {
|
||||
tb.Skip("set SNOWCHIVE_BENCH_ROOT to an archive root to run the page benchmarks")
|
||||
}
|
||||
benchOnce.Do(func() {
|
||||
setRoot(root)
|
||||
devMode = false
|
||||
apiEnabled = true
|
||||
initAssets()
|
||||
if err := parseTemplates(); err != nil {
|
||||
tb.Fatal(err)
|
||||
}
|
||||
loadData()
|
||||
benchMux = withBaseHeaders(newMux())
|
||||
})
|
||||
return benchMux
|
||||
}
|
||||
|
||||
func benchPage(b *testing.B, path string) {
|
||||
h := benchCorpus(b)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
r := httptest.NewRequest("GET", path, nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, r)
|
||||
if w.Code != 200 {
|
||||
b.Fatalf("%s: %d", path, w.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkPageHome(b *testing.B) { benchPage(b, "/") }
|
||||
func BenchmarkPageBrowse(b *testing.B) { benchPage(b, "/browse") }
|
||||
func BenchmarkPageArchiveTab(b *testing.B) { benchPage(b, "/browse?collection=sidtoday-archive") }
|
||||
func BenchmarkPageCategoryList(b *testing.B) { benchPage(b, "/browse/program") }
|
||||
func BenchmarkPageCategoryValue(b *testing.B) { benchPage(b, "/browse/agency/NSA") }
|
||||
func BenchmarkPageDoc(b *testing.B) {
|
||||
h := benchCorpus(b)
|
||||
if len(corpus().archive.Docs) == 0 {
|
||||
b.Skip("empty corpus")
|
||||
}
|
||||
benchPageWith(b, h, "/doc/"+corpus().archive.Docs[0].ID)
|
||||
}
|
||||
func BenchmarkPageExploreIDs(b *testing.B) { benchPage(b, "/explore/ids") }
|
||||
func BenchmarkPageGlossary(b *testing.B) { benchPage(b, "/glossary") }
|
||||
func BenchmarkPageSitemap(b *testing.B) { benchPage(b, "/sitemap.xml") }
|
||||
func BenchmarkPageFacetsAPI(b *testing.B) { benchPage(b, "/api/facets") }
|
||||
func BenchmarkPageSearch(b *testing.B) { benchPage(b, "/search?q=nsa&do_search=1") }
|
||||
|
||||
func benchPageWith(b *testing.B, h http.Handler, path string) {
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
r := httptest.NewRequest("GET", path, nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, r)
|
||||
if w.Code != 200 {
|
||||
b.Fatalf("%s: %d", path, w.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkPageCategoryTag(b *testing.B) { benchPage(b, "/browse/tag") }
|
||||
func BenchmarkPageCategoryPeople(b *testing.B) { benchPage(b, "/browse/people") }
|
||||
func BenchmarkPageCategoryTagSorted(b *testing.B) { benchPage(b, "/browse/tag?sort=count") }
|
||||
func BenchmarkPageExploreMetric(b *testing.B) { benchPage(b, "/explore/pages") }
|
||||
func BenchmarkPageCodewords(b *testing.B) { benchPage(b, "/explore/terms") }
|
||||
func BenchmarkPageOutlets(b *testing.B) { benchPage(b, "/explore/outlets") }
|
||||
func BenchmarkPageClassPatterns(b *testing.B) { benchPage(b, "/explore/classification") }
|
||||
func BenchmarkPageArticles(b *testing.B) { benchPage(b, "/articles") }
|
||||
func BenchmarkPageChangelog(b *testing.B) { benchPage(b, "/changelog") }
|
||||
func BenchmarkPageAbout(b *testing.B) { benchPage(b, "/about") }
|
||||
func BenchmarkPageAPIDocs(b *testing.B) { benchPage(b, "/api/docs") }
|
||||
func BenchmarkPageExploreIndex(b *testing.B) { benchPage(b, "/explore") }
|
||||
func BenchmarkPageContactSheet(b *testing.B) {
|
||||
h := benchCorpus(b)
|
||||
id := ""
|
||||
for i := range corpus().archive.Docs {
|
||||
if len(corpus().archive.Docs[i].Images) > 40 {
|
||||
id = corpus().archive.Docs[i].ID
|
||||
break
|
||||
}
|
||||
}
|
||||
if id == "" {
|
||||
b.Skip("no document with many pages")
|
||||
}
|
||||
benchPageWith(b, h, "/doc/"+id+"/pages")
|
||||
}
|
||||
|
||||
// Pathological documents: the biggest thing in the corpus is what breaks
|
||||
// first. What IS the biggest thing?
|
||||
func TestProbeExtremes(t *testing.T) {
|
||||
if os.Getenv("SNOWCHIVE_BENCH_ROOT") == "" {
|
||||
t.Skip("needs a real corpus")
|
||||
}
|
||||
benchCorpus(t)
|
||||
var maxImgs, maxText, maxPubs, maxVersions *Document
|
||||
for i := range corpus().archive.Docs {
|
||||
d := &corpus().archive.Docs[i]
|
||||
if maxImgs == nil || len(d.Images) > len(maxImgs.Images) {
|
||||
maxImgs = d
|
||||
}
|
||||
if maxText == nil || len(d.FullText) > len(maxText.FullText) {
|
||||
maxText = d
|
||||
}
|
||||
if maxPubs == nil || len(d.Publication) > len(maxPubs.Publication) {
|
||||
maxPubs = d
|
||||
}
|
||||
if maxVersions == nil || len(d.Versions) > len(maxVersions.Versions) {
|
||||
maxVersions = d
|
||||
}
|
||||
}
|
||||
h := benchCorpus(t)
|
||||
timeIt := func(p string) time.Duration {
|
||||
start := time.Now()
|
||||
r := httptest.NewRequest("GET", p, nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, r)
|
||||
return time.Since(start)
|
||||
}
|
||||
fmt.Printf("most pages: %d %s doc %s, contact sheet %s\n", len(maxImgs.Images), trunc30(maxImgs.ID),
|
||||
timeIt("/doc/"+maxImgs.ID).Round(time.Millisecond), timeIt("/doc/"+maxImgs.ID+"/pages").Round(time.Millisecond))
|
||||
fmt.Printf("most text: %d KB %s doc page %s\n", len(maxText.FullText)/1024, trunc30(maxText.ID),
|
||||
timeIt("/doc/"+maxText.ID).Round(time.Millisecond))
|
||||
fmt.Printf("most pubs: %d %s\n", len(maxPubs.Publication), trunc30(maxPubs.ID))
|
||||
fmt.Printf("most copies: %d %s\n", len(maxVersions.Versions), trunc30(maxVersions.ID))
|
||||
}
|
||||
|
||||
// Distribution, not the median: which document pages are slow, and why?
|
||||
func TestProbeDocPageDistribution(t *testing.T) {
|
||||
if os.Getenv("SNOWCHIVE_BENCH_ROOT") == "" {
|
||||
t.Skip("needs a real corpus")
|
||||
}
|
||||
h := benchCorpus(t)
|
||||
totalImgs := 0
|
||||
for i := range corpus().archive.Docs {
|
||||
totalImgs += len(corpus().archive.Docs[i].Images)
|
||||
}
|
||||
var ds []time.Duration
|
||||
for i := 0; i < len(corpus().archive.Docs); i += 7 { // a sample, every 7th
|
||||
d := &corpus().archive.Docs[i]
|
||||
start := time.Now()
|
||||
r := httptest.NewRequest("GET", "/doc/"+d.ID, nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, r)
|
||||
ds = append(ds, time.Since(start))
|
||||
}
|
||||
sort.Slice(ds, func(i, j int) bool { return ds[i] < ds[j] })
|
||||
p := func(q float64) time.Duration { return ds[int(float64(len(ds)-1)*q)] }
|
||||
fmt.Printf("%d page images in the corpus (%d docs sampled)\n", totalImgs, len(ds))
|
||||
fmt.Printf("doc page: p50 %s p90 %s p99 %s max %s\n",
|
||||
p(0.5).Round(time.Microsecond), p(0.9).Round(time.Microsecond),
|
||||
p(0.99).Round(time.Millisecond), ds[len(ds)-1].Round(time.Millisecond))
|
||||
}
|
||||
|
||||
func TestProbeWorstDocSplit(t *testing.T) {
|
||||
if os.Getenv("SNOWCHIVE_BENCH_ROOT") == "" {
|
||||
t.Skip("needs a real corpus")
|
||||
}
|
||||
h := benchCorpus(t)
|
||||
var worst *Document
|
||||
for i := range corpus().archive.Docs {
|
||||
if worst == nil || len(corpus().archive.Docs[i].Images) > len(worst.Images) {
|
||||
worst = &corpus().archive.Docs[i]
|
||||
}
|
||||
}
|
||||
run := func(path string, cookies ...*http.Cookie) (time.Duration, int) {
|
||||
var best time.Duration
|
||||
var size int
|
||||
for i := 0; i < 5; i++ {
|
||||
r := httptest.NewRequest("GET", path, nil)
|
||||
for _, c := range cookies {
|
||||
r.AddCookie(c)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
start := time.Now()
|
||||
h.ServeHTTP(w, r)
|
||||
d := time.Since(start)
|
||||
if i == 0 || d < best {
|
||||
best = d
|
||||
}
|
||||
size = w.Body.Len()
|
||||
}
|
||||
return best, size
|
||||
}
|
||||
full, fullSize := run("/doc/" + worst.ID)
|
||||
textOnly, textSize := run("/doc/"+worst.ID, &http.Cookie{Name: "pref_text_only", Value: "1"})
|
||||
sheet, sheetSize := run("/doc/" + worst.ID + "/pages")
|
||||
fmt.Printf("worst doc: %d images, %d KB transcript\n", len(worst.Images), len(worst.FullText)/1024)
|
||||
fmt.Printf(" full page %6s %d KB\n", full.Round(time.Microsecond), fullSize/1024)
|
||||
fmt.Printf(" text-only %6s %d KB (no image grid, no lightbox)\n", textOnly.Round(time.Microsecond), textSize/1024)
|
||||
fmt.Printf(" contact sheet %6s %d KB\n", sheet.Round(time.Microsecond), sheetSize/1024)
|
||||
}
|
||||
|
||||
func trunc30(s string) string {
|
||||
if len(s) > 30 {
|
||||
return s[:30]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// TestSoak hammers the server for a while and watches its memory. The claim
|
||||
// it checks: nothing accumulates between requests - no cache, no growing
|
||||
// map, no goroutine left behind. Run it against a real corpus:
|
||||
//
|
||||
// SNOWCHIVE_BENCH_ROOT=/path/to/archive tests/run.sh -run TestSoak -timeout 30m
|
||||
//
|
||||
// SNOWCHIVE_SOAK_REQUESTS sets the count (default 20,000).
|
||||
func TestSoak(t *testing.T) {
|
||||
h := benchCorpus(t)
|
||||
n := 20000
|
||||
if v := os.Getenv("SNOWCHIVE_SOAK_REQUESTS"); v != "" {
|
||||
if parsed, err := strconv.Atoi(v); err == nil && parsed > 0 {
|
||||
n = parsed
|
||||
}
|
||||
}
|
||||
paths := []string{"/", "/browse", "/browse?collection=foia", "/browse/program",
|
||||
"/browse/tag?sort=count", "/browse/agency/NSA", "/search?q=nsa&do_search=1",
|
||||
"/search?q=%22upstream+collection%22&do_search=1", "/explore/ids", "/glossary",
|
||||
"/articles", "/api/facets", "/api/search?q=prism", "/sitemap.xml"}
|
||||
cookies := []*http.Cookie{
|
||||
{Name: "pref_theme", Value: "dark"},
|
||||
{Name: "pref_per_page", Value: "100"},
|
||||
{Name: "pref_text_only", Value: "1"},
|
||||
}
|
||||
|
||||
read := func() (heap, sys uint64, goroutines int) {
|
||||
runtime.GC()
|
||||
var m runtime.MemStats
|
||||
runtime.ReadMemStats(&m)
|
||||
return m.HeapAlloc, m.Sys, runtime.NumGoroutine()
|
||||
}
|
||||
// Warm up first: the pre-rendered bodies and the first template
|
||||
// executions allocate once and stay.
|
||||
for i := 0; i < 200; i++ {
|
||||
r := httptest.NewRequest("GET", paths[i%len(paths)], nil)
|
||||
h.ServeHTTP(httptest.NewRecorder(), r)
|
||||
}
|
||||
heap0, sys0, gor0 := read()
|
||||
|
||||
start := time.Now()
|
||||
var wg sync.WaitGroup
|
||||
const workers = 8
|
||||
for w := 0; w < workers; w++ {
|
||||
wg.Add(1)
|
||||
go func(id int) {
|
||||
defer wg.Done()
|
||||
for i := id; i < n; i += workers {
|
||||
r := httptest.NewRequest("GET", paths[i%len(paths)], nil)
|
||||
if i%4 == 0 {
|
||||
r.AddCookie(cookies[i%len(cookies)])
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, r)
|
||||
if rec.Code != 200 {
|
||||
t.Errorf("%s: %d", paths[i%len(paths)], rec.Code)
|
||||
return
|
||||
}
|
||||
}
|
||||
}(w)
|
||||
}
|
||||
wg.Wait()
|
||||
elapsed := time.Since(start)
|
||||
heap1, sys1, gor1 := read()
|
||||
|
||||
mb := func(b uint64) float64 { return float64(b) / (1 << 20) }
|
||||
fmt.Printf("%d requests in %s (%.0f req/s, %d workers)\n",
|
||||
n, elapsed.Round(time.Millisecond), float64(n)/elapsed.Seconds(), workers)
|
||||
fmt.Printf("heap %.1f MB -> %.1f MB (%+.1f)\n", mb(heap0), mb(heap1), mb(heap1)-mb(heap0))
|
||||
fmt.Printf("sys %.1f MB -> %.1f MB (%+.1f)\n", mb(sys0), mb(sys1), mb(sys1)-mb(sys0))
|
||||
fmt.Printf("goroutines %d -> %d\n", gor0, gor1)
|
||||
|
||||
if gor1 > gor0+2 {
|
||||
t.Errorf("goroutines grew from %d to %d", gor0, gor1)
|
||||
}
|
||||
// The heap after a GC should be the corpus and the pre-rendered pages,
|
||||
// not a function of how many requests have been served.
|
||||
if grown := mb(heap1) - mb(heap0); grown > 32 {
|
||||
t.Errorf("heap grew by %.1f MB over %d requests - something is accumulating", grown, n)
|
||||
}
|
||||
}
|
||||
78
tests/bugs_test.go
Normal file
78
tests/bugs_test.go
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A hyphenated term must match its words in sequence, not just the first one.
|
||||
func TestHyphenatedTermMatchesInSequence(t *testing.T) {
|
||||
hit := Document{ID: "hit", FullText: "A cyber attack on the power grid."}
|
||||
miss := Document{ID: "miss", FullText: "Cyber policy, and an attack helicopter."}
|
||||
for _, d := range []*Document{&hit, &miss} {
|
||||
setBodySearchText(d)
|
||||
d.StemmedText, d.SearchText = d.StemmedFull, d.FoldedFull
|
||||
}
|
||||
|
||||
for _, field := range []string{"FT", "TE"} {
|
||||
if !compileQuery([]string{"cyber-attack"}, []string{field}, []string{"and"}).matches(&hit) {
|
||||
t.Errorf("%s: cyber-attack should match text containing the words in sequence", field)
|
||||
}
|
||||
if compileQuery([]string{"cyber-attack"}, []string{field}, []string{"and"}).matches(&miss) {
|
||||
t.Errorf("%s: cyber-attack matched a document where the words are apart", field)
|
||||
}
|
||||
}
|
||||
// The words on their own still match both, as before.
|
||||
if !compileQuery([]string{"cyber"}, []string{"FT"}, []string{"and"}).matches(&miss) {
|
||||
t.Error("a plain word should still match")
|
||||
}
|
||||
}
|
||||
|
||||
// A value page's publication-year bars count every year a document was
|
||||
// published in, because each bar links to a search that matches any of them.
|
||||
func TestPubYearsOf(t *testing.T) {
|
||||
var d Document
|
||||
if err := json.Unmarshal([]byte(`{"publication": [
|
||||
{"publisher": "The Guardian", "date_published": "2013-06-06"},
|
||||
{"publisher": "Le Monde", "date_published": "2014-10-21"},
|
||||
{"publisher": "Der Spiegel", "date_published": "2013-12-30"},
|
||||
{"publisher": "No date"},
|
||||
{"publisher": "Bad date", "date_published": "201"}
|
||||
]}`), &d); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := fmt.Sprint(pubYearsOf(&d)); got != "[2013 2014]" {
|
||||
t.Errorf("pubYearsOf = %s, want [2013 2014]", got)
|
||||
}
|
||||
if got := fmt.Sprint(pubYearsOf(&Document{})); got != "[]" {
|
||||
t.Errorf("pubYearsOf(no publications) = %s, want []", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A saved search's label must name the type filter, or two saved searches
|
||||
// differing only by Reported/SIDtoday are indistinguishable.
|
||||
func TestSearchLabelIncludesTypeFilter(t *testing.T) {
|
||||
entry := func(v url.Values) string { return url.QueryEscape(v.Encode()) }
|
||||
base := url.Values{"fqv": {"xkeyscore"}, "fqf": {"TE"}, "fqc": {"and"}}
|
||||
typed := url.Values{"fqv": {"xkeyscore"}, "fqf": {"TE"}, "fqc": {"and"}, "type": {"REPORTED"}}
|
||||
both := url.Values{"fqv": {"xkeyscore"}, "fqf": {"TE"}, "fqc": {"and"},
|
||||
"type": {"SIDTODAY"}, "coll": {"news"}}
|
||||
|
||||
got := corpus().parseSearchEntries([]string{entry(typed), entry(base), entry(both)})
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("parsed %d entries, want 3", len(got))
|
||||
}
|
||||
if !strings.Contains(got[0].Label, "REPORTED") {
|
||||
t.Errorf("label %q should name the type filter", got[0].Label)
|
||||
}
|
||||
if got[0].Label == got[1].Label {
|
||||
t.Errorf("searches differing only by type share the label %q", got[0].Label)
|
||||
}
|
||||
// Collection and type share one bracket rather than two.
|
||||
if !strings.Contains(got[2].Label, "(news, SIDTODAY)") {
|
||||
t.Errorf("label %q should list collection and type together", got[2].Label)
|
||||
}
|
||||
}
|
||||
538
tests/consistency_test.go
Normal file
538
tests/consistency_test.go
Normal file
|
|
@ -0,0 +1,538 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"html"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A corpus built to exercise the places where one fact used to be computed
|
||||
// twice: search rows and their negations, the page and the API, counts and
|
||||
// the pages they link to. Each document's words are chosen so the expected
|
||||
// result sets can be written down by hand.
|
||||
//
|
||||
// n1 news alpha gamma ZEBRA 2010 The Intercept 2014 + the intercept 2016
|
||||
// n2 news alpha ZEBRA 2011 The Intercept 2014 + Der Spiegel 2014
|
||||
// n3 news beta gamma 2012 Der Spiegel 2015
|
||||
// n4 news beta 2012 Der Spiegel 2015
|
||||
// f1 foia alpha ZEBRA 2010 (court filing: "(c)" list labels)
|
||||
// s1 sidtoday ZEBRA 2011
|
||||
// a1 news "aka_id": null
|
||||
// d1 news date_created "ODNI", date_published "15 May 2015"
|
||||
func writeConsistencyCorpus(t *testing.T) string {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
doc := func(dir, id string, meta map[string]interface{}, text string) {
|
||||
b, err := json.Marshal(meta)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := filepath.Join(root, dir, id)
|
||||
writeFixtureFile(t, filepath.Join(d, "metadata.json"), string(b))
|
||||
writeFixtureFile(t, filepath.Join(d, id+".pdf"), "%PDF-1.4\n")
|
||||
if text != "" {
|
||||
writeFixtureFile(t, filepath.Join(d, id+".md"), text)
|
||||
}
|
||||
}
|
||||
pub := func(publisher, date, link string, reporters ...string) map[string]interface{} {
|
||||
return map[string]interface{}{"publisher": publisher, "date_published": date,
|
||||
"link": link, "reporters": reporters}
|
||||
}
|
||||
meta := func(title, typ, created string, programs []string, pubs ...map[string]interface{}) map[string]interface{} {
|
||||
return map[string]interface{}{"title": title, "type": typ,
|
||||
"document": map[string]interface{}{"creator": "NSA", "date_created": created, "programs": programs},
|
||||
"publication": pubs}
|
||||
}
|
||||
zebra := []string{"ZEBRA"}
|
||||
doc("", "n1", meta("Report one", "REPORTED", "2010-03-01", zebra,
|
||||
pub("The Intercept", "2014-01-01", "https://example.org/a", "Glenn Greenwald"),
|
||||
pub("the intercept", "2016-01-01", "https://example.org/b", "glenn greenwald")),
|
||||
"alpha gamma\n\n(S//SI) a secret line\n(S/SI) the same marking, spelled by OCR\n(U) plain\n")
|
||||
doc("", "n2", meta("Report two", "REPORTED", "2011-03-01", zebra,
|
||||
pub("The Intercept", "2014-02-01", "https://example.org/c"),
|
||||
pub("Der Spiegel", "2014-02-02", "https://example.org/f")),
|
||||
"alpha\n\n(TS//SI//NF) top line\n")
|
||||
doc("", "n3", meta("Report three", "REPORTED", "2012-03-01", nil,
|
||||
pub("Der Spiegel", "2015-01-01", "https://example.org/d")), "beta gamma\n")
|
||||
doc("", "n4", meta("Report four", "REPORTED", "2012-04-01", nil,
|
||||
pub("Der Spiegel", "2015-01-01", "https://example.org/d")), "beta\n")
|
||||
doc("FOIA", "f1", meta("Court filing", "FOIA", "2010-05-01", zebra),
|
||||
"alpha\n\n(c) The Court shall issue an order.\n(s) another subsection\n")
|
||||
doc("SIDtoday-archive", "s1", meta("Issue", "SIDTODAY", "2011-05-01", zebra), "(S) issue line\n")
|
||||
a1 := meta("Null aka", "REPORTED", "", nil)
|
||||
a1["aka_id"] = nil
|
||||
doc("", "a1", a1, "")
|
||||
// A date of no shape the archive reads: served, undated.
|
||||
doc("", "d1", meta("Odd date", "REPORTED", "ODNI", nil, pub("Der Spiegel", "15 May 2015", "https://example.org/e")), "")
|
||||
writeFixtureFile(t, filepath.Join(root, "glossary.json"), `[]`)
|
||||
return root
|
||||
}
|
||||
|
||||
// matchingIDs runs the search engine and returns the matching ids, sorted.
|
||||
func matchingIDs(queries, fields, conjs []string) []string {
|
||||
q, f, c := normalizeClauses(queries, fields, conjs)
|
||||
var ids []string
|
||||
scored, _ := corpus().executeSearch(searchParams{Queries: q, Fields: f, Conjs: c})
|
||||
for _, s := range scored {
|
||||
ids = append(ids, s.Doc.ID)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
return ids
|
||||
}
|
||||
|
||||
// A row is one unit: its -terms are exclusions within it, whatever joins
|
||||
// it to the rows before. And an empty row starts nothing - a blank first
|
||||
// row with "OR x" under it is a search for x.
|
||||
func TestSearchRowsAndNegations(t *testing.T) {
|
||||
withCorpus(t, writeConsistencyCorpus(t))
|
||||
te2 := []string{"TE", "TE"}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
queries, conjs []string
|
||||
want []string
|
||||
}{
|
||||
{"inline negation", []string{"alpha -gamma"}, []string{"and"}, []string{"f1", "n2"}},
|
||||
// alpha AND NOT (beta AND NOT gamma): no alpha document is lost
|
||||
// for mentioning gamma.
|
||||
{"negation inside NOT", []string{"alpha", "beta -gamma"}, []string{"and", "not"}, []string{"f1", "n1", "n2"}},
|
||||
// alpha OR (beta AND NOT gamma): the exclusion stays inside its row.
|
||||
{"negation inside OR", []string{"alpha", "beta -gamma"}, []string{"and", "or"}, []string{"f1", "n1", "n2", "n4"}},
|
||||
{"blank first row, OR", []string{"", "alpha"}, []string{"and", "or"}, []string{"f1", "n1", "n2"}},
|
||||
{"spaces first row, OR", []string{" ", "alpha"}, []string{"and", "or"}, []string{"f1", "n1", "n2"}},
|
||||
{"blank first row, OR nothing", []string{"", "zzzzunmatched"}, []string{"and", "or"}, nil},
|
||||
{"blank first row, NOT", []string{"", "gamma"}, []string{"and", "not"}, []string{"a1", "d1", "f1", "n2", "n4", "s1"}},
|
||||
{"only exclusions", []string{"-gamma -alpha"}, []string{"and"}, []string{"a1", "d1", "n4", "s1"}},
|
||||
} {
|
||||
if got := matchingIDs(c.queries, te2[:len(c.queries)], c.conjs); !reflect.DeepEqual(got, c.want) {
|
||||
t.Errorf("%s: %q %v matched %v, want %v", c.name, c.queries, c.conjs, got, c.want)
|
||||
}
|
||||
}
|
||||
// The scorer and the highlighter read the same rows: an excluded term
|
||||
// is neither rewarded nor marked.
|
||||
if got := positiveTerms([]string{"alpha -gamma", "", "beta"}, []string{"and", "and", "not"}); !reflect.DeepEqual(got, []string{"alpha"}) {
|
||||
t.Errorf("positiveTerms = %q, want [alpha]", got)
|
||||
}
|
||||
if term, _ := snippetAnchor([]string{"-gamma", "alpha -beta"}, te2, []string{"and", "and"}); term != "alpha" {
|
||||
t.Errorf("snippetAnchor = %q, want alpha", term)
|
||||
}
|
||||
}
|
||||
|
||||
// "+ Add row" and "remove" re-run the search: on the same rows, meaning the
|
||||
// same thing, as the Search button would.
|
||||
func TestRowEditsSearchTheSameRows(t *testing.T) {
|
||||
withCorpus(t, writeConsistencyCorpus(t))
|
||||
rows := "do_search=1&fqv=alpha+-gamma&fqf=TE&fqc=and"
|
||||
want := resultTotal(t, "/search?action=search&"+rows, foiaOn)
|
||||
if want != 2 {
|
||||
t.Fatalf("search for alpha -gamma found %d, want 2", want)
|
||||
}
|
||||
for _, action := range []string{"add", "remove-1"} {
|
||||
path := "/search?action=" + action + "&" + rows + "&fqv=beta&fqf=TE&fqc=or"
|
||||
if action == "add" {
|
||||
path = "/search?action=add&" + rows
|
||||
}
|
||||
if got := resultTotal(t, path, foiaOn); got != want {
|
||||
t.Errorf("action=%s: %d results, the same rows searched find %d", action, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The type filter resubmits the search the page shows, not the request: a
|
||||
// removed row stays removed, and the chosen sort is kept.
|
||||
func TestTypeFilterFormCarriesTheEffectiveSearch(t *testing.T) {
|
||||
withCorpus(t, writeConsistencyCorpus(t))
|
||||
body := get(t, "/search?do_search=1&action=remove-1&sort=title"+
|
||||
"&fqv=alpha&fqf=TE&fqc=and&fqv=removedrow&fqf=TE&fqc=or", foiaOn).Body.String()
|
||||
if !strings.Contains(body, `class="type-filter"`) {
|
||||
t.Fatal("no type filter on a result set holding two types")
|
||||
}
|
||||
if strings.Contains(body, "removedrow") {
|
||||
t.Error("the removed row is still in the page - the type filter would bring it back")
|
||||
}
|
||||
if !strings.Contains(body, `name="sort" value="title"`) {
|
||||
t.Error("the type filter drops the chosen sort")
|
||||
}
|
||||
}
|
||||
|
||||
// /api/search runs the search page's pipeline, type filter included. The
|
||||
// API has no preference: it matches the page of a reader who includes FOIA
|
||||
// releases.
|
||||
func TestAPISearchMatchesThePage(t *testing.T) {
|
||||
withCorpus(t, writeConsistencyCorpus(t))
|
||||
for qs, want := range map[string]int{
|
||||
"q=alpha": 3,
|
||||
"q=alpha&type=REPORTED": 2,
|
||||
"q=alpha&type=FOIA&type=REPORTED": 3,
|
||||
"q=alpha&type=reported": 2,
|
||||
"q=zebra&coll=news": 2,
|
||||
"q=zebra&coll=foia&type=FOIA": 1,
|
||||
"do_search=1&fqv=alpha&fqf=TE&fqc=and&fqv=beta+-gamma&fqf=TE&fqc=or": 4,
|
||||
} {
|
||||
page := resultTotal(t, "/search?"+qs, foiaOn)
|
||||
var api struct{ Total int }
|
||||
if err := json.Unmarshal(get(t, "/api/search?"+qs).Body.Bytes(), &api); err != nil {
|
||||
t.Fatalf("%s: %v", qs, err)
|
||||
}
|
||||
if page != want || api.Total != want {
|
||||
t.Errorf("%s: the page finds %d, the API %d, want %d", qs, page, api.Total, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Every count in this corpus is the number its page or search returns.
|
||||
func TestConsistencyCorpusCounts(t *testing.T) {
|
||||
withCorpus(t, writeConsistencyCorpus(t))
|
||||
|
||||
// Facets, with journalist and publisher spellings that differ by case
|
||||
// across one document's publications.
|
||||
for ci := range categories {
|
||||
c := &categories[ci]
|
||||
for val, want := range c.Facet(corpus()) {
|
||||
if got := len(corpus().categoryMatches(c.Slug, val)); got != want {
|
||||
t.Errorf("%s=%q: facet says %d, the page lists %d", c.Slug, val, want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
if js := corpus().archive.Facets["journalist"]; len(js) != 1 || js["Glenn Greenwald"] != 1 {
|
||||
t.Errorf("journalists = %v, want Glenn Greenwald once", js)
|
||||
}
|
||||
if n := corpus().archive.Facets["publisher"]["The Intercept"]; n != 2 {
|
||||
t.Errorf("The Intercept: %d documents, want 2 (%v)", n, corpus().archive.Facets["publisher"])
|
||||
}
|
||||
|
||||
// The value-page timelines and refine links, scoped to a collection:
|
||||
// f1 (FOIA) shares 2010 with n1 and s1 (SIDtoday) shares 2011 with n2,
|
||||
// so an unscoped bar link finds more than its bar counts.
|
||||
for _, p := range []string{"/browse/program/ZEBRA?coll=news", "/browse/program/ZEBRA"} {
|
||||
if n := followLinks(t, p, reBarLink, "timeline bar"); n < 2 {
|
||||
t.Errorf("%s: %d timeline bars checked, want the doc and pub timelines", p, n)
|
||||
}
|
||||
}
|
||||
|
||||
// /explore/outlets: rows, cells and the per-year footer.
|
||||
for coll, table := range corpus().outletTables {
|
||||
for _, row := range table.Rows {
|
||||
if got := resultTotal(t, row.URL); got != row.Total {
|
||||
t.Errorf("outlets[%q] %s: total %d, its page lists %d", coll, row.Name, row.Total, got)
|
||||
}
|
||||
for _, cell := range row.Cells {
|
||||
if cell.URL != "" && resultTotal(t, cell.URL) != cell.Docs {
|
||||
t.Errorf("outlets[%q] %s: a cell says %d, its search finds %d", coll, row.Name, cell.Docs, resultTotal(t, cell.URL))
|
||||
}
|
||||
}
|
||||
}
|
||||
// The footer adds up its column: a document several outlets
|
||||
// published counts once for each of them.
|
||||
for i, y := range table.Years {
|
||||
sum := 0
|
||||
for _, row := range table.Rows {
|
||||
sum += row.Cells[i].Docs
|
||||
}
|
||||
if table.Totals[i] != sum {
|
||||
t.Errorf("outlets[%q] %s: footer %d, its column adds up to %d", coll, y, table.Totals[i], sum)
|
||||
}
|
||||
}
|
||||
}
|
||||
cell := func(outlet, year string) (docs, footer int) {
|
||||
table := corpus().outletTables[""]
|
||||
for i, y := range table.Years {
|
||||
if y != year {
|
||||
continue
|
||||
}
|
||||
footer = table.Totals[i]
|
||||
for _, row := range table.Rows {
|
||||
if row.Name == outlet {
|
||||
docs = row.Cells[i].Docs
|
||||
}
|
||||
}
|
||||
}
|
||||
return docs, footer
|
||||
}
|
||||
// n2: The Intercept and Der Spiegel both published it in 2014, so it
|
||||
// counts for both outlets and twice in the year's footer (n1 is the
|
||||
// third: The Intercept, 2014).
|
||||
if spiegel, footer := cell("Der Spiegel", "2014"); spiegel != 1 || footer != 3 {
|
||||
t.Errorf("2014: Der Spiegel cell %d, footer %d - want 1 and 3 (n2 once per outlet)", spiegel, footer)
|
||||
}
|
||||
if intercept, _ := cell("The Intercept", "2014"); intercept != 2 {
|
||||
t.Errorf("2014: The Intercept cell %d, want 2", intercept)
|
||||
}
|
||||
intercept := 0
|
||||
for _, row := range corpus().outletTables[""].Rows {
|
||||
if row.Name == "The Intercept" {
|
||||
intercept = row.Total
|
||||
}
|
||||
}
|
||||
if intercept != 2 {
|
||||
t.Errorf("The Intercept's outlet total = %d, want 2 documents (n1 counts once for its two years)", intercept)
|
||||
}
|
||||
|
||||
// /explore/portions: lower-case list labels are not markings, and every
|
||||
// row's document count is what its search finds.
|
||||
for _, l := range corpus().portionStats.Levels {
|
||||
if l.Name == "C" {
|
||||
t.Errorf("a CONFIDENTIAL row from the court filing's (c) labels: %+v", l)
|
||||
}
|
||||
}
|
||||
rows := append(append(append([]portionRow(nil), corpus().portionStats.Levels...), corpus().portionStats.Markings...), corpus().portionStats.Controls...)
|
||||
if len(rows) < 6 {
|
||||
t.Fatalf("only %d portion rows", len(rows))
|
||||
}
|
||||
for _, r := range rows {
|
||||
if got := resultTotal(t, string(r.SearchURL)); got != r.Docs {
|
||||
t.Errorf("portion row %s: %d documents, its search finds %d (%s)", r.Name, r.Docs, got, r.SearchURL)
|
||||
}
|
||||
}
|
||||
if got := corpus().docIndex["n1"].PortionMarkings; !reflect.DeepEqual(got, []string{"S//SI", "U"}) {
|
||||
t.Errorf("n1 markings = %v, want [S//SI U]", got)
|
||||
}
|
||||
|
||||
// A malformed date is no date: no "ODNI" year that no year page opens.
|
||||
if d := corpus().docIndex["d1"]; d == nil || d.DocMeta.DateCreated != "" || d.Publication[0].DatePublished != "" {
|
||||
t.Errorf("d1's malformed dates were kept: %+v", d)
|
||||
}
|
||||
if _, ok := corpus().archive.Facets["docdate"]["ODNI"]; ok {
|
||||
t.Error(`"ODNI" counted as a document year`)
|
||||
}
|
||||
|
||||
// aka_id: null is no akas, not a reason to drop the document.
|
||||
if corpus().docIndex["a1"] == nil {
|
||||
t.Error(`a document with "aka_id": null was not loaded`)
|
||||
}
|
||||
}
|
||||
|
||||
// A corpus for the counts that turn on WHICH publication entry carries a
|
||||
// date or a byline, and on the reader's collection scope. Every document
|
||||
// says "alpha".
|
||||
//
|
||||
// p1 news 2010-03 Outlet X 2013 (Able), Outlet Y 2014 (Baker)
|
||||
// p2 news 2010-06 Outlet X 2014 (Able) people: Clerk
|
||||
// p3 news 2011-01 Outlet X 2014 (Baker)
|
||||
// p4 news 2012-03 Outlet Y 2014
|
||||
// p5 news 2012-09 Outlet Y 2014
|
||||
// p6 news 2013-02 Outlet Y 2014, Partner Z 2014 (its own story; reporting only)
|
||||
// f1 foia 2009-07 Outlet Y 2014 (a story of its own) people: Docket
|
||||
func writeEntryCorpus(t *testing.T) string {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
pub := func(publisher, date string, reporters ...string) map[string]interface{} {
|
||||
return map[string]interface{}{"publisher": publisher, "date_published": date, "reporters": reporters}
|
||||
}
|
||||
doc := func(dir, id, typ, created string, people []string, pubs ...map[string]interface{}) {
|
||||
b, err := json.Marshal(map[string]interface{}{"title": "Document " + id, "type": typ,
|
||||
"document": map[string]interface{}{"creator": "NSA", "date_created": created, "people": people},
|
||||
"publication": pubs})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := filepath.Join(root, dir, id)
|
||||
writeFixtureFile(t, filepath.Join(d, "metadata.json"), string(b))
|
||||
writeFixtureFile(t, filepath.Join(d, id+".pdf"), "%PDF-1.4\n")
|
||||
writeFixtureFile(t, filepath.Join(d, id+".md"), "alpha\n")
|
||||
}
|
||||
doc("", "p1", "REPORTED", "2010-03-01", nil,
|
||||
pub("Outlet X", "2013-05-01", "Able, Ann"), pub("Outlet Y", "2014-02-01", "Baker, Bob"))
|
||||
doc("", "p2", "REPORTED", "2010-06-01", []string{"Clerk, Carol"}, pub("Outlet X", "2014-03-01", "Able, Ann"))
|
||||
doc("", "p3", "REPORTED", "2011-01-01", nil, pub("Outlet X", "2014-04-01", "Baker, Bob"))
|
||||
doc("", "p4", "REPORTED", "2012-03-01", nil, pub("Outlet Y", "2014-05-01"))
|
||||
doc("", "p5", "REPORTED", "2012-09-01", nil, pub("Outlet Y", "2014-06-01"))
|
||||
story := func(p map[string]interface{}, title, link string) map[string]interface{} {
|
||||
p["title"], p["link"] = title, link
|
||||
return p
|
||||
}
|
||||
partner := story(pub("Partner Z", "2014-07-02"), "Partner story", "https://example.org/partner")
|
||||
partner["reporting_only"] = true
|
||||
doc("", "p6", "REPORTED", "2013-02-01", nil, pub("Outlet Y", "2014-07-01"), partner)
|
||||
doc("FOIA", "f1", "FOIA", "2009-07-01", []string{"Docket, Dan"},
|
||||
story(pub("Outlet Y", "2014-08-01"), "FOIA story", "https://example.org/foia-story"))
|
||||
writeFixtureFile(t, filepath.Join(root, "glossary.json"), `[]`)
|
||||
return root
|
||||
}
|
||||
|
||||
// Counts that turn on a publication entry's date or byline, or on the
|
||||
// reader's scope, are still the numbers their links return.
|
||||
func TestCountsTiedToEntriesAndScope(t *testing.T) {
|
||||
withCorpus(t, writeEntryCorpus(t))
|
||||
|
||||
// A publication-year page files a document under its first publication,
|
||||
// which no search asks for: p1, first published in 2013, would join the
|
||||
// 2010 bar's search for 2014. So those bars carry no link. The
|
||||
// document-year page's publication bars do, and they are exact.
|
||||
body := get(t, "/browse/published/2014").Body.String()
|
||||
if !strings.Contains(body, "By document date") || strings.Contains(body, `<a class="hist-col"`) {
|
||||
t.Error("/browse/published/2014: want the document-date bars, and no link on them")
|
||||
}
|
||||
if n := followLinks(t, "/browse/docdate/2010", reBarLink, "publication bar"); n != 2 {
|
||||
t.Errorf("/browse/docdate/2010: %d publication bars checked, want 2013 and 2014", n)
|
||||
}
|
||||
|
||||
// Under a publication year a journalist counts for the documents that
|
||||
// byline them on an entry of that year, because that is what the refine
|
||||
// link's search matches: Able's byline on p1 is Outlet X's of 2013, not
|
||||
// Outlet Y's of 2014.
|
||||
for _, p := range []string{
|
||||
"/browse/publisher/Outlet%20Y?year=2014",
|
||||
"/search?q=alpha+published%3A2014",
|
||||
// An OR row unties the dates, and the bylines with them.
|
||||
"/search?do_search=1&fqv=alpha&fqf=TE&fqc=and&fqv=beta&fqf=TE&fqc=or&date_pub_start=2014&date_pub_end=2014",
|
||||
} {
|
||||
if n := followLinks(t, p, reFacetLink, "refine link"); n == 0 {
|
||||
t.Errorf("%s: no refine links to check", p)
|
||||
}
|
||||
}
|
||||
|
||||
// A year bar narrows the search it stands under: in a range that starts
|
||||
// in June 2012, the 2012 bar counts p5 and not p4 of March.
|
||||
if n := followLinks(t, "/search?q=alpha+date%3A2012-06..2013-06", reBarLink, "year bar"); n != 2 {
|
||||
t.Errorf("%d year bars checked, want 2012 and 2013", n)
|
||||
}
|
||||
|
||||
// Docket is named only in the FOIA release and 2009 is only its year:
|
||||
// neither is on the default reader's lists, nor in the boxes' counts.
|
||||
for _, reader := range [][]*http.Cookie{nil, {foiaOn}} {
|
||||
if n := checkCategoryBoxes(t, reader...); n < 10 {
|
||||
t.Errorf("%d category boxes checked", n)
|
||||
}
|
||||
}
|
||||
|
||||
// The home page's publication chart counts first publications, as the
|
||||
// year pages its bars link file them: p1 counts for 2013 and not 2014.
|
||||
body = get(t, "/").Body.String()
|
||||
for year, want := range map[string]string{"2013": "1", "2014": "5"} {
|
||||
if !strings.Contains(body, `href="/browse/published/`+year+`?coll=news" title="`+want+` documents first published in `+year+`"`) {
|
||||
t.Errorf("home: no bar of %s documents first published in %s linking to that year's page", want, year)
|
||||
}
|
||||
}
|
||||
|
||||
// In a search result a publication year links to its year page only
|
||||
// when that page lists the document: p1 is filed under 2013, its first.
|
||||
body = get(t, "/search?q=title%3Ap1").Body.String()
|
||||
if !strings.Contains(body, `href="/browse/published/2013"`) || strings.Contains(body, `href="/browse/published/2014"`) {
|
||||
t.Error("p1's result should link 2013, its first publication year, and not 2014")
|
||||
}
|
||||
|
||||
// A partner that only reported has no publisher page for its story's
|
||||
// byline to link.
|
||||
body = get(t, "/article/"+articleKeyFor("https://example.org/partner")).Body.String()
|
||||
if !strings.Contains(body, "Partner Z") || strings.Contains(body, "/browse/publisher/Partner") {
|
||||
t.Error("the partner's article should name Partner Z without linking a publisher page")
|
||||
}
|
||||
|
||||
// The outlets table of one collection counts the stories that have a
|
||||
// document in it: Outlet Y's 2014 story published only the FOIA release.
|
||||
stories := func(coll string) int {
|
||||
table := corpus().outletTables[coll]
|
||||
for _, row := range table.Rows {
|
||||
if row.Name == "Outlet Y" {
|
||||
return row.Cells[slices.Index(table.Years, "2014")].Articles
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
if all, news := stories(""), stories(collectionNews); all != 1 || news != 0 {
|
||||
t.Errorf("Outlet Y's 2014 stories: %d over every collection and %d reported, want 1 and 0", all, news)
|
||||
}
|
||||
}
|
||||
|
||||
var reCategoryBox = regexp.MustCompile(`<a href="/browse/([a-z]+)" class="cat-box[^"]*"><strong>[^<]*</strong><span class="count">(\d+)</span></a>`)
|
||||
|
||||
// checkCategoryBoxes holds the number on each category box in the browse
|
||||
// nav to the rows of the list its click opens, for the reader with these
|
||||
// cookies.
|
||||
func checkCategoryBoxes(t *testing.T, cookies ...*http.Cookie) int {
|
||||
t.Helper()
|
||||
boxes := reCategoryBox.FindAllStringSubmatch(get(t, "/browse", cookies...).Body.String(), -1)
|
||||
for _, m := range boxes {
|
||||
slug, shown := m[1], mustAtoi(t, m[2])
|
||||
list := get(t, "/browse/"+slug, cookies...).Body.String()
|
||||
if rows := strings.Count(list, `href="/browse/`+slug+`/`); rows != shown {
|
||||
t.Errorf("the %s box says %d, its list has %d rows (%d cookies)", slug, shown, rows, len(cookies))
|
||||
}
|
||||
}
|
||||
return len(boxes)
|
||||
}
|
||||
|
||||
// -check reads the load counters, so the glossary's problems must still be
|
||||
// in them when it does.
|
||||
func TestGlossaryWarningsReachCheck(t *testing.T) {
|
||||
root := writeConsistencyCorpus(t)
|
||||
withCorpus(t, root)
|
||||
clean := loadWarnings
|
||||
writeFixtureFile(t, filepath.Join(root, "glossary.json"), `{not json`)
|
||||
loadData()
|
||||
if loadWarnings != clean+1 {
|
||||
t.Errorf("malformed glossary.json: %d warnings, want %d", loadWarnings, clean+1)
|
||||
}
|
||||
}
|
||||
|
||||
// A 404 under /files must not carry the files' public cache policy.
|
||||
func TestMissingFileIsNotCached(t *testing.T) {
|
||||
w := get(t, "/files/prism-slides/no-such-file.pdf")
|
||||
if w.Code != http.StatusNotFound {
|
||||
t.Fatalf("status %d, want 404", w.Code)
|
||||
}
|
||||
if cc := w.Header().Get("Cache-Control"); strings.Contains(cc, "public") {
|
||||
t.Errorf("404 carries Cache-Control %q", cc)
|
||||
}
|
||||
}
|
||||
|
||||
// -dev serves the on-disk assets so edits show without a restart: no
|
||||
// startup ETag may pin the old content.
|
||||
func TestDevModeStaticRevalidatesEdits(t *testing.T) {
|
||||
buildFixture(t)
|
||||
devMode = true
|
||||
initAssets()
|
||||
t.Cleanup(func() {
|
||||
devMode = false
|
||||
initAssets()
|
||||
})
|
||||
if staticHashes != nil {
|
||||
t.Errorf("dev mode built %d static hashes", len(staticHashes))
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
handleStatic(w, httptest.NewRequest("GET", "/static/style.css", nil))
|
||||
if w.Code != http.StatusOK || w.Header().Get("ETag") != "" || w.Header().Get("Last-Modified") == "" {
|
||||
t.Errorf("dev static: status %d, ETag %q, Last-Modified %q - want 200, none, the file's",
|
||||
w.Code, w.Header().Get("ETag"), w.Header().Get("Last-Modified"))
|
||||
}
|
||||
}
|
||||
|
||||
// What the search page offers to pin is what the pinned page can list.
|
||||
func TestPinnableSearchesAreListed(t *testing.T) {
|
||||
h, _ := buildFixture(t)
|
||||
if label, ok := corpus().searchLabel(mustQuery(t, "/search?do_search=1&fqv=&fqf=TE&fqc=and&date_doc_start=2013&date_doc_end=2013")); !ok || label != "all documents · (doc 2013)" {
|
||||
t.Errorf("date-only search label = %q, %v", label, ok)
|
||||
}
|
||||
if strings.Contains(get(t, "/search?do_search=1&fqv=&fqf=TE&fqc=and").Body.String(), "pin-search-btn") {
|
||||
t.Error("a search with no terms and no filters offers a pin")
|
||||
}
|
||||
if !strings.Contains(get(t, "/search?q=date:2013").Body.String(), "pin-search-btn") {
|
||||
t.Error("a date-only search offers no pin")
|
||||
}
|
||||
|
||||
pin := func(qs string) *httptest.ResponseRecorder {
|
||||
r := httptest.NewRequest("POST", "/pinsearch", strings.NewReader(url.Values{"qs": {qs}, "back": {"/pinned"}}.Encode()))
|
||||
r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
return record(h, r)
|
||||
}
|
||||
if w := pin("do_search=1&fqc=and&fqf=TE&fqv="); w.Code != http.StatusBadRequest {
|
||||
t.Errorf("pinning an empty search: status %d, want 400", w.Code)
|
||||
}
|
||||
w := pin("date_doc_end=2013&date_doc_start=2013&do_search=1&fqc=and&fqf=TE&fqv=")
|
||||
if w.Code != http.StatusSeeOther {
|
||||
t.Fatalf("pinning a date-only search: status %d", w.Code)
|
||||
}
|
||||
body := get(t, "/pinned", w.Result().Cookies()...).Body.String()
|
||||
if !strings.Contains(html.UnescapeString(body), "all documents · (doc 2013)") {
|
||||
t.Error("the pinned date-only search is not listed on /pinned")
|
||||
}
|
||||
}
|
||||
199
tests/corpus_test.go
Normal file
199
tests/corpus_test.go
Normal file
|
|
@ -0,0 +1,199 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"html"
|
||||
"net/http"
|
||||
"os"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Checks against a REAL corpus. Point $SNOWCHIVE_BENCH_DATA at one, as for
|
||||
// the page benchmarks, and run: go test -run TestCorpus -v
|
||||
//
|
||||
// The fixtures are a handful of documents, each built to show one thing. A
|
||||
// count and its link are computed by different code, and whether the two
|
||||
// agree turns on what the documents happen to hold: a second publication in
|
||||
// a later year, a byline on only one of them, a value found only in FOIA
|
||||
// releases. The disagreements these tests walk the archive for were all
|
||||
// found on the full corpus and on no fixture.
|
||||
func TestCorpus(t *testing.T) {
|
||||
root := os.Getenv("SNOWCHIVE_BENCH_DATA")
|
||||
if root == "" {
|
||||
t.Skip("set SNOWCHIVE_BENCH_DATA to a corpus directory to check a real corpus")
|
||||
}
|
||||
withCorpus(t, root)
|
||||
t.Run("counts", corpusCounts)
|
||||
t.Run("FOIA links", corpusFOIALinks)
|
||||
t.Run("inflections", corpusInflections)
|
||||
t.Run("snippets", corpusSnippets)
|
||||
t.Run("index", corpusIndex)
|
||||
}
|
||||
|
||||
// corpusCounts follows the counted links of the loaded corpus, as a reader
|
||||
// who leaves FOIA releases out and as one who includes them.
|
||||
func corpusCounts(t *testing.T) {
|
||||
// The pages whose bars and refine links are followed: the year pages of
|
||||
// both date categories, the three commonest values of every other
|
||||
// category, each cell of the outlets table (a publisher's page for one
|
||||
// publication year) and searches under each kind of date range.
|
||||
var pages []string
|
||||
for ci := range categories {
|
||||
c := &categories[ci]
|
||||
n := 3
|
||||
if c.isDate() {
|
||||
n = -1
|
||||
}
|
||||
for _, v := range rankCounts(c.Facet(corpus()), n) {
|
||||
pages = append(pages, categoryValueURL(c.Slug, v))
|
||||
}
|
||||
}
|
||||
for _, row := range corpus().outletTables[""].Rows {
|
||||
for _, cell := range row.Cells {
|
||||
if cell.URL != "" {
|
||||
pages = append(pages, cell.URL)
|
||||
}
|
||||
}
|
||||
}
|
||||
pages = append(pages,
|
||||
"/search?q=nsa",
|
||||
"/search?q=nsa+date%3A2012-06..2013-06",
|
||||
"/search?q=nsa+published%3A2016",
|
||||
"/search?q=published%3A2014",
|
||||
"/search?do_search=1&fqv=nsa&fqf=TE&fqc=and&date_pub_start=2015-01-01&date_pub_end=2016-06-30",
|
||||
"/search?do_search=1&fqv=nsa&fqf=TE&fqc=and&fqv=gchq&fqf=TE&fqc=or&date_pub_start=2016&date_pub_end=2016",
|
||||
)
|
||||
|
||||
for name, reader := range map[string][]*http.Cookie{"FOIA left out": nil, "FOIA included": {foiaOn}} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
bars, refines := 0, 0
|
||||
for _, p := range pages {
|
||||
bars += followLinks(t, p, reBarLink, "bar", reader...)
|
||||
refines += followLinks(t, p, reFacetLink, "refine link", reader...)
|
||||
}
|
||||
if bars == 0 || refines == 0 {
|
||||
t.Errorf("nothing exercised: %d bars, %d refine links", bars, refines)
|
||||
}
|
||||
boxes := checkCategoryBoxes(t, reader...)
|
||||
t.Logf("%d pages: %d bars, %d refine links and %d category boxes checked", len(pages), bars, refines, boxes)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// corpusFOIALinks opens the page of every tenth FOIA release as a reader
|
||||
// who leaves FOIA out, and follows each category value it links: the page
|
||||
// behind the link has to list the release.
|
||||
func corpusFOIALinks(t *testing.T) {
|
||||
reValueLink := regexp.MustCompile(`href="(/browse/[a-z]+/[^"]+)"`)
|
||||
links := 0
|
||||
for i, d := range corpus().collectionDocs[collectionFOIA] {
|
||||
if i%10 != 0 {
|
||||
continue
|
||||
}
|
||||
// The release's own metadata: the cards under it are other documents.
|
||||
page, _, _ := strings.Cut(get(t, docPath(d.ID)).Body.String(), `class="related-section"`)
|
||||
for _, m := range reValueLink.FindAllStringSubmatch(page, -1) {
|
||||
link := html.UnescapeString(m[1])
|
||||
links++
|
||||
if !strings.Contains(get(t, withQuery(link, "view=table")).Body.String(), `href="`+docPath(d.ID)+`"`) {
|
||||
t.Errorf("%s links %s, which does not list it", d.ID, link)
|
||||
}
|
||||
}
|
||||
}
|
||||
if links == 0 {
|
||||
t.Error("no FOIA release with a value link to follow")
|
||||
}
|
||||
t.Logf("%d value links followed", links)
|
||||
}
|
||||
|
||||
// A word in -ies or -ied stems as the word in -y it is a form of, whatever
|
||||
// rules that word goes through - checked over every such pair the corpus
|
||||
// holds, since a suffix rule that reads one of the two forms and not the
|
||||
// other splits them silently ("capability" from "capabilities").
|
||||
func corpusInflections(t *testing.T) {
|
||||
vocab := map[string]bool{}
|
||||
for i := range corpus().archive.Docs {
|
||||
for _, w := range splitWords(corpus().archive.Docs[i].SearchText) {
|
||||
vocab[w] = true
|
||||
}
|
||||
}
|
||||
pairs := 0
|
||||
for w := range vocab {
|
||||
if len(w) < 5 || !(strings.HasSuffix(w, "ies") || strings.HasSuffix(w, "ied")) {
|
||||
continue
|
||||
}
|
||||
base := w[:len(w)-3] + "y"
|
||||
if !vocab[base] {
|
||||
continue
|
||||
}
|
||||
pairs++
|
||||
if stem(w) != stem(base) {
|
||||
t.Errorf("stem(%q) = %q, stem(%q) = %q; want equal", w, stem(w), base, stem(base))
|
||||
}
|
||||
}
|
||||
if pairs == 0 {
|
||||
t.Error("the corpus holds no -ies/-ied word next to its -y form")
|
||||
}
|
||||
t.Logf("%d pairs checked in a vocabulary of %d words", pairs, len(vocab))
|
||||
}
|
||||
|
||||
// corpusSnippets holds every document's snippets off the fold kept at load
|
||||
// to the ones folding its transcript again gives - the versions' lines sit
|
||||
// behind that fold in the same string - and a page of "all" cards to its
|
||||
// bound.
|
||||
func corpusSnippets(t *testing.T) {
|
||||
for _, q := range []string{"nsa", "cable tapping", `"upstream collection"`, "tap*", "the"} {
|
||||
for i := range corpus().archive.Docs {
|
||||
d := &corpus().archive.Docs[i]
|
||||
if got, want := d.bodySnippets(q, 3), getSnippets(d.FullText, q, 3); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("%s, %q: %q from the loaded fold, %q from the text", d.ID, q, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
all := &http.Cookie{Name: "pref_per_page", Value: "all"}
|
||||
for _, c := range categories {
|
||||
for _, v := range rankCounts(c.Facet(corpus()), 1) {
|
||||
if c.Facet(corpus())[v] <= maxCardsPerPage {
|
||||
continue
|
||||
}
|
||||
body := get(t, categoryValueURL(c.Slug, v), all, foiaOn).Body.String()
|
||||
if n := strings.Count(body, `class="doc-card"`); n != maxCardsPerPage {
|
||||
t.Errorf("%s=%s: %d cards on a page of all, want %d", c.Slug, v, n, maxCardsPerPage)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// corpusIndex holds the shortcut a one-word all-fields clause takes - it
|
||||
// asks the document's entry in the index instead of walking its text - to
|
||||
// the walk, for a spread of the vocabulary over every document.
|
||||
func corpusIndex(t *testing.T) {
|
||||
var vocab []string
|
||||
for w := range corpus().tfidf.DocFreq {
|
||||
vocab = append(vocab, w)
|
||||
}
|
||||
slices.Sort(vocab)
|
||||
checked := 0
|
||||
for i := 0; i < len(vocab); i += 131 {
|
||||
w := vocab[i]
|
||||
docs := 0
|
||||
for di := range corpus().archive.Docs {
|
||||
d := &corpus().archive.Docs[di]
|
||||
_, indexed := d.terms[w]
|
||||
if indexed != containsTokens(d.StemmedText, w) {
|
||||
t.Fatalf("%q in %s: the index says %v, the all-fields stream the opposite", w, d.ID, indexed)
|
||||
}
|
||||
if indexed {
|
||||
docs++
|
||||
}
|
||||
}
|
||||
if docs != corpus().tfidf.DocFreq[w] {
|
||||
t.Errorf("%q: in %d documents' entries, with a document frequency of %d", w, docs, corpus().tfidf.DocFreq[w])
|
||||
}
|
||||
checked++
|
||||
}
|
||||
t.Logf("%d terms checked over %d documents", checked, len(corpus().archive.Docs))
|
||||
}
|
||||
123
tests/data_test.go
Normal file
123
tests/data_test.go
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAkaIDShape(t *testing.T) {
|
||||
for _, s := range []string{
|
||||
`{"title": "no aka_id key"}`,
|
||||
`{"aka_id": []}`,
|
||||
// null is "no akas": not a reason to drop the document.
|
||||
`{"aka_id": null}`,
|
||||
`{"aka_id": [{"id": "a"}, {"id": "b", "doc_source": "https://e.org/d", "doc_url": "https://e.org/b.pdf"}]}`,
|
||||
} {
|
||||
var d Document
|
||||
if err := json.Unmarshal([]byte(s), &d); err != nil {
|
||||
t.Errorf("%s: unexpected error: %v", s, err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, s := range []string{`{"aka_id": "a"}`, `{"aka_id": {"id": "a"}}`, `{"aka_id": ["a", "b"]}`, `{"aka_id": [{"id": "a"}, 3]}`} {
|
||||
var d Document
|
||||
if err := json.Unmarshal([]byte(s), &d); err == nil {
|
||||
t.Errorf("%s: decoded without an error", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeAka(t *testing.T) {
|
||||
d := Document{ID: "x", AkaID: []AkaEntry{
|
||||
{ID: " first ", DocSource: " https://e.org/p "},
|
||||
{ID: " "},
|
||||
{ID: "FIRST"},
|
||||
{ID: "second", DocURL: "https://e.org/s.pdf"},
|
||||
}}
|
||||
got := normalizeAka(&d)
|
||||
if len(got) != 2 || got[0].ID != "first" || got[0].DocSource != "https://e.org/p" || got[1].ID != "second" {
|
||||
t.Errorf("normalizeAka = %+v, want first (trimmed) and second, without the blank id and the repeat", got)
|
||||
}
|
||||
}
|
||||
|
||||
func writeTestFile(t *testing.T, path, content string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadVersions(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
v := filepath.Join(dir, "versions")
|
||||
writeTestFile(t, filepath.Join(v, "lemonde-2013", "copy.pdf"), "%PDF-1.4")
|
||||
writeTestFile(t, filepath.Join(v, "lemonde-2013", "copy.md"), "# Slide 12\n\nOnly in Le Monde's copy.")
|
||||
writeTestFile(t, filepath.Join(v, "lemonde-2013", "images", "page-10.jpg"), "")
|
||||
writeTestFile(t, filepath.Join(v, "lemonde-2013", "images", "page-2.jpg"), "")
|
||||
writeTestFile(t, filepath.Join(v, "guardian", "copy.pdf"), "%PDF-1.4") // no transcript or images: still a version
|
||||
writeTestFile(t, filepath.Join(v, "no-pdf", "copy.md"), "text only") // skipped: no PDF
|
||||
writeTestFile(t, filepath.Join(v, "bad name", "copy.pdf"), "%PDF-1.4") // skipped: name
|
||||
writeTestFile(t, filepath.Join(v, "loose.pdf"), "%PDF-1.4") // skipped: not a folder
|
||||
|
||||
var d Document
|
||||
if err := json.Unmarshal([]byte(`{"publication": [
|
||||
{"publisher": "Le Monde", "version": "lemonde-2013"},
|
||||
{"publisher": "The Guardian"},
|
||||
{"publisher": "Dagbladet", "version": " lemonde-2013 "},
|
||||
{"publisher": "Nobody", "version": "missing"}
|
||||
]}`), &d); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.ID, d.Dir = "prism", dir
|
||||
corpus().loadVersions(&d)
|
||||
|
||||
var slugs []string
|
||||
for _, v := range d.Versions {
|
||||
slugs = append(slugs, v.Slug)
|
||||
}
|
||||
if got := strings.Join(slugs, ","); got != "guardian,lemonde-2013" {
|
||||
t.Fatalf("versions %q, want guardian,lemonde-2013", got)
|
||||
}
|
||||
|
||||
lm := d.VersionFor("lemonde-2013")
|
||||
if lm.PDFFile != "copy.pdf" || !strings.Contains(lm.FullText, "Only in Le Monde") || lm.HTMLContent == "" {
|
||||
t.Errorf("lemonde-2013 not fully loaded: pdf %q, text %q, html %q", lm.PDFFile, lm.FullText, lm.HTMLContent)
|
||||
}
|
||||
if got := strings.Join(lm.Images, ","); got != "page-2.jpg,page-10.jpg" {
|
||||
t.Errorf("images %q, want page-2.jpg,page-10.jpg (page order)", got)
|
||||
}
|
||||
if got := fmt.Sprint(lm.PubIndex); got != "[0 2]" {
|
||||
t.Errorf("PubIndex %s, want [0 2]", got)
|
||||
}
|
||||
if got := d.Publication[2].Version; got != "lemonde-2013" {
|
||||
t.Errorf("publication version not trimmed: %q", got)
|
||||
}
|
||||
if u := d.UnlinkedVersions(); len(u) != 1 || u[0].Slug != "guardian" {
|
||||
t.Errorf("unlinked versions %+v, want only guardian", u)
|
||||
}
|
||||
if d.VersionFor("missing") != nil || d.VersionFor("") != nil {
|
||||
t.Error("VersionFor returned a version that doesn't exist")
|
||||
}
|
||||
}
|
||||
|
||||
// TestBodyFormsAreTheCombinedTails: loadData slices FoldedFull and
|
||||
// StemmedFull out of the ends of SearchText and StemmedText to save memory;
|
||||
// they must still be exactly what setBodySearchText builds.
|
||||
func TestBodyFormsAreTheCombinedTails(t *testing.T) {
|
||||
buildFixture(t)
|
||||
for i := range corpus().archive.Docs {
|
||||
d := &corpus().archive.Docs[i]
|
||||
c := *d
|
||||
setBodySearchText(&c)
|
||||
if c.FoldedFull != d.FoldedFull || c.StemmedFull != d.StemmedFull {
|
||||
t.Errorf("%s: body forms differ from a fresh build", d.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
295
tests/fixture_test.go
Normal file
295
tests/fixture_test.go
Normal file
|
|
@ -0,0 +1,295 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A small corpus on disk plus a running mux: the harness every handler test
|
||||
// uses. The fixture covers the shapes the handlers branch on - one document
|
||||
// per collection, a SIDtoday issue with a series, another published copy in
|
||||
// versions/, a joint investigation with a reporting-only partner, and a
|
||||
// directory with no PDF (which the loader must skip).
|
||||
//
|
||||
// Built once for the package: loadData writes package globals, so tests
|
||||
// share one corpus rather than racing to replace it.
|
||||
|
||||
var (
|
||||
fixtureOnce sync.Once
|
||||
fixtureMux http.Handler
|
||||
fixtureDir string
|
||||
)
|
||||
|
||||
type fixtureDoc struct {
|
||||
id, collection string
|
||||
meta map[string]interface{}
|
||||
text string
|
||||
pages []string
|
||||
versions map[string][]string // slug -> page images
|
||||
noPDF bool
|
||||
}
|
||||
|
||||
// fixtureJPEG is a real 4x3 JPEG, so the loader can read each page image's size.
|
||||
var fixtureJPEG = func() string {
|
||||
var b bytes.Buffer
|
||||
if err := jpeg.Encode(&b, image.NewGray(image.Rect(0, 0, 4, 3)), nil); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return b.String()
|
||||
}()
|
||||
|
||||
func writeFixtureFile(t *testing.T, path, content string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMain removes the shared fixture corpus once every test has run.
|
||||
func TestMain(m *testing.M) {
|
||||
code := m.Run()
|
||||
if fixtureDir != "" {
|
||||
os.RemoveAll(fixtureDir)
|
||||
}
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
func buildFixture(t *testing.T) (http.Handler, string) {
|
||||
t.Helper()
|
||||
fixtureOnce.Do(func() {
|
||||
root, err := os.MkdirTemp("", "snowchive-fixture")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, d := range fixtureDocs() {
|
||||
dir := root
|
||||
switch d.collection {
|
||||
case collectionFOIA:
|
||||
dir = filepath.Join(root, "FOIA")
|
||||
case collectionArchive:
|
||||
dir = filepath.Join(root, "SIDtoday-archive")
|
||||
}
|
||||
dir = filepath.Join(dir, d.id)
|
||||
b, err := json.MarshalIndent(d.meta, "", " ")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeFixtureFile(t, filepath.Join(dir, "metadata.json"), string(b)+"\n")
|
||||
if !d.noPDF {
|
||||
writeFixtureFile(t, filepath.Join(dir, d.id+".pdf"), "%PDF-1.4\nfixture\n")
|
||||
}
|
||||
if d.text != "" {
|
||||
writeFixtureFile(t, filepath.Join(dir, d.id+".md"), d.text)
|
||||
}
|
||||
for _, p := range d.pages {
|
||||
writeFixtureFile(t, filepath.Join(dir, "images", p), fixtureJPEG)
|
||||
}
|
||||
for slug, imgs := range d.versions {
|
||||
vdir := filepath.Join(dir, "versions", slug)
|
||||
writeFixtureFile(t, filepath.Join(vdir, slug+".pdf"), "%PDF-1.4\n")
|
||||
writeFixtureFile(t, filepath.Join(vdir, slug+".md"),
|
||||
"# Copy\n\nA line only this copy has: unredacted FAIRVIEW detail.\n")
|
||||
for _, p := range imgs {
|
||||
writeFixtureFile(t, filepath.Join(vdir, "images", p), fixtureJPEG)
|
||||
}
|
||||
}
|
||||
}
|
||||
writeFixtureFile(t, filepath.Join(root, "glossary.json"), `[
|
||||
{"term": "XKEYSCORE", "definition": "A search system.", "aka": ["XKS"]}
|
||||
]`)
|
||||
|
||||
dataDir, projectDir = root, root
|
||||
devMode = false
|
||||
apiEnabled = true
|
||||
initAssets()
|
||||
if err := parseTemplates(); err != nil {
|
||||
t.Fatalf("templates: %v", err)
|
||||
}
|
||||
loadData()
|
||||
fixtureDir = root
|
||||
fixtureMux = withBaseHeaders(newMux())
|
||||
})
|
||||
if fixtureMux == nil {
|
||||
t.Fatal("fixture not built")
|
||||
}
|
||||
return fixtureMux, fixtureDir
|
||||
}
|
||||
|
||||
func fixtureDocs() []fixtureDoc {
|
||||
return []fixtureDoc{
|
||||
{
|
||||
id: "prism-slides",
|
||||
collection: collectionNews,
|
||||
text: "# PRISM\n\nCollection directly from the servers. XKEYSCORE ingests it.\n" +
|
||||
"Upstream collection feeds the corpus, and analysts were collecting metadata\n" +
|
||||
"from RT-RG dashboards. Müller-Maguhn reviewed the São Paulo cable tapping.\n",
|
||||
pages: []string{"page-1.jpg", "page-2.jpg"},
|
||||
versions: map[string][]string{"lemonde": {"page-1.jpg"}},
|
||||
meta: map[string]interface{}{
|
||||
"title": "PRISM Collection Slides",
|
||||
"type": "REPORTED",
|
||||
"description": "Slides describing PRISM collection.",
|
||||
"tags": []string{"PRISM", "collection", "cable tapping"},
|
||||
"pdf_sha256": strings.Repeat("a", 64),
|
||||
"aka_id": []map[string]string{{"id": "prism-slides-guardian"}},
|
||||
"document": map[string]interface{}{
|
||||
"creator": "NSA", "date_created": "2013-04-01",
|
||||
"programs": []string{"PRISM", "XKEYSCORE"},
|
||||
"classification": []string{"TOP SECRET"}, "dissemination": []string{"NOFORN"},
|
||||
"countries": []string{"United States", "Brazil"},
|
||||
"people": []string{"Snowden, Edward", "Müller-Maguhn, Andy"},
|
||||
"banner": "TOP SECRET//SI//NOFORN", "genre": "slides",
|
||||
},
|
||||
"publication": []map[string]interface{}{
|
||||
{"publisher": "The Guardian", "title": "NSA Prism program",
|
||||
"link": "https://example.org/prism", "date_published": "2013-06-06",
|
||||
"reporters": []string{"Greenwald, Glenn"}},
|
||||
{"publisher": "Le Monde", "link": "https://example.org/prism-fr",
|
||||
"date_published": "2013-06-08", "version": "lemonde"},
|
||||
{"publisher": "NRK", "link": "https://example.org/prism",
|
||||
"date_published": "2013-06-06", "reporting_only": true},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
// Shares the Guardian link with prism-slides, which is what
|
||||
// makes the "published in the same report" section exist.
|
||||
id: "prism-companion", collection: collectionNews,
|
||||
text: "# Companion\n\nA second document from the same story.\n",
|
||||
pages: []string{"page-1.jpg"},
|
||||
meta: map[string]interface{}{
|
||||
"title": "PRISM Companion Slide", "type": "REPORTED",
|
||||
"description": "A second slide from the same story.",
|
||||
"tags": []string{"PRISM"},
|
||||
"document": map[string]interface{}{
|
||||
"creator": "NSA", "date_created": "2013-04-02",
|
||||
"programs": []string{"PRISM"},
|
||||
},
|
||||
"publication": []map[string]interface{}{
|
||||
{"publisher": "The Guardian", "title": "NSA Prism program",
|
||||
"link": "https://example.org/prism", "date_published": "2013-06-06",
|
||||
"reporters": []string{"Greenwald, Glenn"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "2003-05-29_SIDToday_-_Fixture_Issue",
|
||||
collection: collectionArchive,
|
||||
text: "# SIDtoday\n\nThe analyst cockpit and other matters. Upstream tapping was\n" +
|
||||
"discussed at length by the directorate, and the collection of that traffic\n" +
|
||||
"began only the following year.\n",
|
||||
pages: []string{"page-1.jpg"},
|
||||
meta: map[string]interface{}{
|
||||
"title": "A Day in the Life of a Fixture", "type": "SIDTODAY",
|
||||
"description": "An issue of SIDtoday.",
|
||||
"tags": []string{"analysis"},
|
||||
"document": map[string]interface{}{
|
||||
"creator": "NSA", "date_created": "2003-05-29",
|
||||
"classification": []string{"UNCLASSIFIED"}, "distribution": []string{},
|
||||
"series": map[string]interface{}{"name": "(U) A Day in the Life of...", "part": 1},
|
||||
},
|
||||
"publication": []map[string]interface{}{
|
||||
{"publisher": "The Intercept", "link": "https://example.org/sidtoday/1",
|
||||
"date_published": "2016-05-16"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "FISC_-_fixture_order",
|
||||
collection: collectionFOIA,
|
||||
text: "# Order\n\nThe Court finds the application satisfies the statute.\n",
|
||||
meta: map[string]interface{}{
|
||||
"title": "Fixture FISC Order", "type": "FOIA",
|
||||
"description": "A court order released under FOIA.",
|
||||
"document": map[string]interface{}{
|
||||
"creator": "FISC", "date_created": "2013-10-11",
|
||||
"classification": []string{"SECRET"},
|
||||
},
|
||||
"publication": []map[string]interface{}{{"publisher": "ODNI", "link": "https://example.org/foia/1"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "undated-fixture",
|
||||
collection: collectionNews,
|
||||
text: "# Undated\n\nNo document date at all.\n",
|
||||
meta: map[string]interface{}{
|
||||
"title": "Undated Fixture", "type": "REPORTED",
|
||||
"description": "A document with no date.",
|
||||
"document": map[string]interface{}{"creator": "GCHQ"},
|
||||
"publication": []map[string]interface{}{{"publisher": "The Intercept", "link": "https://example.org/undated"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "broken-no-pdf",
|
||||
collection: collectionNews,
|
||||
noPDF: true,
|
||||
meta: map[string]interface{}{"title": "Broken", "type": "REPORTED"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// get runs one request through the real routing table and header wrapper.
|
||||
func get(t *testing.T, path string, cookies ...*http.Cookie) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
mux, _ := buildFixture(t)
|
||||
r := httptest.NewRequest("GET", path, nil)
|
||||
for _, c := range cookies {
|
||||
r.AddCookie(c)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
mux.ServeHTTP(w, r)
|
||||
return w
|
||||
}
|
||||
|
||||
// newPost builds a form POST for the state-changing endpoints.
|
||||
func newPost(path string) *http.Request {
|
||||
r := httptest.NewRequest("POST", path, strings.NewReader("back=/browse"))
|
||||
r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
return r
|
||||
}
|
||||
|
||||
func record(h http.Handler, r *http.Request) *httptest.ResponseRecorder {
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, r)
|
||||
return w
|
||||
}
|
||||
|
||||
// withCorpus loads another corpus for one test and puts the fixture back
|
||||
// afterwards. Only possible because loadData is idempotent - every build*
|
||||
// starts from empty (TestLoadDataIsIdempotent).
|
||||
func withCorpus(t *testing.T, root string) {
|
||||
t.Helper()
|
||||
withDirs(t, root, root)
|
||||
}
|
||||
|
||||
// withDirs is withCorpus with the corpus and the project files apart, as
|
||||
// -root lays them out.
|
||||
func withDirs(t *testing.T, data, project string) {
|
||||
t.Helper()
|
||||
buildFixture(t) // make sure templates and the fixture path exist first
|
||||
prev := dataDir
|
||||
t.Cleanup(func() {
|
||||
dataDir, projectDir = prev, prev
|
||||
loadData()
|
||||
})
|
||||
dataDir, projectDir = data, project
|
||||
loadData()
|
||||
}
|
||||
|
||||
// httptest_NewGet is a plain GET request for the tests that drive a mux
|
||||
// directly rather than through get().
|
||||
func httptest_NewGet(path string) *http.Request {
|
||||
return httptest.NewRequest("GET", path, nil)
|
||||
}
|
||||
167
tests/flags_test.go
Normal file
167
tests/flags_test.go
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
package main
|
||||
|
||||
// flags_test.go - what each command-line flag actually changes. The flags
|
||||
// are parsed in main(), which no test can call, so these exercise the state
|
||||
// the flags set and the behaviour that depends on it: the routes an
|
||||
// instance serves, the URLs it writes, the warnings it prints.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// -no-api: every /api/ path answers a JSON 403, and the UI stops offering
|
||||
// the API. The kill switch is one registration, so a future endpoint
|
||||
// inherits it - that is the property worth testing.
|
||||
func TestFlagNoAPI(t *testing.T) {
|
||||
buildFixture(t)
|
||||
prev := apiEnabled
|
||||
t.Cleanup(func() { apiEnabled = prev; fixtureMux = withBaseHeaders(newMux()) })
|
||||
|
||||
apiEnabled = false
|
||||
mux := withBaseHeaders(newMux())
|
||||
for _, p := range []string{"/api/docs", "/api/doc/prism-slides", "/api/search?q=x",
|
||||
"/api/facets", "/api/something-added-next-year"} {
|
||||
w := record(mux, httptest_NewGet(p))
|
||||
if w.Code != http.StatusForbidden {
|
||||
t.Errorf("%s with -no-api: status %d, want 403", p, w.Code)
|
||||
}
|
||||
var body map[string]string
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil || body["error"] == "" {
|
||||
t.Errorf("%s: body %q is not a JSON error", p, w.Body.String())
|
||||
}
|
||||
if ct := w.Header().Get("Content-Type"); !strings.HasPrefix(ct, "application/json") {
|
||||
t.Errorf("%s: Content-Type %q", p, ct)
|
||||
}
|
||||
}
|
||||
// The visitor's own export is NOT part of the API and stays available.
|
||||
if w := record(mux, httptest_NewGet("/pinned/export")); w.Code != 200 {
|
||||
t.Errorf("/pinned/export with -no-api: %d", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// -base: the absolute URLs in robots.txt and the sitemap follow
|
||||
// the canonical host instead of the request's Host header.
|
||||
func TestFlagBase(t *testing.T) {
|
||||
buildFixture(t)
|
||||
prev, prevMap := canonicalBase, corpus().canonicalSitemap
|
||||
t.Cleanup(func() {
|
||||
canonicalBase, corpus().canonicalSitemap = prev, prevMap
|
||||
corpus().buildCanonicalSitemap()
|
||||
})
|
||||
canonicalBase = "https://archive.example.org"
|
||||
corpus().buildCanonicalSitemap()
|
||||
|
||||
for _, c := range []struct{ path, want string }{
|
||||
{"/robots.txt", "Sitemap: https://archive.example.org/sitemap.xml"},
|
||||
{"/sitemap.xml", "<loc>https://archive.example.org/doc/prism-slides</loc>"},
|
||||
} {
|
||||
if body := get(t, c.path).Body.String(); !strings.Contains(body, c.want) {
|
||||
t.Errorf("%s does not carry %q", c.path, c.want)
|
||||
}
|
||||
}
|
||||
// A hostile Host header cannot reach a response once -base is set.
|
||||
r := httptest_NewGet("/robots.txt")
|
||||
r.Host = "evil.example.net"
|
||||
if body := record(fixtureMux, r).Body.String(); strings.Contains(body, "evil") {
|
||||
t.Errorf("the Host header leaked into robots.txt: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
// -base is validated, not trusted: anything but scheme://host is refused,
|
||||
// so the flag cannot produce URLs that do not resolve.
|
||||
func TestFlagBaseValidation(t *testing.T) {
|
||||
for _, ok := range []string{"https://a.org", "http://a.org", "https://a.org:8443", "https://a.org/"} {
|
||||
if _, err := parseCanonicalBase(ok); err != nil {
|
||||
t.Errorf("%s rejected: %v", ok, err)
|
||||
}
|
||||
}
|
||||
for _, bad := range []string{"", "a.org", "ftp://a.org", "https://", "https://a.org/path",
|
||||
"https://a.org?q=1", "https://a.org#f", "https://user@a.org", "://x"} {
|
||||
if _, err := parseCanonicalBase(bad); err == nil {
|
||||
t.Errorf("%q accepted", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// -check counts what it reports: the fixture holds one document the loader
|
||||
// refuses (no PDF) and one directory that looks like a document.
|
||||
func TestFlagCheckCounts(t *testing.T) {
|
||||
buildFixture(t)
|
||||
if loadErrors == 0 {
|
||||
t.Error("the fixture's PDF-less document should be an error")
|
||||
}
|
||||
if loadErrors+loadWarnings == 0 {
|
||||
t.Error("-check would exit 0 on a corpus with problems")
|
||||
}
|
||||
}
|
||||
|
||||
// -listen warns for anything but loopback, because binding elsewhere puts
|
||||
// the archive in front of the proxy instead of behind it.
|
||||
func TestFlagListenWarning(t *testing.T) {
|
||||
before := loadWarnings
|
||||
for _, quiet := range []string{"127.0.0.1:8080", "localhost:8080", "[::1]:8080", "127.0.0.53:80"} {
|
||||
warnNonLoopback(quiet)
|
||||
if loadWarnings != before {
|
||||
t.Errorf("%s warned", quiet)
|
||||
before = loadWarnings
|
||||
}
|
||||
}
|
||||
for _, loud := range []string{"0.0.0.0:8080", ":8080", "192.168.1.10:8080", "[::]:8080"} {
|
||||
warnNonLoopback(loud)
|
||||
if loadWarnings == before {
|
||||
t.Errorf("%s did not warn", loud)
|
||||
}
|
||||
before = loadWarnings
|
||||
}
|
||||
}
|
||||
|
||||
// -dev: templates come from disk and nothing is pre-rendered, so every page
|
||||
// must still render - the fallback path the pre-rendered blocks rely on.
|
||||
func TestFlagDevMode(t *testing.T) {
|
||||
buildFixture(t)
|
||||
prevDev, prevPre, prevTmpl := devMode, corpus().prerendered, tmplSet
|
||||
t.Cleanup(func() {
|
||||
devMode, corpus().prerendered, tmplSet = prevDev, prevPre, prevTmpl
|
||||
initAssets()
|
||||
})
|
||||
devMode = true
|
||||
initAssets() // templates/ and static/ from the working directory
|
||||
corpus().prerenderPages()
|
||||
if len(corpus().prerendered) != 0 {
|
||||
t.Error("dev mode pre-rendered pages; it must render them live")
|
||||
}
|
||||
for _, p := range []string{"/", "/browse", "/browse/program", "/explore/ids", "/glossary",
|
||||
"/articles", "/explore/terms", "/doc/prism-slides"} {
|
||||
if w := get(t, p); w.Code != 200 {
|
||||
t.Errorf("%s in dev mode: %d", p, w.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// -root: the documents are read from its data/ and the files about the
|
||||
// archive from its project/.
|
||||
func TestRootDir(t *testing.T) {
|
||||
buildFixture(t)
|
||||
prevData, prevProject := dataDir, projectDir
|
||||
t.Cleanup(func() {
|
||||
dataDir, projectDir = prevData, prevProject
|
||||
loadData()
|
||||
})
|
||||
root := t.TempDir()
|
||||
writeFixtureFile(t, filepath.Join(root, "data", "doc-in-root", "metadata.json"), `{"title": "In the root"}`)
|
||||
writeFixtureFile(t, filepath.Join(root, "data", "doc-in-root", "doc-in-root.pdf"), "%PDF-1.4\n")
|
||||
writeFixtureFile(t, filepath.Join(root, "project", "glossary.json"), `[{"term": "INPROJECT", "definition": "Read from project/."}]`)
|
||||
setRoot(root)
|
||||
loadData()
|
||||
|
||||
if corpus().docIndex["doc-in-root"] == nil {
|
||||
t.Error("the document in data/ was not loaded")
|
||||
}
|
||||
if corpus().glossaryLookup("INPROJECT") == nil {
|
||||
t.Error("glossary.json in project/ was not loaded")
|
||||
}
|
||||
}
|
||||
272
tests/foia_test.go
Normal file
272
tests/foia_test.go
Normal file
|
|
@ -0,0 +1,272 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"html"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// foiaOn is the cookie of a reader who opted in to FOIA releases.
|
||||
var foiaOn = &http.Cookie{Name: "pref_include_foia", Value: "1"}
|
||||
|
||||
// In the consistency corpus "alpha" is in n1, n2 and the FOIA release f1.
|
||||
func TestSearchLeavesOutFOIAByDefault(t *testing.T) {
|
||||
withCorpus(t, writeConsistencyCorpus(t))
|
||||
|
||||
if got := resultTotal(t, "/search?q=alpha"); got != 2 {
|
||||
t.Errorf("q=alpha, no preference: %d results, want 2 (FOIA left out)", got)
|
||||
}
|
||||
body := get(t, "/search?q=alpha").Body.String()
|
||||
if strings.Contains(body, "left out") {
|
||||
t.Error("the page counts the FOIA releases it left out")
|
||||
}
|
||||
// The page's own links name the collections it searched, so a copied
|
||||
// link shows the same results to a reader with the other preference.
|
||||
if !strings.Contains(body, "coll=news&coll=sidtoday-archive") {
|
||||
t.Error("the page's links do not carry the default collections")
|
||||
}
|
||||
if got := resultTotal(t, "/search?q=alpha&coll=news&coll=sidtoday-archive&coll=foia"); got != 3 {
|
||||
t.Errorf("q=alpha in every collection: %d results, want 3", got)
|
||||
}
|
||||
|
||||
if got := resultTotal(t, "/search?q=alpha", foiaOn); got != 3 {
|
||||
t.Errorf("q=alpha with FOIA on: %d results, want 3", got)
|
||||
}
|
||||
if strings.Contains(get(t, "/search?q=alpha", foiaOn).Body.String(), "left out") {
|
||||
t.Error("FOIA on, yet the page says something was left out")
|
||||
}
|
||||
// An explicit collection choice wins over the preference.
|
||||
if got := resultTotal(t, "/search?q=alpha&coll=foia"); got != 1 {
|
||||
t.Errorf("q=alpha&coll=foia, no preference: %d results, want 1", got)
|
||||
}
|
||||
if got := resultTotal(t, "/search?q=alpha&coll=news", foiaOn); got != 2 {
|
||||
t.Errorf("q=alpha&coll=news with FOIA on: %d results, want 2", got)
|
||||
}
|
||||
}
|
||||
|
||||
// ZEBRA is on n1, n2, f1 (FOIA) and s1 (SIDtoday).
|
||||
func TestCategoryPagesLeaveOutFOIAByDefault(t *testing.T) {
|
||||
withCorpus(t, writeConsistencyCorpus(t))
|
||||
|
||||
if got := resultTotal(t, "/browse/program/ZEBRA"); got != 3 {
|
||||
t.Errorf("ZEBRA, no preference: %d documents, want 3", got)
|
||||
}
|
||||
if strings.Contains(get(t, "/browse/program/ZEBRA").Body.String(), "left out") {
|
||||
t.Error("the page still carries a FOIA notice")
|
||||
}
|
||||
if got := resultTotal(t, "/browse/program/ZEBRA", foiaOn); got != 4 {
|
||||
t.Errorf("ZEBRA with FOIA on: %d documents, want 4", got)
|
||||
}
|
||||
if strings.Contains(get(t, "/browse/program/ZEBRA", foiaOn).Body.String(), "left out") {
|
||||
t.Error("FOIA on, yet the page says FOIA releases are left out")
|
||||
}
|
||||
}
|
||||
|
||||
// Every total in a category table is the number its row's link lists, for
|
||||
// either preference.
|
||||
func TestCategoryTotalsMatchTheirPages(t *testing.T) {
|
||||
withCorpus(t, writeConsistencyCorpus(t))
|
||||
reRow := regexp.MustCompile(`(?s)<a href="([^"]+)">\s*([^<]+?)\s*</a>.*?</td>\s*<td class="num-col">(\d+)</td>`)
|
||||
for _, cookies := range [][]*http.Cookie{nil, {foiaOn}} {
|
||||
for _, slug := range []string{"program", "agency", "docdate", "published", "publisher"} {
|
||||
body := get(t, "/browse/"+slug, cookies...).Body.String()
|
||||
_, body, _ = strings.Cut(body, "<tbody>")
|
||||
body, _, _ = strings.Cut(body, "</tbody>")
|
||||
rows := reRow.FindAllStringSubmatch(body, -1)
|
||||
if len(rows) == 0 {
|
||||
t.Fatalf("/browse/%s: no rows", slug)
|
||||
}
|
||||
for _, row := range rows {
|
||||
want, _ := strconv.Atoi(row[3])
|
||||
if strings.Contains(row[1], "coll=") {
|
||||
t.Errorf("/browse/%s %s: row links with a collection scope: %s", slug, row[2], row[1])
|
||||
}
|
||||
if got := resultTotal(t, html.UnescapeString(row[1]), cookies...); got != want {
|
||||
t.Errorf("/browse/%s %s (FOIA on: %v): total %d, its page lists %d", slug, row[2], cookies != nil, want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
off, on := get(t, "/browse/program").Body.String(), get(t, "/browse/program", foiaOn).Body.String()
|
||||
if strings.Contains(off, "Totals leave out") || strings.Contains(off, "sort=foia") {
|
||||
t.Error("FOIA off: the table still has a FOIA notice or column")
|
||||
}
|
||||
if !strings.Contains(on, "sort=foia") {
|
||||
t.Error("FOIA on: the table has no FOIA column")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRandomAndRelatedLeaveOutFOIA(t *testing.T) {
|
||||
withCorpus(t, writeConsistencyCorpus(t))
|
||||
|
||||
sawFOIA := false
|
||||
for i := 0; i < 300; i++ {
|
||||
if d := corpus().randomDoc(false); d == nil || d.Collection == collectionFOIA {
|
||||
t.Fatalf("randomDoc(false) = %v", d)
|
||||
}
|
||||
if corpus().randomDoc(true).Collection == collectionFOIA {
|
||||
sawFOIA = true
|
||||
}
|
||||
}
|
||||
if !sawFOIA {
|
||||
t.Error("randomDoc(true) never picked the FOIA release in 300 tries")
|
||||
}
|
||||
|
||||
corpus().archive.RelatedDocs["n2"] = []string{"f1", "n1"}
|
||||
corpus().archive.RelatedDocs["f1"] = []string{"n1"}
|
||||
body := get(t, "/doc/n2").Body.String()
|
||||
if strings.Contains(body, `href="/doc/f1"`) || !strings.Contains(body, `href="/doc/n1"`) {
|
||||
t.Error("/doc/n2, no preference: want n1 related and f1 left out")
|
||||
}
|
||||
if !strings.Contains(get(t, "/doc/n2", foiaOn).Body.String(), `href="/doc/f1"`) {
|
||||
t.Error("/doc/n2 with FOIA on: f1 is not related")
|
||||
}
|
||||
if !strings.Contains(get(t, "/doc/f1").Body.String(), `href="/doc/n1"`) {
|
||||
t.Error("/doc/f1: its related documents are gone")
|
||||
}
|
||||
}
|
||||
|
||||
// The home page counts the Snowden documents apart from the FOIA releases and
|
||||
// says how the preference treats them. The corpus: n1-n4, a1, d1 (news), s1
|
||||
// (SIDtoday archive), f1 (FOIA).
|
||||
func TestHomeStatesTheFOIAScope(t *testing.T) {
|
||||
withCorpus(t, writeConsistencyCorpus(t))
|
||||
body := get(t, "/").Body.String()
|
||||
for _, want := range []string{"holds 7 published Snowden documents. 6 documents are", "1 documents are the NSA newsletter",
|
||||
"also holds 1", "are left out by default but can be included via"} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("home page lacks %q", want)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(get(t, "/", foiaOn).Body.String(), "you have included via") {
|
||||
t.Error("FOIA on: the home page does not say so")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFOIAPreferenceIsSaved(t *testing.T) {
|
||||
withCorpus(t, writeConsistencyCorpus(t))
|
||||
if !strings.Contains(get(t, "/preferences").Body.String(), `name="include_foia"`) {
|
||||
t.Error("/preferences has no FOIA checkbox")
|
||||
}
|
||||
if readPrefs(httptest_NewGet("/")).IncludeFOIA {
|
||||
t.Error("FOIA releases are included by default")
|
||||
}
|
||||
}
|
||||
|
||||
// The form shows the scope the search ran with: FOIA left out by default
|
||||
// is an unchecked box, so submitting the form keeps it out.
|
||||
func TestSearchFormKeepsTheDefaultScope(t *testing.T) {
|
||||
withCorpus(t, writeConsistencyCorpus(t))
|
||||
|
||||
checked := regexp.MustCompile(`value="foia" checked`)
|
||||
if checked.MatchString(get(t, "/search?q=alpha").Body.String()) {
|
||||
t.Error("FOIA left out, yet its box is checked")
|
||||
}
|
||||
if !checked.MatchString(get(t, "/search?q=alpha", foiaOn).Body.String()) {
|
||||
t.Error("FOIA on, yet its box is not checked")
|
||||
}
|
||||
if got := resultTotal(t, "/search?q=alpha&coll=news&coll=sidtoday-archive&do_search=1"); got != 2 {
|
||||
t.Errorf("the form's resubmit finds %d, want 2", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A FOIA release's metadata links name every collection, so its value
|
||||
// pages list it under the default scope.
|
||||
func TestFOIADocLinksIncludeFOIA(t *testing.T) {
|
||||
buildFixture(t)
|
||||
// Every value link printed for the release - on its own page, on its
|
||||
// card and its table row in the FOIA tab, in a search result - opens a
|
||||
// page that lists the release.
|
||||
const id = "FISC_-_fixture_order"
|
||||
reValueLink := regexp.MustCompile(`href="(/browse/[a-z]+/[^"]+)"`)
|
||||
for _, page := range []string{"/doc/" + id, "/browse?collection=foia", "/browse?collection=foia&view=table", "/search?q=court&coll=foia"} {
|
||||
links := reValueLink.FindAllStringSubmatch(get(t, page).Body.String(), -1)
|
||||
if len(links) < 2 {
|
||||
t.Fatalf("%s: %d value links, want the release's agency and classification at least", page, len(links))
|
||||
}
|
||||
for _, m := range links {
|
||||
link := html.UnescapeString(m[1])
|
||||
if !strings.Contains(get(t, withQuery(link, "view=table")).Body.String(), `href="/doc/`+id+`"`) {
|
||||
t.Errorf("%s links %s, which does not list the release", page, link)
|
||||
}
|
||||
}
|
||||
}
|
||||
// /explore/classification counts every collection, and so do its links.
|
||||
for _, m := range regexp.MustCompile(`href="(/browse/[^"]+)"`).FindAllStringSubmatch(get(t, "/explore/classification").Body.String(), -1) {
|
||||
if !strings.Contains(m[1], "coll=foia") {
|
||||
t.Errorf("/explore/classification links %s without its scope", m[1])
|
||||
}
|
||||
}
|
||||
|
||||
body := get(t, "/doc/FISC_-_fixture_order").Body.String()
|
||||
m := regexp.MustCompile(`href="(/browse/agency/FISC[^"]*)"`).FindStringSubmatch(body)
|
||||
if m == nil || !strings.Contains(m[1], "coll=foia") {
|
||||
t.Fatalf("agency link %v does not include FOIA", m)
|
||||
}
|
||||
if got := resultTotal(t, html.UnescapeString(m[1])); got != 1 {
|
||||
t.Errorf("the agency link lists %d documents, want 1", got)
|
||||
}
|
||||
if strings.Contains(get(t, "/doc/FISC_-_fixture_order", foiaOn).Body.String(), "/browse/agency/FISC?") {
|
||||
t.Error("FOIA on: the links need no scope")
|
||||
}
|
||||
}
|
||||
|
||||
// On a publisher page scoped to a publication year, the links built from
|
||||
// its documents carry that year.
|
||||
func TestPublisherYearLinksCarryTheYear(t *testing.T) {
|
||||
buildFixture(t)
|
||||
body := get(t, "/browse/publisher/The%20Intercept?year=2016").Body.String()
|
||||
if !strings.Contains(body, "date_pub_start=2016") {
|
||||
t.Error("no link on the year-scoped page carries the publication year")
|
||||
}
|
||||
}
|
||||
|
||||
// collScope is the one place a page's collection scope is decided: FOIA
|
||||
// releases are opt-in when the URL names no collection, an explicit coll=
|
||||
// wins, and only the search pins a defaulted scope into its links.
|
||||
func TestCollScope(t *testing.T) {
|
||||
off, on := Prefs{}, Prefs{IncludeFOIA: true}
|
||||
none := url.Values{}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
v url.Values
|
||||
p Prefs
|
||||
pin bool
|
||||
foia, def bool
|
||||
qs string
|
||||
}{
|
||||
{"default, browse", none, off, false, false, true, ""},
|
||||
{"default, search", none, off, true, false, true, "coll=news&coll=sidtoday-archive"},
|
||||
{"opted in, browse", none, on, false, true, true, ""},
|
||||
{"opted in, search", none, on, true, true, true, "coll=news&coll=sidtoday-archive&coll=foia"},
|
||||
{"explicit foia", url.Values{"coll": {"foia"}}, off, false, true, false, "coll=foia"},
|
||||
{"explicit news over opt-in", url.Values{"coll": {"news"}}, on, false, false, false, "coll=news"},
|
||||
{"explicit everything", url.Values{"coll": {"news", "foia", "sidtoday-archive"}}, off, false, true, false,
|
||||
"coll=news&coll=sidtoday-archive&coll=foia"},
|
||||
} {
|
||||
s := pageScope(c.v, c.p, c.pin)
|
||||
foia := s.shows(&Document{Collection: collectionFOIA})
|
||||
if foia != c.foia || s.defaulted != c.def || s.qs() != c.qs || s.shown[collectionFOIA] != c.foia {
|
||||
t.Errorf("%s: shows FOIA %v, defaulted %v, qs %q; want %v, %v, %q",
|
||||
c.name, foia, s.defaulted, s.qs(), c.foia, c.def, c.qs)
|
||||
}
|
||||
}
|
||||
|
||||
// A FOIA release's links widen the default to the whole archive and
|
||||
// say so; a reader who opted in needs no widening.
|
||||
if got := pageScope(none, off, false).including(collectionFOIA).suffix(); got != "?coll=news&coll=sidtoday-archive&coll=foia" {
|
||||
t.Errorf("widened default: %q", got)
|
||||
}
|
||||
if got := pageScope(none, on, false).including(collectionFOIA).suffix(); got != "" {
|
||||
t.Errorf("opted in, widened: %q, want no scope", got)
|
||||
}
|
||||
// and adds to the links, not to the documents.
|
||||
s := pageScope(none, off, false).and("date_pub_start=2014")
|
||||
if s.link("/search?q=x") != "/search?q=x&date_pub_start=2014" || s.shows(&Document{Collection: collectionFOIA}) {
|
||||
t.Errorf("and: %q", s.link("/search?q=x"))
|
||||
}
|
||||
}
|
||||
5
tests/go.mod
Normal file
5
tests/go.mod
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
// Keeps this directory out of the snowchive module's ./... patterns. The
|
||||
// files here are package main's tests; tests/run.sh compiles them with it.
|
||||
module snowchive/tests
|
||||
|
||||
go 1.22
|
||||
53
tests/headings_test.go
Normal file
53
tests/headings_test.go
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var reHeadingTag = regexp.MustCompile(`<h([1-6])\b`)
|
||||
|
||||
func TestTranscriptHeadingsNestUnderThePage(t *testing.T) {
|
||||
for src, want := range map[string]string{
|
||||
"# A\n\n## B\n\n#### C\n\n## D": `<h2 class="md-h1">A</h2> <h3 class="md-h2">B</h3> <h4 class="md-h4">C</h4> <h3 class="md-h2">D</h3>`,
|
||||
"### A\n\n### B\n\n#### C": `<h2 class="md-h3">A</h2> <h2 class="md-h3">B</h2> <h3 class="md-h4">C</h3>`,
|
||||
"```\n# not a heading\n```\n\n## A": `<h2 class="md-h2">A</h2>`,
|
||||
} {
|
||||
var got []string
|
||||
for _, line := range strings.Split(Render(src), "\n") {
|
||||
if strings.HasPrefix(line, "<h") {
|
||||
got = append(got, line)
|
||||
}
|
||||
}
|
||||
if g := strings.Join(got, " "); g != want {
|
||||
t.Errorf("Render(%q) headings:\n got %s\nwant %s", src, g, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPagesHaveOneH1AndNoSkippedLevels(t *testing.T) {
|
||||
for _, path := range []string{
|
||||
"/", "/browse", "/browse?view=table", "/browse?collection=sidtoday-archive",
|
||||
"/browse?collection=sidtoday-archive&view=card", "/browse?collection=foia",
|
||||
"/browse/agency/NSA", "/search?q=prism", "/search?q=zzzqqq", "/doc/prism-slides",
|
||||
"/doc/prism-slides/versions/lemonde", "/glossary", "/articles", "/about", "/pinned",
|
||||
} {
|
||||
body := get(t, path).Body.String()
|
||||
h1, prev := 0, 0
|
||||
for _, m := range reHeadingTag.FindAllStringSubmatch(body, -1) {
|
||||
level, _ := strconv.Atoi(m[1])
|
||||
if level == 1 {
|
||||
h1++
|
||||
}
|
||||
if level > prev+1 {
|
||||
t.Errorf("%s: <h%d> follows <h%d>", path, level, prev)
|
||||
}
|
||||
prev = level
|
||||
}
|
||||
if h1 != 1 {
|
||||
t.Errorf("%s: %d <h1>, want 1", path, h1)
|
||||
}
|
||||
}
|
||||
}
|
||||
231
tests/http_test.go
Normal file
231
tests/http_test.go
Normal file
|
|
@ -0,0 +1,231 @@
|
|||
package main
|
||||
|
||||
// http_test.go - protocol behaviour the browser and the proxy rely on:
|
||||
// HEAD, byte ranges, conditional requests, and method handling. net/http
|
||||
// implements most of it, but only if the handlers hand it the right thing -
|
||||
// and a download path that loses Range support breaks resuming a 120 MB PDF
|
||||
// and the page-by-page fetches a PDF viewer makes.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHTTPHead(t *testing.T) {
|
||||
buildFixture(t)
|
||||
for _, p := range []string{"/", "/browse", "/doc/prism-slides",
|
||||
"/files/prism-slides/prism-slides.pdf", "/api/facets", "/static/style.css"} {
|
||||
r := httptest.NewRequest("HEAD", p, nil)
|
||||
w := httptest.NewRecorder()
|
||||
fixtureMux.ServeHTTP(w, r)
|
||||
if w.Code != 200 {
|
||||
t.Errorf("HEAD %s: %d", p, w.Code)
|
||||
continue
|
||||
}
|
||||
// net/http drops the body for HEAD at the transport layer, so the
|
||||
// recorder still sees one; the headers are what matters here.
|
||||
if w.Header().Get("Content-Type") == "" {
|
||||
t.Errorf("HEAD %s: no Content-Type", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Range on a document file. http.ServeFile implements it - this test is
|
||||
// here so a future change to the download path (a buffered writer, a
|
||||
// compressing wrapper, a manual io.Copy) cannot quietly remove it.
|
||||
func TestHTTPRangeOnFiles(t *testing.T) {
|
||||
buildFixture(t)
|
||||
const path = "/files/prism-slides/prism-slides.pdf"
|
||||
full := get(t, path)
|
||||
body := full.Body.String()
|
||||
if full.Header().Get("Accept-Ranges") != "bytes" {
|
||||
t.Fatalf("no Accept-Ranges on a document file: %v", full.Header())
|
||||
}
|
||||
|
||||
req := func(rangeHdr string) *httptest.ResponseRecorder {
|
||||
r := httptest.NewRequest("GET", path, nil)
|
||||
r.Header.Set("Range", rangeHdr)
|
||||
w := httptest.NewRecorder()
|
||||
fixtureMux.ServeHTTP(w, r)
|
||||
return w
|
||||
}
|
||||
|
||||
w := req("bytes=0-3")
|
||||
if w.Code != http.StatusPartialContent {
|
||||
t.Fatalf("Range bytes=0-3: status %d, want 206", w.Code)
|
||||
}
|
||||
if got, want := w.Body.String(), body[:4]; got != want {
|
||||
t.Errorf("range body %q, want %q", got, want)
|
||||
}
|
||||
if cr := w.Header().Get("Content-Range"); cr != fmt.Sprintf("bytes 0-3/%d", len(body)) {
|
||||
t.Errorf("Content-Range %q", cr)
|
||||
}
|
||||
|
||||
// A suffix range: what a resumed download asks for.
|
||||
w = req(fmt.Sprintf("bytes=%d-", len(body)-4))
|
||||
if w.Code != http.StatusPartialContent || w.Body.String() != body[len(body)-4:] {
|
||||
t.Errorf("suffix range: %d %q", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
// Past the end is a 416 with the length, not a 200 of the whole file.
|
||||
w = req(fmt.Sprintf("bytes=%d-%d", len(body)+10, len(body)+20))
|
||||
if w.Code != http.StatusRequestedRangeNotSatisfiable {
|
||||
t.Errorf("unsatisfiable range: status %d, want 416", w.Code)
|
||||
}
|
||||
|
||||
// A malformed Range is refused with 416 - net/http's choice, not this
|
||||
// server's; it is recorded here so a change in behaviour is noticed.
|
||||
if w = req("bytes=abc"); w.Code != http.StatusRequestedRangeNotSatisfiable {
|
||||
t.Errorf("malformed Range: status %d, want 416", w.Code)
|
||||
}
|
||||
// An unknown range unit is refused the same way. RFC 7233 says to
|
||||
// ignore it and serve the whole representation; net/http does not, and
|
||||
// nothing this archive serves sends one. Recorded, not worked around.
|
||||
if w = req("pages=1-2"); w.Code != http.StatusRequestedRangeNotSatisfiable {
|
||||
t.Errorf("unknown range unit: status %d, want net/http's 416", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// Conditional requests: a document file and a static asset both carry
|
||||
// validators, so a reader's second visit is a 304 instead of a re-download.
|
||||
func TestHTTPConditional(t *testing.T) {
|
||||
buildFixture(t)
|
||||
// A document file comes off disk and carries a modification time.
|
||||
const doc = "/files/prism-slides/prism-slides.pdf"
|
||||
lm := get(t, doc).Header().Get("Last-Modified")
|
||||
if lm == "" {
|
||||
t.Fatalf("%s: no Last-Modified", doc)
|
||||
}
|
||||
r := httptest.NewRequest("GET", doc, nil)
|
||||
r.Header.Set("If-Modified-Since", lm)
|
||||
w := httptest.NewRecorder()
|
||||
fixtureMux.ServeHTTP(w, r)
|
||||
if w.Code != http.StatusNotModified || w.Body.Len() != 0 {
|
||||
t.Errorf("%s with If-Modified-Since: status %d, %d bytes", doc, w.Code, w.Body.Len())
|
||||
}
|
||||
|
||||
// An embedded asset has no modification time - embed.FS zeroes it - so
|
||||
// it carries a content ETag instead. Without one, every page view
|
||||
// re-downloads the stylesheet on an unstamped build.
|
||||
for _, asset := range []string{"/static/style.css", "/static/main.js"} {
|
||||
first := get(t, asset)
|
||||
tag := first.Header().Get("ETag")
|
||||
if tag == "" {
|
||||
t.Errorf("%s: no ETag", asset)
|
||||
continue
|
||||
}
|
||||
r := httptest.NewRequest("GET", asset, nil)
|
||||
r.Header.Set("If-None-Match", tag)
|
||||
w := httptest.NewRecorder()
|
||||
fixtureMux.ServeHTTP(w, r)
|
||||
if w.Code != http.StatusNotModified || w.Body.Len() != 0 {
|
||||
t.Errorf("%s with If-None-Match: status %d, %d bytes (want an empty 304)",
|
||||
asset, w.Code, w.Body.Len())
|
||||
}
|
||||
// A different asset's tag must not satisfy it.
|
||||
r = httptest.NewRequest("GET", asset, nil)
|
||||
r.Header.Set("If-None-Match", `"0000000000000000"`)
|
||||
w = httptest.NewRecorder()
|
||||
fixtureMux.ServeHTTP(w, r)
|
||||
if w.Code != 200 {
|
||||
t.Errorf("%s with a stale ETag: status %d, want the asset", asset, w.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Method handling: the routes declare GET or POST, so anything else is a
|
||||
// 405 from the mux rather than a handler running on the wrong verb.
|
||||
func TestHTTPMethods(t *testing.T) {
|
||||
buildFixture(t)
|
||||
for _, c := range []struct {
|
||||
method, path string
|
||||
want int
|
||||
}{
|
||||
// Methods the archive never answers stop at the wrapper.
|
||||
{"PUT", "/api/docs", http.StatusMethodNotAllowed},
|
||||
{"DELETE", "/doc/prism-slides", http.StatusMethodNotAllowed},
|
||||
{"OPTIONS", "/", http.StatusMethodNotAllowed},
|
||||
{"TRACE", "/browse", http.StatusMethodNotAllowed},
|
||||
// A supported method on the wrong route is a 404: the catch-all
|
||||
// route claims every path, which is what keeps unknown paths on the
|
||||
// archive's own 404 page, and it claims them for POST too.
|
||||
{"POST", "/browse", http.StatusNotFound},
|
||||
{"GET", "/preferences/save", http.StatusNotFound},
|
||||
{"GET", "/pin/prism-slides", http.StatusNotFound},
|
||||
} {
|
||||
r := httptest.NewRequest(c.method, c.path, nil)
|
||||
w := httptest.NewRecorder()
|
||||
fixtureMux.ServeHTTP(w, r)
|
||||
if w.Code != c.want {
|
||||
t.Errorf("%s %s: status %d, want %d", c.method, c.path, w.Code, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Odd but legal requests must not break anything: repeated parameters,
|
||||
// unknown parameters, a very long path, an empty query.
|
||||
func TestHTTPOddRequests(t *testing.T) {
|
||||
buildFixture(t)
|
||||
for _, c := range []struct {
|
||||
path string
|
||||
want int
|
||||
}{
|
||||
{"/browse?sort=title&sort=date", 200},
|
||||
{"/browse?unknown=1&another=2", 200},
|
||||
{"/search?q=a&q=b&do_search=1", 200},
|
||||
{"/browse/program/" + strings.Repeat("x", 2000), 200}, // an empty page, not a crash
|
||||
{"/doc/" + strings.Repeat("x", 2000), 404},
|
||||
{"/browse?", 200},
|
||||
{"/search?do_search=1&fqv=&fqv=&fqv=", 200},
|
||||
} {
|
||||
if w := get(t, c.path); w.Code != c.want {
|
||||
t.Errorf("%.60s: status %d, want %d", c.path, w.Code, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// With -accel-files the file is nginx's to send: the handler answers only
|
||||
// the files it would have served itself, with the redirect and no body.
|
||||
func TestAccelFiles(t *testing.T) {
|
||||
buildFixture(t)
|
||||
accelPrefix = "/_files/"
|
||||
defer func() { accelPrefix = "" }()
|
||||
|
||||
w := get(t, "/files/prism-slides/prism-slides.pdf")
|
||||
rel, err := filepath.Rel(dataDir, filepath.Join(corpus().docIndex["prism-slides"].Dir, "prism-slides.pdf"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, want := w.Header().Get("X-Accel-Redirect"), "/_files/"+filepath.ToSlash(rel); got != want {
|
||||
t.Errorf("X-Accel-Redirect %q, want %q", got, want)
|
||||
}
|
||||
if w.Code != http.StatusOK || w.Body.Len() != 0 {
|
||||
t.Errorf("accel answer: status %d, %d body bytes, want an empty 200", w.Code, w.Body.Len())
|
||||
}
|
||||
if w.Header().Get("Cache-Control") == "" {
|
||||
t.Error("accel answer carries no Cache-Control for nginx to pass on")
|
||||
}
|
||||
|
||||
for _, p := range []string{"/files/prism-slides/zz-stray.pdf", "/files/prism-slides/../glossary.json", "/files/no-such-doc/x.pdf"} {
|
||||
w := get(t, p)
|
||||
if w.Header().Get("X-Accel-Redirect") != "" || w.Code == http.StatusOK {
|
||||
t.Errorf("%s: status %d, X-Accel-Redirect %q - a file the server does not serve was handed to nginx", p, w.Code, w.Header().Get("X-Accel-Redirect"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAccelPathEscapes(t *testing.T) {
|
||||
old, oldPrefix := dataDir, accelPrefix
|
||||
dataDir, accelPrefix = "/srv/data", "/_files/"
|
||||
defer func() { dataDir, accelPrefix = old, oldPrefix }()
|
||||
if got, ok := accelPath("/srv/data/FOIA/a doc/50%?.pdf"); !ok || got != "/_files/FOIA/a%20doc/50%25%3F.pdf" {
|
||||
t.Errorf("accelPath = %q, %v", got, ok)
|
||||
}
|
||||
if got, ok := accelPath("/srv/other/x.pdf"); ok {
|
||||
t.Errorf("a file outside the data directory mapped to %q", got)
|
||||
}
|
||||
}
|
||||
99
tests/ids_test.go
Normal file
99
tests/ids_test.go
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The /explore/ids table: one row per document in id order, versions nested
|
||||
// under their document, and a checksum file that sha256sum -c can read from
|
||||
// the corpus root.
|
||||
func TestBuildIDRows(t *testing.T) {
|
||||
oldDocs, oldDir := corpus().archive.Docs, dataDir
|
||||
t.Cleanup(func() {
|
||||
// The id tables are package state the routes serve: rebuild them
|
||||
// from the restored corpus, or the fake rows below leak into
|
||||
// every later test.
|
||||
corpus().archive.Docs, dataDir = oldDocs, oldDir
|
||||
corpus().buildIDRows()
|
||||
})
|
||||
|
||||
dataDir = "/corpus"
|
||||
corpus().archive.Docs = []Document{
|
||||
{
|
||||
ID: "zebra", Dir: filepath.Join(dataDir, "FOIA", "zebra"), Collection: "foia",
|
||||
PDFFile: "zebra.pdf", PDFSize: 2 << 20, Pages: 3,
|
||||
},
|
||||
{
|
||||
ID: "alpha", Dir: filepath.Join(dataDir, "alpha"), Collection: "news",
|
||||
PDFFile: "alpha.pdf", PDFSize: 1500, Pages: 1,
|
||||
PDFSha256: "abc123", AkaID: []AkaEntry{{ID: "alpha-lemonde"}},
|
||||
Versions: []DocVersion{{Slug: "entier", PDFFile: "copy.pdf", PDFSize: 4096, Images: []string{"a", "b"}, SHA: "def456"}},
|
||||
},
|
||||
}
|
||||
corpus().buildIDRows()
|
||||
|
||||
if len(corpus().idRows) != 2 || corpus().idRows[0].ID != "alpha" || corpus().idRows[1].ID != "zebra" {
|
||||
t.Fatalf("rows out of order: %v", corpus().idRows)
|
||||
}
|
||||
if got := corpus().idRows[0].Size; got != "1 KB" {
|
||||
t.Errorf("size %q, want 1 KB", got)
|
||||
}
|
||||
if v := corpus().idRows[0].Versions; len(v) != 1 || v[0].Pages != 2 || v[0].Size != "4 KB" || v[0].SHA != "def456" {
|
||||
t.Errorf("version row %+v", v)
|
||||
}
|
||||
if len(corpus().idRows[1].Versions) != 0 {
|
||||
t.Errorf("zebra has no versions, got %+v", corpus().idRows[1].Versions)
|
||||
}
|
||||
|
||||
// sha256sum -c format: checksum, two spaces, path from the corpus root,
|
||||
// a document's versions after it. Files without a checksum are absent.
|
||||
if corpus().idChecksums != "abc123 alpha/alpha.pdf\ndef456 alpha/versions/entier/copy.pdf\n" {
|
||||
t.Errorf("checksums %q", corpus().idChecksums)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHumanSize(t *testing.T) {
|
||||
for n, want := range map[int64]string{0: "", -1: "", 900: "900 B", 2048: "2 KB",
|
||||
3 << 20: "3.0 MB", 5 << 30: "5.00 GB"} {
|
||||
if got := humanSize(n); got != want {
|
||||
t.Errorf("humanSize(%d) = %q, want %q", n, got, want)
|
||||
}
|
||||
}
|
||||
if !strings.HasSuffix(humanSize(1<<20+1<<19), " MB") {
|
||||
t.Errorf("1.5 MiB should render in MB")
|
||||
}
|
||||
}
|
||||
|
||||
// A version's PDF checksum comes from the metadata's
|
||||
// "versions"/<slug>/pdf_sha256 only; a version without one is warned about
|
||||
// and has no line in ids.txt.
|
||||
func TestVersionChecksums(t *testing.T) {
|
||||
root := assemblyCorpus(t, "")
|
||||
writeAssemblyDoc(t, root, "listed", map[string]interface{}{
|
||||
"title": "Listed",
|
||||
"versions": map[string]interface{}{"copy": map[string]interface{}{"pdf_sha256": " ABC999 "}},
|
||||
}, []string{"page-1.jpg"}, map[string][]string{"copy": {"page-1.jpg"}})
|
||||
var logged bytes.Buffer
|
||||
log.SetOutput(&logged)
|
||||
withCorpus(t, root)
|
||||
log.SetOutput(os.Stderr)
|
||||
|
||||
if v := corpus().docIndex["listed"].VersionFor("copy"); v == nil || v.SHA != "abc999" {
|
||||
t.Errorf("metadata version checksum: %+v, want abc999 from the metadata", v)
|
||||
}
|
||||
if v := corpus().docIndex["deck"].VersionFor("fuller"); v == nil || v.SHA != "" {
|
||||
t.Errorf("a version without pdf_sha256 in the metadata got a checksum: %+v", v)
|
||||
}
|
||||
if !strings.Contains(logged.String(), "deck: versions/fuller has no pdf_sha256") {
|
||||
t.Errorf("no warning for the version without a checksum:\n%s", logged.String())
|
||||
}
|
||||
if !strings.Contains(corpus().idChecksums, "abc999 listed/versions/copy/copy.pdf\n") ||
|
||||
strings.Contains(corpus().idChecksums, "deck/versions/fuller/") {
|
||||
t.Errorf("ids.txt version lines:\n%s", corpus().idChecksums)
|
||||
}
|
||||
}
|
||||
53
tests/image_sizes_test.go
Normal file
53
tests/image_sizes_test.go
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPageImagesCarryTheirSize(t *testing.T) {
|
||||
for _, path := range []string{
|
||||
"/doc/prism-slides",
|
||||
"/doc/prism-slides/pages",
|
||||
"/doc/prism-slides/versions/lemonde",
|
||||
"/doc/prism-slides/versions/lemonde/pages",
|
||||
} {
|
||||
body := get(t, path).Body.String()
|
||||
if n := strings.Count(body, `page-1.jpg" width="4" height="3"`); n < 2 {
|
||||
t.Errorf("%s: %d page-1 images with their size, want the page and its full-screen view", path, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssemblySlidesCarryTheirSize(t *testing.T) {
|
||||
root := assemblyCorpus(t, `[{"id": "sized", "title": "Sized", "slides": [{"doc": "deck", "pages": "1"}]}]`)
|
||||
withCorpus(t, root)
|
||||
for _, path := range []string{"/explore/assemblies", "/explore/assemblies/sized"} {
|
||||
if body := get(t, path).Body.String(); !strings.Contains(body, `page-1.jpg" width="4" height="3"`) {
|
||||
t.Errorf("%s: slide image without its size", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnreadablePageImageWarns(t *testing.T) {
|
||||
root := assemblyCorpus(t, "")
|
||||
writeAssemblyDoc(t, root, "broken", map[string]interface{}{"title": "Broken"}, []string{"page-1.jpg"}, nil)
|
||||
writeFixtureFile(t, filepath.Join(root, "broken", "images", "page-1.jpg"), "not an image")
|
||||
var logged bytes.Buffer
|
||||
log.SetOutput(&logged)
|
||||
withCorpus(t, root)
|
||||
log.SetOutput(os.Stderr)
|
||||
|
||||
if !strings.Contains(logged.String(), "page image unreadable, served without its size: broken/images/page-1.jpg") {
|
||||
t.Errorf("no warning for the unreadable image:\n%s", logged.String())
|
||||
}
|
||||
body := get(t, "/doc/broken").Body.String()
|
||||
if !strings.Contains(body, `<img src="/files/broken/images/page-1.jpg"
|
||||
`) || strings.Contains(body, `page-1.jpg" width=`) {
|
||||
t.Errorf("unreadable image: want it served without width/height")
|
||||
}
|
||||
}
|
||||
277
tests/invariants_test.go
Normal file
277
tests/invariants_test.go
Normal file
|
|
@ -0,0 +1,277 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"html"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The archive's central promise: a number on a link is the number the click
|
||||
// returns. Facet counts, refine boxes and histogram bars all assert it, and
|
||||
// every one of them is computed by different code from the page it opens.
|
||||
// These tests walk the real pages and follow their own links.
|
||||
|
||||
var (
|
||||
reFacetLink = regexp.MustCompile(`<a class="facet-link" href="([^"]+)">.*?<span class="facet-count">(\d+)</span></a>`)
|
||||
reBarLink = regexp.MustCompile(`<a class="hist-col" href="([^"]+)"[^>]*>\s*<span class="hist-count">(\d+)</span>`)
|
||||
reTotal = regexp.MustCompile(`([\d,]+)\s+results?`)
|
||||
reListTotal = regexp.MustCompile(`([\d,]+)\s+documents?`)
|
||||
)
|
||||
|
||||
// resultTotal reads the count a /search or a list page reports.
|
||||
func resultTotal(t *testing.T, path string, cookies ...*http.Cookie) int {
|
||||
t.Helper()
|
||||
body := get(t, path, cookies...).Body.String()
|
||||
re := reTotal
|
||||
if !strings.HasPrefix(path, "/search") {
|
||||
re = reListTotal
|
||||
}
|
||||
m := re.FindStringSubmatch(body)
|
||||
if m == nil {
|
||||
t.Fatalf("%s: no result count in the page", path)
|
||||
}
|
||||
n, err := strconv.Atoi(strings.ReplaceAll(m[1], ",", ""))
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", path, err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// followLinks checks every (link, count) pair a page prints, as the reader
|
||||
// with these cookies sees it.
|
||||
func followLinks(t *testing.T, page string, re *regexp.Regexp, what string, cookies ...*http.Cookie) int {
|
||||
t.Helper()
|
||||
body := get(t, page, cookies...).Body.String()
|
||||
pairs := re.FindAllStringSubmatch(body, -1)
|
||||
for _, p := range pairs {
|
||||
link := html.UnescapeString(p[1])
|
||||
shown, _ := strconv.Atoi(p[2])
|
||||
if got := resultTotal(t, link, cookies...); got != shown {
|
||||
t.Errorf("%s: %s says %d, its link returns %d\n %s", page, what, shown, got, link)
|
||||
}
|
||||
}
|
||||
return len(pairs)
|
||||
}
|
||||
|
||||
// A category page lists exactly as many documents as its facet counts.
|
||||
func TestInvariantFacetCountsMatchPages(t *testing.T) {
|
||||
buildFixture(t)
|
||||
checked := 0
|
||||
for ci := range categories {
|
||||
c := &categories[ci]
|
||||
for val, want := range c.Facet(corpus()) {
|
||||
if got := len(corpus().categoryMatches(c.Slug, val)); got != want {
|
||||
t.Errorf("%s=%q: facet says %d, the page lists %d", c.Slug, val, want, got)
|
||||
}
|
||||
checked++
|
||||
}
|
||||
}
|
||||
if checked == 0 {
|
||||
t.Fatal("no facet values in the fixture")
|
||||
}
|
||||
}
|
||||
|
||||
// Refine links and histogram bars open searches that must return what they
|
||||
// promise - the two places where a count and its click are computed by
|
||||
// different code paths.
|
||||
func TestInvariantRefineAndBarCounts(t *testing.T) {
|
||||
pages := []string{
|
||||
"/browse/agency/NSA",
|
||||
"/browse/program/PRISM",
|
||||
"/browse/classification/TOP%20SECRET",
|
||||
"/search?q=prism&do_search=1",
|
||||
"/search?do_search=1&fqv=NSA&fqf=CR&fqc=and",
|
||||
}
|
||||
facets, bars := 0, 0
|
||||
for _, p := range pages {
|
||||
facets += followLinks(t, p, reFacetLink, "refine link")
|
||||
bars += followLinks(t, p, reBarLink, "histogram bar")
|
||||
}
|
||||
if facets == 0 || bars == 0 {
|
||||
t.Fatalf("nothing exercised: %d refine links, %d bars", facets, bars)
|
||||
}
|
||||
t.Logf("checked %d refine links and %d histogram bars", facets, bars)
|
||||
}
|
||||
|
||||
// The home page is scoped to the reported collection while the category
|
||||
// pages follow the reader's scope, so its bars and lists link to those
|
||||
// pages with the collection named. Every one of them has to be the number
|
||||
// its click returns.
|
||||
func TestInvariantHomeCounts(t *testing.T) {
|
||||
body := get(t, "/").Body.String()
|
||||
pairs := regexp.MustCompile(`<a class="pill" href="([^"]+)">[^<]*<span class="pill-count">(\d+)</span>`).FindAllStringSubmatch(body, -1)
|
||||
lists := regexp.MustCompile(`<li><a href="(/browse/[^"]+)">[^<]*</a> <span class="muted">(\d+)</span>`).FindAllStringSubmatch(body, -1)
|
||||
if len(lists) < 5 {
|
||||
t.Fatalf("only %d counted list links on the home page - the stat lists should link to their browse pages", len(lists))
|
||||
}
|
||||
pairs = append(pairs, lists...)
|
||||
pairs = append(pairs, regexp.MustCompile(`<a class="hist-col" href="([^"]+)" title="(\d+) documents`).FindAllStringSubmatch(body, -1)...)
|
||||
if len(pairs) < 5 {
|
||||
t.Fatalf("only %d counted links on the home page", len(pairs))
|
||||
}
|
||||
for _, m := range pairs {
|
||||
link, shown := html.UnescapeString(m[1]), m[2]
|
||||
if got := resultTotal(t, link); got != mustAtoi(t, shown) {
|
||||
t.Errorf("home says %s, its link returns %d: %s", shown, got, link)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mustAtoi(t *testing.T, s string) int {
|
||||
t.Helper()
|
||||
n, err := strconv.Atoi(s)
|
||||
if err != nil {
|
||||
t.Fatalf("%q: %v", s, err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// The type pills follow the same rule: the count is over the full match set,
|
||||
// and clicking one narrows to exactly that many.
|
||||
func TestInvariantTypePillCounts(t *testing.T) {
|
||||
body := get(t, "/browse").Body.String()
|
||||
re := regexp.MustCompile(`href="([^"]*type=[^"]*)"[^>]*>\s*([A-Z]+)\s*<span[^>]*>\(?(\d+)\)?</span>`)
|
||||
for _, m := range re.FindAllStringSubmatch(body, -1) {
|
||||
link := html.UnescapeString(m[1])
|
||||
shown, _ := strconv.Atoi(m[3])
|
||||
if got := resultTotal(t, link); got != shown {
|
||||
t.Errorf("type pill %s says %d, its link returns %d (%s)", m[2], shown, got, link)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Everything the sitemap advertises must exist: those URLs are built in Go,
|
||||
// away from the templates that normally escape ids.
|
||||
func TestInvariantAdvertisedURLsResolve(t *testing.T) {
|
||||
for _, src := range []string{"/sitemap.xml"} {
|
||||
body := get(t, src).Body.String()
|
||||
locs := regexp.MustCompile(`<loc>([^<]+)</loc>|<link[^>]+href="([^"]+)"`).FindAllStringSubmatch(body, -1)
|
||||
if len(locs) == 0 {
|
||||
t.Fatalf("%s advertises nothing", src)
|
||||
}
|
||||
seen := 0
|
||||
for _, m := range locs {
|
||||
raw := m[1]
|
||||
if raw == "" {
|
||||
raw = m[2]
|
||||
}
|
||||
u, err := url.Parse(html.UnescapeString(raw))
|
||||
if err != nil || u.Path == "" {
|
||||
continue
|
||||
}
|
||||
p := u.Path
|
||||
if u.RawQuery != "" {
|
||||
p += "?" + u.RawQuery
|
||||
}
|
||||
if w := get(t, p); w.Code != 200 {
|
||||
t.Errorf("%s advertises %s, which answers %d", src, p, w.Code)
|
||||
}
|
||||
seen++
|
||||
}
|
||||
t.Logf("%s: %d URLs resolve", src, seen)
|
||||
}
|
||||
}
|
||||
|
||||
// The API's pagination envelope has to describe the page it returned, at
|
||||
// every page size, and per_page=all is the export mode that skips snippets.
|
||||
func TestInvariantAPIPagination(t *testing.T) {
|
||||
type envelope struct {
|
||||
Total int `json:"total"`
|
||||
Page int `json:"page"`
|
||||
PerPage int `json:"per_page"`
|
||||
TotalPages int `json:"total_pages"`
|
||||
Omitted bool `json:"snippets_omitted"`
|
||||
Docs []struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"docs"`
|
||||
Results []struct {
|
||||
ID string `json:"id"`
|
||||
Snippet string `json:"snippet"`
|
||||
} `json:"results"`
|
||||
}
|
||||
read := func(path string) envelope {
|
||||
var e envelope
|
||||
w := get(t, path)
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("%s: status %d", path, w.Code)
|
||||
}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &e); err != nil {
|
||||
t.Fatalf("%s: %v", path, err)
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
for _, path := range []string{"/api/docs", "/api/docs?per_page=1", "/api/docs?per_page=all",
|
||||
"/api/search?q=prism", "/api/search?q=prism&per_page=1", "/api/search?q=prism&per_page=all"} {
|
||||
e := read(path)
|
||||
n := len(e.Docs) + len(e.Results)
|
||||
switch {
|
||||
case e.Page < 1:
|
||||
t.Errorf("%s: page %d", path, e.Page)
|
||||
case e.PerPage < 1:
|
||||
t.Errorf("%s: per_page %d", path, e.PerPage)
|
||||
case n > e.PerPage:
|
||||
t.Errorf("%s: returned %d rows for a page of %d", path, n, e.PerPage)
|
||||
case e.TotalPages < 1:
|
||||
t.Errorf("%s: total_pages %d", path, e.TotalPages)
|
||||
}
|
||||
if want := (e.Total + e.PerPage - 1) / e.PerPage; e.Total > 0 && e.TotalPages != want {
|
||||
t.Errorf("%s: total_pages %d, want %d for %d/%d", path, e.TotalPages, want, e.Total, e.PerPage)
|
||||
}
|
||||
}
|
||||
|
||||
// An out-of-range page clamps rather than 500s or returns junk.
|
||||
if e := read("/api/docs?p=9999"); e.Page != e.TotalPages {
|
||||
t.Errorf("p=9999 gave page %d of %d", e.Page, e.TotalPages)
|
||||
}
|
||||
// Export mode returns everything and says why there are no snippets.
|
||||
all := read("/api/search?q=prism&per_page=all")
|
||||
if !all.Omitted || len(all.Results) != all.Total {
|
||||
t.Errorf("per_page=all: omitted=%v, %d of %d results", all.Omitted, len(all.Results), all.Total)
|
||||
}
|
||||
for _, r := range all.Results {
|
||||
if r.Snippet != "" {
|
||||
t.Errorf("per_page=all returned a snippet for %s", r.ID)
|
||||
}
|
||||
}
|
||||
// A normal page does carry them.
|
||||
paged := read("/api/search?q=prism")
|
||||
if paged.Omitted {
|
||||
t.Error("a paged search should not omit snippets")
|
||||
}
|
||||
}
|
||||
|
||||
// Snippet work is bounded by maxSnippetDocs, not by the page size, and the
|
||||
// page says so when the two differ.
|
||||
func TestInvariantSnippetCap(t *testing.T) {
|
||||
buildFixture(t)
|
||||
doc := corpus().docIndex["prism-slides"]
|
||||
if doc == nil {
|
||||
t.Fatal("fixture document missing")
|
||||
}
|
||||
docs := make([]*Document, maxSnippetDocs+5)
|
||||
for i := range docs {
|
||||
docs[i] = doc
|
||||
}
|
||||
page, capped := decorateResults(docs, []string{"collection"}, []string{"TE"}, []string{"and"})
|
||||
if !capped {
|
||||
t.Error("a page past the budget should report itself capped")
|
||||
}
|
||||
withSnippets := 0
|
||||
for i := range page {
|
||||
if len(page[i].Snippets) > 0 {
|
||||
withSnippets++
|
||||
}
|
||||
if page[i].Title == "" {
|
||||
t.Fatalf("row %d lost its title highlighting", i)
|
||||
}
|
||||
}
|
||||
if withSnippets != maxSnippetDocs {
|
||||
t.Errorf("%d rows got snippets, want %d", withSnippets, maxSnippetDocs)
|
||||
}
|
||||
}
|
||||
166
tests/loader_test.go
Normal file
166
tests/loader_test.go
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// What the loader does with a corpus that is wrong in the ways corpora are
|
||||
// wrong in practice. Every case must leave the rest of the archive intact -
|
||||
// a broken document is skipped, never a reason to serve nothing - and say
|
||||
// so in the tally that -check reads.
|
||||
func TestLoaderRejectsBadDocuments(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
write := func(path, content string) {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
good := func(id string) {
|
||||
write(filepath.Join(root, id, "metadata.json"), `{"title": "`+id+`", "type": "REPORTED"}`)
|
||||
write(filepath.Join(root, id, id+".pdf"), "%PDF-1.4")
|
||||
}
|
||||
good("keeper-one")
|
||||
good("keeper-two")
|
||||
|
||||
// No PDF: this is an archive OF the PDFs, so the document is skipped.
|
||||
write(filepath.Join(root, "no-pdf", "metadata.json"), `{"title": "No PDF"}`)
|
||||
// Invalid JSON: skipped, everything else still loads.
|
||||
write(filepath.Join(root, "bad-json", "metadata.json"), `{"title": "oops"`)
|
||||
write(filepath.Join(root, "bad-json", "x.pdf"), "%PDF-1.4")
|
||||
// aka_id that is not a list of objects: refused, not silently half-read.
|
||||
write(filepath.Join(root, "bad-aka", "metadata.json"), `{"title": "Aka", "aka_id": "other-id"}`)
|
||||
write(filepath.Join(root, "bad-aka", "x.pdf"), "%PDF-1.4")
|
||||
// A directory that is not a document at all: skipped without a warning.
|
||||
write(filepath.Join(root, "notes", "README.txt"), "just notes")
|
||||
// A directory holding a PDF and nothing else: the shape worth warning
|
||||
// about, since it looks like a document nobody described.
|
||||
write(filepath.Join(root, "loose", "orphan.pdf"), "%PDF-1.4")
|
||||
|
||||
withCorpus(t, root)
|
||||
clean := loadWarnings
|
||||
|
||||
// A document in the SIDtoday archive that is not typed SIDTODAY is on
|
||||
// no browse tab: served, and warned about.
|
||||
write(filepath.Join(root, "SIDtoday-archive", "mistyped", "metadata.json"), `{"title": "Mistyped", "type": "REPORTED"}`)
|
||||
write(filepath.Join(root, "SIDtoday-archive", "mistyped", "x.pdf"), "%PDF-1.4")
|
||||
loadData()
|
||||
if corpus().docIndex["mistyped"] == nil || loadWarnings != clean+1 {
|
||||
t.Errorf("a REPORTED document in SIDtoday-archive: loaded %v, %d warnings, want it loaded with %d", corpus().docIndex["mistyped"] != nil, loadWarnings, clean+1)
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(root, "SIDtoday-archive")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loadData()
|
||||
|
||||
if len(corpus().archive.Docs) != 2 {
|
||||
ids := make([]string, 0, len(corpus().archive.Docs))
|
||||
for _, d := range corpus().archive.Docs {
|
||||
ids = append(ids, d.ID)
|
||||
}
|
||||
t.Fatalf("loaded %v, want the two good documents", ids)
|
||||
}
|
||||
if loadErrors < 3 { // no-pdf, bad-aka and bad JSON are errors: not served
|
||||
t.Errorf("loadErrors = %d, want at least 3", loadErrors)
|
||||
}
|
||||
if loadWarnings < 1 { // the loose PDF directory: served nothing, lost nothing
|
||||
t.Errorf("loadWarnings = %d, want at least 1", loadWarnings)
|
||||
}
|
||||
for _, id := range []string{"no-pdf", "bad-json", "bad-aka", "notes", "loose"} {
|
||||
if corpus().docIndex[id] != nil {
|
||||
t.Errorf("%s should not have loaded", id)
|
||||
}
|
||||
}
|
||||
// The archive is coherent: facets, search and the sitemap all see the
|
||||
// same two documents.
|
||||
if got := corpus().archive.CollectionCounts[collectionNews]; got != 2 {
|
||||
t.Errorf("news collection count %d, want 2", got)
|
||||
}
|
||||
if got, _ := corpus().executeSearch(searchParams{Queries: []string{""}, Fields: []string{"TE"}, Conjs: []string{"and"}}); len(got) != 2 {
|
||||
t.Errorf("match-all search found %d, want 2", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
// An empty corpus is a corpus: the server loads, every page renders, and
|
||||
// nothing divides by zero.
|
||||
func TestLoaderEmptyCorpus(t *testing.T) {
|
||||
withCorpus(t, t.TempDir())
|
||||
if len(corpus().archive.Docs) != 0 {
|
||||
t.Fatalf("loaded %d documents from an empty directory", len(corpus().archive.Docs))
|
||||
}
|
||||
for _, p := range []string{"/", "/browse", "/search?q=x&do_search=1", "/explore", "/explore/ids",
|
||||
"/articles", "/api/docs", "/api/facets", "/sitemap.xml"} {
|
||||
if w := get(t, p); w.Code != 200 {
|
||||
t.Errorf("%s on an empty corpus: %d", p, w.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Values are tidied before anything reads them, and a copy the loader skips
|
||||
// leaves no trace - not even a claim on its ID.
|
||||
func TestLoaderTidiesBeforeUse(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
write := func(path, content string) {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// FOIA/ sorts before the flat root, so the PDF-less copy is seen first.
|
||||
write(filepath.Join(root, "FOIA", "dup-doc", "metadata.json"), `{"title": "Shadow"}`)
|
||||
write(filepath.Join(root, "dup-doc", "metadata.json"), `{"title": "Real", "type": "REPORTED"}`)
|
||||
write(filepath.Join(root, "dup-doc", "x.pdf"), "%PDF-1.4")
|
||||
|
||||
write(filepath.Join(root, "padded", "metadata.json"), `{
|
||||
"title": "Padded", "type": "REPORTED",
|
||||
"document": {"creator": " NSA ", "series": {"name": "Series X ", "part": 1}},
|
||||
"publication": [{"publisher": "Le Monde", "date_published": "June 2014", "version": "entier"}]
|
||||
}`)
|
||||
write(filepath.Join(root, "padded", "x.pdf"), "%PDF-1.4")
|
||||
write(filepath.Join(root, "padded", "versions", "entier", "entier.pdf"), "%PDF-1.4")
|
||||
|
||||
withCorpus(t, root)
|
||||
|
||||
if d := corpus().docIndex["dup-doc"]; d == nil || d.Title != "Real" {
|
||||
t.Errorf("dup-doc = %+v, want the copy with a PDF", d)
|
||||
}
|
||||
d := corpus().docIndex["padded"]
|
||||
if d == nil {
|
||||
t.Fatal("padded not loaded")
|
||||
}
|
||||
if d.DocMeta.Creator != "NSA" || d.DocMeta.Series.Name != "Series X" {
|
||||
t.Errorf("creator %q, series %q: want both trimmed", d.DocMeta.Creator, d.DocMeta.Series.Name)
|
||||
}
|
||||
// The malformed date is dropped everywhere, the version label included.
|
||||
if v := d.VersionFor("entier"); v == nil || v.Label != "Le Monde" {
|
||||
t.Errorf("version label %+v, want \"Le Monde\" without the invalid date", v)
|
||||
}
|
||||
}
|
||||
|
||||
// Two page images with one page number show that page twice; the loader
|
||||
// says so instead of linking changed pages to whichever sorted first.
|
||||
func TestLoadPageImagesWarnsOnDuplicatePageNumbers(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(dir, "images"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, name := range []string{"page-1.jpg", "page-2.jpg", "page-02.png"} {
|
||||
if err := os.WriteFile(filepath.Join(dir, "images", name), nil, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
before := loadWarnings
|
||||
imgs := loadPageImages(dir, "doc")
|
||||
if len(imgs) != 3 || imgs[0] != "page-1.jpg" {
|
||||
t.Errorf("images %v, want all three in page order", imgs)
|
||||
}
|
||||
if loadWarnings != before+1 {
|
||||
t.Errorf("%d warnings for one duplicated page number, want 1", loadWarnings-before)
|
||||
}
|
||||
}
|
||||
74
tests/main_test.go
Normal file
74
tests/main_test.go
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestFileWriteBudget: a download slower than the server's WriteTimeout still
|
||||
// completes, because handleFiles replaces that page-sized budget with
|
||||
// fileWriteBudget. The control - the same slow download with the budget cut
|
||||
// short - must be truncated, or the test would not be exercising the deadline
|
||||
// at all.
|
||||
func TestFileWriteBudget(t *testing.T) {
|
||||
const size = 64 << 20 // well past the loopback socket buffers
|
||||
path := filepath.Join(t.TempDir(), "big.pdf")
|
||||
if err := os.WriteFile(path, bytes.Repeat([]byte("snowchive"), size/9+1)[:size], 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old := fileWriteBudget
|
||||
defer func() { fileWriteBudget = old }()
|
||||
|
||||
// The same two lines handleFiles uses, over a server whose page budget is
|
||||
// far shorter than the download takes.
|
||||
ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_ = http.NewResponseController(w).SetWriteDeadline(time.Now().Add(fileWriteBudget))
|
||||
http.ServeFile(w, r, path)
|
||||
}))
|
||||
ts.Config.WriteTimeout = 300 * time.Millisecond
|
||||
ts.Start()
|
||||
defer ts.Close()
|
||||
|
||||
// fetch reads 1 MB at a time with a pause after each chunk, returning the
|
||||
// bytes received before the body ended.
|
||||
fetch := func(pause time.Duration) (int64, time.Duration) {
|
||||
start := time.Now()
|
||||
resp, err := http.Get(ts.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
var got int64
|
||||
buf := make([]byte, 1<<20)
|
||||
for {
|
||||
n, err := io.ReadFull(resp.Body, buf)
|
||||
got += int64(n)
|
||||
if err != nil {
|
||||
return got, time.Since(start)
|
||||
}
|
||||
time.Sleep(pause)
|
||||
}
|
||||
}
|
||||
|
||||
fileWriteBudget = 30 * time.Second
|
||||
got, elapsed := fetch(40 * time.Millisecond)
|
||||
if got != size {
|
||||
t.Errorf("slow download got %d of %d bytes", got, size)
|
||||
}
|
||||
if elapsed < 2*ts.Config.WriteTimeout {
|
||||
t.Errorf("download took %v: too fast to prove it outlasted the %v WriteTimeout",
|
||||
elapsed, ts.Config.WriteTimeout)
|
||||
}
|
||||
|
||||
fileWriteBudget = 200 * time.Millisecond
|
||||
if got, _ := fetch(40 * time.Millisecond); got >= size {
|
||||
t.Errorf("control: a %v budget did not cut the download (got all %d bytes)",
|
||||
fileWriteBudget, got)
|
||||
}
|
||||
}
|
||||
145
tests/mdparser_test.go
Normal file
145
tests/mdparser_test.go
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
// Emphasis follows CommonMark's flanking rules, so the loose asterisks and
|
||||
// underscores of OCR - footnote marks, flattened bullets, star pages,
|
||||
// form blanks - stay text, and real emphasis nests properly.
|
||||
func TestParseInlineEmphasis(t *testing.T) {
|
||||
for in, want := range map[string]string{
|
||||
// Real emphasis.
|
||||
"*it* and **bold**": "<em>it</em> and <strong>bold</strong>",
|
||||
"***bold italic***": "<em><strong>bold italic</strong></em>",
|
||||
"**2** | **61**": "<strong>2</strong> | <strong>61</strong>",
|
||||
"An *n*-clique": "An <em>n</em>-clique",
|
||||
"*a **b** c*": "<em>a <strong>b</strong> c</em>",
|
||||
"**bold *nested* x**": "<strong>bold <em>nested</em> x</strong>",
|
||||
"*`code`*": "<em><code>code</code></em>",
|
||||
"_em_ and __init__.py": "<em>em</em> and <strong>init</strong>.py",
|
||||
`"On Language,"* a *New York Times*`: ""On Language,"* a <em>New York Times</em>",
|
||||
// A closer takes the nearest opener, not the first one on the line.
|
||||
"Inc.*, 321 F.3d; see also *North v. Ashcroft*, 308": "Inc.*, 321 F.3d; see also <em>North v. Ashcroft</em>, 308",
|
||||
// Not emphasis.
|
||||
"PINWALE (Sweet* and Sour* partitions)": "PINWALE (Sweet* and Sour* partitions)",
|
||||
"NRO*, CIA, SCEs* and field sites": "NRO*, CIA, SCEs* and field sites",
|
||||
"works with: * NCSC * TAO": "works with: * NCSC * TAO",
|
||||
"*** Currently Inactive ***": "*** Currently Inactive ***",
|
||||
"a Driving Force ******": "a Driving Force ******",
|
||||
"2013 WL 5741573, at *3-*9": "2013 WL 5741573, at *3-*9",
|
||||
"2 * 3 * 4": "2 * 3 * 4",
|
||||
"snake_case_name": "snake_case_name",
|
||||
"D/DIR____,IG_____,OGC___": "D/DIR____,IG_____,OGC___",
|
||||
"SO ORDERED, this _____ day of July": "SO ORDERED, this _____ day of July",
|
||||
"datastore<br>_ target user<br>_ time": "datastore<br>_ target user<br>_ time",
|
||||
`\*not em\*`: "*not em*",
|
||||
} {
|
||||
if got := parseInline(in); got != want {
|
||||
t.Errorf("parseInline(%q)\n got %q\nwant %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A list never mixes bullets and numbers: a number kept in a <ul> would
|
||||
// lose itself to a bullet, a bullet kept in an <ol> would gain a number.
|
||||
func TestRenderListKindChange(t *testing.T) {
|
||||
for in, want := range map[string]string{
|
||||
"1. Fact:\n* South Korea\n* Japan\n\n2. Existence": "<ol>\n<li>Fact:</li>\n</ol>\n" +
|
||||
"<ul>\n<li>South Korea</li>\n<li>Japan</li>\n</ul>\n" +
|
||||
"<ol>\n<li value=\"2\">Existence</li>\n</ol>\n",
|
||||
"* Insider claims\n1. ↑ one\n2. ↑ two": "<ul>\n<li>Insider claims</li>\n</ul>\n" +
|
||||
"<ol>\n<li>↑ one</li>\n<li>↑ two</li>\n</ol>\n",
|
||||
// Both nested lists stay inside the item they hang from.
|
||||
"- a\n - b\n 1. c": "<ul>\n<li>a\n<ul>\n<li>b</li>\n</ul>\n<ol>\n<li>c</li>\n</ol>\n</li>\n</ul>\n",
|
||||
// Unindented sub-bullets: "5." picks up where "4." left off.
|
||||
"4. Demonstrate:\n* one\n* two\n5. Thank": "<ol>\n<li value=\"4\">Demonstrate:</li>\n</ol>\n" +
|
||||
"<ul>\n<li>one</li>\n<li>two</li>\n</ul>\n" +
|
||||
"<ol>\n<li value=\"5\">Thank</li>\n</ol>\n",
|
||||
} {
|
||||
if got := Render(in); got != want {
|
||||
t.Errorf("Render(%q)\n got %q\nwant %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A number only starts a list right under a line of text when it is a 1
|
||||
// with content (CommonMark), so OCR runover - a year, a section number, a
|
||||
// page cite wrapped onto the next line - stays part of the sentence.
|
||||
func TestRenderRunoverNumbers(t *testing.T) {
|
||||
for in, want := range map[string]string{
|
||||
"as added by the FISA Amendments Act of\n2008.": "<p>as added by the FISA Amendments Act of<br>\n2008.</p>\n",
|
||||
"extended until August\n2013. He will be joined": "<p>extended until August<br>\n2013. He will be joined</p>\n",
|
||||
"* last visited NSA/CSS in May\n2012.": "<ul>\n<li>last visited NSA/CSS in May<br>\n2012.</li>\n</ul>\n",
|
||||
"See Alexander Decl. at 39-\n42. While terrorists": "<p>See Alexander Decl. at 39-<br>\n42. While terrorists</p>\n",
|
||||
"First Events\n+\nLast Events": "<p>First Events<br>\n+<br>\nLast Events</p>\n",
|
||||
// These still start a list.
|
||||
"Steps:\n1. one\n2. two": "<p>Steps:</p>\n<ol>\n<li>one</li>\n<li>two</li>\n</ol>\n",
|
||||
"Steps:\n- one": "<p>Steps:</p>\n<ul>\n<li>one</li>\n</ul>\n",
|
||||
"\n2008. after a blank": "<ol>\n<li value=\"2008\">after a blank</li>\n</ol>\n",
|
||||
"5. five\n6.\n7. seven": "<ol>\n<li value=\"5\">five</li>\n<li></li>\n<li>seven</li>\n</ol>\n",
|
||||
"1. one\n3. gap\n10. ten": "<ol>\n<li>one</li>\n<li value=\"3\">gap</li>\n<li value=\"10\">ten</li>\n</ol>\n",
|
||||
} {
|
||||
if got := Render(in); got != want {
|
||||
t.Errorf("Render(%q)\n got %q\nwant %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// "0." is a number like any other; it used to double as "not ordered" and
|
||||
// render as whatever the running count was.
|
||||
func TestRenderZeroNumber(t *testing.T) {
|
||||
for in, want := range map[string]string{
|
||||
"0. zero\n1. one": "<ol>\n<li value=\"0\">zero</li>\n<li>one</li>\n</ol>\n",
|
||||
"20. a\n10. b\n0.": "<ol>\n<li value=\"20\">a</li>\n<li value=\"10\">b</li>\n<li value=\"0\"></li>\n</ol>\n",
|
||||
} {
|
||||
if got := Render(in); got != want {
|
||||
t.Errorf("Render(%q)\n got %q\nwant %q", in, got, want)
|
||||
}
|
||||
}
|
||||
// Ten digits is text, not a list item.
|
||||
if got := Render("1234567890. x"); got != "<p>1234567890. x</p>\n" {
|
||||
t.Errorf("Render ten-digit number = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Thematic breaks may space out their characters, as the court filings'
|
||||
// asterisms do; such a line is a rule, never a bullet.
|
||||
func TestRenderSpacedRule(t *testing.T) {
|
||||
for _, in := range []string{"* * *", "- - - - -", "_ _ _", " * *\t* ", "***", "---"} {
|
||||
if got := Render(in); got != "<hr>\n" {
|
||||
t.Errorf("Render(%q) = %q, want a rule", in, got)
|
||||
}
|
||||
}
|
||||
if got := Render("1. a\n* * *\n2. b"); got != "<ol>\n<li>a</li>\n</ol>\n<hr>\n<ol>\n<li value=\"2\">b</li>\n</ol>\n" {
|
||||
t.Errorf("a spaced rule inside a list = %q", got)
|
||||
}
|
||||
for _, in := range []string{"* * x", "--", "- -", "*-*"} {
|
||||
if got := Render(in); got == "<hr>\n" {
|
||||
t.Errorf("Render(%q) is a rule", in)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Only the one outer pipe at each end of a row is dropped, so empty edge
|
||||
// cells keep their columns, and "\|" is a pipe inside a cell.
|
||||
func TestSplitCells(t *testing.T) {
|
||||
for in, want := range map[string][]string{
|
||||
"| a | b |": {"a", "b"},
|
||||
"||||| ECIs. |": {"", "", "", "", "ECIs."},
|
||||
"| B. (U) ASSOCIATIONS|||||": {"B. (U) ASSOCIATIONS", "", "", "", ""},
|
||||
"| | |": {"", ""},
|
||||
`| Service \| Rights | x |`: {"Service | Rights", "x"},
|
||||
`| a \|`: {"a |"},
|
||||
} {
|
||||
got := splitCells(in)
|
||||
if len(got) != len(want) {
|
||||
t.Errorf("splitCells(%q) = %q, want %q", in, got, want)
|
||||
continue
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != want[i] {
|
||||
t.Errorf("splitCells(%q) = %q, want %q", in, got, want)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
54
tests/outlets_test.go
Normal file
54
tests/outlets_test.go
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A document counts for an outlet in the years of THAT outlet's entries: The
|
||||
// Guardian's 2013 publication must not put The Intercept's 2014 copy in
|
||||
// The Intercept's 2013 column, and the cell links a page listing exactly the
|
||||
// documents it counts.
|
||||
func TestOutletYearsArePerPublisher(t *testing.T) {
|
||||
root := assemblyCorpus(t, "")
|
||||
writeAssemblyDoc(t, root, "shared", map[string]interface{}{
|
||||
"title": "Shared",
|
||||
"publication": []map[string]interface{}{
|
||||
{"publisher": "The Guardian", "date_published": "2013-06-06"},
|
||||
{"publisher": "The Intercept", "date_published": "2014-02-10"},
|
||||
{"publisher": "Partner", "date_published": "2012-01-01", "reporting_only": true},
|
||||
},
|
||||
}, []string{"page-1.jpg"}, nil)
|
||||
withCorpus(t, root)
|
||||
|
||||
cells := map[string]outletCell{}
|
||||
table := corpus().outletTables[""]
|
||||
for _, row := range table.Rows {
|
||||
for i, y := range table.Years {
|
||||
cells[row.Name+" "+y] = row.Cells[i]
|
||||
}
|
||||
}
|
||||
for k, want := range map[string]int{
|
||||
"The Guardian 2013": 1, "The Guardian 2014": 0,
|
||||
"The Intercept 2013": 0, "The Intercept 2014": 1,
|
||||
} {
|
||||
if got := cells[k].Docs; got != want {
|
||||
t.Errorf("%s: %d documents, want %d", k, got, want)
|
||||
}
|
||||
}
|
||||
for _, y := range table.Years {
|
||||
if y == "2012" {
|
||||
t.Errorf("a reporting-only entry made a 2012 column")
|
||||
}
|
||||
}
|
||||
link := cells["The Intercept 2014"].URL
|
||||
if link != "/browse/publisher/The%20Intercept?year=2014&coll=news&coll=sidtoday-archive&coll=foia" {
|
||||
t.Errorf("cell link %q", link)
|
||||
}
|
||||
if got := resultTotal(t, link); got != 1 {
|
||||
t.Errorf("%s lists %d documents, want 1", link, got)
|
||||
}
|
||||
if body := get(t, "/browse/publisher/The%20Intercept?year=2013").Body.String(); !strings.Contains(body, "0 documents") {
|
||||
t.Errorf("The Intercept's 2013 page is not empty")
|
||||
}
|
||||
}
|
||||
119
tests/portions_test.go
Normal file
119
tests/portions_test.go
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParsePortion(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
canonical string
|
||||
level string
|
||||
ok bool
|
||||
}{
|
||||
{"U", "U", "U", true},
|
||||
{"U//FOUO", "U//FOUO", "U", true},
|
||||
{"TS//SI//NF", "TS//SI//NF", "TS", true},
|
||||
{"S//REL TO USA, FVEY", "S//REL TO USA, FVEY", "S", true},
|
||||
// OCR writes the separator every way; one marking, one canonical form.
|
||||
{"S/NF", "S//NF", "S", true},
|
||||
{"U/FOUO", "U//FOUO", "U", true},
|
||||
{"TS/SI//REL", "TS//SI//REL", "TS", true},
|
||||
{"TS//SI/NF", "TS//SI//NF", "TS", true},
|
||||
{"TS//STLW//SI//OC/NF", "TS//STLW//SI//OC//NF", "TS", true},
|
||||
{"TS//SI// ECI SOL", "TS//SI//ECI SOL", "TS", true},
|
||||
// FOUO alone is a control on unclassified material.
|
||||
{"FOUO", "U//FOUO", "U", true},
|
||||
// Spelled out.
|
||||
{"TOP SECRET//COMINT", "TS//COMINT", "TS", true},
|
||||
// A level with a part no rule covers: recognised as a marking, but
|
||||
// not counted - the Explore page lists these separately.
|
||||
{"S//SI) (DNI collection", "", "S", false},
|
||||
{"S//WIBBLE", "S//WIBBLE", "S", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := parsePortion(c.in)
|
||||
if got.Level != c.level {
|
||||
t.Errorf("parsePortion(%q).Level = %q, want %q", c.in, got.Level, c.level)
|
||||
}
|
||||
if got.OK != c.ok {
|
||||
t.Errorf("parsePortion(%q).OK = %v, want %v", c.in, got.OK, c.ok)
|
||||
}
|
||||
if c.ok && got.Canonical != c.canonical {
|
||||
t.Errorf("parsePortion(%q).Canonical = %q, want %q", c.in, got.Canonical, c.canonical)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The things that merely look like markings: list enumerators, FOIA
|
||||
// exemption citations, asides. Each of these appears hundreds of times.
|
||||
func TestParsePortionRejectsNonMarkings(t *testing.T) {
|
||||
for _, s := range []string{"1", "2", "a", "b", "iii", "d", "Cont'd.", "continued...",
|
||||
"Dollars in Thousands", "b)(1), (b)(3", "A", "", " "} {
|
||||
if got := parsePortion(s); got.Level != "" {
|
||||
t.Errorf("parsePortion(%q) read a level %q - it is not a marking", s, got.Level)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractPortions(t *testing.T) {
|
||||
text := "" +
|
||||
"TOP SECRET//COMINT\n" +
|
||||
"\n" +
|
||||
"(U) Some Tips on Portion Marking\n" +
|
||||
"\n" +
|
||||
"(U//FOUO) The lead-in sentence MUST ALWAYS be portion-marked.\n" +
|
||||
"* (S//SI) a bullet carrying a complete thought\n" +
|
||||
"* an unmarked bullet, covered by its lead-in\n" +
|
||||
"## (TS//SI//NF) A marked heading\n" +
|
||||
"1. (C) a numbered portion\n" +
|
||||
"(1) not a marking, a list number\n" +
|
||||
"text mentioning (U//FOUO) mid-line is not a portion mark\n"
|
||||
got := make([]string, 0, 5)
|
||||
for _, m := range extractPortions(text) {
|
||||
got = append(got, m.Canonical)
|
||||
}
|
||||
want := []string{"U", "U//FOUO", "S//SI", "TS//SI//NF", "C"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("extractPortions = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A document quoting a marking is not marked by it. Fenced blocks and block
|
||||
// quotes are where a transcript puts an example, and they are the curator's
|
||||
// correction surface: wrap the example, and it stops being counted.
|
||||
func TestExtractPortionsSkipsQuotedExamples(t *testing.T) {
|
||||
text := "" +
|
||||
"(U) Wrong way:\n" +
|
||||
"\n" +
|
||||
"> (U) All good men and women should come to the aid of their country through:\n" +
|
||||
"> * (U//FOUO) volunteering for a tdy to the Middle East\n" +
|
||||
"\n" +
|
||||
"```\n" +
|
||||
"(TS//SI//NF) an example inside a fenced block\n" +
|
||||
"```\n" +
|
||||
"(S) a real portion after the block\n"
|
||||
var got []string
|
||||
for _, m := range extractPortions(text) {
|
||||
got = append(got, m.Canonical)
|
||||
}
|
||||
want := []string{"U", "S"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("extractPortions = %v, want %v (quoted and fenced examples must not count)", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// Markings are written in capitals. Court filings enumerate subsections
|
||||
// "(a)", "(b)", "(c)" at the start of a line, and "(c)" is not CONFIDENTIAL.
|
||||
func TestParsePortionNeedsCapitals(t *testing.T) {
|
||||
for _, s := range []string{"c", "s", "u", "ts", "s//si", "Ts", "secret"} {
|
||||
if got := parsePortion(s); got.Level != "" {
|
||||
t.Errorf("parsePortion(%q) read level %q from a lower-case label", s, got.Level)
|
||||
}
|
||||
}
|
||||
// The controls after a capital level are read whatever their case.
|
||||
if got := parsePortion("S//si"); got.Canonical != "S//SI" || !got.OK {
|
||||
t.Errorf("parsePortion(S//si) = %+v, want S//SI", got)
|
||||
}
|
||||
}
|
||||
153
tests/prefs_matrix_test.go
Normal file
153
tests/prefs_matrix_test.go
Normal file
|
|
@ -0,0 +1,153 @@
|
|||
package main
|
||||
|
||||
// prefs_matrix_test.go - every preference, asserted against the markup it
|
||||
// is supposed to change. The cookies are documented in prefs.go and their
|
||||
// effects are spread across four templates; before this, nothing checked
|
||||
// that setting one did anything at all.
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// prefCase is one preference: the cookie that turns it on, a page that
|
||||
// shows it, and a marker that must be present in exactly one of the two
|
||||
// states. present says which state the marker belongs to.
|
||||
type prefCase struct {
|
||||
name string
|
||||
cookie string
|
||||
value string
|
||||
path string
|
||||
marker string
|
||||
present bool // true: the marker appears when the cookie is SET
|
||||
}
|
||||
|
||||
func TestPreferenceMatrix(t *testing.T) {
|
||||
buildFixture(t)
|
||||
cases := []prefCase{
|
||||
// --- card display switches (browse lists, search results) -------
|
||||
{"hide thumbnails", "pref_hide_thumbs", "1", "/browse", `class="thumb-container"`, false},
|
||||
{"hide thumbnails (search)", "pref_hide_thumbs", "1", "/search?q=prism&do_search=1", `class="result-thumb"`, false},
|
||||
{"hide summaries", "pref_hide_summary", "1", "/browse", `class="card-summary"`, false},
|
||||
{"hide classification", "pref_hide_classification", "1", "/browse", `class="doc-classification"`, false},
|
||||
{"hide programs", "pref_hide_programs", "1", "/browse", `class="doc-programs"`, false},
|
||||
{"hide people", "pref_hide_people", "1", "/browse", `class="doc-people"`, false},
|
||||
{"hide journalists", "pref_hide_journalists", "1", "/browse", `class="doc-journalist`, false},
|
||||
{"page preview hook", "pref_page_preview", "1", "/browse", "data-pp-id=", true},
|
||||
|
||||
// --- category tables --------------------------------------------
|
||||
{"collection columns", "pref_show_coll_cols", "1", "/browse/program", `id="coll-cols" class="coll-cols-checkbox" checked`, true},
|
||||
|
||||
// --- the document page ------------------------------------------
|
||||
{"info sidebar", "pref_info_position", "beside", "/doc/prism-slides", `class="doc-layout info-sidebar"`, true},
|
||||
{"scroll panes", "pref_scroll_doc", "1", "/doc/prism-slides", "scroll-panes", true},
|
||||
{"expand more info", "pref_open_info", "1", "/doc/prism-slides", `id="meta-toggle" class="meta-checkbox" checked`, true},
|
||||
{"programs one per line", "pref_programs_list", "1", "/doc/prism-slides", `<ul class="meta-value-list">`, true},
|
||||
{"tags one per line", "pref_tags_list", "1", "/doc/prism-slides", `<ul class="meta-value-list">`, true},
|
||||
{"collapse published-together", "pref_collapse_pub", "1", "/doc/prism-slides", `<details class="collapse-section" open>`, false},
|
||||
|
||||
// --- the viewport picker ----------------------------------------
|
||||
{"default view: text", "pref_doc_view", "text", "/doc/prism-slides", `id="view-text" aria-controls="doc-viewport" checked`, true},
|
||||
{"default view: split", "pref_doc_view", "split", "/doc/prism-slides", `id="view-split" aria-controls="doc-viewport" checked`, true},
|
||||
|
||||
// --- theme -------------------------------------------------------
|
||||
{"dark theme", "pref_theme", "dark", "/", `data-theme="dark"`, true},
|
||||
{"light theme", "pref_theme", "light", "/browse", `data-theme="light"`, true},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
off := get(t, c.path).Body.String()
|
||||
on := get(t, c.path, &http.Cookie{Name: c.cookie, Value: c.value}).Body.String()
|
||||
hasOff, hasOn := strings.Contains(off, c.marker), strings.Contains(on, c.marker)
|
||||
wantOn, wantOff := c.present, !c.present
|
||||
if hasOn != wantOn || hasOff != wantOff {
|
||||
t.Errorf("%s (%s=%s on %s): marker %q present without the cookie=%v, with it=%v; wanted %v / %v",
|
||||
c.name, c.cookie, c.value, c.path, c.marker, hasOff, hasOn, wantOff, wantOn)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Text-only mode is the archive's promise to readers on Tor and metered
|
||||
// links: NO page image, thumbnail or preview, anywhere, whatever else is
|
||||
// set. It also overrides the thumbnail preference rather than being
|
||||
// overridden by it.
|
||||
func TestPreferenceTextOnly(t *testing.T) {
|
||||
buildFixture(t)
|
||||
textOnly := &http.Cookie{Name: "pref_text_only", Value: "1"}
|
||||
thumbsOn := &http.Cookie{Name: "pref_hide_thumbs", Value: "0"}
|
||||
preview := &http.Cookie{Name: "pref_page_preview", Value: "1"}
|
||||
|
||||
for _, p := range []string{"/", "/browse", "/browse?collection=sidtoday-archive&view=card",
|
||||
"/doc/prism-slides", "/doc/prism-slides/pages", "/doc/prism-slides/versions/lemonde",
|
||||
"/search?q=prism&do_search=1", "/pinned", "/articles"} {
|
||||
body := get(t, p, textOnly, thumbsOn, preview).Body.String()
|
||||
if i := strings.Index(body, "<img"); i >= 0 {
|
||||
t.Errorf("%s in text-only mode emits an image: %.80s", p, body[i:])
|
||||
}
|
||||
if strings.Contains(body, "data-pp-imgs") {
|
||||
t.Errorf("%s in text-only mode emits the page-preview payload", p)
|
||||
}
|
||||
}
|
||||
// The document page says so and offers the pages as links instead.
|
||||
doc := get(t, "/doc/prism-slides", textOnly).Body.String()
|
||||
for _, want := range []string{"Text-only mode is on", `class="page-link-list"`} {
|
||||
if !strings.Contains(doc, want) {
|
||||
t.Errorf("the document page in text-only mode lacks %q", want)
|
||||
}
|
||||
}
|
||||
// Without it, that same page does show images.
|
||||
if !strings.Contains(get(t, "/doc/prism-slides").Body.String(), "<img") {
|
||||
t.Error("the document page shows no images even with images on")
|
||||
}
|
||||
}
|
||||
|
||||
// Documents per page drives the list length and the pagination, on browse
|
||||
// and on search alike.
|
||||
func TestPreferencePerPage(t *testing.T) {
|
||||
buildFixture(t)
|
||||
count := func(body string) int { return strings.Count(body, `class="doc-card"`) }
|
||||
all := count(get(t, "/browse", &http.Cookie{Name: "pref_per_page", Value: "all"}).Body.String())
|
||||
if all < 2 {
|
||||
t.Fatalf("the fixture's news collection has %d cards; the test needs at least 2", all)
|
||||
}
|
||||
one := get(t, "/browse", &http.Cookie{Name: "pref_per_page", Value: "10"}).Body.String()
|
||||
if n := count(one); n > 10 {
|
||||
t.Errorf("per_page=10 rendered %d cards", n)
|
||||
}
|
||||
// An unknown value falls back to the default rather than to everything.
|
||||
def := count(get(t, "/browse", &http.Cookie{Name: "pref_per_page", Value: "7"}).Body.String())
|
||||
if def > 20 {
|
||||
t.Errorf("an invalid per_page rendered %d cards, want at most the default 20", def)
|
||||
}
|
||||
}
|
||||
|
||||
// The two history preferences are opt-in: nothing is recorded until they
|
||||
// are on, and turning one off deletes what it recorded.
|
||||
func TestPreferenceHistoryOptIn(t *testing.T) {
|
||||
buildFixture(t)
|
||||
// Off by default: viewing a document sets no history cookie.
|
||||
w := get(t, "/doc/prism-slides")
|
||||
for _, c := range w.Result().Cookies() {
|
||||
if c.Name == recentDocsCookie && c.Value != "" {
|
||||
t.Errorf("a document view recorded history without the preference: %v", c)
|
||||
}
|
||||
}
|
||||
// On: the view is recorded.
|
||||
w = get(t, "/doc/prism-slides", &http.Cookie{Name: "pref_track_recent", Value: "1"})
|
||||
found := false
|
||||
for _, c := range w.Result().Cookies() {
|
||||
if c.Name == recentDocsCookie && strings.Contains(c.Value, "prism-slides") {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Error("with tracking on, a document view was not recorded")
|
||||
}
|
||||
// A page showing history must never be stored by a shared cache.
|
||||
hist := get(t, "/", &http.Cookie{Name: "pref_track_recent", Value: "1"},
|
||||
&http.Cookie{Name: recentDocsCookie, Value: "prism-slides"})
|
||||
if cc := hist.Header().Get("Cache-Control"); !strings.Contains(cc, "no-store") {
|
||||
t.Errorf("the home page showing history: Cache-Control %q", cc)
|
||||
}
|
||||
}
|
||||
117
tests/prefs_test.go
Normal file
117
tests/prefs_test.go
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func reqWithCookies(pairs ...string) *http.Request {
|
||||
r := httptest.NewRequest("GET", "/", nil)
|
||||
for i := 0; i+1 < len(pairs); i += 2 {
|
||||
r.AddCookie(&http.Cookie{Name: pairs[i], Value: pairs[i+1]})
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// Defaults apply when a cookie is absent, and a value the preference does
|
||||
// not accept is a value it does not take.
|
||||
func TestReadPrefsValues(t *testing.T) {
|
||||
p := readPrefs(reqWithCookies())
|
||||
if p.DocView != "images" || p.Theme != "auto" || p.PerPage != 20 || p.InfoSidebar {
|
||||
t.Errorf("defaults wrong: %+v", p)
|
||||
}
|
||||
if !p.ShowThumbs || !p.ShowClassification || p.TrackRecent {
|
||||
t.Errorf("boolean defaults wrong: %+v", p)
|
||||
}
|
||||
|
||||
p = readPrefs(reqWithCookies(
|
||||
"pref_doc_view", "split",
|
||||
"pref_theme", "dark",
|
||||
"pref_per_page", "all",
|
||||
"pref_info_position", "beside",
|
||||
"pref_hide_thumbs", "1",
|
||||
"pref_track_recent", "1",
|
||||
))
|
||||
if p.DocView != "split" || p.Theme != "dark" || p.PerPage != 0 || !p.InfoSidebar {
|
||||
t.Errorf("cookie values not applied: %+v", p)
|
||||
}
|
||||
if p.ShowThumbs || !p.TrackRecent {
|
||||
t.Errorf("boolean cookies not applied: %+v", p)
|
||||
}
|
||||
if p.ThemeAttr() != "dark" {
|
||||
t.Errorf("ThemeAttr = %q", p.ThemeAttr())
|
||||
}
|
||||
|
||||
// Junk in the jar: every value pref falls back to its default.
|
||||
p = readPrefs(reqWithCookies(
|
||||
"pref_doc_view", "sideways",
|
||||
"pref_theme", "neon",
|
||||
"pref_per_page", "7",
|
||||
"pref_info_position", "under",
|
||||
))
|
||||
if p.DocView != "images" || p.Theme != "auto" || p.PerPage != 20 || p.InfoSidebar {
|
||||
t.Errorf("junk cookies not rejected: %+v", p)
|
||||
}
|
||||
if p.ThemeAttr() != "" {
|
||||
t.Errorf("auto must render no data-theme, got %q", p.ThemeAttr())
|
||||
}
|
||||
}
|
||||
|
||||
// Saving writes the cookies a non-default choice needs and clears the rest,
|
||||
// so a default-settings visitor stays cookieless (and their pages stay
|
||||
// publicly cacheable).
|
||||
func TestPreferencesSaveRoundTrip(t *testing.T) {
|
||||
form := url.Values{
|
||||
"doc_view": {"text"}, "theme": {"light"}, "per_page": {"100"},
|
||||
"info_position": {"beside"}, "hide_thumbs": {"1"},
|
||||
}
|
||||
r := httptest.NewRequest("POST", "/preferences/save", strings.NewReader(form.Encode()))
|
||||
r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
r.Header.Set("Sec-Fetch-Site", "same-origin")
|
||||
w := httptest.NewRecorder()
|
||||
handlePreferencesSave(w, r)
|
||||
|
||||
set := map[string]string{}
|
||||
for _, c := range w.Result().Cookies() {
|
||||
set[c.Name] = c.Value
|
||||
if c.MaxAge < 0 {
|
||||
set[c.Name] = "" // deleted
|
||||
}
|
||||
}
|
||||
for name, want := range map[string]string{
|
||||
"pref_doc_view": "text", "pref_theme": "light", "pref_per_page": "100",
|
||||
"pref_info_position": "beside", "pref_hide_thumbs": "1",
|
||||
"pref_track_recent": "", // unchecked: deleted
|
||||
} {
|
||||
if got, ok := set[name]; !ok || got != want {
|
||||
t.Errorf("%s = %q (present %v), want %q", name, got, ok, want)
|
||||
}
|
||||
}
|
||||
|
||||
// Defaults delete rather than store: nothing in the jar means nothing
|
||||
// to vary the cache on.
|
||||
form = url.Values{"doc_view": {"images"}, "theme": {"auto"}, "per_page": {"20"}, "info_position": {"above"}}
|
||||
r = httptest.NewRequest("POST", "/preferences/save", strings.NewReader(form.Encode()))
|
||||
r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
r.Header.Set("Sec-Fetch-Site", "same-origin")
|
||||
w = httptest.NewRecorder()
|
||||
handlePreferencesSave(w, r)
|
||||
for _, c := range w.Result().Cookies() {
|
||||
if c.MaxAge >= 0 {
|
||||
t.Errorf("default settings wrote cookie %s=%q", c.Name, c.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// "All" per page is every card up to maxCardsPerPage: a list longer than
|
||||
// that is paged, however large the reader's page size.
|
||||
func TestCardsPerPage(t *testing.T) {
|
||||
for perPage, want := range map[int]int{0: maxCardsPerPage, 20: 20, 100: 100, maxCardsPerPage + 1: maxCardsPerPage} {
|
||||
if got := (Prefs{PerPage: perPage}).cardsPerPage(); got != want {
|
||||
t.Errorf("PerPage %d: %d cards a page, want %d", perPage, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
126
tests/probes_test.go
Normal file
126
tests/probes_test.go
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
package main
|
||||
|
||||
// probes_test.go - the instruments that catch what reading the code cannot:
|
||||
// data races, the parameter space behind each route, and header injection.
|
||||
// Each of these exists because a review that only read the source missed
|
||||
// something a probe found in seconds.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Concurrency: the archive claims to be read-only while serving. Hammer the
|
||||
// handlers from many goroutines with -race and see.
|
||||
func TestProbeConcurrent(t *testing.T) {
|
||||
if os.Getenv("SNOWCHIVE_BENCH_DATA") == "" {
|
||||
buildFixture(t)
|
||||
}
|
||||
paths := []string{"/", "/browse", "/browse/program", "/browse/tag?sort=count",
|
||||
"/search?q=prism&do_search=1", "/explore/ids", "/api/facets", "/glossary",
|
||||
"/doc/prism-slides", "/articles", "/sitemap.xml"}
|
||||
mux, _ := buildFixture(t)
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 24; i++ {
|
||||
wg.Add(1)
|
||||
go func(n int) {
|
||||
defer wg.Done()
|
||||
for j := 0; j < 20; j++ {
|
||||
p := paths[(n+j)%len(paths)]
|
||||
r := httptest.NewRequest("GET", p, nil)
|
||||
if j%3 == 0 {
|
||||
r.AddCookie(&http.Cookie{Name: "pref_theme", Value: "dark"})
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
mux.ServeHTTP(w, r)
|
||||
if w.Code != 200 {
|
||||
t.Errorf("%s: %d", p, w.Code)
|
||||
return
|
||||
}
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// Input space: every route crossed with the parameters it accepts. The
|
||||
// 80 ms page hid behind ?sort=count - a path-only sweep never saw it.
|
||||
|
||||
// Input space: every route crossed with the parameters it accepts. The
|
||||
// 80 ms page hid behind ?sort=count - a path-only sweep never saw it.
|
||||
func TestProbeParamSweep(t *testing.T) {
|
||||
buildFixture(t)
|
||||
base := []string{"/", "/browse", "/browse/program", "/browse/tag", "/browse/docdate",
|
||||
"/browse/program/PRISM", "/browse/docdate/2013", "/search", "/explore/ids",
|
||||
"/explore/terms", "/explore/pages", "/explore/outlets", "/articles",
|
||||
"/api/docs", "/api/search", "/glossary", "/pinned", "/preferences"}
|
||||
params := []string{"", "?sort=count", "?sort=count&order=asc", "?sort=zzz", "?order=desc",
|
||||
"?view=table", "?view=card", "?p=2", "?p=-1", "?p=99999999999999999999",
|
||||
"?per_page=all", "?per_page=0", "?per_page=-5", "?type=REPORTED", "?type=NOPE",
|
||||
"?coll=news", "?coll=news&coll=foia", "?missing=1", "?q=prism&do_search=1",
|
||||
"?q=%22a+b%22~5&do_search=1", "?q=-prism&do_search=1", "?hist=foia,sidtoday-archive",
|
||||
"?date_doc_start=2013&date_doc_end=2013", "?fqv=x&fqf=ZZ&fqc=xx&do_search=1",
|
||||
"?sort=name&order=desc&p=1&type=FOIA&coll=foia"}
|
||||
type slow struct {
|
||||
path string
|
||||
d time.Duration
|
||||
}
|
||||
var slows []slow
|
||||
bad := 0
|
||||
for _, b := range base {
|
||||
for _, p := range params {
|
||||
u := b + p
|
||||
start := time.Now()
|
||||
w := get(t, u)
|
||||
d := time.Since(start)
|
||||
if w.Code != 200 && w.Code != 400 && w.Code != 404 {
|
||||
bad++
|
||||
t.Errorf("%s: status %d", u, w.Code)
|
||||
}
|
||||
slows = append(slows, slow{u, d})
|
||||
}
|
||||
}
|
||||
sort.Slice(slows, func(i, j int) bool { return slows[i].d > slows[j].d })
|
||||
fmt.Printf("%d combinations, %d unexpected statuses; slowest on the fixture:\n", len(slows), bad)
|
||||
for _, s := range slows[:5] {
|
||||
fmt.Printf(" %8s %s\n", s.d.Round(time.Microsecond), s.path)
|
||||
}
|
||||
}
|
||||
|
||||
// Header injection: the Onion-Location header echoes the request URI.
|
||||
|
||||
// Header injection: the Onion-Location header echoes the request URI.
|
||||
func TestProbeHeaderInjection(t *testing.T) {
|
||||
buildFixture(t)
|
||||
for _, raw := range []string{
|
||||
"/browse?x=1%0d%0aX-Injected:+yes",
|
||||
"/browse?x=%0aSet-Cookie:+a=b",
|
||||
"/doc/" + strings.Repeat("a", 200),
|
||||
} {
|
||||
r := httptest.NewRequest("GET", raw, nil)
|
||||
w := httptest.NewRecorder()
|
||||
mux, _ := buildFixture(t)
|
||||
mux.ServeHTTP(w, r)
|
||||
h := w.Header()
|
||||
if h.Get("X-Injected") != "" || h.Get("Set-Cookie") != "" {
|
||||
t.Errorf("%s: header injected: %v", raw, h)
|
||||
}
|
||||
for k, vs := range h {
|
||||
for _, v := range vs {
|
||||
if strings.ContainsAny(v, "\r\n") {
|
||||
t.Errorf("%s: header %s carries a newline: %q", raw, k, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Pathological documents: the biggest thing in the corpus is what breaks
|
||||
// first. What IS the biggest thing?
|
||||
43
tests/published_together_test.go
Normal file
43
tests/published_together_test.go
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The "published in the same report" box is capped at pubTogetherCap, and
|
||||
// the overflow links point at the article pages: one per article that holds
|
||||
// other documents, in publication order, so each link's number is the
|
||||
// number its page lists.
|
||||
func TestDocArticles(t *testing.T) {
|
||||
saved := corpus().articleByLink
|
||||
defer func() { corpus().articleByLink = saved }()
|
||||
small := &Article{Key: "aaaaaaaaaaaa", DocIDs: []string{"a", "b"}}
|
||||
big := &Article{Key: "bbbbbbbbbbbb", DocIDs: []string{"a", "b", "c", "d"}}
|
||||
alone := &Article{Key: "cccccccccccc", DocIDs: []string{"a"}}
|
||||
corpus().articleByLink = map[string]*Article{
|
||||
"https://small.example/story": small,
|
||||
"https://big.example/appendix": big,
|
||||
"https://alone.example/x": alone,
|
||||
}
|
||||
var doc Document
|
||||
if err := json.Unmarshal([]byte(`{"publication":[
|
||||
{"link": "https://small.example/story"},
|
||||
{"link": "https://alone.example/x"},
|
||||
{"link": "https://big.example/appendix"},
|
||||
{"link": "https://small.example/story"}]}`), &doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := corpus().docArticles(&doc)
|
||||
if len(got) != 2 || got[0] != small || got[1] != big {
|
||||
t.Fatalf("docArticles = %+v, want the two articles with other documents, once each, in order", got)
|
||||
}
|
||||
|
||||
var orphan Document
|
||||
if err := json.Unmarshal([]byte(`{"publication":[{"link": "https://nowhere.example/x"}]}`), &orphan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a := corpus().docArticles(&orphan); a != nil {
|
||||
t.Error("a document whose links have no article must produce no overflow link")
|
||||
}
|
||||
}
|
||||
193
tests/query_test.go
Normal file
193
tests/query_test.go
Normal file
|
|
@ -0,0 +1,193 @@
|
|||
package main
|
||||
|
||||
// query_test.go - what the search engine promises, as queries and the
|
||||
// documents they must return. The unit tests cover the pieces (tokenizer,
|
||||
// wildcards, ranges); this covers the sentence a reader types.
|
||||
//
|
||||
// Results come from /api/search, which runs the same engine as the page, so
|
||||
// a refactor that changes what search FINDS fails here even when every page
|
||||
// still renders.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func searchIDs(t *testing.T, q string) []string {
|
||||
t.Helper()
|
||||
w := get(t, "/api/search?per_page=all&q="+url.QueryEscape(q))
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("%q: status %d", q, w.Code)
|
||||
}
|
||||
var body struct {
|
||||
Results []struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"results"`
|
||||
}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("%q: %v", q, err)
|
||||
}
|
||||
ids := make([]string, 0, len(body.Results))
|
||||
for _, r := range body.Results {
|
||||
ids = append(ids, r.ID)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
return ids
|
||||
}
|
||||
|
||||
func TestQuerySuite(t *testing.T) {
|
||||
buildFixture(t)
|
||||
const (
|
||||
prism = "prism-slides"
|
||||
companion = "prism-companion"
|
||||
sid = "2003-05-29_SIDToday_-_Fixture_Issue"
|
||||
foia = "FISC_-_fixture_order"
|
||||
undated = "undated-fixture"
|
||||
)
|
||||
cases := []struct {
|
||||
query string
|
||||
want []string
|
||||
why string
|
||||
}{
|
||||
// --- words, stems and case --------------------------------------
|
||||
{"xkeyscore", []string{prism}, "a plain word in the transcript"},
|
||||
{"XKEYSCORE", []string{prism}, "matching ignores case"},
|
||||
{"collections", []string{prism, sid}, "stemming: plural finds the singular"},
|
||||
{"analysts", []string{prism, sid}, "plural finds singular"},
|
||||
// The stemmer deliberately leaves -tion alone (see stem), so
|
||||
// "collecting" and "collection" are different words here. This is
|
||||
// the behaviour, recorded so a change to the stemmer is a decision
|
||||
// rather than a surprise.
|
||||
{"collecting", []string{prism}, "-ing does not reach -tion"},
|
||||
{"zzzznotacorpusword", nil, "a word nobody wrote"},
|
||||
|
||||
// --- phrases and proximity ---------------------------------------
|
||||
{`"upstream collection"`, []string{prism}, "a quoted phrase is adjacency, and SIDtoday has the words apart"},
|
||||
{`"upstream collection"~12`, []string{prism, sid}, "proximity widens it to a window"},
|
||||
{`"cable tapping"`, []string{prism}, "a phrase that is also a tag"},
|
||||
|
||||
// --- punctuation and accents --------------------------------------
|
||||
{"RT-RG", []string{prism}, "a hyphenated term matches its parts in sequence"},
|
||||
{"muller", []string{prism}, "accent folding: muller finds Müller-Maguhn"},
|
||||
{"Müller-Maguhn", []string{prism}, "and the accented form finds itself"},
|
||||
{"sao paulo", []string{prism}, "folding works across a phrase too"},
|
||||
|
||||
// --- wildcards -----------------------------------------------------
|
||||
{"*keyscore", []string{prism}, "a leading wildcard"},
|
||||
{"tapp*", []string{prism, sid}, "a trailing wildcard reaches tapping"},
|
||||
{"*ollectio*", []string{prism, sid}, "both ends open"},
|
||||
|
||||
// --- exact values vs substrings -------------------------------------
|
||||
{`classification:="TOP SECRET"`, []string{prism}, "an exact value is the whole value"},
|
||||
{`classification:="SECRET"`, []string{foia}, "so SECRET does not match TOP SECRET"},
|
||||
{"classification:secret", []string{foia, prism}, "unquoted, it is a word match on both"},
|
||||
|
||||
// --- fielded search ---------------------------------------------------
|
||||
{"program:PRISM", []string{companion, prism}, "a metadata field"},
|
||||
{"agency:FISC", []string{foia}, "creator"},
|
||||
{"agency:NSA", []string{companion, prism, sid}, "creator across collections"},
|
||||
{`journalist:"Greenwald, Glenn"`, []string{companion, prism}, "a quoted byline"},
|
||||
{"publisher:NRK", nil, "a reporting-only partner is not a publisher"},
|
||||
{"publisher:\"The Guardian\"", []string{companion, prism}, "the outlet that published it"},
|
||||
{"tag:collection", []string{prism}, "tags are their own field"},
|
||||
{"country:Brazil", []string{prism}, "countries"},
|
||||
{"people:Snowden", []string{prism}, "people, by surname"},
|
||||
{"type:FOIA", []string{foia}, "document type"},
|
||||
{"series:\"(U) A Day in the Life of...\"", []string{sid}, "a series name with punctuation"},
|
||||
|
||||
// --- boolean and negation ---------------------------------------------
|
||||
{"prism -companion", []string{prism}, "inline negation excludes"},
|
||||
{"upstream -xkeyscore", []string{sid}, "negation against a word in the other document"},
|
||||
{"prism -x", []string{companion, prism}, "a single letter is a literal word, not an empty clause that excludes everything"},
|
||||
{"u.s.", nil, "a dotted abbreviation is matched as written, not dropped"},
|
||||
|
||||
// --- a phrase stays inside one value ------------------------------------
|
||||
{`"united states"`, []string{prism}, "a phrase inside one country"},
|
||||
{`"andy united"`, nil, "a phrase cannot run from one metadata value into the next"},
|
||||
{`"slides slides describing"`, nil, "nor from the title into the description"},
|
||||
{"program:PRISM tag:collection", []string{prism}, "two fields are an AND"},
|
||||
|
||||
// --- dates -------------------------------------------------------------
|
||||
{"date:2013", []string{companion, foia, prism}, "document year, across collections"},
|
||||
{"date:2013-04-02", []string{companion}, "an exact document date"},
|
||||
{"date:2003..2013", []string{companion, foia, prism, sid}, "a range"},
|
||||
{"published:2016", []string{sid}, "publication year, not document year"},
|
||||
|
||||
// --- the corner the archive cares about ---------------------------------
|
||||
{"fairview", []string{prism}, "text that exists only in another published copy"},
|
||||
{"guardian", []string{companion, prism}, "an outlet name is searchable as text"},
|
||||
{"prism-slides", []string{prism}, "the document id itself"},
|
||||
{"prism-slides-guardian", []string{prism}, "and the name the same file was published under"},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
got := searchIDs(t, c.query)
|
||||
want := append([]string(nil), c.want...)
|
||||
sort.Strings(want)
|
||||
if strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Errorf("%-34s got %v, want %v (%s)", c.query, got, want, c.why)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Relevance is not just membership: a title hit must outrank a body hit,
|
||||
// and the phrase bonus must beat loose words.
|
||||
func TestQueryRanking(t *testing.T) {
|
||||
buildFixture(t)
|
||||
first := func(q string) string {
|
||||
ids := func() []string {
|
||||
w := get(t, "/api/search?q="+url.QueryEscape(q))
|
||||
var body struct {
|
||||
Results []struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"results"`
|
||||
}
|
||||
json.Unmarshal(w.Body.Bytes(), &body)
|
||||
out := make([]string, 0, len(body.Results))
|
||||
for _, r := range body.Results {
|
||||
out = append(out, r.ID)
|
||||
}
|
||||
return out
|
||||
}()
|
||||
if len(ids) == 0 {
|
||||
t.Fatalf("%q returned nothing", q)
|
||||
}
|
||||
return ids[0]
|
||||
}
|
||||
if got := first("collection"); got != "prism-slides" {
|
||||
t.Errorf("collection: first result %s, want the document with it in the title and the tags", got)
|
||||
}
|
||||
if got := first("fixture"); got == "" {
|
||||
t.Error("fixture: no results")
|
||||
}
|
||||
}
|
||||
|
||||
// TestQueryRowParts: a structured row is read as ?q= reads its text - each
|
||||
// quoted phrase is a part of its own and the bare words are one more, all
|
||||
// ANDed. A row with two phrases used to be one substring search for the
|
||||
// text between the outer quotes, quotes included, and found nothing.
|
||||
func TestQueryRowParts(t *testing.T) {
|
||||
buildFixture(t)
|
||||
for _, c := range []struct {
|
||||
row string
|
||||
want int
|
||||
}{
|
||||
{`"upstream collection" "cable tapping"`, 1},
|
||||
{`"upstream collection" xkeyscore`, 1},
|
||||
{`"collection upstream" xkeyscore`, 0},
|
||||
} {
|
||||
w := get(t, "/api/search?per_page=all&fqf=TE&fqc=and&fqv="+url.QueryEscape(c.row))
|
||||
var body struct {
|
||||
Total int `json:"total"`
|
||||
}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("%s: %v", c.row, err)
|
||||
}
|
||||
if body.Total != c.want {
|
||||
t.Errorf("row %s: %d results, want %d", c.row, body.Total, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
77
tests/related_test.go
Normal file
77
tests/related_test.go
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The regression this guards: a SIDtoday piece called "The Life of an Exec"
|
||||
// used to list five FISC court filings as related, because their titles say
|
||||
// "of" eight times and every occurrence scored like a shared codename. A
|
||||
// shared word is worth only as much as it is rare, and it counts once.
|
||||
func TestRelatedIgnoresFillerTitleWords(t *testing.T) {
|
||||
filler := strings.Repeat("Motion of the Court for Disclosure of Records of the Court of Appeal ", 3)
|
||||
docs := []Document{
|
||||
{ID: "exec", Title: "The Life of an Exec (repost)", Collection: "sidtoday-archive"},
|
||||
{ID: "sibling", Title: "A Day in the Life of an Exec", Collection: "sidtoday-archive"},
|
||||
{ID: "filing", Title: filler, Collection: "foia"},
|
||||
}
|
||||
// Enough other titles saying "of" and "court" that those words are
|
||||
// common - which is exactly what the real corpus looks like.
|
||||
for i := 0; i < 30; i++ {
|
||||
docs = append(docs, Document{
|
||||
ID: fmt.Sprintf("noise-%02d", i),
|
||||
Title: fmt.Sprintf("Order of the Court of Review number %d", i),
|
||||
Collection: "foia",
|
||||
})
|
||||
}
|
||||
|
||||
got := corpus().buildRelatedDocs(docs[:1], docs)["exec"]
|
||||
if len(got) == 0 || got[0] != "sibling" {
|
||||
t.Fatalf("related = %v, want sibling first", got)
|
||||
}
|
||||
for _, id := range got {
|
||||
if id == "filing" || strings.HasPrefix(id, "noise-") {
|
||||
t.Errorf("related = %v, want no documents matched only on filler words", got)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Shared metadata counts each distinct value once: a tag listed twice in one
|
||||
// document is one tag, not two matches.
|
||||
func TestRelatedScoreCountsEachValueOnce(t *testing.T) {
|
||||
a := Document{ID: "a", Title: "Alpha", Tags: []string{"PRISM", "prism", "SSO"}}
|
||||
b := Document{ID: "b", Title: "Beta", Tags: []string{"PRISM", "PRISM"}}
|
||||
if got := corpus().relatedScore(&a, &b, titleWeights{}); got != 2.0 {
|
||||
t.Errorf("score = %v, want 2 (one shared tag)", got)
|
||||
}
|
||||
|
||||
// A shared program, tag, agency and country each still count.
|
||||
a.DocMeta.Programs, a.DocMeta.Creator, a.DocMeta.Countries = []string{"XKEYSCORE"}, "NSA", []string{"Norway"}
|
||||
b.DocMeta.Programs, b.DocMeta.Creator, b.DocMeta.Countries = []string{"xkeyscore"}, "nsa", []string{"NORWAY", "Norway"}
|
||||
if got := corpus().relatedScore(&a, &b, titleWeights{}); got != 2.0+4.0+1.0+0.5 {
|
||||
t.Errorf("score = %v, want 7.5", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A word in most titles is worth almost nothing; one in a couple of titles is
|
||||
// worth nearly full marks.
|
||||
func TestTitleWeights(t *testing.T) {
|
||||
tw := titleWeights{n: 1000, df: map[string]int{"of": 800, "exec": 2}}
|
||||
if w := tw.weight("of"); w > 0.05 {
|
||||
t.Errorf("weight(of) = %v, want near 0", w)
|
||||
}
|
||||
if w := tw.weight("exec"); w < 0.8 {
|
||||
t.Errorf("weight(exec) = %v, want near 1", w)
|
||||
}
|
||||
// An unknown word is treated as unique, and a corpus too small to have
|
||||
// frequencies weighs everything equally.
|
||||
if w := tw.weight("never-seen"); w < 0.99 {
|
||||
t.Errorf("weight(unseen) = %v, want ~1", w)
|
||||
}
|
||||
if w := (titleWeights{}).weight("anything"); w != 1 {
|
||||
t.Errorf("empty weights = %v, want 1", w)
|
||||
}
|
||||
}
|
||||
49
tests/reporting_only_test.go
Normal file
49
tests/reporting_only_test.go
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A reporting-only partner keeps its entry, but is not a source of the file:
|
||||
// publisher: must not match it and the source category must not list it.
|
||||
func TestReportingOnlyIsNotASource(t *testing.T) {
|
||||
var d Document
|
||||
if err := json.Unmarshal([]byte(`{"publication": [
|
||||
{"publisher": "The Intercept", "title": "Norway Used NSA Technology",
|
||||
"date_published": "2018-03-01", "reporters": ["Gallagher, Ryan"]},
|
||||
{"publisher": "NRK", "title": "Antennene som samler inn data",
|
||||
"date_published": "2018-03-01", "reporters": ["Hopland, Standahl"],
|
||||
"reporting_only": true}
|
||||
]}`), &d); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if got := fmt.Sprint(facetValues(&d, "PU")); got != "[The Intercept]" {
|
||||
t.Errorf("publisher values %s, want only [The Intercept]", got)
|
||||
}
|
||||
// The /browse/publisher/{outlet} index is built from the same values, so
|
||||
// the page and the facet agree: the publisher is a source, the partner
|
||||
// that only reported the story is not. The index is package state, so
|
||||
// put the corpus back for whatever runs next.
|
||||
prev := corpus().archive.Docs
|
||||
t.Cleanup(func() { corpus().archive.Docs = prev; corpus().buildCategoryDocs() })
|
||||
corpus().archive.Docs = []Document{d}
|
||||
corpus().buildCategoryDocs()
|
||||
if got := len(corpus().categoryMatches("publisher", "The Intercept")); got != 1 {
|
||||
t.Errorf("the publishing outlet's page lists %d documents, want 1", got)
|
||||
}
|
||||
if got := len(corpus().categoryMatches("publisher", "NRK")); got != 0 {
|
||||
t.Errorf("the reporting-only partner's page lists %d documents, want 0", got)
|
||||
}
|
||||
// The entry itself, its byline and its article survive.
|
||||
if len(d.Publication) != 2 || d.Publication[1].Publisher != "NRK" ||
|
||||
len(d.Publication[1].Journalists) != 1 {
|
||||
t.Errorf("the reporting-only entry lost data: %+v", d.Publication)
|
||||
}
|
||||
if !strings.Contains(fmt.Sprint(d.JournalistsDeduped()), "Hopland") {
|
||||
t.Error("a partner's reporters should still be credited")
|
||||
}
|
||||
}
|
||||
705
tests/routes_test.go
Normal file
705
tests/routes_test.go
Normal file
|
|
@ -0,0 +1,705 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"html/template"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Every route, through the real mux: the status it must answer, and a
|
||||
// fragment of the body that proves the page is the one it should be. The
|
||||
// handlers had no coverage at all before this - a missing template
|
||||
// function, a nil map or a renamed data key reached production as a 500.
|
||||
func TestRoutes(t *testing.T) {
|
||||
cases := []struct {
|
||||
path string
|
||||
code int
|
||||
want string // substring of the body
|
||||
}{
|
||||
{"/", 200, "Snowden"},
|
||||
{"/browse", 200, "Reported"},
|
||||
{"/browse?collection=foia", 200, "FOIA"},
|
||||
{"/browse?collection=sidtoday-archive", 200, "SIDtoday Archive"},
|
||||
{"/browse?collection=sidtoday-archive&view=card", 200, "SIDtoday Archive"},
|
||||
{"/browse?collection=sidtoday-archive&view=table", 200, "SIDtoday Archive"},
|
||||
{"/browse?collection=bogus", 200, "Reported"}, // unknown collection falls back
|
||||
{"/browse/program", 200, "PRISM"},
|
||||
{"/browse/program/PRISM", 200, "PRISM Collection Slides"},
|
||||
{"/browse/program/NOSUCHTHING", 200, "0 document"},
|
||||
{"/browse/tag/PRISM", 200, "PRISM Collection Slides"},
|
||||
{"/browse/agency/NSA", 200, "PRISM Collection Slides"},
|
||||
{"/browse/publisher/The%20Guardian", 200, "PRISM Collection Slides"},
|
||||
{"/browse/publisher/NRK", 200, "0 document"}, // reporting-only: not a publisher
|
||||
{"/browse/journalist/Greenwald%2C%20Glenn", 200, "PRISM"},
|
||||
{"/browse/classification/TOP%20SECRET", 200, "PRISM"},
|
||||
{"/browse/dissemination/NOFORN", 200, "PRISM"},
|
||||
{"/browse/docdate/2013", 200, "PRISM"},
|
||||
{"/browse/docdate/2013-04", 200, "PRISM"},
|
||||
{"/browse/docdate/20", 200, "0 document"}, // not a valid date prefix
|
||||
{"/browse/published/2013", 200, "PRISM"},
|
||||
{"/browse/series", 200, "A Day in the Life"},
|
||||
{"/browse/nosuchcategory", 404, ""},
|
||||
{"/browse/nosuchcategory/value", 404, ""},
|
||||
{"/doc/prism-slides", 200, "PRISM Collection Slides"},
|
||||
{"/doc/prism-slides/pages", 200, "All pages"},
|
||||
{"/doc/prism-slides/versions/lemonde", 200, "lemonde"},
|
||||
{"/doc/prism-slides/versions/lemonde/pages", 200, "lemonde"},
|
||||
{"/doc/prism-slides/versions/nope", 404, ""},
|
||||
{"/doc/broken-no-pdf", 404, ""}, // skipped at load: no PDF
|
||||
{"/doc/nope", 404, ""},
|
||||
{"/search", 200, "Search"},
|
||||
{"/search?q=prism&do_search=1", 200, "Collection Slides"}, // the term itself is <mark>ed
|
||||
{"/search?q=zzzznomatch&do_search=1", 200, "No matches"},
|
||||
{"/search?q=fairview&do_search=1", 200, "Copy:"}, // matched in a version
|
||||
{"/articles", 200, "NSA Prism program"},
|
||||
{"/explore", 200, "Explore"},
|
||||
{"/explore/pages", 200, "Longest"},
|
||||
{"/explore/ids", 200, "prism-slides"},
|
||||
{"/explore/ids.txt", 200, "prism-slides.pdf"},
|
||||
{"/explore/ids?missing=1", 200, "Technical IDs"},
|
||||
{"/explore/outlets", 200, "Guardian"},
|
||||
{"/explore/terms", 200, "PRISM"},
|
||||
{"/explore/classification", 200, "Classification"},
|
||||
{"/explore/nosuchmetric", 404, ""},
|
||||
{"/glossary", 200, "XKEYSCORE"},
|
||||
{"/changelog", 200, ""},
|
||||
{"/about", 200, "About"},
|
||||
{"/preferences", 200, "Preferences"},
|
||||
{"/pinned", 200, "Pinned"},
|
||||
{"/pinned/export", 200, ""},
|
||||
{"/random", 302, ""},
|
||||
{"/robots.txt", 200, "Sitemap"},
|
||||
{"/sitemap.xml", 200, "/doc/prism-slides"},
|
||||
{"/feed.xml", 404, ""},
|
||||
{"/api/docs", 200, `"total"`},
|
||||
{"/api/docs?per_page=all", 200, `"docs"`},
|
||||
{"/api/doc/prism-slides", 200, `"full_text"`},
|
||||
{"/api/doc/nope", 404, `"error"`},
|
||||
{"/api/search?q=prism", 200, `"results"`},
|
||||
{"/api/search", 400, `"error"`},
|
||||
{"/api/facets", 200, `"programs"`},
|
||||
{"/api/nosuch", 404, `"error"`},
|
||||
{"/static/style.css", 200, "--"},
|
||||
{"/static/nosuch.css", 404, ""},
|
||||
{"/files/prism-slides/prism-slides.pdf", 200, "%PDF"},
|
||||
{"/nosuchpage", 404, ""},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
w := get(t, c.path)
|
||||
if w.Code != c.code {
|
||||
t.Errorf("%s: status %d, want %d", c.path, w.Code, c.code)
|
||||
continue
|
||||
}
|
||||
if c.want != "" && !strings.Contains(w.Body.String(), c.want) {
|
||||
t.Errorf("%s: body does not contain %q", c.path, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The document file handler is the one place a URL reaches the filesystem.
|
||||
func TestFileHandlerContainment(t *testing.T) {
|
||||
for _, path := range []string{
|
||||
"/files/prism-slides/images/page-1.jpg",
|
||||
"/files/prism-slides/versions/lemonde/lemonde.pdf",
|
||||
"/files/prism-slides/versions/lemonde/images/page-1.jpg",
|
||||
} {
|
||||
if w := get(t, path); w.Code != 200 {
|
||||
t.Fatalf("%s: a file of the document should serve, got %d", path, w.Code)
|
||||
}
|
||||
}
|
||||
// A file the loader never indexed is not served, whatever its type: a
|
||||
// document directory gathers backups and second copies.
|
||||
_, root := buildFixture(t)
|
||||
dir := filepath.Join(root, "prism-slides")
|
||||
for _, stray := range []string{"zz-stray.pdf", "images/notes/scan.jpg", "old-versions/draft.pdf"} {
|
||||
writeFixtureFile(t, filepath.Join(dir, filepath.FromSlash(stray)), "%PDF-1.4\n")
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
for _, stray := range []string{"zz-stray.pdf", "images/notes", "old-versions"} {
|
||||
os.RemoveAll(filepath.Join(dir, stray))
|
||||
}
|
||||
})
|
||||
for _, path := range []string{
|
||||
"/files/prism-slides/zz-stray.pdf", // a second PDF
|
||||
"/files/prism-slides/images/notes/scan.jpg", // not a page image
|
||||
"/files/prism-slides/old-versions/draft.pdf",
|
||||
"/files/prism-slides/versions/nosuch/x.pdf",
|
||||
"/files/prism-slides/thumb.jpg", // the fixture has no thumbnail
|
||||
"/files/prism-slides/../../../etc/passwd",
|
||||
"/files/prism-slides/..%2f..%2fetc%2fpasswd",
|
||||
"/files/prism-slides/versions/lemonde/metadata.json", // a version has none
|
||||
"/files/prism-slides/glossary.json",
|
||||
"/files/prism-slides/", // no file
|
||||
"/files/prism-slides/nope.pdf", // inside the doc, missing
|
||||
"/files/nosuchdoc/x.pdf", // unknown id
|
||||
"/files/prism-slides/versions/lemonde/", // directory
|
||||
} {
|
||||
if w := get(t, path); w.Code == 200 {
|
||||
t.Errorf("%s: served 200, want a refusal", path)
|
||||
}
|
||||
}
|
||||
// The curated metadata is a served file like the PDF it describes.
|
||||
if m := get(t, "/files/prism-slides/metadata.json"); m.Code != 200 || !strings.HasPrefix(m.Header().Get("Content-Type"), "application/json") {
|
||||
t.Errorf("metadata.json: status %d, Content-Type %q, want 200 application/json", m.Code, m.Header().Get("Content-Type"))
|
||||
}
|
||||
// A .md transcript is served as plain text, never as HTML.
|
||||
w := get(t, "/files/prism-slides/prism-slides.md")
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("transcript status %d", w.Code)
|
||||
}
|
||||
if ct := w.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") {
|
||||
t.Errorf("transcript Content-Type %q, want text/plain", ct)
|
||||
}
|
||||
}
|
||||
|
||||
// Cacheability is decided by one rule - a request carrying any cookie is
|
||||
// private - and personal pages go further with no-store. The hosting story
|
||||
// (a shared microcache in front) rests on this.
|
||||
func TestCacheHeaders(t *testing.T) {
|
||||
anon := get(t, "/browse")
|
||||
if cc := anon.Header().Get("Cache-Control"); cc != "public, max-age=10" {
|
||||
t.Errorf("cookieless page: Cache-Control %q", cc)
|
||||
}
|
||||
if v := anon.Header().Get("Vary"); v != "Cookie" {
|
||||
t.Errorf("cookieless page: Vary %q, want Cookie", v)
|
||||
}
|
||||
|
||||
cookied := get(t, "/browse", &http.Cookie{Name: "pref_theme", Value: "dark"})
|
||||
if cc := cookied.Header().Get("Cache-Control"); cc != "private, no-cache" {
|
||||
t.Errorf("cookied page: Cache-Control %q", cc)
|
||||
}
|
||||
|
||||
for _, p := range []string{"/preferences", "/pinned", "/pinned/export"} {
|
||||
if cc := get(t, p).Header().Get("Cache-Control"); !strings.Contains(cc, "no-store") {
|
||||
t.Errorf("%s: Cache-Control %q, want no-store", p, cc)
|
||||
}
|
||||
}
|
||||
|
||||
if cc := get(t, "/files/prism-slides/prism-slides.pdf").Header().Get("Cache-Control"); cc != "public, max-age=86400" {
|
||||
t.Errorf("document file: Cache-Control %q", cc)
|
||||
}
|
||||
// A 404 must never inherit the immutable static cache.
|
||||
if cc := get(t, "/static/nosuch.css?v=abc123").Header().Get("Cache-Control"); strings.Contains(cc, "immutable") {
|
||||
t.Errorf("missing static asset marked immutable: %q", cc)
|
||||
}
|
||||
}
|
||||
|
||||
// Every response carries the baseline security headers, whatever it is.
|
||||
func TestSecurityHeaders(t *testing.T) {
|
||||
for _, p := range []string{"/", "/doc/prism-slides", "/api/facets", "/nosuchpage", "/files/prism-slides/prism-slides.pdf"} {
|
||||
h := get(t, p).Header()
|
||||
for _, want := range []string{"Content-Security-Policy", "X-Frame-Options", "X-Content-Type-Options", "Referrer-Policy"} {
|
||||
if h.Get(want) == "" {
|
||||
t.Errorf("%s: missing %s", p, want)
|
||||
}
|
||||
}
|
||||
if csp := h.Get("Content-Security-Policy"); !strings.Contains(csp, "default-src 'self'") {
|
||||
t.Errorf("%s: CSP %q", p, csp)
|
||||
}
|
||||
}
|
||||
// The method gate's 405 too: scanner traffic is what it answers.
|
||||
mux, _ := buildFixture(t)
|
||||
w := record(mux, httptest.NewRequest("DELETE", "/", nil))
|
||||
if w.Code != http.StatusMethodNotAllowed || w.Header().Get("Content-Security-Policy") == "" {
|
||||
t.Errorf("DELETE: status %d, CSP %q; want 405 with the headers", w.Code, w.Header().Get("Content-Security-Policy"))
|
||||
}
|
||||
}
|
||||
|
||||
// State-changing POSTs are cookie-backed with no login, so a cross-site
|
||||
// submission must be refused rather than silently flipping preferences.
|
||||
func TestCrossSitePOSTRejected(t *testing.T) {
|
||||
mux, _ := buildFixture(t)
|
||||
for _, p := range []string{"/preferences/save", "/pin/prism-slides", "/history/clear", "/pinsearch"} {
|
||||
// same-site is a sibling subdomain: another origin, not this site.
|
||||
for _, site := range []string{"cross-site", "same-site"} {
|
||||
r := newPost(p)
|
||||
r.Header.Set("Sec-Fetch-Site", site)
|
||||
if w := record(mux, r); w.Code != http.StatusForbidden {
|
||||
t.Errorf("%s from %s: status %d, want 403", p, site, w.Code)
|
||||
}
|
||||
}
|
||||
r := newPost(p)
|
||||
r.Header.Set("Sec-Fetch-Site", "same-origin")
|
||||
if w := record(mux, r); w.Code == http.StatusForbidden {
|
||||
t.Errorf("%s from this site: rejected", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Loading is idempotent: every build* starts from empty, so a second load
|
||||
// (a test with another corpus, a future reload) produces the same archive
|
||||
// rather than a doubled one. Several builders used to append to package
|
||||
// slices that nothing reset.
|
||||
func TestLoadDataIsIdempotent(t *testing.T) {
|
||||
buildFixture(t)
|
||||
type snapshot struct {
|
||||
docs, timeline, codewords, groups, titled, glossary, sitemap, articles int
|
||||
}
|
||||
take := func() snapshot {
|
||||
years := 0
|
||||
for _, y := range corpus().archiveTimeline {
|
||||
years += len(y.Months)
|
||||
}
|
||||
return snapshot{
|
||||
docs: len(corpus().archive.Docs), timeline: years, codewords: len(corpus().codewords),
|
||||
groups: len(corpus().articleGroups), titled: corpus().articlesTitled,
|
||||
glossary: len(corpus().glossaryIndex) + len(corpus().glossaryGroups), sitemap: len(corpus().sitemapPaths), articles: len(corpus().articles),
|
||||
}
|
||||
}
|
||||
before := take()
|
||||
loadData()
|
||||
if after := take(); after != before {
|
||||
t.Errorf("a second load changed the archive:\n before %+v\n after %+v", before, after)
|
||||
}
|
||||
}
|
||||
|
||||
// robots.txt keeps crawlers off the routes that cost a corpus scan, and the
|
||||
// sitemap never advertises a path robots.txt disallows.
|
||||
func TestRobotsAndSitemapAgree(t *testing.T) {
|
||||
robots := get(t, "/robots.txt").Body.String()
|
||||
for _, want := range []string{"Disallow: /search", "Disallow: /api/", "Sitemap: "} {
|
||||
if !strings.Contains(robots, want) {
|
||||
t.Errorf("robots.txt lacks %q:\n%s", want, robots)
|
||||
}
|
||||
}
|
||||
var disallowed []string
|
||||
for _, line := range strings.Split(robots, "\n") {
|
||||
if v := strings.TrimPrefix(line, "Disallow:"); v != line {
|
||||
disallowed = append(disallowed, strings.TrimSpace(v))
|
||||
}
|
||||
}
|
||||
sitemap := get(t, "/sitemap.xml").Body.String()
|
||||
for _, m := range regexp.MustCompile(`<loc>([^<]+)</loc>`).FindAllStringSubmatch(sitemap, -1) {
|
||||
path := m[1]
|
||||
if i := strings.Index(path[len("https://"):], "/"); i >= 0 {
|
||||
path = path[len("https://")+i:]
|
||||
}
|
||||
for _, d := range disallowed {
|
||||
if d != "" && strings.HasPrefix(path, d) {
|
||||
t.Errorf("sitemap advertises %s, which robots.txt disallows (%s)", path, d)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// With a canonical base configured, /sitemap.xml is marshaled once at load
|
||||
// rather than per request - and still says the same thing.
|
||||
func TestCanonicalSitemapPrebuilt(t *testing.T) {
|
||||
buildFixture(t)
|
||||
prev := canonicalBase
|
||||
t.Cleanup(func() {
|
||||
canonicalBase = prev
|
||||
corpus().canonicalSitemap = nil
|
||||
corpus().buildCanonicalSitemap()
|
||||
})
|
||||
canonicalBase = "https://archive.example.org"
|
||||
corpus().buildCanonicalSitemap()
|
||||
if corpus().canonicalSitemap == nil {
|
||||
t.Fatal("no sitemap was built for the canonical base")
|
||||
}
|
||||
body := get(t, "/sitemap.xml").Body.String()
|
||||
if !strings.Contains(body, "<loc>https://archive.example.org/doc/prism-slides</loc>") {
|
||||
t.Errorf("sitemap does not use the canonical base:\n%s", body[:200])
|
||||
}
|
||||
if body != string(corpus().canonicalSitemap) {
|
||||
t.Error("the served sitemap is not the one built at load")
|
||||
}
|
||||
}
|
||||
|
||||
// Without a canonical base the sitemap is written from parts cut at load,
|
||||
// with the request's base between them: the document marshalSitemap gives
|
||||
// for that base, to the byte - a base that needs escaping included.
|
||||
func TestSitemapFromParts(t *testing.T) {
|
||||
h, _ := buildFixture(t)
|
||||
if canonicalBase != "" || corpus().sitemapParts == nil {
|
||||
t.Fatalf("the fixture should serve its sitemap from parts (base %q, %d parts)", canonicalBase, len(corpus().sitemapParts))
|
||||
}
|
||||
for _, host := range []string{"archive.example.org", "odd&host<x>.example"} {
|
||||
r := httptest.NewRequest("GET", "/sitemap.xml", nil)
|
||||
r.Host = host
|
||||
want, err := corpus().marshalSitemap(requestBase(r))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := record(h, r).Body.String(); got != string(want) {
|
||||
t.Errorf("host %q: the sitemap written from parts differs from marshalSitemap", host)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The big visitor-independent page bodies are executed once at startup
|
||||
// (prerenderPages). The page must look exactly the same whether it used
|
||||
// that copy or rendered its rows inline - which is what dev mode does, and
|
||||
// what every one of these pages falls back to if a block ever fails to
|
||||
// pre-render.
|
||||
func TestPrerenderedMatchesLive(t *testing.T) {
|
||||
buildFixture(t)
|
||||
if len(corpus().prerendered) == 0 {
|
||||
t.Fatal("nothing was pre-rendered")
|
||||
}
|
||||
pages := []string{
|
||||
"/explore/ids",
|
||||
"/explore/ids?missing=1",
|
||||
"/glossary",
|
||||
"/browse?collection=sidtoday-archive",
|
||||
"/browse/program",
|
||||
"/browse/tag",
|
||||
// Category orderings are ASSEMBLED from pre-rendered rows rather
|
||||
// than pre-rendered whole, so each one has to match too.
|
||||
"/browse/tag?sort=count",
|
||||
"/browse/program?sort=count&order=asc",
|
||||
"/browse/agency?sort=name&order=desc",
|
||||
"/browse/people?sort=foia",
|
||||
// So are the doc tables, out of pre-rendered document rows.
|
||||
"/browse?view=table",
|
||||
"/browse?collection=sidtoday-archive&view=table&sort=title&order=desc",
|
||||
"/browse/program/PRISM?view=table",
|
||||
}
|
||||
was, wasRows, wasAZ, wasDocRows := corpus().prerendered, corpus().categoryRowHTML, corpus().azRowHTML, corpus().docRowHTML
|
||||
t.Cleanup(func() {
|
||||
corpus().prerendered, corpus().categoryRowHTML, corpus().azRowHTML, corpus().docRowHTML = was, wasRows, wasAZ, wasDocRows
|
||||
})
|
||||
|
||||
live := make([]string, len(pages))
|
||||
corpus().prerendered = map[string]template.HTML{} // force the inline path
|
||||
corpus().categoryRowHTML, corpus().azRowHTML, corpus().docRowHTML = nil, nil, nil
|
||||
for i, p := range pages {
|
||||
live[i] = get(t, p).Body.String()
|
||||
}
|
||||
corpus().prerendered, corpus().categoryRowHTML, corpus().azRowHTML, corpus().docRowHTML = was, wasRows, wasAZ, wasDocRows
|
||||
for i, p := range pages {
|
||||
if got := get(t, p).Body.String(); got != live[i] {
|
||||
t.Errorf("%s: pre-rendered output differs from the live one", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The table view's headings are its sorters, and the one marked active is
|
||||
// the order the rows are actually in - the page's default when the URL
|
||||
// names none (the SIDtoday tab sorts by date, not title).
|
||||
func TestDocTableActiveHeading(t *testing.T) {
|
||||
for path, want := range map[string]string{
|
||||
"/browse?collection=sidtoday-archive&view=table": "Doc Date ▲",
|
||||
"/browse?view=table": "Title ▲",
|
||||
"/browse?view=table&sort=country&order=desc": "Countries ▼",
|
||||
"/browse?view=table&sort=bogus": "Title ▲",
|
||||
} {
|
||||
body := get(t, path).Body.String()
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("%s: no active heading %q", path, want)
|
||||
}
|
||||
if n := strings.Count(body, "▲") + strings.Count(body, "▼"); n != 1 {
|
||||
t.Errorf("%s: %d active headings, want 1", path, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A reporter credited by two co-publishing outlets is one name in the
|
||||
// table's Journalists column, as on the cards.
|
||||
func TestDocTableJournalistsDeduped(t *testing.T) {
|
||||
buildFixture(t)
|
||||
d := testDoc(t, `{"id": "x", "publication": [
|
||||
{"publisher": "A", "reporters": ["Greenwald, Glenn"]},
|
||||
{"publisher": "B", "reporters": ["Greenwald, Glenn", "Poitras, Laura"]}]}`)
|
||||
var b strings.Builder
|
||||
err := tmplSet["list.html"].ExecuteTemplate(&b, "doc-table", listView{Docs: []*Document{&d}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := strings.Count(b.String(), "/browse/journalist/Greenwald"); n != 1 {
|
||||
t.Errorf("Greenwald linked %d times, want once", n)
|
||||
}
|
||||
if !strings.Contains(b.String(), "/browse/journalist/Poitras") {
|
||||
t.Error("Poitras missing")
|
||||
}
|
||||
}
|
||||
|
||||
// Without fetch metadata the Origin decides, and it omits a default port
|
||||
// the Host header may carry.
|
||||
func TestOriginIsHost(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
origin, host string
|
||||
want bool
|
||||
}{
|
||||
{"https://example.org", "example.org", true},
|
||||
{"https://example.org", "example.org:443", true},
|
||||
{"https://EXAMPLE.org", "example.org", true},
|
||||
{"http://127.0.0.1:8080", "127.0.0.1:8080", true},
|
||||
{"http://[::1]:8080", "[::1]:8080", true},
|
||||
{"https://example.org", "example.org:8443", false},
|
||||
{"https://example.org:8443", "example.org", false},
|
||||
{"http://example.org", "example.org:443", false},
|
||||
{"https://evil.example", "example.org", false},
|
||||
} {
|
||||
u, err := url.Parse(c.origin)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := originIsHost(u, c.host); got != c.want {
|
||||
t.Errorf("originIsHost(%s, %s) = %v, want %v", c.origin, c.host, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnvBool(t *testing.T) {
|
||||
for v, want := range map[string]bool{"": false, "1": true, "true": true, "TRUE": true, "0": false, "false": false} {
|
||||
t.Setenv("SNOWCHIVE_TEST_BOOL", v)
|
||||
if got := envBool("SNOWCHIVE_TEST_BOOL"); got != want {
|
||||
t.Errorf("envBool(%q) = %v, want %v", v, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestBrowseValueSpelling: a value matches ignoring case, so another
|
||||
// spelling of it redirects to the one the lists link instead of serving a
|
||||
// second page for the same documents; the query string goes along.
|
||||
func TestBrowseValueSpelling(t *testing.T) {
|
||||
w := get(t, "/browse/program/prism?view=table")
|
||||
if w.Code != http.StatusMovedPermanently || w.Header().Get("Location") != "/browse/program/PRISM?view=table" {
|
||||
t.Errorf("/browse/program/prism: %d to %q, want 301 to /browse/program/PRISM?view=table", w.Code, w.Header().Get("Location"))
|
||||
}
|
||||
if w := get(t, "/browse/program/NOSUCHTHING"); w.Code != 200 {
|
||||
t.Errorf("an unknown value: %d, want the empty page", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCardHighlightNeedsTheCategory: the cards on a value's page mark the
|
||||
// links to that value - not every link that happens to read the same, such
|
||||
// as the document date 2013 on the publication-year page for 2013.
|
||||
func TestCardHighlightNeedsTheCategory(t *testing.T) {
|
||||
if body := get(t, "/browse/published/2013").Body.String(); strings.Contains(body, "doc-created active") {
|
||||
t.Error("/browse/published/2013 marks the document-date links")
|
||||
}
|
||||
if body := get(t, "/browse/docdate/2013").Body.String(); !strings.Contains(body, "doc-created active") {
|
||||
t.Error("/browse/docdate/2013 does not mark the document-date links")
|
||||
}
|
||||
if body := get(t, "/browse/program/PRISM").Body.String(); !strings.Contains(body, `class="active">PRISM</a>`) {
|
||||
t.Error("/browse/program/PRISM does not mark the program links")
|
||||
}
|
||||
}
|
||||
|
||||
// TestUnknownSortIsNotEchoed: a sort key no list has is the default sort,
|
||||
// and stays out of the page's links and labels.
|
||||
func TestUnknownSortIsNotEchoed(t *testing.T) {
|
||||
for _, path := range []string{"/browse?sort=bogus", "/browse/program/PRISM?sort=bogus", "/search?q=prism&do_search=1&sort=bogus"} {
|
||||
body := get(t, path).Body.String()
|
||||
if strings.Contains(body, "sort=bogus") || strings.Contains(body, "Bogus") {
|
||||
t.Errorf("%s: the unknown sort is echoed", path)
|
||||
}
|
||||
}
|
||||
if body := get(t, "/search?q=prism&do_search=1").Body.String(); strings.Contains(body, "sort=relevance&order=desc") {
|
||||
t.Error("relevance offers a direction it does not have")
|
||||
}
|
||||
}
|
||||
|
||||
// TestOutletYearLinksKeepTheScope: on the reported-only outlets table the
|
||||
// year headings open the reported-only year, like the rows and cells do.
|
||||
func TestOutletYearLinksKeepTheScope(t *testing.T) {
|
||||
body := get(t, "/explore/outlets?coll=news").Body.String()
|
||||
if !strings.Contains(body, `href="/browse/published/2013?coll=news"`) {
|
||||
t.Error("the 2013 heading drops coll=news")
|
||||
}
|
||||
}
|
||||
|
||||
// pinSearch posts qs to /pinsearch from this site, with the given pinned
|
||||
// searches already in the cookie.
|
||||
func pinSearch(t *testing.T, qs string, pinned []string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
mux, _ := buildFixture(t)
|
||||
r := httptest.NewRequest("POST", "/pinsearch", strings.NewReader(url.Values{"qs": {qs}, "back": {"/search"}}.Encode()))
|
||||
r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
r.Header.Set("Sec-Fetch-Site", "same-origin")
|
||||
if len(pinned) > 0 {
|
||||
r.AddCookie(&http.Cookie{Name: pinnedSearchesCookie, Value: strings.Join(pinned, "|")})
|
||||
}
|
||||
return record(mux, r)
|
||||
}
|
||||
|
||||
// TestPinListFull: a pin the cookie has no room for is refused, and the
|
||||
// reader lands on /pinned, which says so, rather than back on a page whose
|
||||
// button still reads "Pin" as though nothing happened.
|
||||
func TestPinListFull(t *testing.T) {
|
||||
qs := url.Values{"do_search": {"1"}, "fqc": {"and"}, "fqf": {"TE"}, "fqv": {"prism"}}.Encode()
|
||||
if w := pinSearch(t, qs, nil); w.Code != http.StatusSeeOther || w.Header().Get("Location") != "/search" {
|
||||
t.Fatalf("pinning into an empty list: %d to %q", w.Code, w.Header().Get("Location"))
|
||||
}
|
||||
// Filler searches up to the byte budget: stored entries are not
|
||||
// checked on read, so any escaped query string stands in for one.
|
||||
var fill []string
|
||||
for i := 0; ; i++ {
|
||||
e := url.QueryEscape(fmt.Sprintf("fqv=filler%02d", i) + strings.Repeat("x", 180))
|
||||
if len(strings.Join(append(fill, e), "|")) > cookieValueMax {
|
||||
break
|
||||
}
|
||||
fill = append(fill, e)
|
||||
}
|
||||
// The last filler takes the cookie to exactly its budget.
|
||||
fill[len(fill)-1] += strings.Repeat("x", cookieValueMax-len(strings.Join(fill, "|")))
|
||||
w := pinSearch(t, qs, fill)
|
||||
if w.Code != http.StatusSeeOther || w.Header().Get("Location") != "/pinned?full=1" {
|
||||
t.Errorf("pinning into a full list: %d to %q, want 303 to /pinned?full=1", w.Code, w.Header().Get("Location"))
|
||||
}
|
||||
if body := get(t, "/pinned?full=1").Body.String(); !strings.Contains(body, "Your pinned list is full") {
|
||||
t.Error("/pinned?full=1 does not say the list is full")
|
||||
}
|
||||
}
|
||||
|
||||
// TestSavableSearchMatchesPinSearch: the search page offers the pin button
|
||||
// for exactly the searches /pinsearch accepts - one size limit for both.
|
||||
func TestSavableSearchMatchesPinSearch(t *testing.T) {
|
||||
long := strings.TrimSpace(strings.Repeat("abcdefgh ", 58)) // 58 words, under the word cap
|
||||
q := url.Values{"do_search": {"1"}, "fqc": {"and"}, "fqf": {"TE"}, "fqv": {long}}
|
||||
if len(q.Encode()) <= searchQSMax {
|
||||
t.Fatalf("test query is %d bytes, not over the %d limit", len(q.Encode()), searchQSMax)
|
||||
}
|
||||
if body := get(t, "/search?"+q.Encode()).Body.String(); strings.Contains(body, `action="/pinsearch"`) {
|
||||
t.Error("the search page offers to pin a search /pinsearch refuses")
|
||||
}
|
||||
if w := pinSearch(t, q.Encode(), nil); w.Code != http.StatusBadRequest {
|
||||
t.Errorf("/pinsearch with an over-long search: %d, want 400", w.Code)
|
||||
}
|
||||
short := url.Values{"do_search": {"1"}, "fqc": {"and"}, "fqf": {"TE"}, "fqv": {"prism"}}
|
||||
if body := get(t, "/search?"+short.Encode()).Body.String(); !strings.Contains(body, `action="/pinsearch"`) {
|
||||
t.Error("the search page does not offer to pin a short search")
|
||||
}
|
||||
}
|
||||
|
||||
// TestAssetStamp: pages link each asset by its content hash; that URL is
|
||||
// cached for a year, and any other stamp only revalidates.
|
||||
func TestAssetStamp(t *testing.T) {
|
||||
body := get(t, "/").Body.String()
|
||||
want := "/static/style.css?v=" + staticHashes["style.css"]
|
||||
if !strings.Contains(body, `href="`+want+`"`) {
|
||||
t.Fatalf("home page does not link %s", want)
|
||||
}
|
||||
if cc := get(t, want).Header().Get("Cache-Control"); !strings.Contains(cc, "immutable") {
|
||||
t.Errorf("%s: Cache-Control %q, want immutable", want, cc)
|
||||
}
|
||||
w := get(t, "/static/style.css?v=0000000000000000")
|
||||
if cc := w.Header().Get("Cache-Control"); strings.Contains(cc, "immutable") || w.Header().Get("ETag") == "" {
|
||||
t.Errorf("a stale stamp: Cache-Control %q, ETag %q; want revalidation", cc, w.Header().Get("ETag"))
|
||||
}
|
||||
}
|
||||
|
||||
// TestSitemapIgnoresFile: the sitemap is always the one built from the
|
||||
// loaded corpus - a sitemap.xml lying in the data or project directory is
|
||||
// never served, with -base or without it.
|
||||
func TestSitemapIgnoresFile(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeFixtureFile(t, filepath.Join(root, "sitemap.xml"), xmlHeaderForTest+"<urlset><url><loc>https://stale.example/</loc></url></urlset>\n")
|
||||
withCorpus(t, root)
|
||||
prev := canonicalBase
|
||||
t.Cleanup(func() { canonicalBase = prev })
|
||||
for _, base := range []string{"", "https://archive.example.org"} {
|
||||
canonicalBase = base
|
||||
loadData()
|
||||
if body := get(t, "/sitemap.xml").Body.String(); strings.Contains(body, "stale.example") {
|
||||
t.Errorf("base %q: a sitemap.xml file was served", base)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const xmlHeaderForTest = `<?xml version="1.0" encoding="UTF-8"?>` + "\n"
|
||||
|
||||
// TestPinnedSearchListed: /pinned lists a pinned search with its label,
|
||||
// its link and the unpin form carrying the canonical query string.
|
||||
func TestPinnedSearchListed(t *testing.T) {
|
||||
qs := url.Values{"do_search": {"1"}, "fqc": {"and"}, "fqf": {"TE"}, "fqv": {"prism"}}.Encode()
|
||||
body := get(t, "/pinned", &http.Cookie{Name: pinnedSearchesCookie, Value: url.QueryEscape(qs)}).Body.String()
|
||||
for _, want := range []string{`href="/search?` + template.HTMLEscapeString(qs) + `"`, `name="qs" value="` + template.HTMLEscapeString(qs) + `"`, ">prism<"} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("/pinned lacks %s", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestPageTitles: every page names itself in the tab, not just the site.
|
||||
func TestPageTitles(t *testing.T) {
|
||||
buildFixture(t) // the article key comes from the loaded corpus
|
||||
for path, want := range map[string]string{
|
||||
"/browse/program/PRISM": "<title>Program: PRISM - Snowden Archive</title>",
|
||||
"/browse/program": "<title>Browse by Programs - Snowden Archive</title>",
|
||||
"/search?q=prism&do_search=1": "<title>prism - Search - Snowden Archive</title>",
|
||||
"/search": "<title>Search - Snowden Archive</title>",
|
||||
"/browse?collection=foia": "<title>FOIA documents - Snowden Archive</title>",
|
||||
"/pinned": "<title>Pinned - Snowden Archive</title>",
|
||||
"/nosuchpage": "<title>Page not found - Snowden Archive</title>",
|
||||
"/explore/pages": "- Snowden Archive</title>",
|
||||
"/article/" + corpus().articleByLink["https://example.org/prism"].Key: "<title>NSA Prism program - Snowden Archive</title>",
|
||||
} {
|
||||
if body := get(t, path).Body.String(); !strings.Contains(body, want) {
|
||||
t.Errorf("%s: no %q", path, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestNoBlockContentInHeadings: a heading holds text and inline controls
|
||||
// only - a form or a disclosure inside one is invalid HTML and lands in
|
||||
// the heading's accessible name. Forms tie their buttons from outside.
|
||||
func TestNoBlockContentInHeadings(t *testing.T) {
|
||||
heading := regexp.MustCompile(`(?s)<h([1-6])\b[^>]*>(.*?)</h[1-6]>`)
|
||||
block := regexp.MustCompile(`<(form|details|div|p|ul|ol|table)\b`)
|
||||
cookie := &http.Cookie{Name: pinnedCookie, Value: "prism-slides"}
|
||||
recent := []*http.Cookie{{Name: "pref_track_recent", Value: "1"}, {Name: recentDocsCookie, Value: "prism-slides"}}
|
||||
pages := map[string][]*http.Cookie{
|
||||
"/pinned": {cookie},
|
||||
"/": recent,
|
||||
"/search?q=prism&do_search=1": nil,
|
||||
"/doc/prism-slides": nil,
|
||||
}
|
||||
for path, cookies := range pages {
|
||||
body := get(t, path, cookies...).Body.String()
|
||||
for _, m := range heading.FindAllStringSubmatch(body, -1) {
|
||||
if b := block.FindString(m[2]); b != "" {
|
||||
t.Errorf("%s: %s inside <h%s>", path, b, m[1])
|
||||
}
|
||||
}
|
||||
}
|
||||
if body := get(t, "/pinned", cookie).Body.String(); !strings.Contains(body, `form="unpin-prism-slides"`) || !strings.Contains(body, `id="unpin-prism-slides"`) {
|
||||
t.Error("/pinned: the unpin button is not tied to its form")
|
||||
}
|
||||
}
|
||||
|
||||
// TestGlossarySeeAlsoResolves: a see-also name links to the entry it
|
||||
// names - through an alias or another case too - and a name no entry has
|
||||
// is plain text rather than a link to nowhere.
|
||||
func TestGlossarySeeAlsoResolves(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeFixtureFile(t, filepath.Join(root, "glossary.json"), `[
|
||||
{"term": "XKEYSCORE", "definition": "A search system.", "aka": ["XKS"], "see": ["rt-rg", "TURMOIL", "NOSUCHTERM"]},
|
||||
{"term": "RT-RG", "definition": "Real time regional gateway."},
|
||||
{"term": "TURMOIL", "definition": "Passive collection.", "see": ["xks"]}
|
||||
]`)
|
||||
withCorpus(t, root)
|
||||
body := get(t, "/glossary").Body.String()
|
||||
for _, want := range []string{`<a href="#term-RT-RG">rt-rg</a>`, `<a href="#term-TURMOIL">TURMOIL</a>`, `<a href="#term-XKEYSCORE">xks</a>`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("/glossary lacks %s", want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(body, `href="#term-NOSUCHTERM"`) {
|
||||
t.Error("/glossary links a see-also name no entry has")
|
||||
}
|
||||
}
|
||||
|
||||
// A pager or sort link appends its parameter to queryBase, with nothing
|
||||
// left over when the page has no query of its own.
|
||||
func TestQueryBase(t *testing.T) {
|
||||
for want, q := range map[string]url.Values{
|
||||
"/browse?": {},
|
||||
"/browse?view=table&": {"view": {"table"}},
|
||||
"/browse?coll=foia&view=card&": {"view": {"card"}, "coll": {"foia"}},
|
||||
} {
|
||||
if got := queryBase("/browse", q); got != want {
|
||||
t.Errorf("queryBase(%v) = %q, want %q", q, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
22
tests/run.sh
Executable file
22
tests/run.sh
Executable file
|
|
@ -0,0 +1,22 @@
|
|||
#!/bin/sh
|
||||
# Runs the tests in this directory. They are package main's own tests, and Go
|
||||
# only compiles those from the package's directory, so an overlay presents
|
||||
# them to the go tool as if they were there. Arguments go to "go test":
|
||||
#
|
||||
# tests/run.sh
|
||||
# tests/run.sh -run TestCorpus -v
|
||||
set -e
|
||||
cd "$(dirname "$0")/.."
|
||||
root=$(pwd)
|
||||
overlay=$(mktemp)
|
||||
trap 'rm -f "$overlay"' EXIT
|
||||
{
|
||||
printf '{"Replace":{'
|
||||
sep=
|
||||
for f in tests/*_test.go; do
|
||||
printf '%s"%s/%s":"%s/%s"' "$sep" "$root" "${f#tests/}" "$root" "$f"
|
||||
sep=,
|
||||
done
|
||||
printf '}}\n'
|
||||
} >"$overlay"
|
||||
go test -overlay "$overlay" "$@" .
|
||||
1000
tests/search_test.go
Normal file
1000
tests/search_test.go
Normal file
File diff suppressed because it is too large
Load diff
58
tests/sidtoday_scope_test.go
Normal file
58
tests/sidtoday_scope_test.go
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The home figures describe the Reported collection; /?sidtoday=1 adds the
|
||||
// SIDtoday archive, and the note above them links between the two. In the
|
||||
// consistency corpus only s1 (SIDtoday archive) is dated 2011-05.
|
||||
func TestHomeStatsWithSIDtoday(t *testing.T) {
|
||||
withCorpus(t, writeConsistencyCorpus(t))
|
||||
plain := get(t, "/").Body.String()
|
||||
if !strings.Contains(plain, "exclude SIDtoday archive documents") || !strings.Contains(plain, `href="/?sidtoday=1"`) {
|
||||
t.Error("/: no note linking to the figures with the SIDtoday archive")
|
||||
}
|
||||
if strings.Contains(plain, "coll=sidtoday-archive") {
|
||||
t.Error("/: a figure links into the SIDtoday archive")
|
||||
}
|
||||
with := get(t, "/?sidtoday=1").Body.String()
|
||||
if !strings.Contains(with, "include SIDtoday archive documents") {
|
||||
t.Error("/?sidtoday=1: the note does not say the figures include the SIDtoday archive")
|
||||
}
|
||||
if !strings.Contains(with, `href="/browse/docdate/2011?coll=news&coll=sidtoday-archive"`) {
|
||||
t.Error("/?sidtoday=1: the document-date bars do not link with both collections")
|
||||
}
|
||||
}
|
||||
|
||||
// The "reported" badge tells an individually reported SIDtoday issue apart
|
||||
// from the bulk release, so only the SIDtoday tab shows it: every view of
|
||||
// that tab, and no other page.
|
||||
func TestReportedBadgeOnlyOnSIDtodayTab(t *testing.T) {
|
||||
root := writeConsistencyCorpus(t)
|
||||
d := filepath.Join(root, "r1")
|
||||
writeFixtureFile(t, filepath.Join(d, "r1.pdf"), "%PDF-1.4\n")
|
||||
writeFixtureFile(t, filepath.Join(d, "metadata.json"), `{"title": "Reported issue", "type": "SIDTODAY",
|
||||
"document": {"date_created": "2011-06-01"},
|
||||
"publication": [{"publisher": "The Intercept", "date_published": "2014-03-01", "link": "https://example.org/r"}]}`)
|
||||
withCorpus(t, root)
|
||||
|
||||
const badge = `class="issue-reported" title=`
|
||||
for _, p := range []string{"/browse?collection=sidtoday-archive", "/browse?collection=sidtoday-archive&view=card",
|
||||
"/browse?collection=sidtoday-archive&view=table"} {
|
||||
if !strings.Contains(get(t, p).Body.String(), badge) {
|
||||
t.Errorf("%s: no reported badge", p)
|
||||
}
|
||||
}
|
||||
for _, p := range []string{"/browse", "/browse?view=table", "/search?q=reported&do_search=1", "/browse/publisher/The%20Intercept"} {
|
||||
body := get(t, p).Body.String()
|
||||
if !strings.Contains(body, `href="/doc/r1"`) {
|
||||
t.Errorf("%s does not list r1", p)
|
||||
}
|
||||
if strings.Contains(body, badge) {
|
||||
t.Errorf("%s shows the reported badge", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
464
tests/tmplcheck_test.go
Normal file
464
tests/tmplcheck_test.go
Normal file
|
|
@ -0,0 +1,464 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"text/template/parse"
|
||||
)
|
||||
|
||||
// Every template is checked against the struct it is executed with, without
|
||||
// executing it: each field or method it reads must exist on the type it is
|
||||
// read from - in every branch, including the ones no test request takes.
|
||||
// html/template only finds a wrong name when the action runs.
|
||||
//
|
||||
// The checker follows the data through range, with, variables and template
|
||||
// calls. Where a type is not known statically it does not guess: a value
|
||||
// behind an interface{} (a dict's missing key, the argument of a partial
|
||||
// called with a literal) is accepted whatever is read from it.
|
||||
func TestTemplatesMatchPages(t *testing.T) {
|
||||
buildFixture(t)
|
||||
for _, p := range pageTypes {
|
||||
name := p.templateName()
|
||||
set, err := parsePage(name) // a fresh parse: executing rewrites the trees
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
c := &tmplCheck{seen: map[string]bool{}, records: map[reflect.Type]bool{}}
|
||||
c.lookup = func(n string) *parse.Tree {
|
||||
if tt := set.Lookup(n); tt != nil {
|
||||
return tt.Tree
|
||||
}
|
||||
return nil
|
||||
}
|
||||
c.call(name, reflect.TypeOf(p), 0)
|
||||
sort.Strings(c.errs)
|
||||
for _, e := range c.errs {
|
||||
t.Errorf("%s (%T): %s", name, p, e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Each template file is some page's, and each page's template exists.
|
||||
func TestEveryTemplateHasAPage(t *testing.T) {
|
||||
buildFixture(t)
|
||||
files, err := fs.Glob(templateFS, "*.html")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
byName := map[string]bool{}
|
||||
for _, p := range pageTypes {
|
||||
if byName[p.templateName()] {
|
||||
t.Errorf("%s is the template of two page types", p.templateName())
|
||||
}
|
||||
byName[p.templateName()] = true
|
||||
}
|
||||
for _, f := range files {
|
||||
if f == "base.html" || f == "partials.html" {
|
||||
continue
|
||||
}
|
||||
if !byName[f] {
|
||||
t.Errorf("%s has no page type in pageTypes", f)
|
||||
}
|
||||
delete(byName, f)
|
||||
}
|
||||
for n := range byName {
|
||||
t.Errorf("page type for %s, which is not a template file", n)
|
||||
}
|
||||
}
|
||||
|
||||
// The checker itself: a misspelled field is found in a branch that would
|
||||
// not run, behind a range, a variable and a partial; a dict's keys carry
|
||||
// their types into the partial they are passed to.
|
||||
func TestTemplateCheckerFindsWrongNames(t *testing.T) {
|
||||
type row struct{ Name string }
|
||||
type page struct {
|
||||
Rows []row
|
||||
Doc *Document
|
||||
M map[string]int
|
||||
}
|
||||
for src, want := range map[string]string{
|
||||
`{{ if .Nope }}x{{ end }}`: "Nope",
|
||||
`{{ if false }}{{ range .Rows }}{{ .Nmae }}{{ end }}{{ end }}`: "Nmae",
|
||||
`{{ range $i, $r := .Rows }}{{ $r.Name }}{{ $.Rowz }}{{ end }}`: "Rowz",
|
||||
`{{ with .Doc }}{{ .Title }}{{ .DocMeta.Creatr }}{{ end }}`: "Creatr",
|
||||
`{{ $d := .Doc }}{{ $d.Titel }}`: "Titel",
|
||||
`{{ template "p" .Doc }}{{ define "p" }}{{ .ID }}{{ .Idd }}{{ end }}`: "Idd",
|
||||
`{{ template "p" (dict "D" .Doc) }}{{ define "p" }}{{ .D.Idd }}{{ end }}`: "Idd",
|
||||
`{{ index .M "a" }}{{ (index .Rows 0).Nmae }}`: "Nmae",
|
||||
`{{ .Rows.Name }}`: "Name",
|
||||
`{{ range .Rows }}{{ .Name }}{{ else }}{{ .Name }}{{ end }}`: "Name",
|
||||
`{{ template "p" (dict "D" .Doc) }}{{ define "p" }}{{ .Missing.X }}{{ end }}`: "",
|
||||
`{{ range .Rows }}{{ .Name }}{{ end }}{{ .Doc.Title }}{{ len .M }}`: "",
|
||||
`{{ $x := "" }}{{ if .Doc }}{{ $x = .Doc }}{{ end }}{{ $x.Anything }}`: "",
|
||||
} {
|
||||
trees, err := parse.Parse("t", src, "", "", checkerFuncs())
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", src, err)
|
||||
}
|
||||
c := &tmplCheck{seen: map[string]bool{}, records: map[reflect.Type]bool{}}
|
||||
c.lookup = func(n string) *parse.Tree { return trees[n] }
|
||||
c.call("t", reflect.TypeOf(page{}), 0)
|
||||
got := strings.Join(c.errs, "; ")
|
||||
if (want == "") != (got == "") || !strings.Contains(got, want) {
|
||||
t.Errorf("%s\n errors: %q, want one naming %q", src, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// checkerFuncs is funcMap as the parser wants it, with the builtins the
|
||||
// templates use.
|
||||
func checkerFuncs() map[string]any {
|
||||
m := map[string]any{}
|
||||
for k, v := range funcMap {
|
||||
m[k] = v
|
||||
}
|
||||
for _, b := range []string{"and", "or", "not", "eq", "ne", "lt", "le", "gt", "ge", "len", "index",
|
||||
"slice", "print", "printf", "println", "html", "js", "urlquery", "call"} {
|
||||
m[b] = true
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
type tmplCheck struct {
|
||||
lookup func(name string) *parse.Tree
|
||||
seen map[string]bool // template name + argument type, already walked
|
||||
records map[reflect.Type]bool // types made from a dict call: a missing key is nil, not an error
|
||||
errs []string
|
||||
where string // the template being walked
|
||||
}
|
||||
|
||||
type tmplVar struct {
|
||||
name string
|
||||
typ reflect.Type // nil: not known
|
||||
}
|
||||
|
||||
var (
|
||||
stringType = reflect.TypeOf("")
|
||||
intType = reflect.TypeOf(0)
|
||||
boolType = reflect.TypeOf(false)
|
||||
floatType = reflect.TypeOf(0.0)
|
||||
anyType = reflect.TypeOf((*any)(nil)).Elem()
|
||||
)
|
||||
|
||||
func (c *tmplCheck) errorf(n parse.Node, format string, a ...any) {
|
||||
loc := c.where
|
||||
if tree := c.lookup(c.where); tree != nil {
|
||||
loc, _ = tree.ErrorContext(n)
|
||||
}
|
||||
c.errs = append(c.errs, fmt.Sprintf("%s (in %q): %s", loc, c.where, fmt.Sprintf(format, a...)))
|
||||
}
|
||||
|
||||
// call walks the named template with dot of type arg.
|
||||
func (c *tmplCheck) call(name string, arg reflect.Type, depth int) {
|
||||
key := name + "|"
|
||||
if arg != nil {
|
||||
key += arg.String()
|
||||
if c.records[arg] {
|
||||
key += fmt.Sprintf("|%p", arg) // records print alike only when they are alike; keep them apart anyway
|
||||
}
|
||||
}
|
||||
if c.seen[key] || depth > 20 {
|
||||
return
|
||||
}
|
||||
c.seen[key] = true
|
||||
tree := c.lookup(name)
|
||||
if tree == nil || tree.Root == nil {
|
||||
c.errs = append(c.errs, fmt.Sprintf("template %q is not defined", name))
|
||||
return
|
||||
}
|
||||
prev := c.where
|
||||
c.where = name
|
||||
vars := []tmplVar{{"$", arg}}
|
||||
c.walk(tree.Root, arg, &vars, depth)
|
||||
c.where = prev
|
||||
}
|
||||
|
||||
func (c *tmplCheck) walk(n parse.Node, dot reflect.Type, vars *[]tmplVar, depth int) {
|
||||
switch n := n.(type) {
|
||||
case *parse.ListNode:
|
||||
if n == nil {
|
||||
return
|
||||
}
|
||||
for _, x := range n.Nodes {
|
||||
c.walk(x, dot, vars, depth)
|
||||
}
|
||||
case *parse.ActionNode:
|
||||
c.pipe(n.Pipe, dot, vars)
|
||||
case *parse.IfNode:
|
||||
mark := len(*vars)
|
||||
c.pipe(n.Pipe, dot, vars)
|
||||
c.walk(n.List, dot, vars, depth)
|
||||
c.walk(n.ElseList, dot, vars, depth)
|
||||
*vars = (*vars)[:mark]
|
||||
case *parse.WithNode:
|
||||
mark := len(*vars)
|
||||
t := c.pipe(n.Pipe, dot, vars)
|
||||
c.walk(n.List, t, vars, depth)
|
||||
c.walk(n.ElseList, dot, vars, depth)
|
||||
*vars = (*vars)[:mark]
|
||||
case *parse.RangeNode:
|
||||
mark := len(*vars)
|
||||
t := c.cmds(n.Pipe, dot, vars)
|
||||
key, elem := rangeTypes(t)
|
||||
switch len(n.Pipe.Decl) {
|
||||
case 1:
|
||||
*vars = append(*vars, tmplVar{n.Pipe.Decl[0].Ident[0], elem})
|
||||
case 2:
|
||||
*vars = append(*vars, tmplVar{n.Pipe.Decl[0].Ident[0], key}, tmplVar{n.Pipe.Decl[1].Ident[0], elem})
|
||||
}
|
||||
c.walk(n.List, elem, vars, depth)
|
||||
*vars = (*vars)[:mark]
|
||||
c.walk(n.ElseList, dot, vars, depth)
|
||||
case *parse.TemplateNode:
|
||||
var arg reflect.Type
|
||||
if n.Pipe != nil {
|
||||
arg = c.pipe(n.Pipe, dot, vars)
|
||||
}
|
||||
c.call(n.Name, arg, depth+1)
|
||||
}
|
||||
}
|
||||
|
||||
func rangeTypes(t reflect.Type) (key, elem reflect.Type) {
|
||||
t = indirectType(t)
|
||||
if t == nil {
|
||||
return nil, nil
|
||||
}
|
||||
switch t.Kind() {
|
||||
case reflect.Slice, reflect.Array:
|
||||
return intType, known(t.Elem())
|
||||
case reflect.Map:
|
||||
return known(t.Key()), known(t.Elem())
|
||||
case reflect.Int:
|
||||
return intType, intType
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// known is t, or nil when t says nothing about the value (an interface).
|
||||
func known(t reflect.Type) reflect.Type {
|
||||
if t != nil && t.Kind() == reflect.Interface {
|
||||
return nil
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
func indirectType(t reflect.Type) reflect.Type {
|
||||
for t != nil && t.Kind() == reflect.Pointer {
|
||||
t = t.Elem()
|
||||
}
|
||||
return known(t)
|
||||
}
|
||||
|
||||
// pipe is the type a pipeline yields, declaring or assigning its variable.
|
||||
func (c *tmplCheck) pipe(p *parse.PipeNode, dot reflect.Type, vars *[]tmplVar) reflect.Type {
|
||||
if p == nil {
|
||||
return nil
|
||||
}
|
||||
t := c.cmds(p, dot, vars)
|
||||
for _, d := range p.Decl {
|
||||
name := d.Ident[0]
|
||||
if !p.IsAssign {
|
||||
*vars = append(*vars, tmplVar{name, t})
|
||||
continue
|
||||
}
|
||||
for i := len(*vars) - 1; i >= 0; i-- {
|
||||
if (*vars)[i].name == name {
|
||||
if (*vars)[i].typ != t {
|
||||
(*vars)[i].typ = nil // assigned two types: whichever branch ran decides
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
func (c *tmplCheck) cmds(p *parse.PipeNode, dot reflect.Type, vars *[]tmplVar) reflect.Type {
|
||||
var t reflect.Type
|
||||
for i, cmd := range p.Cmds {
|
||||
t = c.cmd(cmd, dot, vars, i > 0, t)
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
func (c *tmplCheck) cmd(cmd *parse.CommandNode, dot reflect.Type, vars *[]tmplVar, piped bool, prev reflect.Type) reflect.Type {
|
||||
first := cmd.Args[0]
|
||||
if id, ok := first.(*parse.IdentifierNode); ok {
|
||||
if id.Ident == "dict" {
|
||||
return c.dict(cmd, dot, vars)
|
||||
}
|
||||
args := make([]reflect.Type, 0, len(cmd.Args))
|
||||
for _, a := range cmd.Args[1:] {
|
||||
args = append(args, c.arg(a, dot, vars))
|
||||
}
|
||||
if piped {
|
||||
args = append(args, prev)
|
||||
}
|
||||
return c.fn(id, args)
|
||||
}
|
||||
for _, a := range cmd.Args[1:] { // a method's arguments
|
||||
c.arg(a, dot, vars)
|
||||
}
|
||||
return c.arg(first, dot, vars)
|
||||
}
|
||||
|
||||
func (c *tmplCheck) arg(n parse.Node, dot reflect.Type, vars *[]tmplVar) reflect.Type {
|
||||
switch n := n.(type) {
|
||||
case *parse.DotNode:
|
||||
return dot
|
||||
case *parse.FieldNode:
|
||||
return c.fields(n, dot, n.Ident)
|
||||
case *parse.VariableNode:
|
||||
for i := len(*vars) - 1; i >= 0; i-- {
|
||||
if (*vars)[i].name == n.Ident[0] {
|
||||
return c.fields(n, (*vars)[i].typ, n.Ident[1:])
|
||||
}
|
||||
}
|
||||
c.errorf(n, "undefined variable %s", n.Ident[0])
|
||||
case *parse.ChainNode:
|
||||
return c.fields(n, c.arg(n.Node, dot, vars), n.Field)
|
||||
case *parse.PipeNode:
|
||||
return c.pipe(n, dot, vars)
|
||||
case *parse.IdentifierNode:
|
||||
return c.fn(n, nil)
|
||||
case *parse.StringNode:
|
||||
return stringType
|
||||
case *parse.BoolNode:
|
||||
return boolType
|
||||
case *parse.NumberNode:
|
||||
if n.IsInt {
|
||||
return intType
|
||||
}
|
||||
return floatType
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// fields follows a chain of names from t.
|
||||
func (c *tmplCheck) fields(n parse.Node, t reflect.Type, names []string) reflect.Type {
|
||||
for _, name := range names {
|
||||
if t == nil {
|
||||
return nil
|
||||
}
|
||||
// A method, on the value or on its address.
|
||||
recv := t
|
||||
if recv.Kind() != reflect.Pointer && recv.Kind() != reflect.Interface {
|
||||
recv = reflect.PointerTo(recv)
|
||||
}
|
||||
if m, ok := recv.MethodByName(name); ok {
|
||||
t = nil
|
||||
if m.Type.NumOut() > 0 {
|
||||
t = known(m.Type.Out(0))
|
||||
}
|
||||
continue
|
||||
}
|
||||
s := indirectType(t)
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
switch s.Kind() {
|
||||
case reflect.Struct:
|
||||
f, ok := s.FieldByName(name)
|
||||
if !ok || !f.IsExported() {
|
||||
if c.records[s] {
|
||||
return nil // a key this dict call did not pass
|
||||
}
|
||||
c.errorf(n, "no field or method %s in %s", name, s)
|
||||
return nil
|
||||
}
|
||||
t = known(f.Type)
|
||||
case reflect.Map:
|
||||
if s.Key().Kind() != reflect.String {
|
||||
c.errorf(n, "%s read from %s, a map without string keys", name, s)
|
||||
return nil
|
||||
}
|
||||
t = known(s.Elem())
|
||||
default:
|
||||
c.errorf(n, "%s read from %s, which has no fields", name, s)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// dict is the record a dict call builds: each literal key with the type of
|
||||
// the value passed for it.
|
||||
func (c *tmplCheck) dict(cmd *parse.CommandNode, dot reflect.Type, vars *[]tmplVar) reflect.Type {
|
||||
var fs []reflect.StructField
|
||||
args := cmd.Args[1:]
|
||||
for i := 0; i+1 < len(args); i += 2 {
|
||||
k, ok := args[i].(*parse.StringNode)
|
||||
t := c.arg(args[i+1], dot, vars)
|
||||
if !ok {
|
||||
return nil // a computed key: nothing is known about the record
|
||||
}
|
||||
if t == nil {
|
||||
t = anyType
|
||||
}
|
||||
fs = append(fs, reflect.StructField{Name: k.Text, Type: t})
|
||||
}
|
||||
defer func() { recover() }() // a key that is not a Go identifier: no record
|
||||
rt := reflect.StructOf(fs)
|
||||
c.records[rt] = true
|
||||
return rt
|
||||
}
|
||||
|
||||
// fn is the type a function call yields.
|
||||
func (c *tmplCheck) fn(id *parse.IdentifierNode, args []reflect.Type) reflect.Type {
|
||||
switch id.Ident {
|
||||
case "not", "eq", "ne", "lt", "le", "gt", "ge":
|
||||
return boolType
|
||||
case "len":
|
||||
return intType
|
||||
case "print", "printf", "println", "html", "js", "urlquery":
|
||||
return stringType
|
||||
case "and", "or":
|
||||
for _, a := range args {
|
||||
if a == nil || a != args[0] {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if len(args) > 0 {
|
||||
return args[0]
|
||||
}
|
||||
return nil
|
||||
case "slice":
|
||||
if len(args) > 0 {
|
||||
return args[0]
|
||||
}
|
||||
return nil
|
||||
case "index":
|
||||
if len(args) == 0 {
|
||||
return nil
|
||||
}
|
||||
t := args[0]
|
||||
for range args[1:] {
|
||||
t = indirectType(t)
|
||||
if t == nil {
|
||||
return nil
|
||||
}
|
||||
switch t.Kind() {
|
||||
case reflect.Map, reflect.Slice, reflect.Array:
|
||||
t = known(t.Elem())
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return t
|
||||
case "call":
|
||||
return nil
|
||||
}
|
||||
f, ok := funcMap[id.Ident]
|
||||
if !ok {
|
||||
return nil // the parser has already refused an unknown function
|
||||
}
|
||||
ft := reflect.TypeOf(f)
|
||||
if ft.NumOut() == 0 {
|
||||
return nil
|
||||
}
|
||||
return known(ft.Out(0))
|
||||
}
|
||||
59
tests/versions_pages_test.go
Normal file
59
tests/versions_pages_test.go
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestChangedPages(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
v := filepath.Join(dir, "versions", "entier")
|
||||
writeTestFile(t, filepath.Join(v, "copy.pdf"), "%PDF-1.4")
|
||||
for _, n := range []string{"1", "2", "7", "8"} {
|
||||
writeTestFile(t, filepath.Join(v, "images", "image-"+n+".jpg"), "")
|
||||
}
|
||||
|
||||
var d Document
|
||||
if err := json.Unmarshal([]byte(`{
|
||||
"publication": [{"publisher": "Le Monde", "version": "entier"}],
|
||||
"versions": {"entier": {"changed_pages": [8, 7, 12, 7]}, "ghost": {"changed_pages": [1]}}
|
||||
}`), &d); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.ID, d.Dir = "prism", dir
|
||||
corpus().loadVersions(&d)
|
||||
|
||||
e := d.VersionFor("entier")
|
||||
if e == nil {
|
||||
t.Fatal("version entier not loaded")
|
||||
}
|
||||
// Numbers come from the filenames; positions are where those images sit
|
||||
// in the page grid (image-7 is the 3rd image). 12 has no image: kept,
|
||||
// without a position.
|
||||
if got := fmt.Sprint(e.ChangedPages); got != "[{7 3} {8 4} {12 0}]" {
|
||||
t.Errorf("changed pages %s, want [{7 3} {8 4} {12 0}]", got)
|
||||
}
|
||||
for pos, want := range map[int]bool{0: false, 1: false, 2: false, 3: true, 4: true} {
|
||||
if e.PageChanged(pos) != want {
|
||||
t.Errorf("PageChanged(%d) = %v, want %v", pos, !want, want)
|
||||
}
|
||||
}
|
||||
if e.DocID != "prism" {
|
||||
t.Errorf("DocID %q, want prism", e.DocID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPageNumber(t *testing.T) {
|
||||
for name, want := range map[string]int{"page-12.jpg": 12, "image-07.png": 7} {
|
||||
if n, err := pageNumber(name); err != nil || n != want {
|
||||
t.Errorf("pageNumber(%q) = %d, %v; want %d", name, n, err, want)
|
||||
}
|
||||
}
|
||||
for _, name := range []string{"cover.jpg", "page-a.jpg"} {
|
||||
if _, err := pageNumber(name); err == nil {
|
||||
t.Errorf("pageNumber(%q) should fail", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
101
tests/versions_search_test.go
Normal file
101
tests/versions_search_test.go
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func versionedDoc(t *testing.T, main string, versions map[string]string, pubsJSON string) Document {
|
||||
t.Helper()
|
||||
var d Document
|
||||
if pubsJSON != "" {
|
||||
if err := json.Unmarshal([]byte(pubsJSON), &d); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
d.ID, d.FullText = "doc", main
|
||||
for _, slug := range []string{"lemonde", "spiegel"} {
|
||||
if text, ok := versions[slug]; ok {
|
||||
d.Versions = append(d.Versions, DocVersion{Slug: slug, FullText: text})
|
||||
}
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func TestVersionNovelText(t *testing.T) {
|
||||
d := versionedDoc(t, "PRISM Overview\nShared line\n", map[string]string{
|
||||
"lemonde": "shared LINE\r\nNew in Le Monde\n---\nIn both copies\nNew in Le Monde\n",
|
||||
"spiegel": "In both copies\nOnly Der Spiegel\n",
|
||||
}, "")
|
||||
combined := versionNovelText(&d)
|
||||
|
||||
if got := d.Versions[0].NovelText; got != "New in Le Monde\nIn both copies\nNew in Le Monde" {
|
||||
t.Errorf("lemonde new lines %q", got)
|
||||
}
|
||||
if got := d.Versions[1].NovelText; got != "In both copies\nOnly Der Spiegel" {
|
||||
t.Errorf("spiegel new lines %q", got)
|
||||
}
|
||||
if combined != "New in Le Monde\nIn both copies\nOnly Der Spiegel" {
|
||||
t.Errorf("combined new lines %q, want each line once", combined)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVersionLabel(t *testing.T) {
|
||||
d := versionedDoc(t, "", map[string]string{"lemonde": "", "spiegel": ""}, `{"publication": [
|
||||
{"publisher": "Le Monde", "date_published": "2013-10-21"},
|
||||
{"publisher": "Le Monde", "date_published": "2013-10-25"},
|
||||
{"publisher": "The Guardian", "date_published": "2013-10-20"}
|
||||
]}`)
|
||||
d.Versions[0].PubIndex = []int{0, 1, 2}
|
||||
if got := versionLabel(&d, &d.Versions[0]); got != "Le Monde / The Guardian, 2013-10-20" {
|
||||
t.Errorf("label %q", got)
|
||||
}
|
||||
if got := versionLabel(&d, &d.Versions[1]); got != "spiegel" {
|
||||
t.Errorf("label of a version no publication names %q, want its folder name", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVersionTextIsSearchable(t *testing.T) {
|
||||
d := versionedDoc(t, "The overview ends with alpha", map[string]string{
|
||||
"lemonde": "The overview ends with alpha\nbeta starts the Wanadoo slide",
|
||||
}, "")
|
||||
setBodySearchText(&d)
|
||||
d.StemmedText = d.StemmedFull
|
||||
d.SearchText = d.FoldedFull
|
||||
|
||||
match := func(q, field string) bool {
|
||||
return compileQuery([]string{q}, []string{field}, []string{"and"}).matches(&d)
|
||||
}
|
||||
if !match("wanadoo", "FT") || !match(`"wanadoo slide"`, "FT") || !match("=wanadoo", "FT") || !match("wanad*", "FT") {
|
||||
t.Error("a term only in a version's new lines should match the document")
|
||||
}
|
||||
if match(`"alpha beta"`, "FT") {
|
||||
t.Error("a quoted phrase matched across the join between the main text and a version's lines")
|
||||
}
|
||||
if tok := tokenize("overview")[0]; strings.Count(" "+d.StemmedFull+" ", " "+tok+" ") != 1 {
|
||||
t.Errorf("a line shared with the main text was indexed twice: %q", d.StemmedFull)
|
||||
}
|
||||
|
||||
// buildTFIDF skips the separator only as a token of its own; glued to a
|
||||
// word it would mint terms and hide that word from phrase matching.
|
||||
for _, tok := range strings.Fields(d.StemmedFull) {
|
||||
if tok != valueSep && strings.Contains(tok, valueSep) {
|
||||
t.Errorf("the join separator is glued to a term in StemmedFull: %q", tok)
|
||||
}
|
||||
}
|
||||
idx := buildTFIDF([]Document{d})
|
||||
for w := range idx.DocFreq {
|
||||
if strings.Contains(w, valueSep) {
|
||||
t.Errorf("the join separator was indexed in the TF-IDF term %q", w)
|
||||
}
|
||||
}
|
||||
|
||||
hits := versionHits(&d, "wanadoo", 3)
|
||||
if len(hits) != 1 || hits[0].Version.Slug != "lemonde" || !strings.Contains(hits[0].Snippets[0], "Wanadoo") {
|
||||
t.Errorf("versionHits = %+v, want one lemonde hit with a Wanadoo snippet", hits)
|
||||
}
|
||||
if hits := versionHits(&d, "overview", 3); len(hits) != 0 {
|
||||
t.Errorf("versionHits found %+v for a word only in lines shared with the main text", hits)
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue