295 lines
9.6 KiB
Go
295 lines
9.6 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"image"
|
|
"image/jpeg"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
)
|
|
|
|
// A small corpus on disk plus a running mux: the harness every handler test
|
|
// uses. The fixture covers the shapes the handlers branch on - one document
|
|
// per collection, a SIDtoday issue with a series, another published copy in
|
|
// versions/, a joint investigation with a reporting-only partner, and a
|
|
// directory with no PDF (which the loader must skip).
|
|
//
|
|
// Built once for the package: loadData writes package globals, so tests
|
|
// share one corpus rather than racing to replace it.
|
|
|
|
var (
|
|
fixtureOnce sync.Once
|
|
fixtureMux http.Handler
|
|
fixtureDir string
|
|
)
|
|
|
|
type fixtureDoc struct {
|
|
id, collection string
|
|
meta map[string]interface{}
|
|
text string
|
|
pages []string
|
|
versions map[string][]string // slug -> page images
|
|
noPDF bool
|
|
}
|
|
|
|
// fixtureJPEG is a real 4x3 JPEG, so the loader can read each page image's size.
|
|
var fixtureJPEG = func() string {
|
|
var b bytes.Buffer
|
|
if err := jpeg.Encode(&b, image.NewGray(image.Rect(0, 0, 4, 3)), nil); err != nil {
|
|
panic(err)
|
|
}
|
|
return b.String()
|
|
}()
|
|
|
|
func writeFixtureFile(t *testing.T, path, content string) {
|
|
t.Helper()
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// TestMain removes the shared fixture corpus once every test has run.
|
|
func TestMain(m *testing.M) {
|
|
code := m.Run()
|
|
if fixtureDir != "" {
|
|
os.RemoveAll(fixtureDir)
|
|
}
|
|
os.Exit(code)
|
|
}
|
|
|
|
func buildFixture(t *testing.T) (http.Handler, string) {
|
|
t.Helper()
|
|
fixtureOnce.Do(func() {
|
|
root, err := os.MkdirTemp("", "snowchive-fixture")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, d := range fixtureDocs() {
|
|
dir := root
|
|
switch d.collection {
|
|
case collectionFOIA:
|
|
dir = filepath.Join(root, "FOIA")
|
|
case collectionArchive:
|
|
dir = filepath.Join(root, "SIDtoday-archive")
|
|
}
|
|
dir = filepath.Join(dir, d.id)
|
|
b, err := json.MarshalIndent(d.meta, "", " ")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
writeFixtureFile(t, filepath.Join(dir, "metadata.json"), string(b)+"\n")
|
|
if !d.noPDF {
|
|
writeFixtureFile(t, filepath.Join(dir, d.id+".pdf"), "%PDF-1.4\nfixture\n")
|
|
}
|
|
if d.text != "" {
|
|
writeFixtureFile(t, filepath.Join(dir, d.id+".md"), d.text)
|
|
}
|
|
for _, p := range d.pages {
|
|
writeFixtureFile(t, filepath.Join(dir, "images", p), fixtureJPEG)
|
|
}
|
|
for slug, imgs := range d.versions {
|
|
vdir := filepath.Join(dir, "versions", slug)
|
|
writeFixtureFile(t, filepath.Join(vdir, slug+".pdf"), "%PDF-1.4\n")
|
|
writeFixtureFile(t, filepath.Join(vdir, slug+".md"),
|
|
"# Copy\n\nA line only this copy has: unredacted FAIRVIEW detail.\n")
|
|
for _, p := range imgs {
|
|
writeFixtureFile(t, filepath.Join(vdir, "images", p), fixtureJPEG)
|
|
}
|
|
}
|
|
}
|
|
writeFixtureFile(t, filepath.Join(root, "glossary.json"), `[
|
|
{"term": "XKEYSCORE", "definition": "A search system.", "aka": ["XKS"]}
|
|
]`)
|
|
|
|
dataDir, projectDir = root, root
|
|
devMode = false
|
|
apiEnabled = true
|
|
initAssets()
|
|
if err := parseTemplates(); err != nil {
|
|
t.Fatalf("templates: %v", err)
|
|
}
|
|
loadData()
|
|
fixtureDir = root
|
|
fixtureMux = withBaseHeaders(newMux())
|
|
})
|
|
if fixtureMux == nil {
|
|
t.Fatal("fixture not built")
|
|
}
|
|
return fixtureMux, fixtureDir
|
|
}
|
|
|
|
func fixtureDocs() []fixtureDoc {
|
|
return []fixtureDoc{
|
|
{
|
|
id: "prism-slides",
|
|
collection: collectionNews,
|
|
text: "# PRISM\n\nCollection directly from the servers. XKEYSCORE ingests it.\n" +
|
|
"Upstream collection feeds the corpus, and analysts were collecting metadata\n" +
|
|
"from RT-RG dashboards. Müller-Maguhn reviewed the São Paulo cable tapping.\n",
|
|
pages: []string{"page-1.jpg", "page-2.jpg"},
|
|
versions: map[string][]string{"lemonde": {"page-1.jpg"}},
|
|
meta: map[string]interface{}{
|
|
"title": "PRISM Collection Slides",
|
|
"type": "REPORTED",
|
|
"description": "Slides describing PRISM collection.",
|
|
"tags": []string{"PRISM", "collection", "cable tapping"},
|
|
"pdf_sha256": strings.Repeat("a", 64),
|
|
"aka_id": []map[string]string{{"id": "prism-slides-guardian"}},
|
|
"document": map[string]interface{}{
|
|
"creator": "NSA", "date_created": "2013-04-01",
|
|
"programs": []string{"PRISM", "XKEYSCORE"},
|
|
"classification": []string{"TOP SECRET"}, "dissemination": []string{"NOFORN"},
|
|
"countries": []string{"United States", "Brazil"},
|
|
"people": []string{"Snowden, Edward", "Müller-Maguhn, Andy"},
|
|
"banner": "TOP SECRET//SI//NOFORN", "genre": "slides",
|
|
},
|
|
"publication": []map[string]interface{}{
|
|
{"publisher": "The Guardian", "title": "NSA Prism program",
|
|
"link": "https://example.org/prism", "date_published": "2013-06-06",
|
|
"reporters": []string{"Greenwald, Glenn"}},
|
|
{"publisher": "Le Monde", "link": "https://example.org/prism-fr",
|
|
"date_published": "2013-06-08", "version": "lemonde"},
|
|
{"publisher": "NRK", "link": "https://example.org/prism",
|
|
"date_published": "2013-06-06", "reporting_only": true},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
// Shares the Guardian link with prism-slides, which is what
|
|
// makes the "published in the same report" section exist.
|
|
id: "prism-companion", collection: collectionNews,
|
|
text: "# Companion\n\nA second document from the same story.\n",
|
|
pages: []string{"page-1.jpg"},
|
|
meta: map[string]interface{}{
|
|
"title": "PRISM Companion Slide", "type": "REPORTED",
|
|
"description": "A second slide from the same story.",
|
|
"tags": []string{"PRISM"},
|
|
"document": map[string]interface{}{
|
|
"creator": "NSA", "date_created": "2013-04-02",
|
|
"programs": []string{"PRISM"},
|
|
},
|
|
"publication": []map[string]interface{}{
|
|
{"publisher": "The Guardian", "title": "NSA Prism program",
|
|
"link": "https://example.org/prism", "date_published": "2013-06-06",
|
|
"reporters": []string{"Greenwald, Glenn"}},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
id: "2003-05-29_SIDToday_-_Fixture_Issue",
|
|
collection: collectionArchive,
|
|
text: "# SIDtoday\n\nThe analyst cockpit and other matters. Upstream tapping was\n" +
|
|
"discussed at length by the directorate, and the collection of that traffic\n" +
|
|
"began only the following year.\n",
|
|
pages: []string{"page-1.jpg"},
|
|
meta: map[string]interface{}{
|
|
"title": "A Day in the Life of a Fixture", "type": "SIDTODAY",
|
|
"description": "An issue of SIDtoday.",
|
|
"tags": []string{"analysis"},
|
|
"document": map[string]interface{}{
|
|
"creator": "NSA", "date_created": "2003-05-29",
|
|
"classification": []string{"UNCLASSIFIED"}, "distribution": []string{},
|
|
"series": map[string]interface{}{"name": "(U) A Day in the Life of...", "part": 1},
|
|
},
|
|
"publication": []map[string]interface{}{
|
|
{"publisher": "The Intercept", "link": "https://example.org/sidtoday/1",
|
|
"date_published": "2016-05-16"},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
id: "FISC_-_fixture_order",
|
|
collection: collectionFOIA,
|
|
text: "# Order\n\nThe Court finds the application satisfies the statute.\n",
|
|
meta: map[string]interface{}{
|
|
"title": "Fixture FISC Order", "type": "FOIA",
|
|
"description": "A court order released under FOIA.",
|
|
"document": map[string]interface{}{
|
|
"creator": "FISC", "date_created": "2013-10-11",
|
|
"classification": []string{"SECRET"},
|
|
},
|
|
"publication": []map[string]interface{}{{"publisher": "ODNI", "link": "https://example.org/foia/1"}},
|
|
},
|
|
},
|
|
{
|
|
id: "undated-fixture",
|
|
collection: collectionNews,
|
|
text: "# Undated\n\nNo document date at all.\n",
|
|
meta: map[string]interface{}{
|
|
"title": "Undated Fixture", "type": "REPORTED",
|
|
"description": "A document with no date.",
|
|
"document": map[string]interface{}{"creator": "GCHQ"},
|
|
"publication": []map[string]interface{}{{"publisher": "The Intercept", "link": "https://example.org/undated"}},
|
|
},
|
|
},
|
|
{
|
|
id: "broken-no-pdf",
|
|
collection: collectionNews,
|
|
noPDF: true,
|
|
meta: map[string]interface{}{"title": "Broken", "type": "REPORTED"},
|
|
},
|
|
}
|
|
}
|
|
|
|
// get runs one request through the real routing table and header wrapper.
|
|
func get(t *testing.T, path string, cookies ...*http.Cookie) *httptest.ResponseRecorder {
|
|
t.Helper()
|
|
mux, _ := buildFixture(t)
|
|
r := httptest.NewRequest("GET", path, nil)
|
|
for _, c := range cookies {
|
|
r.AddCookie(c)
|
|
}
|
|
w := httptest.NewRecorder()
|
|
mux.ServeHTTP(w, r)
|
|
return w
|
|
}
|
|
|
|
// newPost builds a form POST for the state-changing endpoints.
|
|
func newPost(path string) *http.Request {
|
|
r := httptest.NewRequest("POST", path, strings.NewReader("back=/browse"))
|
|
r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
|
return r
|
|
}
|
|
|
|
func record(h http.Handler, r *http.Request) *httptest.ResponseRecorder {
|
|
w := httptest.NewRecorder()
|
|
h.ServeHTTP(w, r)
|
|
return w
|
|
}
|
|
|
|
// withCorpus loads another corpus for one test and puts the fixture back
|
|
// afterwards. Only possible because loadData is idempotent - every build*
|
|
// starts from empty (TestLoadDataIsIdempotent).
|
|
func withCorpus(t *testing.T, root string) {
|
|
t.Helper()
|
|
withDirs(t, root, root)
|
|
}
|
|
|
|
// withDirs is withCorpus with the corpus and the project files apart, as
|
|
// -root lays them out.
|
|
func withDirs(t *testing.T, data, project string) {
|
|
t.Helper()
|
|
buildFixture(t) // make sure templates and the fixture path exist first
|
|
prev := dataDir
|
|
t.Cleanup(func() {
|
|
dataDir, projectDir = prev, prev
|
|
loadData()
|
|
})
|
|
dataDir, projectDir = data, project
|
|
loadData()
|
|
}
|
|
|
|
// httptest_NewGet is a plain GET request for the tests that drive a mux
|
|
// directly rather than through get().
|
|
func httptest_NewGet(path string) *http.Request {
|
|
return httptest.NewRequest("GET", path, nil)
|
|
}
|