166 lines
6.6 KiB
Go
166 lines
6.6 KiB
Go
package main
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
)
|
|
|
|
// What the loader does with a corpus that is wrong in the ways corpora are
|
|
// wrong in practice. Every case must leave the rest of the archive intact -
|
|
// a broken document is skipped, never a reason to serve nothing - and say
|
|
// so in the tally that -check reads.
|
|
func TestLoaderRejectsBadDocuments(t *testing.T) {
|
|
root := t.TempDir()
|
|
write := func(path, content string) {
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
good := func(id string) {
|
|
write(filepath.Join(root, id, "metadata.json"), `{"title": "`+id+`", "type": "REPORTED"}`)
|
|
write(filepath.Join(root, id, id+".pdf"), "%PDF-1.4")
|
|
}
|
|
good("keeper-one")
|
|
good("keeper-two")
|
|
|
|
// No PDF: this is an archive OF the PDFs, so the document is skipped.
|
|
write(filepath.Join(root, "no-pdf", "metadata.json"), `{"title": "No PDF"}`)
|
|
// Invalid JSON: skipped, everything else still loads.
|
|
write(filepath.Join(root, "bad-json", "metadata.json"), `{"title": "oops"`)
|
|
write(filepath.Join(root, "bad-json", "x.pdf"), "%PDF-1.4")
|
|
// aka_id that is not a list of objects: refused, not silently half-read.
|
|
write(filepath.Join(root, "bad-aka", "metadata.json"), `{"title": "Aka", "aka_id": "other-id"}`)
|
|
write(filepath.Join(root, "bad-aka", "x.pdf"), "%PDF-1.4")
|
|
// A directory that is not a document at all: skipped without a warning.
|
|
write(filepath.Join(root, "notes", "README.txt"), "just notes")
|
|
// A directory holding a PDF and nothing else: the shape worth warning
|
|
// about, since it looks like a document nobody described.
|
|
write(filepath.Join(root, "loose", "orphan.pdf"), "%PDF-1.4")
|
|
|
|
withCorpus(t, root)
|
|
clean := loadWarnings
|
|
|
|
// A document in the SIDtoday archive that is not typed SIDTODAY is on
|
|
// no browse tab: served, and warned about.
|
|
write(filepath.Join(root, "SIDtoday-archive", "mistyped", "metadata.json"), `{"title": "Mistyped", "type": "REPORTED"}`)
|
|
write(filepath.Join(root, "SIDtoday-archive", "mistyped", "x.pdf"), "%PDF-1.4")
|
|
loadData()
|
|
if corpus().docIndex["mistyped"] == nil || loadWarnings != clean+1 {
|
|
t.Errorf("a REPORTED document in SIDtoday-archive: loaded %v, %d warnings, want it loaded with %d", corpus().docIndex["mistyped"] != nil, loadWarnings, clean+1)
|
|
}
|
|
if err := os.RemoveAll(filepath.Join(root, "SIDtoday-archive")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
loadData()
|
|
|
|
if len(corpus().archive.Docs) != 2 {
|
|
ids := make([]string, 0, len(corpus().archive.Docs))
|
|
for _, d := range corpus().archive.Docs {
|
|
ids = append(ids, d.ID)
|
|
}
|
|
t.Fatalf("loaded %v, want the two good documents", ids)
|
|
}
|
|
if loadErrors < 3 { // no-pdf, bad-aka and bad JSON are errors: not served
|
|
t.Errorf("loadErrors = %d, want at least 3", loadErrors)
|
|
}
|
|
if loadWarnings < 1 { // the loose PDF directory: served nothing, lost nothing
|
|
t.Errorf("loadWarnings = %d, want at least 1", loadWarnings)
|
|
}
|
|
for _, id := range []string{"no-pdf", "bad-json", "bad-aka", "notes", "loose"} {
|
|
if corpus().docIndex[id] != nil {
|
|
t.Errorf("%s should not have loaded", id)
|
|
}
|
|
}
|
|
// The archive is coherent: facets, search and the sitemap all see the
|
|
// same two documents.
|
|
if got := corpus().archive.CollectionCounts[collectionNews]; got != 2 {
|
|
t.Errorf("news collection count %d, want 2", got)
|
|
}
|
|
if got, _ := corpus().executeSearch(searchParams{Queries: []string{""}, Fields: []string{"TE"}, Conjs: []string{"and"}}); len(got) != 2 {
|
|
t.Errorf("match-all search found %d, want 2", len(got))
|
|
}
|
|
}
|
|
|
|
// An empty corpus is a corpus: the server loads, every page renders, and
|
|
// nothing divides by zero.
|
|
func TestLoaderEmptyCorpus(t *testing.T) {
|
|
withCorpus(t, t.TempDir())
|
|
if len(corpus().archive.Docs) != 0 {
|
|
t.Fatalf("loaded %d documents from an empty directory", len(corpus().archive.Docs))
|
|
}
|
|
for _, p := range []string{"/", "/browse", "/search?q=x&do_search=1", "/explore", "/explore/ids",
|
|
"/articles", "/api/docs", "/api/facets", "/sitemap.xml"} {
|
|
if w := get(t, p); w.Code != 200 {
|
|
t.Errorf("%s on an empty corpus: %d", p, w.Code)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Values are tidied before anything reads them, and a copy the loader skips
|
|
// leaves no trace - not even a claim on its ID.
|
|
func TestLoaderTidiesBeforeUse(t *testing.T) {
|
|
root := t.TempDir()
|
|
write := func(path, content string) {
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
// FOIA/ sorts before the flat root, so the PDF-less copy is seen first.
|
|
write(filepath.Join(root, "FOIA", "dup-doc", "metadata.json"), `{"title": "Shadow"}`)
|
|
write(filepath.Join(root, "dup-doc", "metadata.json"), `{"title": "Real", "type": "REPORTED"}`)
|
|
write(filepath.Join(root, "dup-doc", "x.pdf"), "%PDF-1.4")
|
|
|
|
write(filepath.Join(root, "padded", "metadata.json"), `{
|
|
"title": "Padded", "type": "REPORTED",
|
|
"document": {"creator": " NSA ", "series": {"name": "Series X ", "part": 1}},
|
|
"publication": [{"publisher": "Le Monde", "date_published": "June 2014", "version": "entier"}]
|
|
}`)
|
|
write(filepath.Join(root, "padded", "x.pdf"), "%PDF-1.4")
|
|
write(filepath.Join(root, "padded", "versions", "entier", "entier.pdf"), "%PDF-1.4")
|
|
|
|
withCorpus(t, root)
|
|
|
|
if d := corpus().docIndex["dup-doc"]; d == nil || d.Title != "Real" {
|
|
t.Errorf("dup-doc = %+v, want the copy with a PDF", d)
|
|
}
|
|
d := corpus().docIndex["padded"]
|
|
if d == nil {
|
|
t.Fatal("padded not loaded")
|
|
}
|
|
if d.DocMeta.Creator != "NSA" || d.DocMeta.Series.Name != "Series X" {
|
|
t.Errorf("creator %q, series %q: want both trimmed", d.DocMeta.Creator, d.DocMeta.Series.Name)
|
|
}
|
|
// The malformed date is dropped everywhere, the version label included.
|
|
if v := d.VersionFor("entier"); v == nil || v.Label != "Le Monde" {
|
|
t.Errorf("version label %+v, want \"Le Monde\" without the invalid date", v)
|
|
}
|
|
}
|
|
|
|
// Two page images with one page number show that page twice; the loader
|
|
// says so instead of linking changed pages to whichever sorted first.
|
|
func TestLoadPageImagesWarnsOnDuplicatePageNumbers(t *testing.T) {
|
|
dir := t.TempDir()
|
|
if err := os.MkdirAll(filepath.Join(dir, "images"), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, name := range []string{"page-1.jpg", "page-2.jpg", "page-02.png"} {
|
|
if err := os.WriteFile(filepath.Join(dir, "images", name), nil, 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
before := loadWarnings
|
|
imgs := loadPageImages(dir, "doc")
|
|
if len(imgs) != 3 || imgs[0] != "page-1.jpg" {
|
|
t.Errorf("images %v, want all three in page order", imgs)
|
|
}
|
|
if loadWarnings != before+1 {
|
|
t.Errorf("%d warnings for one duplicated page number, want 1", loadWarnings-before)
|
|
}
|
|
}
|