snowchive-server/tests/loader_test.go
2026-11-19 04:40:59 +02:00

166 lines
6.6 KiB
Go

package main
import (
"os"
"path/filepath"
"testing"
)
// What the loader does with a corpus that is wrong in the ways corpora are
// wrong in practice. Every case must leave the rest of the archive intact -
// a broken document is skipped, never a reason to serve nothing - and say
// so in the tally that -check reads.
func TestLoaderRejectsBadDocuments(t *testing.T) {
root := t.TempDir()
write := func(path, content string) {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
t.Fatal(err)
}
}
good := func(id string) {
write(filepath.Join(root, id, "metadata.json"), `{"title": "`+id+`", "type": "REPORTED"}`)
write(filepath.Join(root, id, id+".pdf"), "%PDF-1.4")
}
good("keeper-one")
good("keeper-two")
// No PDF: this is an archive OF the PDFs, so the document is skipped.
write(filepath.Join(root, "no-pdf", "metadata.json"), `{"title": "No PDF"}`)
// Invalid JSON: skipped, everything else still loads.
write(filepath.Join(root, "bad-json", "metadata.json"), `{"title": "oops"`)
write(filepath.Join(root, "bad-json", "x.pdf"), "%PDF-1.4")
// aka_id that is not a list of objects: refused, not silently half-read.
write(filepath.Join(root, "bad-aka", "metadata.json"), `{"title": "Aka", "aka_id": "other-id"}`)
write(filepath.Join(root, "bad-aka", "x.pdf"), "%PDF-1.4")
// A directory that is not a document at all: skipped without a warning.
write(filepath.Join(root, "notes", "README.txt"), "just notes")
// A directory holding a PDF and nothing else: the shape worth warning
// about, since it looks like a document nobody described.
write(filepath.Join(root, "loose", "orphan.pdf"), "%PDF-1.4")
withCorpus(t, root)
clean := loadWarnings
// A document in the SIDtoday archive that is not typed SIDTODAY is on
// no browse tab: served, and warned about.
write(filepath.Join(root, "SIDtoday-archive", "mistyped", "metadata.json"), `{"title": "Mistyped", "type": "REPORTED"}`)
write(filepath.Join(root, "SIDtoday-archive", "mistyped", "x.pdf"), "%PDF-1.4")
loadData()
if corpus().docIndex["mistyped"] == nil || loadWarnings != clean+1 {
t.Errorf("a REPORTED document in SIDtoday-archive: loaded %v, %d warnings, want it loaded with %d", corpus().docIndex["mistyped"] != nil, loadWarnings, clean+1)
}
if err := os.RemoveAll(filepath.Join(root, "SIDtoday-archive")); err != nil {
t.Fatal(err)
}
loadData()
if len(corpus().archive.Docs) != 2 {
ids := make([]string, 0, len(corpus().archive.Docs))
for _, d := range corpus().archive.Docs {
ids = append(ids, d.ID)
}
t.Fatalf("loaded %v, want the two good documents", ids)
}
if loadErrors < 3 { // no-pdf, bad-aka and bad JSON are errors: not served
t.Errorf("loadErrors = %d, want at least 3", loadErrors)
}
if loadWarnings < 1 { // the loose PDF directory: served nothing, lost nothing
t.Errorf("loadWarnings = %d, want at least 1", loadWarnings)
}
for _, id := range []string{"no-pdf", "bad-json", "bad-aka", "notes", "loose"} {
if corpus().docIndex[id] != nil {
t.Errorf("%s should not have loaded", id)
}
}
// The archive is coherent: facets, search and the sitemap all see the
// same two documents.
if got := corpus().archive.CollectionCounts[collectionNews]; got != 2 {
t.Errorf("news collection count %d, want 2", got)
}
if got, _ := corpus().executeSearch(searchParams{Queries: []string{""}, Fields: []string{"TE"}, Conjs: []string{"and"}}); len(got) != 2 {
t.Errorf("match-all search found %d, want 2", len(got))
}
}
// An empty corpus is a corpus: the server loads, every page renders, and
// nothing divides by zero.
func TestLoaderEmptyCorpus(t *testing.T) {
withCorpus(t, t.TempDir())
if len(corpus().archive.Docs) != 0 {
t.Fatalf("loaded %d documents from an empty directory", len(corpus().archive.Docs))
}
for _, p := range []string{"/", "/browse", "/search?q=x&do_search=1", "/explore", "/explore/ids",
"/articles", "/api/docs", "/api/facets", "/sitemap.xml"} {
if w := get(t, p); w.Code != 200 {
t.Errorf("%s on an empty corpus: %d", p, w.Code)
}
}
}
// Values are tidied before anything reads them, and a copy the loader skips
// leaves no trace - not even a claim on its ID.
func TestLoaderTidiesBeforeUse(t *testing.T) {
root := t.TempDir()
write := func(path, content string) {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
t.Fatal(err)
}
}
// FOIA/ sorts before the flat root, so the PDF-less copy is seen first.
write(filepath.Join(root, "FOIA", "dup-doc", "metadata.json"), `{"title": "Shadow"}`)
write(filepath.Join(root, "dup-doc", "metadata.json"), `{"title": "Real", "type": "REPORTED"}`)
write(filepath.Join(root, "dup-doc", "x.pdf"), "%PDF-1.4")
write(filepath.Join(root, "padded", "metadata.json"), `{
"title": "Padded", "type": "REPORTED",
"document": {"creator": " NSA ", "series": {"name": "Series X ", "part": 1}},
"publication": [{"publisher": "Le Monde", "date_published": "June 2014", "version": "entier"}]
}`)
write(filepath.Join(root, "padded", "x.pdf"), "%PDF-1.4")
write(filepath.Join(root, "padded", "versions", "entier", "entier.pdf"), "%PDF-1.4")
withCorpus(t, root)
if d := corpus().docIndex["dup-doc"]; d == nil || d.Title != "Real" {
t.Errorf("dup-doc = %+v, want the copy with a PDF", d)
}
d := corpus().docIndex["padded"]
if d == nil {
t.Fatal("padded not loaded")
}
if d.DocMeta.Creator != "NSA" || d.DocMeta.Series.Name != "Series X" {
t.Errorf("creator %q, series %q: want both trimmed", d.DocMeta.Creator, d.DocMeta.Series.Name)
}
// The malformed date is dropped everywhere, the version label included.
if v := d.VersionFor("entier"); v == nil || v.Label != "Le Monde" {
t.Errorf("version label %+v, want \"Le Monde\" without the invalid date", v)
}
}
// Two page images with one page number show that page twice; the loader
// says so instead of linking changed pages to whichever sorted first.
func TestLoadPageImagesWarnsOnDuplicatePageNumbers(t *testing.T) {
dir := t.TempDir()
if err := os.MkdirAll(filepath.Join(dir, "images"), 0o755); err != nil {
t.Fatal(err)
}
for _, name := range []string{"page-1.jpg", "page-2.jpg", "page-02.png"} {
if err := os.WriteFile(filepath.Join(dir, "images", name), nil, 0o644); err != nil {
t.Fatal(err)
}
}
before := loadWarnings
imgs := loadPageImages(dir, "doc")
if len(imgs) != 3 || imgs[0] != "page-1.jpg" {
t.Errorf("images %v, want all three in page order", imgs)
}
if loadWarnings != before+1 {
t.Errorf("%d warnings for one duplicated page number, want 1", loadWarnings-before)
}
}