package main import ( "os" "path/filepath" "testing" ) // What the loader does with a corpus that is wrong in the ways corpora are // wrong in practice. Every case must leave the rest of the archive intact - // a broken document is skipped, never a reason to serve nothing - and say // so in the tally that -check reads. func TestLoaderRejectsBadDocuments(t *testing.T) { root := t.TempDir() write := func(path, content string) { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(path, []byte(content), 0o644); err != nil { t.Fatal(err) } } good := func(id string) { write(filepath.Join(root, id, "metadata.json"), `{"title": "`+id+`", "type": "REPORTED"}`) write(filepath.Join(root, id, id+".pdf"), "%PDF-1.4") } good("keeper-one") good("keeper-two") // No PDF: this is an archive OF the PDFs, so the document is skipped. write(filepath.Join(root, "no-pdf", "metadata.json"), `{"title": "No PDF"}`) // Invalid JSON: skipped, everything else still loads. write(filepath.Join(root, "bad-json", "metadata.json"), `{"title": "oops"`) write(filepath.Join(root, "bad-json", "x.pdf"), "%PDF-1.4") // aka_id that is not a list of objects: refused, not silently half-read. write(filepath.Join(root, "bad-aka", "metadata.json"), `{"title": "Aka", "aka_id": "other-id"}`) write(filepath.Join(root, "bad-aka", "x.pdf"), "%PDF-1.4") // A directory that is not a document at all: skipped without a warning. write(filepath.Join(root, "notes", "README.txt"), "just notes") // A directory holding a PDF and nothing else: the shape worth warning // about, since it looks like a document nobody described. write(filepath.Join(root, "loose", "orphan.pdf"), "%PDF-1.4") withCorpus(t, root) clean := loadWarnings // A document in the SIDtoday archive that is not typed SIDTODAY is on // no browse tab: served, and warned about. write(filepath.Join(root, "SIDtoday-archive", "mistyped", "metadata.json"), `{"title": "Mistyped", "type": "REPORTED"}`) write(filepath.Join(root, "SIDtoday-archive", "mistyped", "x.pdf"), "%PDF-1.4") loadData() if corpus().docIndex["mistyped"] == nil || loadWarnings != clean+1 { t.Errorf("a REPORTED document in SIDtoday-archive: loaded %v, %d warnings, want it loaded with %d", corpus().docIndex["mistyped"] != nil, loadWarnings, clean+1) } if err := os.RemoveAll(filepath.Join(root, "SIDtoday-archive")); err != nil { t.Fatal(err) } loadData() if len(corpus().archive.Docs) != 2 { ids := make([]string, 0, len(corpus().archive.Docs)) for _, d := range corpus().archive.Docs { ids = append(ids, d.ID) } t.Fatalf("loaded %v, want the two good documents", ids) } if loadErrors < 3 { // no-pdf, bad-aka and bad JSON are errors: not served t.Errorf("loadErrors = %d, want at least 3", loadErrors) } if loadWarnings < 1 { // the loose PDF directory: served nothing, lost nothing t.Errorf("loadWarnings = %d, want at least 1", loadWarnings) } for _, id := range []string{"no-pdf", "bad-json", "bad-aka", "notes", "loose"} { if corpus().docIndex[id] != nil { t.Errorf("%s should not have loaded", id) } } // The archive is coherent: facets, search and the sitemap all see the // same two documents. if got := corpus().archive.CollectionCounts[collectionNews]; got != 2 { t.Errorf("news collection count %d, want 2", got) } if got, _ := corpus().executeSearch(searchParams{Queries: []string{""}, Fields: []string{"TE"}, Conjs: []string{"and"}}); len(got) != 2 { t.Errorf("match-all search found %d, want 2", len(got)) } } // An empty corpus is a corpus: the server loads, every page renders, and // nothing divides by zero. func TestLoaderEmptyCorpus(t *testing.T) { withCorpus(t, t.TempDir()) if len(corpus().archive.Docs) != 0 { t.Fatalf("loaded %d documents from an empty directory", len(corpus().archive.Docs)) } for _, p := range []string{"/", "/browse", "/search?q=x&do_search=1", "/explore", "/explore/ids", "/articles", "/api/docs", "/api/facets", "/sitemap.xml"} { if w := get(t, p); w.Code != 200 { t.Errorf("%s on an empty corpus: %d", p, w.Code) } } } // Values are tidied before anything reads them, and a copy the loader skips // leaves no trace - not even a claim on its ID. func TestLoaderTidiesBeforeUse(t *testing.T) { root := t.TempDir() write := func(path, content string) { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(path, []byte(content), 0o644); err != nil { t.Fatal(err) } } // FOIA/ sorts before the flat root, so the PDF-less copy is seen first. write(filepath.Join(root, "FOIA", "dup-doc", "metadata.json"), `{"title": "Shadow"}`) write(filepath.Join(root, "dup-doc", "metadata.json"), `{"title": "Real", "type": "REPORTED"}`) write(filepath.Join(root, "dup-doc", "x.pdf"), "%PDF-1.4") write(filepath.Join(root, "padded", "metadata.json"), `{ "title": "Padded", "type": "REPORTED", "document": {"creator": " NSA ", "series": {"name": "Series X ", "part": 1}}, "publication": [{"publisher": "Le Monde", "date_published": "June 2014", "version": "entier"}] }`) write(filepath.Join(root, "padded", "x.pdf"), "%PDF-1.4") write(filepath.Join(root, "padded", "versions", "entier", "entier.pdf"), "%PDF-1.4") withCorpus(t, root) if d := corpus().docIndex["dup-doc"]; d == nil || d.Title != "Real" { t.Errorf("dup-doc = %+v, want the copy with a PDF", d) } d := corpus().docIndex["padded"] if d == nil { t.Fatal("padded not loaded") } if d.DocMeta.Creator != "NSA" || d.DocMeta.Series.Name != "Series X" { t.Errorf("creator %q, series %q: want both trimmed", d.DocMeta.Creator, d.DocMeta.Series.Name) } // The malformed date is dropped everywhere, the version label included. if v := d.VersionFor("entier"); v == nil || v.Label != "Le Monde" { t.Errorf("version label %+v, want \"Le Monde\" without the invalid date", v) } } // Two page images with one page number show that page twice; the loader // says so instead of linking changed pages to whichever sorted first. func TestLoadPageImagesWarnsOnDuplicatePageNumbers(t *testing.T) { dir := t.TempDir() if err := os.MkdirAll(filepath.Join(dir, "images"), 0o755); err != nil { t.Fatal(err) } for _, name := range []string{"page-1.jpg", "page-2.jpg", "page-02.png"} { if err := os.WriteFile(filepath.Join(dir, "images", name), nil, 0o644); err != nil { t.Fatal(err) } } before := loadWarnings imgs := loadPageImages(dir, "doc") if len(imgs) != 3 || imgs[0] != "page-1.jpg" { t.Errorf("images %v, want all three in page order", imgs) } if loadWarnings != before+1 { t.Errorf("%d warnings for one duplicated page number, want 1", loadWarnings-before) } }