package main import ( "encoding/json" "fmt" "os" "path/filepath" "strings" "testing" ) func TestAkaIDShape(t *testing.T) { for _, s := range []string{ `{"title": "no aka_id key"}`, `{"aka_id": []}`, // null is "no akas": not a reason to drop the document. `{"aka_id": null}`, `{"aka_id": [{"id": "a"}, {"id": "b", "doc_source": "https://e.org/d", "doc_url": "https://e.org/b.pdf"}]}`, } { var d Document if err := json.Unmarshal([]byte(s), &d); err != nil { t.Errorf("%s: unexpected error: %v", s, err) } } for _, s := range []string{`{"aka_id": "a"}`, `{"aka_id": {"id": "a"}}`, `{"aka_id": ["a", "b"]}`, `{"aka_id": [{"id": "a"}, 3]}`} { var d Document if err := json.Unmarshal([]byte(s), &d); err == nil { t.Errorf("%s: decoded without an error", s) } } } func TestNormalizeAka(t *testing.T) { d := Document{ID: "x", AkaID: []AkaEntry{ {ID: " first ", DocSource: " https://e.org/p "}, {ID: " "}, {ID: "FIRST"}, {ID: "second", DocURL: "https://e.org/s.pdf"}, }} got := normalizeAka(&d) if len(got) != 2 || got[0].ID != "first" || got[0].DocSource != "https://e.org/p" || got[1].ID != "second" { t.Errorf("normalizeAka = %+v, want first (trimmed) and second, without the blank id and the repeat", got) } } func writeTestFile(t *testing.T, path, content string) { t.Helper() if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(path, []byte(content), 0o644); err != nil { t.Fatal(err) } } func TestLoadVersions(t *testing.T) { dir := t.TempDir() v := filepath.Join(dir, "versions") writeTestFile(t, filepath.Join(v, "lemonde-2013", "copy.pdf"), "%PDF-1.4") writeTestFile(t, filepath.Join(v, "lemonde-2013", "copy.md"), "# Slide 12\n\nOnly in Le Monde's copy.") writeTestFile(t, filepath.Join(v, "lemonde-2013", "images", "page-10.jpg"), "") writeTestFile(t, filepath.Join(v, "lemonde-2013", "images", "page-2.jpg"), "") writeTestFile(t, filepath.Join(v, "guardian", "copy.pdf"), "%PDF-1.4") // no transcript or images: still a version writeTestFile(t, filepath.Join(v, "no-pdf", "copy.md"), "text only") // skipped: no PDF writeTestFile(t, filepath.Join(v, "bad name", "copy.pdf"), "%PDF-1.4") // skipped: name writeTestFile(t, filepath.Join(v, "loose.pdf"), "%PDF-1.4") // skipped: not a folder var d Document if err := json.Unmarshal([]byte(`{"publication": [ {"publisher": "Le Monde", "version": "lemonde-2013"}, {"publisher": "The Guardian"}, {"publisher": "Dagbladet", "version": " lemonde-2013 "}, {"publisher": "Nobody", "version": "missing"} ]}`), &d); err != nil { t.Fatal(err) } d.ID, d.Dir = "prism", dir corpus().loadVersions(&d) var slugs []string for _, v := range d.Versions { slugs = append(slugs, v.Slug) } if got := strings.Join(slugs, ","); got != "guardian,lemonde-2013" { t.Fatalf("versions %q, want guardian,lemonde-2013", got) } lm := d.VersionFor("lemonde-2013") if lm.PDFFile != "copy.pdf" || !strings.Contains(lm.FullText, "Only in Le Monde") || lm.HTMLContent == "" { t.Errorf("lemonde-2013 not fully loaded: pdf %q, text %q, html %q", lm.PDFFile, lm.FullText, lm.HTMLContent) } if got := strings.Join(lm.Images, ","); got != "page-2.jpg,page-10.jpg" { t.Errorf("images %q, want page-2.jpg,page-10.jpg (page order)", got) } if got := fmt.Sprint(lm.PubIndex); got != "[0 2]" { t.Errorf("PubIndex %s, want [0 2]", got) } if got := d.Publication[2].Version; got != "lemonde-2013" { t.Errorf("publication version not trimmed: %q", got) } if u := d.UnlinkedVersions(); len(u) != 1 || u[0].Slug != "guardian" { t.Errorf("unlinked versions %+v, want only guardian", u) } if d.VersionFor("missing") != nil || d.VersionFor("") != nil { t.Error("VersionFor returned a version that doesn't exist") } } // TestBodyFormsAreTheCombinedTails: loadData slices FoldedFull and // StemmedFull out of the ends of SearchText and StemmedText to save memory; // they must still be exactly what setBodySearchText builds. func TestBodyFormsAreTheCombinedTails(t *testing.T) { buildFixture(t) for i := range corpus().archive.Docs { d := &corpus().archive.Docs[i] c := *d setBodySearchText(&c) if c.FoldedFull != d.FoldedFull || c.StemmedFull != d.StemmedFull { t.Errorf("%s: body forms differ from a fresh build", d.ID) } } }