first commit
This commit is contained in:
commit
8c4dcae475
89 changed files with 24623 additions and 0 deletions
123
tests/data_test.go
Normal file
123
tests/data_test.go
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAkaIDShape(t *testing.T) {
|
||||
for _, s := range []string{
|
||||
`{"title": "no aka_id key"}`,
|
||||
`{"aka_id": []}`,
|
||||
// null is "no akas": not a reason to drop the document.
|
||||
`{"aka_id": null}`,
|
||||
`{"aka_id": [{"id": "a"}, {"id": "b", "doc_source": "https://e.org/d", "doc_url": "https://e.org/b.pdf"}]}`,
|
||||
} {
|
||||
var d Document
|
||||
if err := json.Unmarshal([]byte(s), &d); err != nil {
|
||||
t.Errorf("%s: unexpected error: %v", s, err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, s := range []string{`{"aka_id": "a"}`, `{"aka_id": {"id": "a"}}`, `{"aka_id": ["a", "b"]}`, `{"aka_id": [{"id": "a"}, 3]}`} {
|
||||
var d Document
|
||||
if err := json.Unmarshal([]byte(s), &d); err == nil {
|
||||
t.Errorf("%s: decoded without an error", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeAka(t *testing.T) {
|
||||
d := Document{ID: "x", AkaID: []AkaEntry{
|
||||
{ID: " first ", DocSource: " https://e.org/p "},
|
||||
{ID: " "},
|
||||
{ID: "FIRST"},
|
||||
{ID: "second", DocURL: "https://e.org/s.pdf"},
|
||||
}}
|
||||
got := normalizeAka(&d)
|
||||
if len(got) != 2 || got[0].ID != "first" || got[0].DocSource != "https://e.org/p" || got[1].ID != "second" {
|
||||
t.Errorf("normalizeAka = %+v, want first (trimmed) and second, without the blank id and the repeat", got)
|
||||
}
|
||||
}
|
||||
|
||||
func writeTestFile(t *testing.T, path, content string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadVersions(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
v := filepath.Join(dir, "versions")
|
||||
writeTestFile(t, filepath.Join(v, "lemonde-2013", "copy.pdf"), "%PDF-1.4")
|
||||
writeTestFile(t, filepath.Join(v, "lemonde-2013", "copy.md"), "# Slide 12\n\nOnly in Le Monde's copy.")
|
||||
writeTestFile(t, filepath.Join(v, "lemonde-2013", "images", "page-10.jpg"), "")
|
||||
writeTestFile(t, filepath.Join(v, "lemonde-2013", "images", "page-2.jpg"), "")
|
||||
writeTestFile(t, filepath.Join(v, "guardian", "copy.pdf"), "%PDF-1.4") // no transcript or images: still a version
|
||||
writeTestFile(t, filepath.Join(v, "no-pdf", "copy.md"), "text only") // skipped: no PDF
|
||||
writeTestFile(t, filepath.Join(v, "bad name", "copy.pdf"), "%PDF-1.4") // skipped: name
|
||||
writeTestFile(t, filepath.Join(v, "loose.pdf"), "%PDF-1.4") // skipped: not a folder
|
||||
|
||||
var d Document
|
||||
if err := json.Unmarshal([]byte(`{"publication": [
|
||||
{"publisher": "Le Monde", "version": "lemonde-2013"},
|
||||
{"publisher": "The Guardian"},
|
||||
{"publisher": "Dagbladet", "version": " lemonde-2013 "},
|
||||
{"publisher": "Nobody", "version": "missing"}
|
||||
]}`), &d); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.ID, d.Dir = "prism", dir
|
||||
corpus().loadVersions(&d)
|
||||
|
||||
var slugs []string
|
||||
for _, v := range d.Versions {
|
||||
slugs = append(slugs, v.Slug)
|
||||
}
|
||||
if got := strings.Join(slugs, ","); got != "guardian,lemonde-2013" {
|
||||
t.Fatalf("versions %q, want guardian,lemonde-2013", got)
|
||||
}
|
||||
|
||||
lm := d.VersionFor("lemonde-2013")
|
||||
if lm.PDFFile != "copy.pdf" || !strings.Contains(lm.FullText, "Only in Le Monde") || lm.HTMLContent == "" {
|
||||
t.Errorf("lemonde-2013 not fully loaded: pdf %q, text %q, html %q", lm.PDFFile, lm.FullText, lm.HTMLContent)
|
||||
}
|
||||
if got := strings.Join(lm.Images, ","); got != "page-2.jpg,page-10.jpg" {
|
||||
t.Errorf("images %q, want page-2.jpg,page-10.jpg (page order)", got)
|
||||
}
|
||||
if got := fmt.Sprint(lm.PubIndex); got != "[0 2]" {
|
||||
t.Errorf("PubIndex %s, want [0 2]", got)
|
||||
}
|
||||
if got := d.Publication[2].Version; got != "lemonde-2013" {
|
||||
t.Errorf("publication version not trimmed: %q", got)
|
||||
}
|
||||
if u := d.UnlinkedVersions(); len(u) != 1 || u[0].Slug != "guardian" {
|
||||
t.Errorf("unlinked versions %+v, want only guardian", u)
|
||||
}
|
||||
if d.VersionFor("missing") != nil || d.VersionFor("") != nil {
|
||||
t.Error("VersionFor returned a version that doesn't exist")
|
||||
}
|
||||
}
|
||||
|
||||
// TestBodyFormsAreTheCombinedTails: loadData slices FoldedFull and
|
||||
// StemmedFull out of the ends of SearchText and StemmedText to save memory;
|
||||
// they must still be exactly what setBodySearchText builds.
|
||||
func TestBodyFormsAreTheCombinedTails(t *testing.T) {
|
||||
buildFixture(t)
|
||||
for i := range corpus().archive.Docs {
|
||||
d := &corpus().archive.Docs[i]
|
||||
c := *d
|
||||
setBodySearchText(&c)
|
||||
if c.FoldedFull != d.FoldedFull || c.StemmedFull != d.StemmedFull {
|
||||
t.Errorf("%s: body forms differ from a fresh build", d.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue