package main import ( "bytes" "log" "os" "path/filepath" "strings" "testing" ) // The /explore/ids table: one row per document in id order, versions nested // under their document, and a checksum file that sha256sum -c can read from // the corpus root. func TestBuildIDRows(t *testing.T) { oldDocs, oldDir := corpus().archive.Docs, dataDir t.Cleanup(func() { // The id tables are package state the routes serve: rebuild them // from the restored corpus, or the fake rows below leak into // every later test. corpus().archive.Docs, dataDir = oldDocs, oldDir corpus().buildIDRows() }) dataDir = "/corpus" corpus().archive.Docs = []Document{ { ID: "zebra", Dir: filepath.Join(dataDir, "FOIA", "zebra"), Collection: "foia", PDFFile: "zebra.pdf", PDFSize: 2 << 20, Pages: 3, }, { ID: "alpha", Dir: filepath.Join(dataDir, "alpha"), Collection: "news", PDFFile: "alpha.pdf", PDFSize: 1500, Pages: 1, PDFSha256: "abc123", AkaID: []AkaEntry{{ID: "alpha-lemonde"}}, Versions: []DocVersion{{Slug: "entier", PDFFile: "copy.pdf", PDFSize: 4096, Images: []string{"a", "b"}, SHA: "def456"}}, }, } corpus().buildIDRows() if len(corpus().idRows) != 2 || corpus().idRows[0].ID != "alpha" || corpus().idRows[1].ID != "zebra" { t.Fatalf("rows out of order: %v", corpus().idRows) } if got := corpus().idRows[0].Size; got != "1 KB" { t.Errorf("size %q, want 1 KB", got) } if v := corpus().idRows[0].Versions; len(v) != 1 || v[0].Pages != 2 || v[0].Size != "4 KB" || v[0].SHA != "def456" { t.Errorf("version row %+v", v) } if len(corpus().idRows[1].Versions) != 0 { t.Errorf("zebra has no versions, got %+v", corpus().idRows[1].Versions) } // sha256sum -c format: checksum, two spaces, path from the corpus root, // a document's versions after it. Files without a checksum are absent. if corpus().idChecksums != "abc123 alpha/alpha.pdf\ndef456 alpha/versions/entier/copy.pdf\n" { t.Errorf("checksums %q", corpus().idChecksums) } } func TestHumanSize(t *testing.T) { for n, want := range map[int64]string{0: "", -1: "", 900: "900 B", 2048: "2 KB", 3 << 20: "3.0 MB", 5 << 30: "5.00 GB"} { if got := humanSize(n); got != want { t.Errorf("humanSize(%d) = %q, want %q", n, got, want) } } if !strings.HasSuffix(humanSize(1<<20+1<<19), " MB") { t.Errorf("1.5 MiB should render in MB") } } // A version's PDF checksum comes from the metadata's // "versions"//pdf_sha256 only; a version without one is warned about // and has no line in ids.txt. func TestVersionChecksums(t *testing.T) { root := assemblyCorpus(t, "") writeAssemblyDoc(t, root, "listed", map[string]interface{}{ "title": "Listed", "versions": map[string]interface{}{"copy": map[string]interface{}{"pdf_sha256": " ABC999 "}}, }, []string{"page-1.jpg"}, map[string][]string{"copy": {"page-1.jpg"}}) var logged bytes.Buffer log.SetOutput(&logged) withCorpus(t, root) log.SetOutput(os.Stderr) if v := corpus().docIndex["listed"].VersionFor("copy"); v == nil || v.SHA != "abc999" { t.Errorf("metadata version checksum: %+v, want abc999 from the metadata", v) } if v := corpus().docIndex["deck"].VersionFor("fuller"); v == nil || v.SHA != "" { t.Errorf("a version without pdf_sha256 in the metadata got a checksum: %+v", v) } if !strings.Contains(logged.String(), "deck: versions/fuller has no pdf_sha256") { t.Errorf("no warning for the version without a checksum:\n%s", logged.String()) } if !strings.Contains(corpus().idChecksums, "abc999 listed/versions/copy/copy.pdf\n") || strings.Contains(corpus().idChecksums, "deck/versions/fuller/") { t.Errorf("ids.txt version lines:\n%s", corpus().idChecksums) } }