101 lines
3.6 KiB
Go
101 lines
3.6 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func versionedDoc(t *testing.T, main string, versions map[string]string, pubsJSON string) Document {
|
|
t.Helper()
|
|
var d Document
|
|
if pubsJSON != "" {
|
|
if err := json.Unmarshal([]byte(pubsJSON), &d); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
d.ID, d.FullText = "doc", main
|
|
for _, slug := range []string{"lemonde", "spiegel"} {
|
|
if text, ok := versions[slug]; ok {
|
|
d.Versions = append(d.Versions, DocVersion{Slug: slug, FullText: text})
|
|
}
|
|
}
|
|
return d
|
|
}
|
|
|
|
func TestVersionNovelText(t *testing.T) {
|
|
d := versionedDoc(t, "PRISM Overview\nShared line\n", map[string]string{
|
|
"lemonde": "shared LINE\r\nNew in Le Monde\n---\nIn both copies\nNew in Le Monde\n",
|
|
"spiegel": "In both copies\nOnly Der Spiegel\n",
|
|
}, "")
|
|
combined := versionNovelText(&d)
|
|
|
|
if got := d.Versions[0].NovelText; got != "New in Le Monde\nIn both copies\nNew in Le Monde" {
|
|
t.Errorf("lemonde new lines %q", got)
|
|
}
|
|
if got := d.Versions[1].NovelText; got != "In both copies\nOnly Der Spiegel" {
|
|
t.Errorf("spiegel new lines %q", got)
|
|
}
|
|
if combined != "New in Le Monde\nIn both copies\nOnly Der Spiegel" {
|
|
t.Errorf("combined new lines %q, want each line once", combined)
|
|
}
|
|
}
|
|
|
|
func TestVersionLabel(t *testing.T) {
|
|
d := versionedDoc(t, "", map[string]string{"lemonde": "", "spiegel": ""}, `{"publication": [
|
|
{"publisher": "Le Monde", "date_published": "2013-10-21"},
|
|
{"publisher": "Le Monde", "date_published": "2013-10-25"},
|
|
{"publisher": "The Guardian", "date_published": "2013-10-20"}
|
|
]}`)
|
|
d.Versions[0].PubIndex = []int{0, 1, 2}
|
|
if got := versionLabel(&d, &d.Versions[0]); got != "Le Monde / The Guardian, 2013-10-20" {
|
|
t.Errorf("label %q", got)
|
|
}
|
|
if got := versionLabel(&d, &d.Versions[1]); got != "spiegel" {
|
|
t.Errorf("label of a version no publication names %q, want its folder name", got)
|
|
}
|
|
}
|
|
|
|
func TestVersionTextIsSearchable(t *testing.T) {
|
|
d := versionedDoc(t, "The overview ends with alpha", map[string]string{
|
|
"lemonde": "The overview ends with alpha\nbeta starts the Wanadoo slide",
|
|
}, "")
|
|
setBodySearchText(&d)
|
|
d.StemmedText = d.StemmedFull
|
|
d.SearchText = d.FoldedFull
|
|
|
|
match := func(q, field string) bool {
|
|
return compileQuery([]string{q}, []string{field}, []string{"and"}).matches(&d)
|
|
}
|
|
if !match("wanadoo", "FT") || !match(`"wanadoo slide"`, "FT") || !match("=wanadoo", "FT") || !match("wanad*", "FT") {
|
|
t.Error("a term only in a version's new lines should match the document")
|
|
}
|
|
if match(`"alpha beta"`, "FT") {
|
|
t.Error("a quoted phrase matched across the join between the main text and a version's lines")
|
|
}
|
|
if tok := tokenize("overview")[0]; strings.Count(" "+d.StemmedFull+" ", " "+tok+" ") != 1 {
|
|
t.Errorf("a line shared with the main text was indexed twice: %q", d.StemmedFull)
|
|
}
|
|
|
|
// buildTFIDF skips the separator only as a token of its own; glued to a
|
|
// word it would mint terms and hide that word from phrase matching.
|
|
for _, tok := range strings.Fields(d.StemmedFull) {
|
|
if tok != valueSep && strings.Contains(tok, valueSep) {
|
|
t.Errorf("the join separator is glued to a term in StemmedFull: %q", tok)
|
|
}
|
|
}
|
|
idx := buildTFIDF([]Document{d})
|
|
for w := range idx.DocFreq {
|
|
if strings.Contains(w, valueSep) {
|
|
t.Errorf("the join separator was indexed in the TF-IDF term %q", w)
|
|
}
|
|
}
|
|
|
|
hits := versionHits(&d, "wanadoo", 3)
|
|
if len(hits) != 1 || hits[0].Version.Slug != "lemonde" || !strings.Contains(hits[0].Snippets[0], "Wanadoo") {
|
|
t.Errorf("versionHits = %+v, want one lemonde hit with a Wanadoo snippet", hits)
|
|
}
|
|
if hits := versionHits(&d, "overview", 3); len(hits) != 0 {
|
|
t.Errorf("versionHits found %+v for a word only in lines shared with the main text", hits)
|
|
}
|
|
}
|