first commit
This commit is contained in:
commit
8c4dcae475
89 changed files with 24623 additions and 0 deletions
101
tests/versions_search_test.go
Normal file
101
tests/versions_search_test.go
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func versionedDoc(t *testing.T, main string, versions map[string]string, pubsJSON string) Document {
|
||||
t.Helper()
|
||||
var d Document
|
||||
if pubsJSON != "" {
|
||||
if err := json.Unmarshal([]byte(pubsJSON), &d); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
d.ID, d.FullText = "doc", main
|
||||
for _, slug := range []string{"lemonde", "spiegel"} {
|
||||
if text, ok := versions[slug]; ok {
|
||||
d.Versions = append(d.Versions, DocVersion{Slug: slug, FullText: text})
|
||||
}
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func TestVersionNovelText(t *testing.T) {
|
||||
d := versionedDoc(t, "PRISM Overview\nShared line\n", map[string]string{
|
||||
"lemonde": "shared LINE\r\nNew in Le Monde\n---\nIn both copies\nNew in Le Monde\n",
|
||||
"spiegel": "In both copies\nOnly Der Spiegel\n",
|
||||
}, "")
|
||||
combined := versionNovelText(&d)
|
||||
|
||||
if got := d.Versions[0].NovelText; got != "New in Le Monde\nIn both copies\nNew in Le Monde" {
|
||||
t.Errorf("lemonde new lines %q", got)
|
||||
}
|
||||
if got := d.Versions[1].NovelText; got != "In both copies\nOnly Der Spiegel" {
|
||||
t.Errorf("spiegel new lines %q", got)
|
||||
}
|
||||
if combined != "New in Le Monde\nIn both copies\nOnly Der Spiegel" {
|
||||
t.Errorf("combined new lines %q, want each line once", combined)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVersionLabel(t *testing.T) {
|
||||
d := versionedDoc(t, "", map[string]string{"lemonde": "", "spiegel": ""}, `{"publication": [
|
||||
{"publisher": "Le Monde", "date_published": "2013-10-21"},
|
||||
{"publisher": "Le Monde", "date_published": "2013-10-25"},
|
||||
{"publisher": "The Guardian", "date_published": "2013-10-20"}
|
||||
]}`)
|
||||
d.Versions[0].PubIndex = []int{0, 1, 2}
|
||||
if got := versionLabel(&d, &d.Versions[0]); got != "Le Monde / The Guardian, 2013-10-20" {
|
||||
t.Errorf("label %q", got)
|
||||
}
|
||||
if got := versionLabel(&d, &d.Versions[1]); got != "spiegel" {
|
||||
t.Errorf("label of a version no publication names %q, want its folder name", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVersionTextIsSearchable(t *testing.T) {
|
||||
d := versionedDoc(t, "The overview ends with alpha", map[string]string{
|
||||
"lemonde": "The overview ends with alpha\nbeta starts the Wanadoo slide",
|
||||
}, "")
|
||||
setBodySearchText(&d)
|
||||
d.StemmedText = d.StemmedFull
|
||||
d.SearchText = d.FoldedFull
|
||||
|
||||
match := func(q, field string) bool {
|
||||
return compileQuery([]string{q}, []string{field}, []string{"and"}).matches(&d)
|
||||
}
|
||||
if !match("wanadoo", "FT") || !match(`"wanadoo slide"`, "FT") || !match("=wanadoo", "FT") || !match("wanad*", "FT") {
|
||||
t.Error("a term only in a version's new lines should match the document")
|
||||
}
|
||||
if match(`"alpha beta"`, "FT") {
|
||||
t.Error("a quoted phrase matched across the join between the main text and a version's lines")
|
||||
}
|
||||
if tok := tokenize("overview")[0]; strings.Count(" "+d.StemmedFull+" ", " "+tok+" ") != 1 {
|
||||
t.Errorf("a line shared with the main text was indexed twice: %q", d.StemmedFull)
|
||||
}
|
||||
|
||||
// buildTFIDF skips the separator only as a token of its own; glued to a
|
||||
// word it would mint terms and hide that word from phrase matching.
|
||||
for _, tok := range strings.Fields(d.StemmedFull) {
|
||||
if tok != valueSep && strings.Contains(tok, valueSep) {
|
||||
t.Errorf("the join separator is glued to a term in StemmedFull: %q", tok)
|
||||
}
|
||||
}
|
||||
idx := buildTFIDF([]Document{d})
|
||||
for w := range idx.DocFreq {
|
||||
if strings.Contains(w, valueSep) {
|
||||
t.Errorf("the join separator was indexed in the TF-IDF term %q", w)
|
||||
}
|
||||
}
|
||||
|
||||
hits := versionHits(&d, "wanadoo", 3)
|
||||
if len(hits) != 1 || hits[0].Version.Slug != "lemonde" || !strings.Contains(hits[0].Snippets[0], "Wanadoo") {
|
||||
t.Errorf("versionHits = %+v, want one lemonde hit with a Wanadoo snippet", hits)
|
||||
}
|
||||
if hits := versionHits(&d, "overview", 3); len(hits) != 0 {
|
||||
t.Errorf("versionHits found %+v for a word only in lines shared with the main text", hits)
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue