package main import ( "encoding/json" "strings" "testing" ) func versionedDoc(t *testing.T, main string, versions map[string]string, pubsJSON string) Document { t.Helper() var d Document if pubsJSON != "" { if err := json.Unmarshal([]byte(pubsJSON), &d); err != nil { t.Fatal(err) } } d.ID, d.FullText = "doc", main for _, slug := range []string{"lemonde", "spiegel"} { if text, ok := versions[slug]; ok { d.Versions = append(d.Versions, DocVersion{Slug: slug, FullText: text}) } } return d } func TestVersionNovelText(t *testing.T) { d := versionedDoc(t, "PRISM Overview\nShared line\n", map[string]string{ "lemonde": "shared LINE\r\nNew in Le Monde\n---\nIn both copies\nNew in Le Monde\n", "spiegel": "In both copies\nOnly Der Spiegel\n", }, "") combined := versionNovelText(&d) if got := d.Versions[0].NovelText; got != "New in Le Monde\nIn both copies\nNew in Le Monde" { t.Errorf("lemonde new lines %q", got) } if got := d.Versions[1].NovelText; got != "In both copies\nOnly Der Spiegel" { t.Errorf("spiegel new lines %q", got) } if combined != "New in Le Monde\nIn both copies\nOnly Der Spiegel" { t.Errorf("combined new lines %q, want each line once", combined) } } func TestVersionLabel(t *testing.T) { d := versionedDoc(t, "", map[string]string{"lemonde": "", "spiegel": ""}, `{"publication": [ {"publisher": "Le Monde", "date_published": "2013-10-21"}, {"publisher": "Le Monde", "date_published": "2013-10-25"}, {"publisher": "The Guardian", "date_published": "2013-10-20"} ]}`) d.Versions[0].PubIndex = []int{0, 1, 2} if got := versionLabel(&d, &d.Versions[0]); got != "Le Monde / The Guardian, 2013-10-20" { t.Errorf("label %q", got) } if got := versionLabel(&d, &d.Versions[1]); got != "spiegel" { t.Errorf("label of a version no publication names %q, want its folder name", got) } } func TestVersionTextIsSearchable(t *testing.T) { d := versionedDoc(t, "The overview ends with alpha", map[string]string{ "lemonde": "The overview ends with alpha\nbeta starts the Wanadoo slide", }, "") setBodySearchText(&d) d.StemmedText = d.StemmedFull d.SearchText = d.FoldedFull match := func(q, field string) bool { return compileQuery([]string{q}, []string{field}, []string{"and"}).matches(&d) } if !match("wanadoo", "FT") || !match(`"wanadoo slide"`, "FT") || !match("=wanadoo", "FT") || !match("wanad*", "FT") { t.Error("a term only in a version's new lines should match the document") } if match(`"alpha beta"`, "FT") { t.Error("a quoted phrase matched across the join between the main text and a version's lines") } if tok := tokenize("overview")[0]; strings.Count(" "+d.StemmedFull+" ", " "+tok+" ") != 1 { t.Errorf("a line shared with the main text was indexed twice: %q", d.StemmedFull) } // buildTFIDF skips the separator only as a token of its own; glued to a // word it would mint terms and hide that word from phrase matching. for _, tok := range strings.Fields(d.StemmedFull) { if tok != valueSep && strings.Contains(tok, valueSep) { t.Errorf("the join separator is glued to a term in StemmedFull: %q", tok) } } idx := buildTFIDF([]Document{d}) for w := range idx.DocFreq { if strings.Contains(w, valueSep) { t.Errorf("the join separator was indexed in the TF-IDF term %q", w) } } hits := versionHits(&d, "wanadoo", 3) if len(hits) != 1 || hits[0].Version.Slug != "lemonde" || !strings.Contains(hits[0].Snippets[0], "Wanadoo") { t.Errorf("versionHits = %+v, want one lemonde hit with a Wanadoo snippet", hits) } if hits := versionHits(&d, "overview", 3); len(hits) != 0 { t.Errorf("versionHits found %+v for a word only in lines shared with the main text", hits) } }