1000 lines
38 KiB
Go
1000 lines
38 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"net/url"
|
|
"reflect"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func mustQuery(t *testing.T, u string) url.Values {
|
|
t.Helper()
|
|
parsed, err := url.Parse(u)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return parsed.Query()
|
|
}
|
|
|
|
// TestStemUnifiesForms pins the stemmer's contract: the inflections a
|
|
// reader is likely to type must land on the same token as the base word,
|
|
// because the index and the query both go through stem().
|
|
func TestStemUnifiesForms(t *testing.T) {
|
|
groups := [][]string{
|
|
{"operation", "operations", "operational"},
|
|
{"communication", "communications"},
|
|
{"collection", "collections"},
|
|
{"collecting", "collected"},
|
|
{"agency", "agencies"},
|
|
{"surveillance", "surveilling"},
|
|
{"target", "targets", "targeting", "targeted"},
|
|
{"meeting", "meetings"},
|
|
{"computer", "computers"},
|
|
{"process", "processes"},
|
|
{"study", "studies"},
|
|
{"tie", "ties", "tied"},
|
|
{"lie", "lies", "lied"},
|
|
{"spy", "spies", "spied", "spying"},
|
|
{"fly", "flies", "flying", "flyer"},
|
|
{"study", "studying", "studied"},
|
|
{"control", "controlled", "controlling", "controls"},
|
|
{"install", "installed", "installing"},
|
|
{"cancel", "cancelled", "canceled"},
|
|
{"fall", "falling", "falls"},
|
|
{"kill", "killed", "killing"},
|
|
// A plural in -ies takes its singular's path through the -ity and
|
|
// -ly rules, and a word in -ie meets its plural.
|
|
{"capability", "capabilities"},
|
|
{"vulnerability", "vulnerabilities"},
|
|
{"activity", "activities"},
|
|
{"family", "families"},
|
|
{"anomaly", "anomalies"},
|
|
{"cookie", "cookies"},
|
|
// Verbs in -ply are not adverbs.
|
|
{"apply", "applies", "applied", "applying"},
|
|
{"supply", "supplies", "supplied", "supplying", "supplier"},
|
|
{"comply", "complies", "complied", "complying", "compliance"},
|
|
{"reply", "replies", "replied"},
|
|
}
|
|
for _, g := range groups {
|
|
want := stem(g[0])
|
|
for _, w := range g[1:] {
|
|
if got := stem(w); got != want {
|
|
t.Errorf("stem(%q) = %q, stem(%q) = %q; want equal", w, got, g[0], want)
|
|
}
|
|
}
|
|
}
|
|
// Words a stripped ending must not turn into each other.
|
|
for _, p := range [][2]string{
|
|
{"apply", "app"}, {"reply", "rep"}, {"supply", "supp"}, {"early", "ear"},
|
|
{"italy", "ita"}, {"july", "julie"},
|
|
} {
|
|
if stem(p[0]) == stem(p[1]) {
|
|
t.Errorf("stem(%q) = stem(%q) = %q; want them apart", p[0], p[1], stem(p[0]))
|
|
}
|
|
}
|
|
// Rules that only exist for suffixes ending in "s"/"ss" must be
|
|
// reachable: the plural pass no longer returns early.
|
|
if stem("happiness") == "happiness" || stem("useless") == "useless" {
|
|
t.Errorf("-ness/-less rules unreachable: %q %q", stem("happiness"), stem("useless"))
|
|
}
|
|
// Domain terms the conservative stemmer must leave recognizable.
|
|
for w, want := range map[string]string{"sigint": "sigint", "nsa": "nsa", "class": "class", "yes": "yes", "tdy": "tdy", "tdi": "tdi"} {
|
|
if got := stem(w); got != want {
|
|
t.Errorf("stem(%q) = %q, want %q", w, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestTokenizeSplitsLikeConcatenation(t *testing.T) {
|
|
// loadData relies on tokenize(a+" "+b) == tokenize(a) ++ tokenize(b).
|
|
a, b := "Müller-Maguhn's XKEYSCORE deck, 2013", "collecting the metadata of targets"
|
|
joined := tokenize(a + " " + b)
|
|
parts := append(tokenize(a), tokenize(b)...)
|
|
if !reflect.DeepEqual(joined, parts) {
|
|
t.Errorf("tokenize(a+b) = %v, parts = %v", joined, parts)
|
|
}
|
|
}
|
|
|
|
func TestParseSimpleQuery(t *testing.T) {
|
|
q, f, c, d := parseSimpleQuery(`program:XKEYSCORE -tempora "cable tapping"~5 date:2013..2015 upstream`)
|
|
wantQ := []string{"XKEYSCORE", `"cable tapping"~5`, "upstream", "tempora"}
|
|
wantF := []string{"PR", "TE", "TE", "TE"}
|
|
wantC := []string{"and", "and", "and", "not"}
|
|
if !reflect.DeepEqual(q, wantQ) || !reflect.DeepEqual(f, wantF) || !reflect.DeepEqual(c, wantC) {
|
|
t.Errorf("got %v %v %v, want %v %v %v", q, f, c, wantQ, wantF, wantC)
|
|
}
|
|
if d.DocStart != "2013" || d.DocEnd != "2015-12-31" {
|
|
t.Errorf("dates = %+v", d)
|
|
}
|
|
// Pure-negative queries get a match-all anchor.
|
|
q, _, c, _ = parseSimpleQuery("-draft")
|
|
if !reflect.DeepEqual(q, []string{"", "draft"}) || !reflect.DeepEqual(c, []string{"and", "not"}) {
|
|
t.Errorf("pure negative: %v %v", q, c)
|
|
}
|
|
}
|
|
|
|
func TestNormalizeClauses(t *testing.T) {
|
|
q, f, c := normalizeClauses([]string{"a", "b"}, []string{"TI"}, nil)
|
|
if !reflect.DeepEqual(f, []string{"TI", "TE"}) || !reflect.DeepEqual(c, []string{"and", "and"}) || len(q) != 2 {
|
|
t.Errorf("got %v %v %v", q, f, c)
|
|
}
|
|
many := make([]string, maxSearchClauses+10)
|
|
q, f, c = normalizeClauses(many, nil, nil)
|
|
if len(q) != maxSearchClauses || len(f) != maxSearchClauses || len(c) != maxSearchClauses {
|
|
t.Errorf("cap not applied: %d %d %d", len(q), len(f), len(c))
|
|
}
|
|
}
|
|
|
|
func TestPaginate(t *testing.T) {
|
|
cases := []struct {
|
|
total, page, perPage int
|
|
start, end, wantPage, wantTotalPages int
|
|
}{
|
|
{45, 1, 20, 0, 20, 1, 3},
|
|
{45, 3, 20, 40, 45, 3, 3},
|
|
{45, 99, 20, 40, 45, 3, 3},
|
|
{45, -5, 20, 0, 20, 1, 3},
|
|
{0, 7, 20, 0, 0, 1, 0},
|
|
{45, 2, 0, 0, 45, 1, 1},
|
|
}
|
|
for _, c := range cases {
|
|
s, e, p, tp := paginate(c.total, c.page, c.perPage)
|
|
if s != c.start || e != c.end || p != c.wantPage || tp != c.wantTotalPages {
|
|
t.Errorf("paginate(%d,%d,%d) = %d,%d,%d,%d; want %d,%d,%d,%d",
|
|
c.total, c.page, c.perPage, s, e, p, tp, c.start, c.end, c.wantPage, c.wantTotalPages)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSafeBackPath(t *testing.T) {
|
|
for in, want := range map[string]string{
|
|
"/doc/x": "/doc/x",
|
|
"//evil.com": "/",
|
|
"/\\evil.com": "/",
|
|
"https://evil": "/",
|
|
"": "/",
|
|
"/search?q=a\r\n": "/",
|
|
} {
|
|
if got := safeBackPath(in); got != want {
|
|
t.Errorf("safeBackPath(%q) = %q, want %q", in, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Tags are curated in the metadata itself, so the
|
|
// server does nothing to them beyond the normalization every list field
|
|
// gets: trim, drop empties, fold case-duplicates keeping the first casing.
|
|
func TestNormalizeListOnTags(t *testing.T) {
|
|
got := normalizeList([]string{" XKS ", "xks", "", "PRISM", "prism"})
|
|
want := []string{"XKS", "PRISM"}
|
|
if !reflect.DeepEqual(got, want) {
|
|
t.Errorf("normalizeList = %v, want %v", got, want)
|
|
}
|
|
if normalizeList(nil) != nil {
|
|
t.Error("nil in should stay nil (JSON null, not [])")
|
|
}
|
|
}
|
|
|
|
func TestClassSortKeyFoldsAliases(t *testing.T) {
|
|
if classSortKey([]string{"TS", "SI", "TK"}) != classSortKey([]string{"TOP SECRET", "COMINT", "Talent Keyhole"}) {
|
|
t.Error("abbreviations and full names must rank identically")
|
|
}
|
|
if classSortKey([]string{"TOP SECRET", "SI"}) <= classSortKey([]string{"TOP SECRET"}) {
|
|
t.Error("a level with SI must outrank the bare level")
|
|
}
|
|
if classSortKey([]string{"NOFORN"}) != "" {
|
|
t.Error("only unranked components must sort with the empties")
|
|
}
|
|
}
|
|
|
|
func TestMarkTermsEscapes(t *testing.T) {
|
|
hl := buildHighlightTerms([]string{"müller says"}, []string{"and"})
|
|
out := string(markTerms(`Müller says <b>hi</b>`, hl))
|
|
if out != `<mark>Müller</mark> <mark>says</mark> <b>hi</b>` {
|
|
t.Errorf("markTerms = %s", out)
|
|
}
|
|
}
|
|
|
|
func TestHeadingLevel(t *testing.T) {
|
|
for in, want := range map[string]int{
|
|
"# Title": 1, "### Deep": 3, "######\tSix": 6, "#": 1,
|
|
"#1": 0, "#hashtag": 0, "####### seven": 0, "no": 0,
|
|
"##########################": 0,
|
|
} {
|
|
if got := headingLevel(in); got != want {
|
|
t.Errorf("headingLevel(%q) = %d, want %d", in, got, want)
|
|
}
|
|
}
|
|
if got := Render("#1 priority\n#2 next"); got != "<p>#1 priority<br>\n#2 next</p>\n" {
|
|
t.Errorf("Render = %q", got)
|
|
}
|
|
}
|
|
|
|
// <br> is the transcription convention for a line break inside a table
|
|
// cell - the one place the table syntax cannot express one. It survives as
|
|
// a break, everything else in the tag vocabulary stays escaped text, and a
|
|
// code span keeps it literal.
|
|
func TestRenderHardBreak(t *testing.T) {
|
|
got := Render("| a | b |\n| --- | --- |\n| one<br>two | <br/>x<br />y |\n")
|
|
for _, want := range []string{"<td>one<br>two</td>", "<td><br>x<br>y</td>"} {
|
|
if !strings.Contains(got, want) {
|
|
t.Errorf("Render table = %q, want %q in it", got, want)
|
|
}
|
|
}
|
|
if got := Render("text <b>bold</b> and <script>x</script>"); strings.Contains(got, "<b>") || strings.Contains(got, "<script>") {
|
|
t.Errorf("only <br> is unescaped, got %q", got)
|
|
}
|
|
if got := Render("a `<br>` b"); !strings.Contains(got, "<code><br></code>") {
|
|
t.Errorf("a code span keeps it literal, got %q", got)
|
|
}
|
|
}
|
|
|
|
func TestDocPathEscapes(t *testing.T) {
|
|
if got := docPath("CBC Cyber Threat/Detection"); got != "/doc/CBC%20Cyber%20Threat%2FDetection" {
|
|
t.Errorf("docPath = %q", got)
|
|
}
|
|
if got := filePath("a b", "c d.pdf"); got != "/files/a%20b/c%20d.pdf" {
|
|
t.Errorf("filePath = %q", got)
|
|
}
|
|
}
|
|
|
|
func TestDateRangeNote(t *testing.T) {
|
|
for in, want := range map[[2]string]string{
|
|
{"", ""}: "",
|
|
{"2013", "2013-12-31"}: "(doc 2013)",
|
|
{"2013", "2015-12-31"}: "(doc 2013 to 2015-12-31)",
|
|
{"", "2014"}: "(doc to 2014)",
|
|
{"2013-06", "2013-06-31"}: "(doc 2013-06)",
|
|
} {
|
|
if got := dateRangeNote("doc", in[0], in[1]); got != want {
|
|
t.Errorf("dateRangeNote(%q, %q) = %q, want %q", in[0], in[1], got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestCollectionFilter(t *testing.T) {
|
|
if collectionFilter(nil) != nil {
|
|
t.Error("no coll params must mean no filter")
|
|
}
|
|
all := map[string][]string{"coll": {"news", "foia", "sidtoday-archive", "bogus"}}
|
|
if collectionFilter(all) != nil {
|
|
t.Error("every collection selected must mean no filter")
|
|
}
|
|
sel := collectionFilter(map[string][]string{"coll": {"foia", "bogus"}})
|
|
if len(sel) != 1 || !sel["foia"] {
|
|
t.Errorf("sel = %v", sel)
|
|
}
|
|
}
|
|
|
|
func TestAgencyName(t *testing.T) {
|
|
for in, want := range map[string]string{
|
|
"CSE": "Communications Security Establishment (Canada)",
|
|
"DoD": "Department of Defense (US)",
|
|
"Booz Allen Hamilton": "",
|
|
"NSA, GCHQ": "National Security Agency (US), Government Communications Headquarters (UK)",
|
|
"DOJ/ODNI": "Department of Justice (US) / Office of the Director of National Intelligence (US)",
|
|
"NSA/Unknown Corp": "National Security Agency (US) / Unknown Corp",
|
|
"": "",
|
|
} {
|
|
if got := agencyName(in); got != want {
|
|
t.Errorf("agencyName(%q) = %q, want %q", in, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestGlossarySearchURL(t *testing.T) {
|
|
e := GlossaryEntry{Term: "AMBASSADORSRECEPTION", Aka: []string{"AMBASSADORS RECEPTION", "X-KEYSCORE"}}
|
|
got := e.SearchURL()
|
|
want := `/search?action=search&do_search=1&fqc=and&fqc=or&fqc=or&fqf=TE&fqf=TE&fqf=TE&fqv=AMBASSADORSRECEPTION&fqv=%22AMBASSADORS+RECEPTION%22&fqv=%22X-KEYSCORE%22`
|
|
if got != want {
|
|
t.Errorf("SearchURL =\n%s\nwant\n%s", got, want)
|
|
}
|
|
// The clauses must round-trip through the search intake as term OR alias.
|
|
sp := readSearchParams(mustQuery(t, got))
|
|
q, f, c := sp.Queries, sp.Fields, sp.Conjs
|
|
if len(q) != 3 || c[1] != "or" || f[2] != "TE" || q[1] != `"AMBASSADORS RECEPTION"` {
|
|
t.Errorf("round-trip: %v %v %v", q, f, c)
|
|
}
|
|
}
|
|
|
|
func testDoc(t *testing.T, js string) Document {
|
|
t.Helper()
|
|
var d Document
|
|
if err := json.Unmarshal([]byte(js), &d); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return d
|
|
}
|
|
|
|
func TestIndexWord(t *testing.T) {
|
|
cases := []struct {
|
|
s, needle string
|
|
from, want int
|
|
}{
|
|
{"the guardian and ard", "ard", 0, 17},
|
|
{"guardian", "ard", 0, -1},
|
|
{"(ard)", "ard", 0, 1},
|
|
{"ardent ard", "ard", 0, 7},
|
|
{"müller ard", "ard", 0, 8},
|
|
{"u.s.a. policy", "u.s.", 0, 0},
|
|
{"ard ard", "ard", 1, 4},
|
|
{"", "ard", 0, -1},
|
|
}
|
|
for _, c := range cases {
|
|
if got := indexWord(c.s, c.needle, c.from); got != c.want {
|
|
t.Errorf("indexWord(%q, %q, %d) = %d, want %d", c.s, c.needle, c.from, got, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestQuotedTextAndExactWords pins the two literal forms: quoted text is
|
|
// found anywhere, like grep -i; an exact value on a text field is whole
|
|
// words only.
|
|
func TestQuotedTextAndExactWords(t *testing.T) {
|
|
for _, c := range []struct {
|
|
content, q string
|
|
want bool
|
|
}{
|
|
{"ing software\" (Mustermann)", `"mann"`, true},
|
|
{"The Guardian", `"ARD"`, true},
|
|
{"cable tappings", `"cable tapping"`, true},
|
|
{"The Guardian", `=ARD`, false},
|
|
{"NDR/ARD", `=ARD`, true},
|
|
{"strap, taps", `=tap`, false},
|
|
{"a tap here", `=tap`, true},
|
|
{"cable tappings", `="cable tapping"`, false},
|
|
{"Cable Tapping programs", `="cable tapping"`, true},
|
|
} {
|
|
// As a text field reads it - and, for quoted text, a metadata
|
|
// field the same way.
|
|
got := fieldByCode["TI"].compile(c.q)(&Document{Title: c.content})
|
|
if _, exact := exactValue(c.q); !exact {
|
|
if asValue := fieldByCode["TA"].compile(c.q)(&Document{Tags: []string{c.content}}); asValue != got {
|
|
t.Errorf("%s in %q: %v as a tag but %v as a title", c.q, c.content, asValue, got)
|
|
}
|
|
}
|
|
if got != c.want {
|
|
t.Errorf("%s in %q = %v, want %v", c.q, c.content, got, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestExactValues pins = on every field kind: one whole value on metadata
|
|
// fields, whole words on the text fields; and quoted text for contrast.
|
|
func TestExactValues(t *testing.T) {
|
|
d := testDoc(t, `{"title": "Guardian Story", "type": "REPORTED",
|
|
"tags": ["CNE implants", "SIGINT", "cable", "tapping"],
|
|
"document": {"creator": "NSA, GCHQ", "classification": ["TOP SECRET", "SI"]},
|
|
"publication": [{"publisher": "ARD", "reporters": ["Poitras, Laura"]}, {"publisher": "The Guardian"}]}`)
|
|
d.SearchText = normFoldString(d.Title)
|
|
for _, c := range []struct {
|
|
field, q string
|
|
want bool
|
|
}{
|
|
{"CR", `="NSA"`, false},
|
|
{"CR", `="nsa, gchq"`, true},
|
|
{"CR", `NSA`, true},
|
|
{"CL", `="SECRET"`, false},
|
|
{"CL", `"SECRET"`, true},
|
|
{"CL", `=si`, true},
|
|
{"TA", `="CNE"`, false},
|
|
{"TA", `"CNE imp"`, true},
|
|
{"TA", `=sigint`, true},
|
|
{"TA", `"cable tapping"`, false}, // quoted text never runs across two tags
|
|
{"PU", `="ARD"`, true},
|
|
{"PU", `="Guardian"`, false},
|
|
{"PU", `"uardi"`, true},
|
|
{"RE", `="Poitras, Laura"`, true},
|
|
{"TY", `=reported`, true},
|
|
{"TI", `="guardian story"`, true},
|
|
{"TI", `=guardian`, true}, // the title is text: whole words
|
|
{"TI", `=guard`, false},
|
|
{"TI", `"guard"`, true},
|
|
{"TE", `="guardian story"`, true},
|
|
{"TE", `=stor`, false},
|
|
{"TE", `"stor"`, true},
|
|
} {
|
|
if got := fieldByCode[c.field].compile(c.q)(&d); got != c.want {
|
|
t.Errorf("%s %s = %v, want %v", c.field, c.q, got, c.want)
|
|
}
|
|
}
|
|
for in, want := range map[string]string{`="a b"`: "a b", "=NSA": "NSA", `= "x" `: "x", "=a b": "", "NSA": "", "=": ""} {
|
|
if got, _ := exactValue(in); got != want {
|
|
t.Errorf("exactValue(%q) = %q, want %q", in, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestParseSimpleQueryExact(t *testing.T) {
|
|
q, f, c, _ := parseSimpleQuery(`publisher:="De Standaard" -tag:=CNE ="all fields" nsa`)
|
|
wantQ := []string{`="De Standaard"`, `="all fields"`, "nsa", "=CNE"}
|
|
wantF := []string{"PU", "TE", "TE", "TA"}
|
|
wantC := []string{"and", "and", "and", "not"}
|
|
if !reflect.DeepEqual(q, wantQ) || !reflect.DeepEqual(f, wantF) || !reflect.DeepEqual(c, wantC) {
|
|
t.Errorf("got %v %v %v, want %v %v %v", q, f, c, wantQ, wantF, wantC)
|
|
}
|
|
pos, negs := rowTerms(`="a b" -"c d" -=e`)
|
|
if !reflect.DeepEqual(pos, []string{`="a b"`}) || !reflect.DeepEqual(negs, []string{`"c d"`, "=e"}) {
|
|
t.Errorf("rowTerms = %q %v", pos, negs)
|
|
}
|
|
}
|
|
|
|
// TestQueryValuesSplitOnAnyWhitespace: a newline (a crafted ?q=) ends a
|
|
// value as a space does, so it cannot carry the next operator or -term
|
|
// inside it.
|
|
func TestQueryValuesSplitOnAnyWhitespace(t *testing.T) {
|
|
q, f, c, _ := parseSimpleQuery("program:XKEYSCORE\ncountry:Germany\r\n-tempora")
|
|
wantQ := []string{"XKEYSCORE", "Germany", "tempora"}
|
|
wantF := []string{"PR", "CO", "TE"}
|
|
wantC := []string{"and", "and", "not"}
|
|
if !reflect.DeepEqual(q, wantQ) || !reflect.DeepEqual(f, wantF) || !reflect.DeepEqual(c, wantC) {
|
|
t.Errorf("got %v %v %v, want %v %v %v", q, f, c, wantQ, wantF, wantC)
|
|
}
|
|
if pos, negs := rowTerms("cable\n-tapping"); !reflect.DeepEqual(pos, []string{"cable"}) || !reflect.DeepEqual(negs, []string{"tapping"}) {
|
|
t.Errorf("rowTerms = %q %v", pos, negs)
|
|
}
|
|
if _, ok := exactValue("=cable\ntapping"); ok {
|
|
t.Error("an unquoted exact value spanning a newline is two words, not one value")
|
|
}
|
|
}
|
|
|
|
func TestHighlightsAndSnippets(t *testing.T) {
|
|
hl := buildHighlightTerms([]string{`"ard"`}, []string{"and"})
|
|
if got := string(markTerms("The Guardian and ARD", hl)); got != "The Gu<mark>ard</mark>ian and <mark>ARD</mark>" {
|
|
t.Errorf("quoted highlight = %s", got)
|
|
}
|
|
hl = buildHighlightTerms([]string{`=ard`}, []string{"and"})
|
|
if got := string(markTerms("The Guardian and ARD", hl)); got != "The Guardian and <mark>ARD</mark>" {
|
|
t.Errorf("exact highlight = %s", got)
|
|
}
|
|
body := strings.Repeat("standard ", 20) + "ARD"
|
|
if s := getSnippets(body, `=ard`, 3); len(s) != 1 || !strings.HasPrefix(s[0], "... ") || !strings.HasSuffix(s[0], "ARD") {
|
|
t.Errorf("exact snippets = %q", s)
|
|
}
|
|
if s := getSnippets(body, `"ard"`, 3); len(s) == 0 || strings.HasPrefix(s[0], "... ") {
|
|
t.Errorf("quoted snippets must anchor on the first substring hit: %q", s)
|
|
}
|
|
}
|
|
|
|
// TestFacetGroupsCountCaseInsensitively: the refine links match ignoring
|
|
// case, so spellings that differ only by case must be one count.
|
|
func TestFacetGroupsCountCaseInsensitively(t *testing.T) {
|
|
docs := []*Document{
|
|
ptr(testDoc(t, `{"document": {"classification": ["TOP SECRET"]}}`)),
|
|
ptr(testDoc(t, `{"document": {"classification": ["Top Secret", "SI"]}}`)),
|
|
ptr(testDoc(t, `{"document": {"classification": ["SI"]}}`)),
|
|
}
|
|
got := facetGroups(docs, []facetGroup{{"Classification", "CL"}}, "/search?", facetValues)
|
|
if len(got) != 1 || len(got[0].Values) != 2 {
|
|
t.Fatalf("facetGroups = %+v", got)
|
|
}
|
|
for _, v := range got[0].Values {
|
|
if v.Count != 2 {
|
|
t.Errorf("%s counted %d, want 2", v.Name, v.Count)
|
|
}
|
|
}
|
|
if v := got[0].Values; !strings.Contains(v[0].URL+v[1].URL, "fqv=%3D%22TOP+SECRET%22") {
|
|
t.Errorf("links = %q %q", v[0].URL, v[1].URL)
|
|
}
|
|
}
|
|
|
|
func TestWildcard(t *testing.T) {
|
|
for _, c := range []struct {
|
|
w string
|
|
frag string
|
|
openStart, openEnd bool
|
|
ok bool
|
|
}{
|
|
{"*mann", "mann", true, false, true},
|
|
{"tap*", "tap", false, true, true},
|
|
{"*Keys*", "keys", true, true, true},
|
|
{"ab*", "ab", false, true, true},
|
|
{"*ab", "", true, false, true},
|
|
{"*", "", true, true, true},
|
|
{"a*b", "", false, false, false},
|
|
{"tap", "", false, false, false},
|
|
} {
|
|
frag, s, e, ok := wildcard(c.w)
|
|
if frag != c.frag || s != c.openStart || e != c.openEnd || ok != c.ok {
|
|
t.Errorf("wildcard(%q) = %q %v %v %v", c.w, frag, s, e, ok)
|
|
}
|
|
}
|
|
for _, c := range []struct {
|
|
s, frag string
|
|
openStart, openEnd bool
|
|
want int
|
|
}{
|
|
{"mustermann and mannheim", "mann", true, false, 6},
|
|
{"mannheim", "mann", true, false, -1},
|
|
{"mann", "mann", true, false, 0},
|
|
{"tapping", "tap", false, true, 0},
|
|
{"strap", "tap", false, true, -1},
|
|
{"x keyscores", "keys", true, true, 2},
|
|
{"x-keyscore", "key", false, true, 2},
|
|
} {
|
|
if got := indexWild(c.s, c.frag, 0, c.openStart, c.openEnd); got != c.want {
|
|
t.Errorf("indexWild(%q, %q, %v, %v) = %d, want %d", c.s, c.frag, c.openStart, c.openEnd, got, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestWildcardSearch: wildcards match the unstemmed text on every field,
|
|
// and highlighting and snippets follow the same rule.
|
|
func TestWildcardSearch(t *testing.T) {
|
|
d := testDoc(t, `{"title": "Report", "publication": [{"publisher": "Der Spiegel", "reporters": ["Mustermann, Erika"]}]}`)
|
|
text := "Erika Mustermann reports on cable tapping"
|
|
d.SearchText = normFoldString(text)
|
|
d.StemmedText = strings.Join(tokenize(text), " ")
|
|
d.FoldedFull, d.StemmedFull = d.SearchText, d.StemmedText
|
|
for _, c := range []struct {
|
|
field, q string
|
|
want bool
|
|
}{
|
|
{"TE", "*mann", true},
|
|
{"TE", "mann", false},
|
|
{"TE", "tapping*", true},
|
|
{"TE", "tapped*", false},
|
|
{"TE", "*ping", true},
|
|
{"TE", "*sterm*", true},
|
|
{"TE", "cable *mann", true},
|
|
{"TE", "spiegel *mann", false},
|
|
{"TE", "*ab", false}, // too short to search: matches nothing
|
|
{"FT", "*mann", true},
|
|
{"RE", "*mann", true},
|
|
{"RE", "*man", false},
|
|
{"RE", "muster*", true},
|
|
{"PU", "*egel", true},
|
|
{"PU", "spieg*", true},
|
|
} {
|
|
if got := fieldByCode[c.field].compile(c.q)(&d); got != c.want {
|
|
t.Errorf("%s %s = %v, want %v", c.field, c.q, got, c.want)
|
|
}
|
|
}
|
|
hl := buildHighlightTerms([]string{"*mann tap*"}, []string{"and"})
|
|
if got := string(markTerms("Mustermann on tapping, not strap", hl)); got != "<mark>Mustermann</mark> on <mark>tapping</mark>, not strap" {
|
|
t.Errorf("wildcard highlight = %s", got)
|
|
}
|
|
body := strings.Repeat("filler ", 30) + "Mustermann"
|
|
if s := getSnippets(body, "*mann", 3); len(s) != 1 || !strings.HasSuffix(s[0], "Mustermann") {
|
|
t.Errorf("snippets = %q", s)
|
|
}
|
|
if s := getSnippets("a cab rides about", "*ab", 3); s != nil {
|
|
t.Errorf("a wildcard that matches nothing anchored snippets: %q", s)
|
|
}
|
|
if s := getSnippets(strings.Repeat("filler ", 30)+"strap and tapping", "tap*", 3); len(s) != 1 || !strings.HasSuffix(s[0], "tapping") {
|
|
t.Errorf("a wildcard anchors where it matches, not inside strap: %q", s)
|
|
}
|
|
if corpus().tfidf.scorer("*mann")(&d) != 0 { // no index entry for the test doc
|
|
t.Error("a document missing from the index must score 0")
|
|
}
|
|
}
|
|
|
|
func TestSortDocsKeysStableAndEmptiesLast(t *testing.T) {
|
|
docs := []*Document{ptr(testDoc(t, `{"title": "beta"}`)), ptr(testDoc(t, `{"title": "Alpha"}`)), ptr(testDoc(t, `{"title": "alpha"}`)), ptr(testDoc(t, `{"title": "Gamma"}`))}
|
|
sortDocs(docs, "title", "")
|
|
if got := []string{docs[0].Title, docs[1].Title, docs[2].Title, docs[3].Title}; !reflect.DeepEqual(got, []string{"Alpha", "alpha", "beta", "Gamma"}) {
|
|
t.Errorf("title sort = %v", got)
|
|
}
|
|
dated := []*Document{ptr(testDoc(t, `{"document": {"date_created": ""}}`)), ptr(testDoc(t, `{"document": {"date_created": "2010"}}`)), ptr(testDoc(t, `{"document": {"date_created": "2013"}}`))}
|
|
sortDocs(dated, "date", "desc")
|
|
if got := []string{dated[0].DocMeta.DateCreated, dated[1].DocMeta.DateCreated, dated[2].DocMeta.DateCreated}; !reflect.DeepEqual(got, []string{"2013", "2010", ""}) {
|
|
t.Errorf("date desc = %v", got)
|
|
}
|
|
}
|
|
|
|
func TestTopFacets(t *testing.T) {
|
|
got := topFacets(map[string]int{"b": 2, "A": 2, "c": 5, "d": 1}, 3)
|
|
if want := []facetEntry{{Name: "c", Count: 5}, {Name: "A", Count: 2}, {Name: "b", Count: 2}}; !reflect.DeepEqual(got, want) {
|
|
t.Errorf("topFacets = %v, want %v", got, want)
|
|
}
|
|
}
|
|
|
|
// Every ranked list breaks count ties the same way: accents and case
|
|
// folded, so "Åkesson" sits with the A's, then the raw spelling.
|
|
func TestRankCounts(t *testing.T) {
|
|
m := map[string]int{"Zeta": 1, "Åkesson": 1, "bates": 1, "Bates": 1, "top": 3}
|
|
if got, want := rankCounts(m, -1), []string{"top", "Åkesson", "Bates", "bates", "Zeta"}; !reflect.DeepEqual(got, want) {
|
|
t.Errorf("rankCounts = %v, want %v", got, want)
|
|
}
|
|
if got := rankCounts(m, 2); len(got) != 2 {
|
|
t.Errorf("rankCounts(m, 2) kept %d", len(got))
|
|
}
|
|
}
|
|
|
|
// A sortable heading links to its own first direction, and the active one
|
|
// to the flip; a fixed column never flips; a plain heading has no link.
|
|
func TestSortHeads(t *testing.T) {
|
|
cols := []sortCol{
|
|
{Key: "title", Label: "Title"},
|
|
{Key: "count", Label: "Count", DescFirst: true},
|
|
{Key: "rank", Label: "Rank", DescFirst: true, Fixed: true},
|
|
{Label: "Notes"},
|
|
}
|
|
heads := sortHeads("/x?", cols, "title", false)
|
|
for i, want := range []string{"/x?sort=title&order=desc", "/x?sort=count&order=desc", "/x?sort=rank", ""} {
|
|
if heads[i].URL != want {
|
|
t.Errorf("heading %d links %q, want %q", i, heads[i].URL, want)
|
|
}
|
|
}
|
|
if !heads[0].Active || heads[0].Desc || heads[1].Active {
|
|
t.Errorf("title asc: active flags %+v", heads[:2])
|
|
}
|
|
heads = sortHeads("/x?", cols, "count", true)
|
|
if !heads[1].Active || !heads[1].Desc || heads[1].URL != "/x?sort=count&order=asc" {
|
|
t.Errorf("count desc: %+v", heads[1])
|
|
}
|
|
heads = sortHeads("/x?", cols, "rank", true)
|
|
if !heads[2].Active || heads[2].URL != "/x?sort=rank" {
|
|
t.Errorf("fixed column: %+v", heads[2])
|
|
}
|
|
}
|
|
|
|
// A document found only through its metadata - here a reporter's name -
|
|
// still gets a relevance score.
|
|
func TestTFIDFScoresAllMetadata(t *testing.T) {
|
|
d := testDoc(t, `{"id": "a", "title": "Slides", "publication": [{"publisher": "X", "reporters": ["Poitras, Laura"]}],
|
|
"document": {"classification": ["TOP SECRET"]}, "long_description": "A cablegram summary."}`)
|
|
other := testDoc(t, `{"id": "b", "title": "Other"}`)
|
|
for _, x := range []*Document{&d, &other} {
|
|
x.StemmedTitle = strings.Join(tokenize(x.Title), " ")
|
|
x.StemmedDesc = strings.Join(tokenize(x.Description+" "+x.LongDescription), " ")
|
|
}
|
|
idx := buildTFIDF([]Document{d, other})
|
|
for _, q := range []string{"poitras", "secret", "cablegram"} {
|
|
if idx.scorer(q)(&d) <= 0 {
|
|
t.Errorf("score(%q) = 0 for a document whose metadata holds it", q)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSortCategoryEntries(t *testing.T) {
|
|
names := func(es []categoryEntry) []string {
|
|
var out []string
|
|
for _, e := range es {
|
|
out = append(out, e.Name)
|
|
}
|
|
return out
|
|
}
|
|
es := []categoryEntry{{Name: "Zeta", Count: 1, fold: "zeta"}, {Name: "Émile", Count: 3, fold: "emile"}, {Name: "alpha", Count: 3, fold: "alpha"}}
|
|
for _, c := range []struct {
|
|
sortBy string
|
|
desc bool
|
|
want []string
|
|
}{
|
|
{"name", false, []string{"alpha", "Émile", "Zeta"}},
|
|
{"name", true, []string{"Zeta", "Émile", "alpha"}},
|
|
{"count", true, []string{"alpha", "Émile", "Zeta"}},
|
|
{"count", false, []string{"Zeta", "alpha", "Émile"}},
|
|
} {
|
|
sortCategoryEntries(es, c.sortBy, c.desc)
|
|
if got := names(es); !reflect.DeepEqual(got, c.want) {
|
|
t.Errorf("%s desc=%v = %v, want %v", c.sortBy, c.desc, got, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Browse and search sort by the same rules: one key function, one
|
|
// empties-last set. The regression this guards is the split brain that
|
|
// existed before - search knew four fields and ignored empties-last, so
|
|
// /search?sort=date led with undated documents while /browse?sort=date put
|
|
// them last.
|
|
func TestSortKeysShared(t *testing.T) {
|
|
dated := Document{ID: "a", Title: "Alpha"}
|
|
dated.DocMeta.DateCreated = "2013-06-01"
|
|
undated := Document{ID: "b", Title: "Bravo"}
|
|
|
|
for _, order := range []string{"asc", "desc"} {
|
|
docs := []*Document{&undated, &dated}
|
|
sortDocs(docs, "date", order)
|
|
if docs[0].ID != "a" {
|
|
t.Errorf("sortDocs date %s: undated first", order)
|
|
}
|
|
scored := []ScoredDoc{{Doc: &undated}, {Doc: &dated}}
|
|
got := sortScored(scored, "date", order)
|
|
if got[0].Doc.ID != "a" {
|
|
t.Errorf("search sort date %s: undated first", order)
|
|
}
|
|
}
|
|
|
|
// Every field a list page can sort by is a field search understands.
|
|
for _, col := range docTableCols {
|
|
if col.Key != "" && docSortByKey[col.Key] == nil {
|
|
t.Errorf("table column %q has no sort key", col.Key)
|
|
}
|
|
}
|
|
for _, o := range sortOptions {
|
|
if o.Key != "relevance" && docSortByKey[o.Key] == nil {
|
|
t.Errorf("sort option %q has no sort key", o.Key)
|
|
}
|
|
}
|
|
}
|
|
|
|
// ptr is the tests' shorthand for taking the address of a fresh document.
|
|
func ptr(d Document) *Document { return &d }
|
|
|
|
// containsTokens is the whole-token search the matcher and the scorer share.
|
|
// It must behave exactly like the padded strings.Contains it replaced -
|
|
// including at the ends of the stream, where the padding used to help.
|
|
func TestContainsTokens(t *testing.T) {
|
|
const stream = "cabl tap program alpha"
|
|
for _, phrase := range []string{"cabl", "alpha", "tap program", "cabl tap program alpha", ""} {
|
|
if !containsTokens(stream, phrase) {
|
|
t.Errorf("containsTokens(%q) = false, want true", phrase)
|
|
}
|
|
}
|
|
for _, phrase := range []string{"cab", "lpha", "ap", "program tap", "cabl program", "zeta"} {
|
|
if containsTokens(stream, phrase) {
|
|
t.Errorf("containsTokens(%q) = true, want false", phrase)
|
|
}
|
|
}
|
|
// The old implementation, as an oracle, over a few awkward streams.
|
|
padded := func(stream, phrase string) bool {
|
|
return strings.Contains(" "+stream+" ", " "+phrase+" ")
|
|
}
|
|
for _, s := range []string{"", "a", "a b", "aa aa", "x aa aa x", "aa"} {
|
|
for _, p := range []string{"a", "aa", "a b", "aa aa", "b"} {
|
|
if got, want := containsTokens(s, p), padded(s, p); got != want {
|
|
t.Errorf("containsTokens(%q, %q) = %v, oracle says %v", s, p, got, want)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// A search over the clause cap is refused, never truncated: cutting from
|
|
// the end dropped the negations and the newest row, so the results silently
|
|
// broadened.
|
|
func TestSearchClauseLimitRefused(t *testing.T) {
|
|
rows := func(n int) string {
|
|
v := url.Values{}
|
|
for i := 0; i < n; i++ {
|
|
v.Add("fqv", "prism")
|
|
}
|
|
return v.Encode()
|
|
}
|
|
phrases := strings.Repeat(`"a b" `, maxSearchClauses) + "cable -snowden" // 24 phrases + words + negation
|
|
|
|
for _, path := range []string{
|
|
"/api/search?" + rows(maxSearchClauses+1),
|
|
"/api/search?q=" + url.QueryEscape(phrases),
|
|
} {
|
|
if w := get(t, path); w.Code != 400 || !strings.Contains(w.Body.String(), "too many clauses") {
|
|
t.Errorf("%s: %d %s, want 400 too many clauses", path[:20], w.Code, w.Body.String())
|
|
}
|
|
}
|
|
if w := get(t, "/api/search?"+rows(maxSearchClauses)); w.Code != 200 {
|
|
t.Errorf("a search at the cap: %d, want 200", w.Code)
|
|
}
|
|
|
|
page := get(t, "/search?do_search=1&"+rows(maxSearchClauses+1)).Body.String()
|
|
if !strings.Contains(page, "Search not run") || strings.Contains(page, "results-heading") {
|
|
t.Error("search page ran an over-cap search, or did not say it was refused")
|
|
}
|
|
if n := strings.Count(page, `name="fqv"`); n != maxSearchClauses {
|
|
t.Errorf("refused search renders %d rows, want the first %d", n, maxSearchClauses)
|
|
}
|
|
|
|
// At the cap, "+ Add row" is neither offered nor honoured.
|
|
full := get(t, "/search?action=add&"+rows(maxSearchClauses)).Body.String()
|
|
if n := strings.Count(full, `name="fqv"`); n != maxSearchClauses {
|
|
t.Errorf("add at the cap renders %d rows, want %d", n, maxSearchClauses)
|
|
}
|
|
if strings.Contains(full, `value="add"`) {
|
|
t.Error("+ Add row offered at the clause cap")
|
|
}
|
|
if !strings.Contains(get(t, "/search?"+rows(1)).Body.String(), `value="add"`) {
|
|
t.Error("+ Add row missing below the cap")
|
|
}
|
|
}
|
|
|
|
// The clause cap alone did not bound a search's work: one row can carry
|
|
// thousands of -terms. Words are capped across the whole search.
|
|
func TestSearchWordLimitRefused(t *testing.T) {
|
|
words := func(n int, prefix string) string {
|
|
ws := make([]string, n)
|
|
for i := range ws {
|
|
ws[i] = prefix + "zzq" + strconv.Itoa(i)
|
|
}
|
|
return strings.Join(ws, " ")
|
|
}
|
|
row := func(v string) string { return url.Values{"fqv": {v}}.Encode() }
|
|
over, at := words(maxSearchWords+1, "-"), words(maxSearchWords, "-")
|
|
|
|
for _, path := range []string{
|
|
"/api/search?" + row(over),
|
|
"/api/search?q=" + url.QueryEscape(`"`+words(maxSearchWords, "")+`" cable`),
|
|
} {
|
|
if w := get(t, path); w.Code != 400 || !strings.Contains(w.Body.String(), "too many words") {
|
|
t.Errorf("%s: %d %s, want 400 too many words", path[:20], w.Code, w.Body.String())
|
|
}
|
|
}
|
|
if w := get(t, "/api/search?"+row(at)); w.Code != 200 {
|
|
t.Errorf("a search at the word cap: %d, want 200", w.Code)
|
|
}
|
|
page := get(t, "/search?do_search=1&"+row(over)).Body.String()
|
|
if !strings.Contains(page, "Search not run") || strings.Contains(page, "results-heading") {
|
|
t.Error("search page ran an over-cap search, or did not say it was refused")
|
|
}
|
|
|
|
// Refine links that would take the search past the cap are not offered.
|
|
facets := []searchFacet{
|
|
{Label: "Programs", Values: []facetEntry{{Name: "PRISM"}, {Name: "Poitras, Laura"}}},
|
|
{Label: "Journalists", Values: []facetEntry{{Name: "Greenwald, Glenn"}}},
|
|
}
|
|
got := facetsWithinWords(facets, 1)
|
|
if len(got) != 1 || len(got[0].Values) != 1 || got[0].Values[0].Name != "PRISM" {
|
|
t.Errorf("facetsWithinWords(room 1) = %+v, want only PRISM", got)
|
|
}
|
|
}
|
|
|
|
// TestSnippetAnchorsOnWords: a bare word anchors where the matcher and the
|
|
// highlighter find it - the whole word or another form of it - never inside
|
|
// a longer word; stopwords and single letters anchor only when nothing else
|
|
// in the query can.
|
|
func TestSnippetAnchorsOnWords(t *testing.T) {
|
|
pad := strings.Repeat("filler ", 30)
|
|
for _, c := range []struct{ body, q, want string }{
|
|
{"especially special " + pad + "the CIA", "cia", "the CIA"},
|
|
{"the " + pad + "NSA", "the nsa", "NSA"},
|
|
{"sport " + pad + "the spied on", "spies", "spied on"},
|
|
{pad + "the U.S. said", "u.s.", "U.S. said"},
|
|
} {
|
|
s := getSnippets(c.body, c.q, 1)
|
|
if len(s) != 1 || !strings.HasSuffix(s[0], c.want) {
|
|
t.Errorf("%q: snippets %q, want one ending %q", c.q, s, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A publication date is tied to each journalist or publisher row on its
|
|
// own: two journalists bylined on different entries both match a range
|
|
// that covers both entries, and a proximity value reads as its phrase, as
|
|
// it does without the date.
|
|
func TestPubDateTiesEachRow(t *testing.T) {
|
|
var d Document
|
|
if err := json.Unmarshal([]byte(`{"publication": [
|
|
{"publisher": "Der Spiegel", "date_published": "2014-01-10", "reporters": ["Laura Poitras"]},
|
|
{"publisher": "The Intercept", "date_published": "2014-02-10", "reporters": ["Glenn Greenwald"]},
|
|
{"publisher": "Le Monde", "date_published": "2019-05-01", "reporters": ["Jacques Follorou"]}
|
|
]}`), &d); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, c := range []struct {
|
|
queries, fields, conjs []string
|
|
start, end string
|
|
want bool
|
|
}{
|
|
{[]string{"Poitras", "Greenwald"}, []string{"RE", "RE"}, []string{"and", "and"}, "2014", "2014-12-31", true},
|
|
{[]string{"Poitras", "Intercept"}, []string{"RE", "PU"}, []string{"and", "and"}, "2014", "2014-12-31", true},
|
|
{[]string{"Poitras", "Follorou"}, []string{"RE", "RE"}, []string{"and", "and"}, "2014", "2014-12-31", false},
|
|
{[]string{"Follorou"}, []string{"RE"}, []string{"and"}, "2014", "2014-12-31", false},
|
|
{[]string{`"laura poitras"~3`}, []string{"RE"}, []string{"and"}, "2014", "2014-12-31", true},
|
|
{[]string{`"laura poitras"~3`}, []string{"RE"}, []string{"and"}, "2019", "2019-12-31", false},
|
|
} {
|
|
got := matchDates(&d, "", "", c.start, c.end, entryRows(c.queries, c.fields, c.conjs))
|
|
if got != c.want {
|
|
t.Errorf("%v %v %s..%s: %v, want %v", c.fields, c.queries, c.start, c.end, got, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// foldedLen is the length of a rune's normal form, which the snippet walk
|
|
// uses to map offsets in the folded text back to the original.
|
|
func TestFoldedLen(t *testing.T) {
|
|
for r := rune(0); r <= 0x2FFFF; r++ {
|
|
if r >= 0xD800 && r <= 0xDFFF {
|
|
continue // surrogates: never the result of ranging over a string
|
|
}
|
|
if got, want := foldedLen(r), len(foldRune(r)); got != want {
|
|
t.Fatalf("foldedLen(%U) = %d, foldRune gives %d bytes", r, got, want)
|
|
}
|
|
}
|
|
// The normal form built rune by rune is normFoldString, and its
|
|
// position map points at the rune each byte came from.
|
|
const s = "Müller-Maguhn's ÆSIR ẞtraße, SÃO paulo"
|
|
norm, pos := normFold(s)
|
|
if norm != normFoldString(s) || len(pos) != len(norm)+1 || pos[len(norm)] != len(s) {
|
|
t.Errorf("normFold(%q) = %q with %d positions; normFoldString gives %q", s, norm, len(pos), normFoldString(s))
|
|
}
|
|
}
|
|
|
|
// A document's transcript is folded once, at load: its snippets come off
|
|
// that fold and are the ones folding it again would give.
|
|
func TestBodySnippetsUseTheLoadedFold(t *testing.T) {
|
|
buildFixture(t)
|
|
checked := 0
|
|
for i := range corpus().archive.Docs {
|
|
d := &corpus().archive.Docs[i]
|
|
for _, q := range []string{"collection", "the", `"upstream"`, "müller", "tap*", "unredacted"} {
|
|
if got, want := d.bodySnippets(q, 3), getSnippets(d.FullText, q, 3); !reflect.DeepEqual(got, want) {
|
|
t.Errorf("%s, %q: %q from the loaded fold, %q from the text", d.ID, q, got, want)
|
|
}
|
|
checked++
|
|
}
|
|
if d.FullText != "" && d.foldedBody() != normFoldString(d.FullText) {
|
|
t.Errorf("%s: foldedBody is not the fold of its text", d.ID)
|
|
}
|
|
}
|
|
if checked == 0 {
|
|
t.Fatal("no fixture documents")
|
|
}
|
|
// A document the loader never saw folds on demand.
|
|
loose := Document{FullText: "Some LOOSE text"}
|
|
if loose.foldedBody() != "some loose text" {
|
|
t.Errorf("foldedBody of a hand-built document = %q", loose.foldedBody())
|
|
}
|
|
}
|
|
|
|
// One rule says when two spellings are one metadata value: valueKey. The
|
|
// allocation-free comparison must agree with it.
|
|
func TestSameValue(t *testing.T) {
|
|
for _, c := range []struct {
|
|
a, b string
|
|
want bool
|
|
}{
|
|
{"NSA", "nsa", true},
|
|
{" The Intercept ", "the intercept", true},
|
|
{"The Intercept", "The Guardian", false},
|
|
{"NSA", "NSAW", false},
|
|
{"Müller-Maguhn, Andy", "müller-maguhn, andy", true},
|
|
{"Müller", "Muller", false}, // a value's accents are part of it
|
|
{"", " ", true},
|
|
} {
|
|
if got := sameValue(c.a, c.b); got != c.want || got != (valueKey(c.a) == valueKey(c.b)) {
|
|
t.Errorf("sameValue(%q, %q) = %v, want %v; the keys are %q and %q", c.a, c.b, got, c.want, valueKey(c.a), valueKey(c.b))
|
|
}
|
|
}
|
|
}
|
|
|
|
// What the loader keeps for the matchers is what reading the document
|
|
// gives: each field's values, the document's terms as its all-fields
|
|
// stream holds them, and the folds of its title and descriptions.
|
|
func TestLoadedFieldsAreTheDocuments(t *testing.T) {
|
|
buildFixture(t)
|
|
for i := range corpus().archive.Docs {
|
|
d := &corpus().archive.Docs[i]
|
|
if d.fields == nil || d.terms == nil {
|
|
t.Fatalf("%s: loaded without its fields or its terms", d.ID)
|
|
}
|
|
for fi := range searchFields {
|
|
f := &searchFields[fi]
|
|
if f.read == nil {
|
|
continue
|
|
}
|
|
want := f.read(d)
|
|
if got := f.Values(d); !reflect.DeepEqual(got, want) && len(got)+len(want) > 0 {
|
|
t.Errorf("%s %s: kept %q, the document reads %q", d.ID, f.Code, got, want)
|
|
}
|
|
if got, want := d.fields.folded[fi], normFoldString(strings.Join(want, valueSep)); got != want {
|
|
t.Errorf("%s %s: kept the fold %q, want %q", d.ID, f.Code, got, want)
|
|
}
|
|
}
|
|
if d.titleFold() != normFoldString(d.Title) || d.descFold() != normFoldString(d.Description+valueSep+d.LongDescription) {
|
|
t.Errorf("%s: the kept title or description fold is not the fold of the text", d.ID)
|
|
}
|
|
// The index entry holds exactly the tokens of the all-fields stream,
|
|
// which is what lets a one-word all-fields clause ask it.
|
|
stream := map[string]bool{}
|
|
eachToken(d.StemmedText, func(w string) { stream[w] = w != valueSep })
|
|
for w := range d.terms {
|
|
if !stream[w] {
|
|
t.Errorf("%s: %q is in the index entry and not in the all-fields stream", d.ID, w)
|
|
}
|
|
}
|
|
for w, term := range stream {
|
|
if _, ok := d.terms[w]; term && !ok {
|
|
t.Errorf("%s: %q is in the all-fields stream and not in the index entry", d.ID, w)
|
|
}
|
|
}
|
|
}
|
|
// A document the loader never saw is read as it stands.
|
|
loose := testDoc(t, `{"title": "Loose", "tags": ["One", "two"], "publication": [{"publisher": "Outlet", "reporters": ["A", "a", "B"]}]}`)
|
|
if got := loose.JournalistsDeduped(); !reflect.DeepEqual(got, []string{"A", "B"}) {
|
|
t.Errorf("JournalistsDeduped of a hand-built document = %q", got)
|
|
}
|
|
if !fieldByCode["TA"].compile(`="one"`)(&loose) || fieldByCode["TA"].compile(`=three`)(&loose) {
|
|
t.Error("a hand-built document's tags should match by whole value")
|
|
}
|
|
}
|