snowchive-server/tests/search_test.go
2026-11-19 04:40:59 +02:00

1000 lines
38 KiB
Go

package main
import (
"encoding/json"
"net/url"
"reflect"
"strconv"
"strings"
"testing"
)
func mustQuery(t *testing.T, u string) url.Values {
t.Helper()
parsed, err := url.Parse(u)
if err != nil {
t.Fatal(err)
}
return parsed.Query()
}
// TestStemUnifiesForms pins the stemmer's contract: the inflections a
// reader is likely to type must land on the same token as the base word,
// because the index and the query both go through stem().
func TestStemUnifiesForms(t *testing.T) {
groups := [][]string{
{"operation", "operations", "operational"},
{"communication", "communications"},
{"collection", "collections"},
{"collecting", "collected"},
{"agency", "agencies"},
{"surveillance", "surveilling"},
{"target", "targets", "targeting", "targeted"},
{"meeting", "meetings"},
{"computer", "computers"},
{"process", "processes"},
{"study", "studies"},
{"tie", "ties", "tied"},
{"lie", "lies", "lied"},
{"spy", "spies", "spied", "spying"},
{"fly", "flies", "flying", "flyer"},
{"study", "studying", "studied"},
{"control", "controlled", "controlling", "controls"},
{"install", "installed", "installing"},
{"cancel", "cancelled", "canceled"},
{"fall", "falling", "falls"},
{"kill", "killed", "killing"},
// A plural in -ies takes its singular's path through the -ity and
// -ly rules, and a word in -ie meets its plural.
{"capability", "capabilities"},
{"vulnerability", "vulnerabilities"},
{"activity", "activities"},
{"family", "families"},
{"anomaly", "anomalies"},
{"cookie", "cookies"},
// Verbs in -ply are not adverbs.
{"apply", "applies", "applied", "applying"},
{"supply", "supplies", "supplied", "supplying", "supplier"},
{"comply", "complies", "complied", "complying", "compliance"},
{"reply", "replies", "replied"},
}
for _, g := range groups {
want := stem(g[0])
for _, w := range g[1:] {
if got := stem(w); got != want {
t.Errorf("stem(%q) = %q, stem(%q) = %q; want equal", w, got, g[0], want)
}
}
}
// Words a stripped ending must not turn into each other.
for _, p := range [][2]string{
{"apply", "app"}, {"reply", "rep"}, {"supply", "supp"}, {"early", "ear"},
{"italy", "ita"}, {"july", "julie"},
} {
if stem(p[0]) == stem(p[1]) {
t.Errorf("stem(%q) = stem(%q) = %q; want them apart", p[0], p[1], stem(p[0]))
}
}
// Rules that only exist for suffixes ending in "s"/"ss" must be
// reachable: the plural pass no longer returns early.
if stem("happiness") == "happiness" || stem("useless") == "useless" {
t.Errorf("-ness/-less rules unreachable: %q %q", stem("happiness"), stem("useless"))
}
// Domain terms the conservative stemmer must leave recognizable.
for w, want := range map[string]string{"sigint": "sigint", "nsa": "nsa", "class": "class", "yes": "yes", "tdy": "tdy", "tdi": "tdi"} {
if got := stem(w); got != want {
t.Errorf("stem(%q) = %q, want %q", w, got, want)
}
}
}
func TestTokenizeSplitsLikeConcatenation(t *testing.T) {
// loadData relies on tokenize(a+" "+b) == tokenize(a) ++ tokenize(b).
a, b := "Müller-Maguhn's XKEYSCORE deck, 2013", "collecting the metadata of targets"
joined := tokenize(a + " " + b)
parts := append(tokenize(a), tokenize(b)...)
if !reflect.DeepEqual(joined, parts) {
t.Errorf("tokenize(a+b) = %v, parts = %v", joined, parts)
}
}
func TestParseSimpleQuery(t *testing.T) {
q, f, c, d := parseSimpleQuery(`program:XKEYSCORE -tempora "cable tapping"~5 date:2013..2015 upstream`)
wantQ := []string{"XKEYSCORE", `"cable tapping"~5`, "upstream", "tempora"}
wantF := []string{"PR", "TE", "TE", "TE"}
wantC := []string{"and", "and", "and", "not"}
if !reflect.DeepEqual(q, wantQ) || !reflect.DeepEqual(f, wantF) || !reflect.DeepEqual(c, wantC) {
t.Errorf("got %v %v %v, want %v %v %v", q, f, c, wantQ, wantF, wantC)
}
if d.DocStart != "2013" || d.DocEnd != "2015-12-31" {
t.Errorf("dates = %+v", d)
}
// Pure-negative queries get a match-all anchor.
q, _, c, _ = parseSimpleQuery("-draft")
if !reflect.DeepEqual(q, []string{"", "draft"}) || !reflect.DeepEqual(c, []string{"and", "not"}) {
t.Errorf("pure negative: %v %v", q, c)
}
}
func TestNormalizeClauses(t *testing.T) {
q, f, c := normalizeClauses([]string{"a", "b"}, []string{"TI"}, nil)
if !reflect.DeepEqual(f, []string{"TI", "TE"}) || !reflect.DeepEqual(c, []string{"and", "and"}) || len(q) != 2 {
t.Errorf("got %v %v %v", q, f, c)
}
many := make([]string, maxSearchClauses+10)
q, f, c = normalizeClauses(many, nil, nil)
if len(q) != maxSearchClauses || len(f) != maxSearchClauses || len(c) != maxSearchClauses {
t.Errorf("cap not applied: %d %d %d", len(q), len(f), len(c))
}
}
func TestPaginate(t *testing.T) {
cases := []struct {
total, page, perPage int
start, end, wantPage, wantTotalPages int
}{
{45, 1, 20, 0, 20, 1, 3},
{45, 3, 20, 40, 45, 3, 3},
{45, 99, 20, 40, 45, 3, 3},
{45, -5, 20, 0, 20, 1, 3},
{0, 7, 20, 0, 0, 1, 0},
{45, 2, 0, 0, 45, 1, 1},
}
for _, c := range cases {
s, e, p, tp := paginate(c.total, c.page, c.perPage)
if s != c.start || e != c.end || p != c.wantPage || tp != c.wantTotalPages {
t.Errorf("paginate(%d,%d,%d) = %d,%d,%d,%d; want %d,%d,%d,%d",
c.total, c.page, c.perPage, s, e, p, tp, c.start, c.end, c.wantPage, c.wantTotalPages)
}
}
}
func TestSafeBackPath(t *testing.T) {
for in, want := range map[string]string{
"/doc/x": "/doc/x",
"//evil.com": "/",
"/\\evil.com": "/",
"https://evil": "/",
"": "/",
"/search?q=a\r\n": "/",
} {
if got := safeBackPath(in); got != want {
t.Errorf("safeBackPath(%q) = %q, want %q", in, got, want)
}
}
}
// Tags are curated in the metadata itself, so the
// server does nothing to them beyond the normalization every list field
// gets: trim, drop empties, fold case-duplicates keeping the first casing.
func TestNormalizeListOnTags(t *testing.T) {
got := normalizeList([]string{" XKS ", "xks", "", "PRISM", "prism"})
want := []string{"XKS", "PRISM"}
if !reflect.DeepEqual(got, want) {
t.Errorf("normalizeList = %v, want %v", got, want)
}
if normalizeList(nil) != nil {
t.Error("nil in should stay nil (JSON null, not [])")
}
}
func TestClassSortKeyFoldsAliases(t *testing.T) {
if classSortKey([]string{"TS", "SI", "TK"}) != classSortKey([]string{"TOP SECRET", "COMINT", "Talent Keyhole"}) {
t.Error("abbreviations and full names must rank identically")
}
if classSortKey([]string{"TOP SECRET", "SI"}) <= classSortKey([]string{"TOP SECRET"}) {
t.Error("a level with SI must outrank the bare level")
}
if classSortKey([]string{"NOFORN"}) != "" {
t.Error("only unranked components must sort with the empties")
}
}
func TestMarkTermsEscapes(t *testing.T) {
hl := buildHighlightTerms([]string{"müller says"}, []string{"and"})
out := string(markTerms(`Müller says <b>hi</b>`, hl))
if out != `<mark>Müller</mark> <mark>says</mark> &lt;b&gt;hi&lt;/b&gt;` {
t.Errorf("markTerms = %s", out)
}
}
func TestHeadingLevel(t *testing.T) {
for in, want := range map[string]int{
"# Title": 1, "### Deep": 3, "######\tSix": 6, "#": 1,
"#1": 0, "#hashtag": 0, "####### seven": 0, "no": 0,
"##########################": 0,
} {
if got := headingLevel(in); got != want {
t.Errorf("headingLevel(%q) = %d, want %d", in, got, want)
}
}
if got := Render("#1 priority\n#2 next"); got != "<p>#1 priority<br>\n#2 next</p>\n" {
t.Errorf("Render = %q", got)
}
}
// <br> is the transcription convention for a line break inside a table
// cell - the one place the table syntax cannot express one. It survives as
// a break, everything else in the tag vocabulary stays escaped text, and a
// code span keeps it literal.
func TestRenderHardBreak(t *testing.T) {
got := Render("| a | b |\n| --- | --- |\n| one<br>two | <br/>x<br />y |\n")
for _, want := range []string{"<td>one<br>two</td>", "<td><br>x<br>y</td>"} {
if !strings.Contains(got, want) {
t.Errorf("Render table = %q, want %q in it", got, want)
}
}
if got := Render("text <b>bold</b> and <script>x</script>"); strings.Contains(got, "<b>") || strings.Contains(got, "<script>") {
t.Errorf("only <br> is unescaped, got %q", got)
}
if got := Render("a `<br>` b"); !strings.Contains(got, "<code>&lt;br&gt;</code>") {
t.Errorf("a code span keeps it literal, got %q", got)
}
}
func TestDocPathEscapes(t *testing.T) {
if got := docPath("CBC Cyber Threat/Detection"); got != "/doc/CBC%20Cyber%20Threat%2FDetection" {
t.Errorf("docPath = %q", got)
}
if got := filePath("a b", "c d.pdf"); got != "/files/a%20b/c%20d.pdf" {
t.Errorf("filePath = %q", got)
}
}
func TestDateRangeNote(t *testing.T) {
for in, want := range map[[2]string]string{
{"", ""}: "",
{"2013", "2013-12-31"}: "(doc 2013)",
{"2013", "2015-12-31"}: "(doc 2013 to 2015-12-31)",
{"", "2014"}: "(doc to 2014)",
{"2013-06", "2013-06-31"}: "(doc 2013-06)",
} {
if got := dateRangeNote("doc", in[0], in[1]); got != want {
t.Errorf("dateRangeNote(%q, %q) = %q, want %q", in[0], in[1], got, want)
}
}
}
func TestCollectionFilter(t *testing.T) {
if collectionFilter(nil) != nil {
t.Error("no coll params must mean no filter")
}
all := map[string][]string{"coll": {"news", "foia", "sidtoday-archive", "bogus"}}
if collectionFilter(all) != nil {
t.Error("every collection selected must mean no filter")
}
sel := collectionFilter(map[string][]string{"coll": {"foia", "bogus"}})
if len(sel) != 1 || !sel["foia"] {
t.Errorf("sel = %v", sel)
}
}
func TestAgencyName(t *testing.T) {
for in, want := range map[string]string{
"CSE": "Communications Security Establishment (Canada)",
"DoD": "Department of Defense (US)",
"Booz Allen Hamilton": "",
"NSA, GCHQ": "National Security Agency (US), Government Communications Headquarters (UK)",
"DOJ/ODNI": "Department of Justice (US) / Office of the Director of National Intelligence (US)",
"NSA/Unknown Corp": "National Security Agency (US) / Unknown Corp",
"": "",
} {
if got := agencyName(in); got != want {
t.Errorf("agencyName(%q) = %q, want %q", in, got, want)
}
}
}
func TestGlossarySearchURL(t *testing.T) {
e := GlossaryEntry{Term: "AMBASSADORSRECEPTION", Aka: []string{"AMBASSADORS RECEPTION", "X-KEYSCORE"}}
got := e.SearchURL()
want := `/search?action=search&do_search=1&fqc=and&fqc=or&fqc=or&fqf=TE&fqf=TE&fqf=TE&fqv=AMBASSADORSRECEPTION&fqv=%22AMBASSADORS+RECEPTION%22&fqv=%22X-KEYSCORE%22`
if got != want {
t.Errorf("SearchURL =\n%s\nwant\n%s", got, want)
}
// The clauses must round-trip through the search intake as term OR alias.
sp := readSearchParams(mustQuery(t, got))
q, f, c := sp.Queries, sp.Fields, sp.Conjs
if len(q) != 3 || c[1] != "or" || f[2] != "TE" || q[1] != `"AMBASSADORS RECEPTION"` {
t.Errorf("round-trip: %v %v %v", q, f, c)
}
}
func testDoc(t *testing.T, js string) Document {
t.Helper()
var d Document
if err := json.Unmarshal([]byte(js), &d); err != nil {
t.Fatal(err)
}
return d
}
func TestIndexWord(t *testing.T) {
cases := []struct {
s, needle string
from, want int
}{
{"the guardian and ard", "ard", 0, 17},
{"guardian", "ard", 0, -1},
{"(ard)", "ard", 0, 1},
{"ardent ard", "ard", 0, 7},
{"müller ard", "ard", 0, 8},
{"u.s.a. policy", "u.s.", 0, 0},
{"ard ard", "ard", 1, 4},
{"", "ard", 0, -1},
}
for _, c := range cases {
if got := indexWord(c.s, c.needle, c.from); got != c.want {
t.Errorf("indexWord(%q, %q, %d) = %d, want %d", c.s, c.needle, c.from, got, c.want)
}
}
}
// TestQuotedTextAndExactWords pins the two literal forms: quoted text is
// found anywhere, like grep -i; an exact value on a text field is whole
// words only.
func TestQuotedTextAndExactWords(t *testing.T) {
for _, c := range []struct {
content, q string
want bool
}{
{"ing software\" (Mustermann)", `"mann"`, true},
{"The Guardian", `"ARD"`, true},
{"cable tappings", `"cable tapping"`, true},
{"The Guardian", `=ARD`, false},
{"NDR/ARD", `=ARD`, true},
{"strap, taps", `=tap`, false},
{"a tap here", `=tap`, true},
{"cable tappings", `="cable tapping"`, false},
{"Cable Tapping programs", `="cable tapping"`, true},
} {
// As a text field reads it - and, for quoted text, a metadata
// field the same way.
got := fieldByCode["TI"].compile(c.q)(&Document{Title: c.content})
if _, exact := exactValue(c.q); !exact {
if asValue := fieldByCode["TA"].compile(c.q)(&Document{Tags: []string{c.content}}); asValue != got {
t.Errorf("%s in %q: %v as a tag but %v as a title", c.q, c.content, asValue, got)
}
}
if got != c.want {
t.Errorf("%s in %q = %v, want %v", c.q, c.content, got, c.want)
}
}
}
// TestExactValues pins = on every field kind: one whole value on metadata
// fields, whole words on the text fields; and quoted text for contrast.
func TestExactValues(t *testing.T) {
d := testDoc(t, `{"title": "Guardian Story", "type": "REPORTED",
"tags": ["CNE implants", "SIGINT", "cable", "tapping"],
"document": {"creator": "NSA, GCHQ", "classification": ["TOP SECRET", "SI"]},
"publication": [{"publisher": "ARD", "reporters": ["Poitras, Laura"]}, {"publisher": "The Guardian"}]}`)
d.SearchText = normFoldString(d.Title)
for _, c := range []struct {
field, q string
want bool
}{
{"CR", `="NSA"`, false},
{"CR", `="nsa, gchq"`, true},
{"CR", `NSA`, true},
{"CL", `="SECRET"`, false},
{"CL", `"SECRET"`, true},
{"CL", `=si`, true},
{"TA", `="CNE"`, false},
{"TA", `"CNE imp"`, true},
{"TA", `=sigint`, true},
{"TA", `"cable tapping"`, false}, // quoted text never runs across two tags
{"PU", `="ARD"`, true},
{"PU", `="Guardian"`, false},
{"PU", `"uardi"`, true},
{"RE", `="Poitras, Laura"`, true},
{"TY", `=reported`, true},
{"TI", `="guardian story"`, true},
{"TI", `=guardian`, true}, // the title is text: whole words
{"TI", `=guard`, false},
{"TI", `"guard"`, true},
{"TE", `="guardian story"`, true},
{"TE", `=stor`, false},
{"TE", `"stor"`, true},
} {
if got := fieldByCode[c.field].compile(c.q)(&d); got != c.want {
t.Errorf("%s %s = %v, want %v", c.field, c.q, got, c.want)
}
}
for in, want := range map[string]string{`="a b"`: "a b", "=NSA": "NSA", `= "x" `: "x", "=a b": "", "NSA": "", "=": ""} {
if got, _ := exactValue(in); got != want {
t.Errorf("exactValue(%q) = %q, want %q", in, got, want)
}
}
}
func TestParseSimpleQueryExact(t *testing.T) {
q, f, c, _ := parseSimpleQuery(`publisher:="De Standaard" -tag:=CNE ="all fields" nsa`)
wantQ := []string{`="De Standaard"`, `="all fields"`, "nsa", "=CNE"}
wantF := []string{"PU", "TE", "TE", "TA"}
wantC := []string{"and", "and", "and", "not"}
if !reflect.DeepEqual(q, wantQ) || !reflect.DeepEqual(f, wantF) || !reflect.DeepEqual(c, wantC) {
t.Errorf("got %v %v %v, want %v %v %v", q, f, c, wantQ, wantF, wantC)
}
pos, negs := rowTerms(`="a b" -"c d" -=e`)
if !reflect.DeepEqual(pos, []string{`="a b"`}) || !reflect.DeepEqual(negs, []string{`"c d"`, "=e"}) {
t.Errorf("rowTerms = %q %v", pos, negs)
}
}
// TestQueryValuesSplitOnAnyWhitespace: a newline (a crafted ?q=) ends a
// value as a space does, so it cannot carry the next operator or -term
// inside it.
func TestQueryValuesSplitOnAnyWhitespace(t *testing.T) {
q, f, c, _ := parseSimpleQuery("program:XKEYSCORE\ncountry:Germany\r\n-tempora")
wantQ := []string{"XKEYSCORE", "Germany", "tempora"}
wantF := []string{"PR", "CO", "TE"}
wantC := []string{"and", "and", "not"}
if !reflect.DeepEqual(q, wantQ) || !reflect.DeepEqual(f, wantF) || !reflect.DeepEqual(c, wantC) {
t.Errorf("got %v %v %v, want %v %v %v", q, f, c, wantQ, wantF, wantC)
}
if pos, negs := rowTerms("cable\n-tapping"); !reflect.DeepEqual(pos, []string{"cable"}) || !reflect.DeepEqual(negs, []string{"tapping"}) {
t.Errorf("rowTerms = %q %v", pos, negs)
}
if _, ok := exactValue("=cable\ntapping"); ok {
t.Error("an unquoted exact value spanning a newline is two words, not one value")
}
}
func TestHighlightsAndSnippets(t *testing.T) {
hl := buildHighlightTerms([]string{`"ard"`}, []string{"and"})
if got := string(markTerms("The Guardian and ARD", hl)); got != "The Gu<mark>ard</mark>ian and <mark>ARD</mark>" {
t.Errorf("quoted highlight = %s", got)
}
hl = buildHighlightTerms([]string{`=ard`}, []string{"and"})
if got := string(markTerms("The Guardian and ARD", hl)); got != "The Guardian and <mark>ARD</mark>" {
t.Errorf("exact highlight = %s", got)
}
body := strings.Repeat("standard ", 20) + "ARD"
if s := getSnippets(body, `=ard`, 3); len(s) != 1 || !strings.HasPrefix(s[0], "... ") || !strings.HasSuffix(s[0], "ARD") {
t.Errorf("exact snippets = %q", s)
}
if s := getSnippets(body, `"ard"`, 3); len(s) == 0 || strings.HasPrefix(s[0], "... ") {
t.Errorf("quoted snippets must anchor on the first substring hit: %q", s)
}
}
// TestFacetGroupsCountCaseInsensitively: the refine links match ignoring
// case, so spellings that differ only by case must be one count.
func TestFacetGroupsCountCaseInsensitively(t *testing.T) {
docs := []*Document{
ptr(testDoc(t, `{"document": {"classification": ["TOP SECRET"]}}`)),
ptr(testDoc(t, `{"document": {"classification": ["Top Secret", "SI"]}}`)),
ptr(testDoc(t, `{"document": {"classification": ["SI"]}}`)),
}
got := facetGroups(docs, []facetGroup{{"Classification", "CL"}}, "/search?", facetValues)
if len(got) != 1 || len(got[0].Values) != 2 {
t.Fatalf("facetGroups = %+v", got)
}
for _, v := range got[0].Values {
if v.Count != 2 {
t.Errorf("%s counted %d, want 2", v.Name, v.Count)
}
}
if v := got[0].Values; !strings.Contains(v[0].URL+v[1].URL, "fqv=%3D%22TOP+SECRET%22") {
t.Errorf("links = %q %q", v[0].URL, v[1].URL)
}
}
func TestWildcard(t *testing.T) {
for _, c := range []struct {
w string
frag string
openStart, openEnd bool
ok bool
}{
{"*mann", "mann", true, false, true},
{"tap*", "tap", false, true, true},
{"*Keys*", "keys", true, true, true},
{"ab*", "ab", false, true, true},
{"*ab", "", true, false, true},
{"*", "", true, true, true},
{"a*b", "", false, false, false},
{"tap", "", false, false, false},
} {
frag, s, e, ok := wildcard(c.w)
if frag != c.frag || s != c.openStart || e != c.openEnd || ok != c.ok {
t.Errorf("wildcard(%q) = %q %v %v %v", c.w, frag, s, e, ok)
}
}
for _, c := range []struct {
s, frag string
openStart, openEnd bool
want int
}{
{"mustermann and mannheim", "mann", true, false, 6},
{"mannheim", "mann", true, false, -1},
{"mann", "mann", true, false, 0},
{"tapping", "tap", false, true, 0},
{"strap", "tap", false, true, -1},
{"x keyscores", "keys", true, true, 2},
{"x-keyscore", "key", false, true, 2},
} {
if got := indexWild(c.s, c.frag, 0, c.openStart, c.openEnd); got != c.want {
t.Errorf("indexWild(%q, %q, %v, %v) = %d, want %d", c.s, c.frag, c.openStart, c.openEnd, got, c.want)
}
}
}
// TestWildcardSearch: wildcards match the unstemmed text on every field,
// and highlighting and snippets follow the same rule.
func TestWildcardSearch(t *testing.T) {
d := testDoc(t, `{"title": "Report", "publication": [{"publisher": "Der Spiegel", "reporters": ["Mustermann, Erika"]}]}`)
text := "Erika Mustermann reports on cable tapping"
d.SearchText = normFoldString(text)
d.StemmedText = strings.Join(tokenize(text), " ")
d.FoldedFull, d.StemmedFull = d.SearchText, d.StemmedText
for _, c := range []struct {
field, q string
want bool
}{
{"TE", "*mann", true},
{"TE", "mann", false},
{"TE", "tapping*", true},
{"TE", "tapped*", false},
{"TE", "*ping", true},
{"TE", "*sterm*", true},
{"TE", "cable *mann", true},
{"TE", "spiegel *mann", false},
{"TE", "*ab", false}, // too short to search: matches nothing
{"FT", "*mann", true},
{"RE", "*mann", true},
{"RE", "*man", false},
{"RE", "muster*", true},
{"PU", "*egel", true},
{"PU", "spieg*", true},
} {
if got := fieldByCode[c.field].compile(c.q)(&d); got != c.want {
t.Errorf("%s %s = %v, want %v", c.field, c.q, got, c.want)
}
}
hl := buildHighlightTerms([]string{"*mann tap*"}, []string{"and"})
if got := string(markTerms("Mustermann on tapping, not strap", hl)); got != "<mark>Mustermann</mark> on <mark>tapping</mark>, not strap" {
t.Errorf("wildcard highlight = %s", got)
}
body := strings.Repeat("filler ", 30) + "Mustermann"
if s := getSnippets(body, "*mann", 3); len(s) != 1 || !strings.HasSuffix(s[0], "Mustermann") {
t.Errorf("snippets = %q", s)
}
if s := getSnippets("a cab rides about", "*ab", 3); s != nil {
t.Errorf("a wildcard that matches nothing anchored snippets: %q", s)
}
if s := getSnippets(strings.Repeat("filler ", 30)+"strap and tapping", "tap*", 3); len(s) != 1 || !strings.HasSuffix(s[0], "tapping") {
t.Errorf("a wildcard anchors where it matches, not inside strap: %q", s)
}
if corpus().tfidf.scorer("*mann")(&d) != 0 { // no index entry for the test doc
t.Error("a document missing from the index must score 0")
}
}
func TestSortDocsKeysStableAndEmptiesLast(t *testing.T) {
docs := []*Document{ptr(testDoc(t, `{"title": "beta"}`)), ptr(testDoc(t, `{"title": "Alpha"}`)), ptr(testDoc(t, `{"title": "alpha"}`)), ptr(testDoc(t, `{"title": "Gamma"}`))}
sortDocs(docs, "title", "")
if got := []string{docs[0].Title, docs[1].Title, docs[2].Title, docs[3].Title}; !reflect.DeepEqual(got, []string{"Alpha", "alpha", "beta", "Gamma"}) {
t.Errorf("title sort = %v", got)
}
dated := []*Document{ptr(testDoc(t, `{"document": {"date_created": ""}}`)), ptr(testDoc(t, `{"document": {"date_created": "2010"}}`)), ptr(testDoc(t, `{"document": {"date_created": "2013"}}`))}
sortDocs(dated, "date", "desc")
if got := []string{dated[0].DocMeta.DateCreated, dated[1].DocMeta.DateCreated, dated[2].DocMeta.DateCreated}; !reflect.DeepEqual(got, []string{"2013", "2010", ""}) {
t.Errorf("date desc = %v", got)
}
}
func TestTopFacets(t *testing.T) {
got := topFacets(map[string]int{"b": 2, "A": 2, "c": 5, "d": 1}, 3)
if want := []facetEntry{{Name: "c", Count: 5}, {Name: "A", Count: 2}, {Name: "b", Count: 2}}; !reflect.DeepEqual(got, want) {
t.Errorf("topFacets = %v, want %v", got, want)
}
}
// Every ranked list breaks count ties the same way: accents and case
// folded, so "Åkesson" sits with the A's, then the raw spelling.
func TestRankCounts(t *testing.T) {
m := map[string]int{"Zeta": 1, "Åkesson": 1, "bates": 1, "Bates": 1, "top": 3}
if got, want := rankCounts(m, -1), []string{"top", "Åkesson", "Bates", "bates", "Zeta"}; !reflect.DeepEqual(got, want) {
t.Errorf("rankCounts = %v, want %v", got, want)
}
if got := rankCounts(m, 2); len(got) != 2 {
t.Errorf("rankCounts(m, 2) kept %d", len(got))
}
}
// A sortable heading links to its own first direction, and the active one
// to the flip; a fixed column never flips; a plain heading has no link.
func TestSortHeads(t *testing.T) {
cols := []sortCol{
{Key: "title", Label: "Title"},
{Key: "count", Label: "Count", DescFirst: true},
{Key: "rank", Label: "Rank", DescFirst: true, Fixed: true},
{Label: "Notes"},
}
heads := sortHeads("/x?", cols, "title", false)
for i, want := range []string{"/x?sort=title&order=desc", "/x?sort=count&order=desc", "/x?sort=rank", ""} {
if heads[i].URL != want {
t.Errorf("heading %d links %q, want %q", i, heads[i].URL, want)
}
}
if !heads[0].Active || heads[0].Desc || heads[1].Active {
t.Errorf("title asc: active flags %+v", heads[:2])
}
heads = sortHeads("/x?", cols, "count", true)
if !heads[1].Active || !heads[1].Desc || heads[1].URL != "/x?sort=count&order=asc" {
t.Errorf("count desc: %+v", heads[1])
}
heads = sortHeads("/x?", cols, "rank", true)
if !heads[2].Active || heads[2].URL != "/x?sort=rank" {
t.Errorf("fixed column: %+v", heads[2])
}
}
// A document found only through its metadata - here a reporter's name -
// still gets a relevance score.
func TestTFIDFScoresAllMetadata(t *testing.T) {
d := testDoc(t, `{"id": "a", "title": "Slides", "publication": [{"publisher": "X", "reporters": ["Poitras, Laura"]}],
"document": {"classification": ["TOP SECRET"]}, "long_description": "A cablegram summary."}`)
other := testDoc(t, `{"id": "b", "title": "Other"}`)
for _, x := range []*Document{&d, &other} {
x.StemmedTitle = strings.Join(tokenize(x.Title), " ")
x.StemmedDesc = strings.Join(tokenize(x.Description+" "+x.LongDescription), " ")
}
idx := buildTFIDF([]Document{d, other})
for _, q := range []string{"poitras", "secret", "cablegram"} {
if idx.scorer(q)(&d) <= 0 {
t.Errorf("score(%q) = 0 for a document whose metadata holds it", q)
}
}
}
func TestSortCategoryEntries(t *testing.T) {
names := func(es []categoryEntry) []string {
var out []string
for _, e := range es {
out = append(out, e.Name)
}
return out
}
es := []categoryEntry{{Name: "Zeta", Count: 1, fold: "zeta"}, {Name: "Émile", Count: 3, fold: "emile"}, {Name: "alpha", Count: 3, fold: "alpha"}}
for _, c := range []struct {
sortBy string
desc bool
want []string
}{
{"name", false, []string{"alpha", "Émile", "Zeta"}},
{"name", true, []string{"Zeta", "Émile", "alpha"}},
{"count", true, []string{"alpha", "Émile", "Zeta"}},
{"count", false, []string{"Zeta", "alpha", "Émile"}},
} {
sortCategoryEntries(es, c.sortBy, c.desc)
if got := names(es); !reflect.DeepEqual(got, c.want) {
t.Errorf("%s desc=%v = %v, want %v", c.sortBy, c.desc, got, c.want)
}
}
}
// Browse and search sort by the same rules: one key function, one
// empties-last set. The regression this guards is the split brain that
// existed before - search knew four fields and ignored empties-last, so
// /search?sort=date led with undated documents while /browse?sort=date put
// them last.
func TestSortKeysShared(t *testing.T) {
dated := Document{ID: "a", Title: "Alpha"}
dated.DocMeta.DateCreated = "2013-06-01"
undated := Document{ID: "b", Title: "Bravo"}
for _, order := range []string{"asc", "desc"} {
docs := []*Document{&undated, &dated}
sortDocs(docs, "date", order)
if docs[0].ID != "a" {
t.Errorf("sortDocs date %s: undated first", order)
}
scored := []ScoredDoc{{Doc: &undated}, {Doc: &dated}}
got := sortScored(scored, "date", order)
if got[0].Doc.ID != "a" {
t.Errorf("search sort date %s: undated first", order)
}
}
// Every field a list page can sort by is a field search understands.
for _, col := range docTableCols {
if col.Key != "" && docSortByKey[col.Key] == nil {
t.Errorf("table column %q has no sort key", col.Key)
}
}
for _, o := range sortOptions {
if o.Key != "relevance" && docSortByKey[o.Key] == nil {
t.Errorf("sort option %q has no sort key", o.Key)
}
}
}
// ptr is the tests' shorthand for taking the address of a fresh document.
func ptr(d Document) *Document { return &d }
// containsTokens is the whole-token search the matcher and the scorer share.
// It must behave exactly like the padded strings.Contains it replaced -
// including at the ends of the stream, where the padding used to help.
func TestContainsTokens(t *testing.T) {
const stream = "cabl tap program alpha"
for _, phrase := range []string{"cabl", "alpha", "tap program", "cabl tap program alpha", ""} {
if !containsTokens(stream, phrase) {
t.Errorf("containsTokens(%q) = false, want true", phrase)
}
}
for _, phrase := range []string{"cab", "lpha", "ap", "program tap", "cabl program", "zeta"} {
if containsTokens(stream, phrase) {
t.Errorf("containsTokens(%q) = true, want false", phrase)
}
}
// The old implementation, as an oracle, over a few awkward streams.
padded := func(stream, phrase string) bool {
return strings.Contains(" "+stream+" ", " "+phrase+" ")
}
for _, s := range []string{"", "a", "a b", "aa aa", "x aa aa x", "aa"} {
for _, p := range []string{"a", "aa", "a b", "aa aa", "b"} {
if got, want := containsTokens(s, p), padded(s, p); got != want {
t.Errorf("containsTokens(%q, %q) = %v, oracle says %v", s, p, got, want)
}
}
}
}
// A search over the clause cap is refused, never truncated: cutting from
// the end dropped the negations and the newest row, so the results silently
// broadened.
func TestSearchClauseLimitRefused(t *testing.T) {
rows := func(n int) string {
v := url.Values{}
for i := 0; i < n; i++ {
v.Add("fqv", "prism")
}
return v.Encode()
}
phrases := strings.Repeat(`"a b" `, maxSearchClauses) + "cable -snowden" // 24 phrases + words + negation
for _, path := range []string{
"/api/search?" + rows(maxSearchClauses+1),
"/api/search?q=" + url.QueryEscape(phrases),
} {
if w := get(t, path); w.Code != 400 || !strings.Contains(w.Body.String(), "too many clauses") {
t.Errorf("%s: %d %s, want 400 too many clauses", path[:20], w.Code, w.Body.String())
}
}
if w := get(t, "/api/search?"+rows(maxSearchClauses)); w.Code != 200 {
t.Errorf("a search at the cap: %d, want 200", w.Code)
}
page := get(t, "/search?do_search=1&"+rows(maxSearchClauses+1)).Body.String()
if !strings.Contains(page, "Search not run") || strings.Contains(page, "results-heading") {
t.Error("search page ran an over-cap search, or did not say it was refused")
}
if n := strings.Count(page, `name="fqv"`); n != maxSearchClauses {
t.Errorf("refused search renders %d rows, want the first %d", n, maxSearchClauses)
}
// At the cap, "+ Add row" is neither offered nor honoured.
full := get(t, "/search?action=add&"+rows(maxSearchClauses)).Body.String()
if n := strings.Count(full, `name="fqv"`); n != maxSearchClauses {
t.Errorf("add at the cap renders %d rows, want %d", n, maxSearchClauses)
}
if strings.Contains(full, `value="add"`) {
t.Error("+ Add row offered at the clause cap")
}
if !strings.Contains(get(t, "/search?"+rows(1)).Body.String(), `value="add"`) {
t.Error("+ Add row missing below the cap")
}
}
// The clause cap alone did not bound a search's work: one row can carry
// thousands of -terms. Words are capped across the whole search.
func TestSearchWordLimitRefused(t *testing.T) {
words := func(n int, prefix string) string {
ws := make([]string, n)
for i := range ws {
ws[i] = prefix + "zzq" + strconv.Itoa(i)
}
return strings.Join(ws, " ")
}
row := func(v string) string { return url.Values{"fqv": {v}}.Encode() }
over, at := words(maxSearchWords+1, "-"), words(maxSearchWords, "-")
for _, path := range []string{
"/api/search?" + row(over),
"/api/search?q=" + url.QueryEscape(`"`+words(maxSearchWords, "")+`" cable`),
} {
if w := get(t, path); w.Code != 400 || !strings.Contains(w.Body.String(), "too many words") {
t.Errorf("%s: %d %s, want 400 too many words", path[:20], w.Code, w.Body.String())
}
}
if w := get(t, "/api/search?"+row(at)); w.Code != 200 {
t.Errorf("a search at the word cap: %d, want 200", w.Code)
}
page := get(t, "/search?do_search=1&"+row(over)).Body.String()
if !strings.Contains(page, "Search not run") || strings.Contains(page, "results-heading") {
t.Error("search page ran an over-cap search, or did not say it was refused")
}
// Refine links that would take the search past the cap are not offered.
facets := []searchFacet{
{Label: "Programs", Values: []facetEntry{{Name: "PRISM"}, {Name: "Poitras, Laura"}}},
{Label: "Journalists", Values: []facetEntry{{Name: "Greenwald, Glenn"}}},
}
got := facetsWithinWords(facets, 1)
if len(got) != 1 || len(got[0].Values) != 1 || got[0].Values[0].Name != "PRISM" {
t.Errorf("facetsWithinWords(room 1) = %+v, want only PRISM", got)
}
}
// TestSnippetAnchorsOnWords: a bare word anchors where the matcher and the
// highlighter find it - the whole word or another form of it - never inside
// a longer word; stopwords and single letters anchor only when nothing else
// in the query can.
func TestSnippetAnchorsOnWords(t *testing.T) {
pad := strings.Repeat("filler ", 30)
for _, c := range []struct{ body, q, want string }{
{"especially special " + pad + "the CIA", "cia", "the CIA"},
{"the " + pad + "NSA", "the nsa", "NSA"},
{"sport " + pad + "the spied on", "spies", "spied on"},
{pad + "the U.S. said", "u.s.", "U.S. said"},
} {
s := getSnippets(c.body, c.q, 1)
if len(s) != 1 || !strings.HasSuffix(s[0], c.want) {
t.Errorf("%q: snippets %q, want one ending %q", c.q, s, c.want)
}
}
}
// A publication date is tied to each journalist or publisher row on its
// own: two journalists bylined on different entries both match a range
// that covers both entries, and a proximity value reads as its phrase, as
// it does without the date.
func TestPubDateTiesEachRow(t *testing.T) {
var d Document
if err := json.Unmarshal([]byte(`{"publication": [
{"publisher": "Der Spiegel", "date_published": "2014-01-10", "reporters": ["Laura Poitras"]},
{"publisher": "The Intercept", "date_published": "2014-02-10", "reporters": ["Glenn Greenwald"]},
{"publisher": "Le Monde", "date_published": "2019-05-01", "reporters": ["Jacques Follorou"]}
]}`), &d); err != nil {
t.Fatal(err)
}
for _, c := range []struct {
queries, fields, conjs []string
start, end string
want bool
}{
{[]string{"Poitras", "Greenwald"}, []string{"RE", "RE"}, []string{"and", "and"}, "2014", "2014-12-31", true},
{[]string{"Poitras", "Intercept"}, []string{"RE", "PU"}, []string{"and", "and"}, "2014", "2014-12-31", true},
{[]string{"Poitras", "Follorou"}, []string{"RE", "RE"}, []string{"and", "and"}, "2014", "2014-12-31", false},
{[]string{"Follorou"}, []string{"RE"}, []string{"and"}, "2014", "2014-12-31", false},
{[]string{`"laura poitras"~3`}, []string{"RE"}, []string{"and"}, "2014", "2014-12-31", true},
{[]string{`"laura poitras"~3`}, []string{"RE"}, []string{"and"}, "2019", "2019-12-31", false},
} {
got := matchDates(&d, "", "", c.start, c.end, entryRows(c.queries, c.fields, c.conjs))
if got != c.want {
t.Errorf("%v %v %s..%s: %v, want %v", c.fields, c.queries, c.start, c.end, got, c.want)
}
}
}
// foldedLen is the length of a rune's normal form, which the snippet walk
// uses to map offsets in the folded text back to the original.
func TestFoldedLen(t *testing.T) {
for r := rune(0); r <= 0x2FFFF; r++ {
if r >= 0xD800 && r <= 0xDFFF {
continue // surrogates: never the result of ranging over a string
}
if got, want := foldedLen(r), len(foldRune(r)); got != want {
t.Fatalf("foldedLen(%U) = %d, foldRune gives %d bytes", r, got, want)
}
}
// The normal form built rune by rune is normFoldString, and its
// position map points at the rune each byte came from.
const s = "Müller-Maguhn's ÆSIR ẞtraße, SÃO paulo"
norm, pos := normFold(s)
if norm != normFoldString(s) || len(pos) != len(norm)+1 || pos[len(norm)] != len(s) {
t.Errorf("normFold(%q) = %q with %d positions; normFoldString gives %q", s, norm, len(pos), normFoldString(s))
}
}
// A document's transcript is folded once, at load: its snippets come off
// that fold and are the ones folding it again would give.
func TestBodySnippetsUseTheLoadedFold(t *testing.T) {
buildFixture(t)
checked := 0
for i := range corpus().archive.Docs {
d := &corpus().archive.Docs[i]
for _, q := range []string{"collection", "the", `"upstream"`, "müller", "tap*", "unredacted"} {
if got, want := d.bodySnippets(q, 3), getSnippets(d.FullText, q, 3); !reflect.DeepEqual(got, want) {
t.Errorf("%s, %q: %q from the loaded fold, %q from the text", d.ID, q, got, want)
}
checked++
}
if d.FullText != "" && d.foldedBody() != normFoldString(d.FullText) {
t.Errorf("%s: foldedBody is not the fold of its text", d.ID)
}
}
if checked == 0 {
t.Fatal("no fixture documents")
}
// A document the loader never saw folds on demand.
loose := Document{FullText: "Some LOOSE text"}
if loose.foldedBody() != "some loose text" {
t.Errorf("foldedBody of a hand-built document = %q", loose.foldedBody())
}
}
// One rule says when two spellings are one metadata value: valueKey. The
// allocation-free comparison must agree with it.
func TestSameValue(t *testing.T) {
for _, c := range []struct {
a, b string
want bool
}{
{"NSA", "nsa", true},
{" The Intercept ", "the intercept", true},
{"The Intercept", "The Guardian", false},
{"NSA", "NSAW", false},
{"Müller-Maguhn, Andy", "müller-maguhn, andy", true},
{"Müller", "Muller", false}, // a value's accents are part of it
{"", " ", true},
} {
if got := sameValue(c.a, c.b); got != c.want || got != (valueKey(c.a) == valueKey(c.b)) {
t.Errorf("sameValue(%q, %q) = %v, want %v; the keys are %q and %q", c.a, c.b, got, c.want, valueKey(c.a), valueKey(c.b))
}
}
}
// What the loader keeps for the matchers is what reading the document
// gives: each field's values, the document's terms as its all-fields
// stream holds them, and the folds of its title and descriptions.
func TestLoadedFieldsAreTheDocuments(t *testing.T) {
buildFixture(t)
for i := range corpus().archive.Docs {
d := &corpus().archive.Docs[i]
if d.fields == nil || d.terms == nil {
t.Fatalf("%s: loaded without its fields or its terms", d.ID)
}
for fi := range searchFields {
f := &searchFields[fi]
if f.read == nil {
continue
}
want := f.read(d)
if got := f.Values(d); !reflect.DeepEqual(got, want) && len(got)+len(want) > 0 {
t.Errorf("%s %s: kept %q, the document reads %q", d.ID, f.Code, got, want)
}
if got, want := d.fields.folded[fi], normFoldString(strings.Join(want, valueSep)); got != want {
t.Errorf("%s %s: kept the fold %q, want %q", d.ID, f.Code, got, want)
}
}
if d.titleFold() != normFoldString(d.Title) || d.descFold() != normFoldString(d.Description+valueSep+d.LongDescription) {
t.Errorf("%s: the kept title or description fold is not the fold of the text", d.ID)
}
// The index entry holds exactly the tokens of the all-fields stream,
// which is what lets a one-word all-fields clause ask it.
stream := map[string]bool{}
eachToken(d.StemmedText, func(w string) { stream[w] = w != valueSep })
for w := range d.terms {
if !stream[w] {
t.Errorf("%s: %q is in the index entry and not in the all-fields stream", d.ID, w)
}
}
for w, term := range stream {
if _, ok := d.terms[w]; term && !ok {
t.Errorf("%s: %q is in the all-fields stream and not in the index entry", d.ID, w)
}
}
}
// A document the loader never saw is read as it stands.
loose := testDoc(t, `{"title": "Loose", "tags": ["One", "two"], "publication": [{"publisher": "Outlet", "reporters": ["A", "a", "B"]}]}`)
if got := loose.JournalistsDeduped(); !reflect.DeepEqual(got, []string{"A", "B"}) {
t.Errorf("JournalistsDeduped of a hand-built document = %q", got)
}
if !fieldByCode["TA"].compile(`="one"`)(&loose) || fieldByCode["TA"].compile(`=three`)(&loose) {
t.Error("a hand-built document's tags should match by whole value")
}
}