snowchive-server/tests/query_test.go
2026-11-19 04:40:59 +02:00

193 lines
7.7 KiB
Go

package main
// query_test.go - what the search engine promises, as queries and the
// documents they must return. The unit tests cover the pieces (tokenizer,
// wildcards, ranges); this covers the sentence a reader types.
//
// Results come from /api/search, which runs the same engine as the page, so
// a refactor that changes what search FINDS fails here even when every page
// still renders.
import (
"encoding/json"
"net/url"
"sort"
"strings"
"testing"
)
func searchIDs(t *testing.T, q string) []string {
t.Helper()
w := get(t, "/api/search?per_page=all&q="+url.QueryEscape(q))
if w.Code != 200 {
t.Fatalf("%q: status %d", q, w.Code)
}
var body struct {
Results []struct {
ID string `json:"id"`
} `json:"results"`
}
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
t.Fatalf("%q: %v", q, err)
}
ids := make([]string, 0, len(body.Results))
for _, r := range body.Results {
ids = append(ids, r.ID)
}
sort.Strings(ids)
return ids
}
func TestQuerySuite(t *testing.T) {
buildFixture(t)
const (
prism = "prism-slides"
companion = "prism-companion"
sid = "2003-05-29_SIDToday_-_Fixture_Issue"
foia = "FISC_-_fixture_order"
undated = "undated-fixture"
)
cases := []struct {
query string
want []string
why string
}{
// --- words, stems and case --------------------------------------
{"xkeyscore", []string{prism}, "a plain word in the transcript"},
{"XKEYSCORE", []string{prism}, "matching ignores case"},
{"collections", []string{prism, sid}, "stemming: plural finds the singular"},
{"analysts", []string{prism, sid}, "plural finds singular"},
// The stemmer deliberately leaves -tion alone (see stem), so
// "collecting" and "collection" are different words here. This is
// the behaviour, recorded so a change to the stemmer is a decision
// rather than a surprise.
{"collecting", []string{prism}, "-ing does not reach -tion"},
{"zzzznotacorpusword", nil, "a word nobody wrote"},
// --- phrases and proximity ---------------------------------------
{`"upstream collection"`, []string{prism}, "a quoted phrase is adjacency, and SIDtoday has the words apart"},
{`"upstream collection"~12`, []string{prism, sid}, "proximity widens it to a window"},
{`"cable tapping"`, []string{prism}, "a phrase that is also a tag"},
// --- punctuation and accents --------------------------------------
{"RT-RG", []string{prism}, "a hyphenated term matches its parts in sequence"},
{"muller", []string{prism}, "accent folding: muller finds Müller-Maguhn"},
{"Müller-Maguhn", []string{prism}, "and the accented form finds itself"},
{"sao paulo", []string{prism}, "folding works across a phrase too"},
// --- wildcards -----------------------------------------------------
{"*keyscore", []string{prism}, "a leading wildcard"},
{"tapp*", []string{prism, sid}, "a trailing wildcard reaches tapping"},
{"*ollectio*", []string{prism, sid}, "both ends open"},
// --- exact values vs substrings -------------------------------------
{`classification:="TOP SECRET"`, []string{prism}, "an exact value is the whole value"},
{`classification:="SECRET"`, []string{foia}, "so SECRET does not match TOP SECRET"},
{"classification:secret", []string{foia, prism}, "unquoted, it is a word match on both"},
// --- fielded search ---------------------------------------------------
{"program:PRISM", []string{companion, prism}, "a metadata field"},
{"agency:FISC", []string{foia}, "creator"},
{"agency:NSA", []string{companion, prism, sid}, "creator across collections"},
{`journalist:"Greenwald, Glenn"`, []string{companion, prism}, "a quoted byline"},
{"publisher:NRK", nil, "a reporting-only partner is not a publisher"},
{"publisher:\"The Guardian\"", []string{companion, prism}, "the outlet that published it"},
{"tag:collection", []string{prism}, "tags are their own field"},
{"country:Brazil", []string{prism}, "countries"},
{"people:Snowden", []string{prism}, "people, by surname"},
{"type:FOIA", []string{foia}, "document type"},
{"series:\"(U) A Day in the Life of...\"", []string{sid}, "a series name with punctuation"},
// --- boolean and negation ---------------------------------------------
{"prism -companion", []string{prism}, "inline negation excludes"},
{"upstream -xkeyscore", []string{sid}, "negation against a word in the other document"},
{"prism -x", []string{companion, prism}, "a single letter is a literal word, not an empty clause that excludes everything"},
{"u.s.", nil, "a dotted abbreviation is matched as written, not dropped"},
// --- a phrase stays inside one value ------------------------------------
{`"united states"`, []string{prism}, "a phrase inside one country"},
{`"andy united"`, nil, "a phrase cannot run from one metadata value into the next"},
{`"slides slides describing"`, nil, "nor from the title into the description"},
{"program:PRISM tag:collection", []string{prism}, "two fields are an AND"},
// --- dates -------------------------------------------------------------
{"date:2013", []string{companion, foia, prism}, "document year, across collections"},
{"date:2013-04-02", []string{companion}, "an exact document date"},
{"date:2003..2013", []string{companion, foia, prism, sid}, "a range"},
{"published:2016", []string{sid}, "publication year, not document year"},
// --- the corner the archive cares about ---------------------------------
{"fairview", []string{prism}, "text that exists only in another published copy"},
{"guardian", []string{companion, prism}, "an outlet name is searchable as text"},
{"prism-slides", []string{prism}, "the document id itself"},
{"prism-slides-guardian", []string{prism}, "and the name the same file was published under"},
}
for _, c := range cases {
got := searchIDs(t, c.query)
want := append([]string(nil), c.want...)
sort.Strings(want)
if strings.Join(got, ",") != strings.Join(want, ",") {
t.Errorf("%-34s got %v, want %v (%s)", c.query, got, want, c.why)
}
}
}
// Relevance is not just membership: a title hit must outrank a body hit,
// and the phrase bonus must beat loose words.
func TestQueryRanking(t *testing.T) {
buildFixture(t)
first := func(q string) string {
ids := func() []string {
w := get(t, "/api/search?q="+url.QueryEscape(q))
var body struct {
Results []struct {
ID string `json:"id"`
} `json:"results"`
}
json.Unmarshal(w.Body.Bytes(), &body)
out := make([]string, 0, len(body.Results))
for _, r := range body.Results {
out = append(out, r.ID)
}
return out
}()
if len(ids) == 0 {
t.Fatalf("%q returned nothing", q)
}
return ids[0]
}
if got := first("collection"); got != "prism-slides" {
t.Errorf("collection: first result %s, want the document with it in the title and the tags", got)
}
if got := first("fixture"); got == "" {
t.Error("fixture: no results")
}
}
// TestQueryRowParts: a structured row is read as ?q= reads its text - each
// quoted phrase is a part of its own and the bare words are one more, all
// ANDed. A row with two phrases used to be one substring search for the
// text between the outer quotes, quotes included, and found nothing.
func TestQueryRowParts(t *testing.T) {
buildFixture(t)
for _, c := range []struct {
row string
want int
}{
{`"upstream collection" "cable tapping"`, 1},
{`"upstream collection" xkeyscore`, 1},
{`"collection upstream" xkeyscore`, 0},
} {
w := get(t, "/api/search?per_page=all&fqf=TE&fqc=and&fqv="+url.QueryEscape(c.row))
var body struct {
Total int `json:"total"`
}
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
t.Fatalf("%s: %v", c.row, err)
}
if body.Total != c.want {
t.Errorf("row %s: %d results, want %d", c.row, body.Total, c.want)
}
}
}