193 lines
7.7 KiB
Go
193 lines
7.7 KiB
Go
package main
|
|
|
|
// query_test.go - what the search engine promises, as queries and the
|
|
// documents they must return. The unit tests cover the pieces (tokenizer,
|
|
// wildcards, ranges); this covers the sentence a reader types.
|
|
//
|
|
// Results come from /api/search, which runs the same engine as the page, so
|
|
// a refactor that changes what search FINDS fails here even when every page
|
|
// still renders.
|
|
|
|
import (
|
|
"encoding/json"
|
|
"net/url"
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func searchIDs(t *testing.T, q string) []string {
|
|
t.Helper()
|
|
w := get(t, "/api/search?per_page=all&q="+url.QueryEscape(q))
|
|
if w.Code != 200 {
|
|
t.Fatalf("%q: status %d", q, w.Code)
|
|
}
|
|
var body struct {
|
|
Results []struct {
|
|
ID string `json:"id"`
|
|
} `json:"results"`
|
|
}
|
|
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
|
|
t.Fatalf("%q: %v", q, err)
|
|
}
|
|
ids := make([]string, 0, len(body.Results))
|
|
for _, r := range body.Results {
|
|
ids = append(ids, r.ID)
|
|
}
|
|
sort.Strings(ids)
|
|
return ids
|
|
}
|
|
|
|
func TestQuerySuite(t *testing.T) {
|
|
buildFixture(t)
|
|
const (
|
|
prism = "prism-slides"
|
|
companion = "prism-companion"
|
|
sid = "2003-05-29_SIDToday_-_Fixture_Issue"
|
|
foia = "FISC_-_fixture_order"
|
|
undated = "undated-fixture"
|
|
)
|
|
cases := []struct {
|
|
query string
|
|
want []string
|
|
why string
|
|
}{
|
|
// --- words, stems and case --------------------------------------
|
|
{"xkeyscore", []string{prism}, "a plain word in the transcript"},
|
|
{"XKEYSCORE", []string{prism}, "matching ignores case"},
|
|
{"collections", []string{prism, sid}, "stemming: plural finds the singular"},
|
|
{"analysts", []string{prism, sid}, "plural finds singular"},
|
|
// The stemmer deliberately leaves -tion alone (see stem), so
|
|
// "collecting" and "collection" are different words here. This is
|
|
// the behaviour, recorded so a change to the stemmer is a decision
|
|
// rather than a surprise.
|
|
{"collecting", []string{prism}, "-ing does not reach -tion"},
|
|
{"zzzznotacorpusword", nil, "a word nobody wrote"},
|
|
|
|
// --- phrases and proximity ---------------------------------------
|
|
{`"upstream collection"`, []string{prism}, "a quoted phrase is adjacency, and SIDtoday has the words apart"},
|
|
{`"upstream collection"~12`, []string{prism, sid}, "proximity widens it to a window"},
|
|
{`"cable tapping"`, []string{prism}, "a phrase that is also a tag"},
|
|
|
|
// --- punctuation and accents --------------------------------------
|
|
{"RT-RG", []string{prism}, "a hyphenated term matches its parts in sequence"},
|
|
{"muller", []string{prism}, "accent folding: muller finds Müller-Maguhn"},
|
|
{"Müller-Maguhn", []string{prism}, "and the accented form finds itself"},
|
|
{"sao paulo", []string{prism}, "folding works across a phrase too"},
|
|
|
|
// --- wildcards -----------------------------------------------------
|
|
{"*keyscore", []string{prism}, "a leading wildcard"},
|
|
{"tapp*", []string{prism, sid}, "a trailing wildcard reaches tapping"},
|
|
{"*ollectio*", []string{prism, sid}, "both ends open"},
|
|
|
|
// --- exact values vs substrings -------------------------------------
|
|
{`classification:="TOP SECRET"`, []string{prism}, "an exact value is the whole value"},
|
|
{`classification:="SECRET"`, []string{foia}, "so SECRET does not match TOP SECRET"},
|
|
{"classification:secret", []string{foia, prism}, "unquoted, it is a word match on both"},
|
|
|
|
// --- fielded search ---------------------------------------------------
|
|
{"program:PRISM", []string{companion, prism}, "a metadata field"},
|
|
{"agency:FISC", []string{foia}, "creator"},
|
|
{"agency:NSA", []string{companion, prism, sid}, "creator across collections"},
|
|
{`journalist:"Greenwald, Glenn"`, []string{companion, prism}, "a quoted byline"},
|
|
{"publisher:NRK", nil, "a reporting-only partner is not a publisher"},
|
|
{"publisher:\"The Guardian\"", []string{companion, prism}, "the outlet that published it"},
|
|
{"tag:collection", []string{prism}, "tags are their own field"},
|
|
{"country:Brazil", []string{prism}, "countries"},
|
|
{"people:Snowden", []string{prism}, "people, by surname"},
|
|
{"type:FOIA", []string{foia}, "document type"},
|
|
{"series:\"(U) A Day in the Life of...\"", []string{sid}, "a series name with punctuation"},
|
|
|
|
// --- boolean and negation ---------------------------------------------
|
|
{"prism -companion", []string{prism}, "inline negation excludes"},
|
|
{"upstream -xkeyscore", []string{sid}, "negation against a word in the other document"},
|
|
{"prism -x", []string{companion, prism}, "a single letter is a literal word, not an empty clause that excludes everything"},
|
|
{"u.s.", nil, "a dotted abbreviation is matched as written, not dropped"},
|
|
|
|
// --- a phrase stays inside one value ------------------------------------
|
|
{`"united states"`, []string{prism}, "a phrase inside one country"},
|
|
{`"andy united"`, nil, "a phrase cannot run from one metadata value into the next"},
|
|
{`"slides slides describing"`, nil, "nor from the title into the description"},
|
|
{"program:PRISM tag:collection", []string{prism}, "two fields are an AND"},
|
|
|
|
// --- dates -------------------------------------------------------------
|
|
{"date:2013", []string{companion, foia, prism}, "document year, across collections"},
|
|
{"date:2013-04-02", []string{companion}, "an exact document date"},
|
|
{"date:2003..2013", []string{companion, foia, prism, sid}, "a range"},
|
|
{"published:2016", []string{sid}, "publication year, not document year"},
|
|
|
|
// --- the corner the archive cares about ---------------------------------
|
|
{"fairview", []string{prism}, "text that exists only in another published copy"},
|
|
{"guardian", []string{companion, prism}, "an outlet name is searchable as text"},
|
|
{"prism-slides", []string{prism}, "the document id itself"},
|
|
{"prism-slides-guardian", []string{prism}, "and the name the same file was published under"},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
got := searchIDs(t, c.query)
|
|
want := append([]string(nil), c.want...)
|
|
sort.Strings(want)
|
|
if strings.Join(got, ",") != strings.Join(want, ",") {
|
|
t.Errorf("%-34s got %v, want %v (%s)", c.query, got, want, c.why)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Relevance is not just membership: a title hit must outrank a body hit,
|
|
// and the phrase bonus must beat loose words.
|
|
func TestQueryRanking(t *testing.T) {
|
|
buildFixture(t)
|
|
first := func(q string) string {
|
|
ids := func() []string {
|
|
w := get(t, "/api/search?q="+url.QueryEscape(q))
|
|
var body struct {
|
|
Results []struct {
|
|
ID string `json:"id"`
|
|
} `json:"results"`
|
|
}
|
|
json.Unmarshal(w.Body.Bytes(), &body)
|
|
out := make([]string, 0, len(body.Results))
|
|
for _, r := range body.Results {
|
|
out = append(out, r.ID)
|
|
}
|
|
return out
|
|
}()
|
|
if len(ids) == 0 {
|
|
t.Fatalf("%q returned nothing", q)
|
|
}
|
|
return ids[0]
|
|
}
|
|
if got := first("collection"); got != "prism-slides" {
|
|
t.Errorf("collection: first result %s, want the document with it in the title and the tags", got)
|
|
}
|
|
if got := first("fixture"); got == "" {
|
|
t.Error("fixture: no results")
|
|
}
|
|
}
|
|
|
|
// TestQueryRowParts: a structured row is read as ?q= reads its text - each
|
|
// quoted phrase is a part of its own and the bare words are one more, all
|
|
// ANDed. A row with two phrases used to be one substring search for the
|
|
// text between the outer quotes, quotes included, and found nothing.
|
|
func TestQueryRowParts(t *testing.T) {
|
|
buildFixture(t)
|
|
for _, c := range []struct {
|
|
row string
|
|
want int
|
|
}{
|
|
{`"upstream collection" "cable tapping"`, 1},
|
|
{`"upstream collection" xkeyscore`, 1},
|
|
{`"collection upstream" xkeyscore`, 0},
|
|
} {
|
|
w := get(t, "/api/search?per_page=all&fqf=TE&fqc=and&fqv="+url.QueryEscape(c.row))
|
|
var body struct {
|
|
Total int `json:"total"`
|
|
}
|
|
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
|
|
t.Fatalf("%s: %v", c.row, err)
|
|
}
|
|
if body.Total != c.want {
|
|
t.Errorf("row %s: %d results, want %d", c.row, body.Total, c.want)
|
|
}
|
|
}
|
|
}
|