first commit
This commit is contained in:
commit
8c4dcae475
89 changed files with 24623 additions and 0 deletions
193
tests/query_test.go
Normal file
193
tests/query_test.go
Normal file
|
|
@ -0,0 +1,193 @@
|
|||
package main
|
||||
|
||||
// query_test.go - what the search engine promises, as queries and the
|
||||
// documents they must return. The unit tests cover the pieces (tokenizer,
|
||||
// wildcards, ranges); this covers the sentence a reader types.
|
||||
//
|
||||
// Results come from /api/search, which runs the same engine as the page, so
|
||||
// a refactor that changes what search FINDS fails here even when every page
|
||||
// still renders.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func searchIDs(t *testing.T, q string) []string {
|
||||
t.Helper()
|
||||
w := get(t, "/api/search?per_page=all&q="+url.QueryEscape(q))
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("%q: status %d", q, w.Code)
|
||||
}
|
||||
var body struct {
|
||||
Results []struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"results"`
|
||||
}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("%q: %v", q, err)
|
||||
}
|
||||
ids := make([]string, 0, len(body.Results))
|
||||
for _, r := range body.Results {
|
||||
ids = append(ids, r.ID)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
return ids
|
||||
}
|
||||
|
||||
func TestQuerySuite(t *testing.T) {
|
||||
buildFixture(t)
|
||||
const (
|
||||
prism = "prism-slides"
|
||||
companion = "prism-companion"
|
||||
sid = "2003-05-29_SIDToday_-_Fixture_Issue"
|
||||
foia = "FISC_-_fixture_order"
|
||||
undated = "undated-fixture"
|
||||
)
|
||||
cases := []struct {
|
||||
query string
|
||||
want []string
|
||||
why string
|
||||
}{
|
||||
// --- words, stems and case --------------------------------------
|
||||
{"xkeyscore", []string{prism}, "a plain word in the transcript"},
|
||||
{"XKEYSCORE", []string{prism}, "matching ignores case"},
|
||||
{"collections", []string{prism, sid}, "stemming: plural finds the singular"},
|
||||
{"analysts", []string{prism, sid}, "plural finds singular"},
|
||||
// The stemmer deliberately leaves -tion alone (see stem), so
|
||||
// "collecting" and "collection" are different words here. This is
|
||||
// the behaviour, recorded so a change to the stemmer is a decision
|
||||
// rather than a surprise.
|
||||
{"collecting", []string{prism}, "-ing does not reach -tion"},
|
||||
{"zzzznotacorpusword", nil, "a word nobody wrote"},
|
||||
|
||||
// --- phrases and proximity ---------------------------------------
|
||||
{`"upstream collection"`, []string{prism}, "a quoted phrase is adjacency, and SIDtoday has the words apart"},
|
||||
{`"upstream collection"~12`, []string{prism, sid}, "proximity widens it to a window"},
|
||||
{`"cable tapping"`, []string{prism}, "a phrase that is also a tag"},
|
||||
|
||||
// --- punctuation and accents --------------------------------------
|
||||
{"RT-RG", []string{prism}, "a hyphenated term matches its parts in sequence"},
|
||||
{"muller", []string{prism}, "accent folding: muller finds Müller-Maguhn"},
|
||||
{"Müller-Maguhn", []string{prism}, "and the accented form finds itself"},
|
||||
{"sao paulo", []string{prism}, "folding works across a phrase too"},
|
||||
|
||||
// --- wildcards -----------------------------------------------------
|
||||
{"*keyscore", []string{prism}, "a leading wildcard"},
|
||||
{"tapp*", []string{prism, sid}, "a trailing wildcard reaches tapping"},
|
||||
{"*ollectio*", []string{prism, sid}, "both ends open"},
|
||||
|
||||
// --- exact values vs substrings -------------------------------------
|
||||
{`classification:="TOP SECRET"`, []string{prism}, "an exact value is the whole value"},
|
||||
{`classification:="SECRET"`, []string{foia}, "so SECRET does not match TOP SECRET"},
|
||||
{"classification:secret", []string{foia, prism}, "unquoted, it is a word match on both"},
|
||||
|
||||
// --- fielded search ---------------------------------------------------
|
||||
{"program:PRISM", []string{companion, prism}, "a metadata field"},
|
||||
{"agency:FISC", []string{foia}, "creator"},
|
||||
{"agency:NSA", []string{companion, prism, sid}, "creator across collections"},
|
||||
{`journalist:"Greenwald, Glenn"`, []string{companion, prism}, "a quoted byline"},
|
||||
{"publisher:NRK", nil, "a reporting-only partner is not a publisher"},
|
||||
{"publisher:\"The Guardian\"", []string{companion, prism}, "the outlet that published it"},
|
||||
{"tag:collection", []string{prism}, "tags are their own field"},
|
||||
{"country:Brazil", []string{prism}, "countries"},
|
||||
{"people:Snowden", []string{prism}, "people, by surname"},
|
||||
{"type:FOIA", []string{foia}, "document type"},
|
||||
{"series:\"(U) A Day in the Life of...\"", []string{sid}, "a series name with punctuation"},
|
||||
|
||||
// --- boolean and negation ---------------------------------------------
|
||||
{"prism -companion", []string{prism}, "inline negation excludes"},
|
||||
{"upstream -xkeyscore", []string{sid}, "negation against a word in the other document"},
|
||||
{"prism -x", []string{companion, prism}, "a single letter is a literal word, not an empty clause that excludes everything"},
|
||||
{"u.s.", nil, "a dotted abbreviation is matched as written, not dropped"},
|
||||
|
||||
// --- a phrase stays inside one value ------------------------------------
|
||||
{`"united states"`, []string{prism}, "a phrase inside one country"},
|
||||
{`"andy united"`, nil, "a phrase cannot run from one metadata value into the next"},
|
||||
{`"slides slides describing"`, nil, "nor from the title into the description"},
|
||||
{"program:PRISM tag:collection", []string{prism}, "two fields are an AND"},
|
||||
|
||||
// --- dates -------------------------------------------------------------
|
||||
{"date:2013", []string{companion, foia, prism}, "document year, across collections"},
|
||||
{"date:2013-04-02", []string{companion}, "an exact document date"},
|
||||
{"date:2003..2013", []string{companion, foia, prism, sid}, "a range"},
|
||||
{"published:2016", []string{sid}, "publication year, not document year"},
|
||||
|
||||
// --- the corner the archive cares about ---------------------------------
|
||||
{"fairview", []string{prism}, "text that exists only in another published copy"},
|
||||
{"guardian", []string{companion, prism}, "an outlet name is searchable as text"},
|
||||
{"prism-slides", []string{prism}, "the document id itself"},
|
||||
{"prism-slides-guardian", []string{prism}, "and the name the same file was published under"},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
got := searchIDs(t, c.query)
|
||||
want := append([]string(nil), c.want...)
|
||||
sort.Strings(want)
|
||||
if strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Errorf("%-34s got %v, want %v (%s)", c.query, got, want, c.why)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Relevance is not just membership: a title hit must outrank a body hit,
|
||||
// and the phrase bonus must beat loose words.
|
||||
func TestQueryRanking(t *testing.T) {
|
||||
buildFixture(t)
|
||||
first := func(q string) string {
|
||||
ids := func() []string {
|
||||
w := get(t, "/api/search?q="+url.QueryEscape(q))
|
||||
var body struct {
|
||||
Results []struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"results"`
|
||||
}
|
||||
json.Unmarshal(w.Body.Bytes(), &body)
|
||||
out := make([]string, 0, len(body.Results))
|
||||
for _, r := range body.Results {
|
||||
out = append(out, r.ID)
|
||||
}
|
||||
return out
|
||||
}()
|
||||
if len(ids) == 0 {
|
||||
t.Fatalf("%q returned nothing", q)
|
||||
}
|
||||
return ids[0]
|
||||
}
|
||||
if got := first("collection"); got != "prism-slides" {
|
||||
t.Errorf("collection: first result %s, want the document with it in the title and the tags", got)
|
||||
}
|
||||
if got := first("fixture"); got == "" {
|
||||
t.Error("fixture: no results")
|
||||
}
|
||||
}
|
||||
|
||||
// TestQueryRowParts: a structured row is read as ?q= reads its text - each
|
||||
// quoted phrase is a part of its own and the bare words are one more, all
|
||||
// ANDed. A row with two phrases used to be one substring search for the
|
||||
// text between the outer quotes, quotes included, and found nothing.
|
||||
func TestQueryRowParts(t *testing.T) {
|
||||
buildFixture(t)
|
||||
for _, c := range []struct {
|
||||
row string
|
||||
want int
|
||||
}{
|
||||
{`"upstream collection" "cable tapping"`, 1},
|
||||
{`"upstream collection" xkeyscore`, 1},
|
||||
{`"collection upstream" xkeyscore`, 0},
|
||||
} {
|
||||
w := get(t, "/api/search?per_page=all&fqf=TE&fqc=and&fqv="+url.QueryEscape(c.row))
|
||||
var body struct {
|
||||
Total int `json:"total"`
|
||||
}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("%s: %v", c.row, err)
|
||||
}
|
||||
if body.Total != c.want {
|
||||
t.Errorf("row %s: %d results, want %d", c.row, body.Total, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue