snowchive-server/search.go
2026-11-19 04:40:59 +02:00

2644 lines
75 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package main
import (
"cmp"
"html/template"
"log"
"math"
"net/http"
"net/url"
"slices"
"sort"
"strconv"
"strings"
"unicode"
"unicode/utf8"
)
type TFIDFIndex struct {
DocStats map[string]map[string]float64
DocFreq map[string]int
N int
}
// stopWords are excluded from tokenization to avoid recreating the map on
// every tokenize() call.
var stopWords = map[string]bool{
"the": true, "and": true, "for": true, "that": true,
"this": true, "with": true, "from": true, "are": true,
"was": true, "were": true, "has": true, "have": true,
"not": true, "but": true, "they": true, "their": true,
"will": true, "been": true, "its": true, "also": true,
"into": true, "which": true, "such": true, "other": true,
}
// --- Stemmer ---
// Porter-lite: suffix stripping for English. Applied at both index time and
// query time so "surveillance" and "surveilling" map to the same token.
// Intentionally conservative - only strips high-confidence suffixes to avoid
// over-stemming domain-specific terms like "SIGINT" or "collection".
func stem(w string) string {
// The last two steps apply whichever suffix rule returned: "flying" and
// "flyer" leave "fly", which must meet "fly" itself.
s := yToI(stripSuffixes(w))
// fixStem keeps a doubled l ("falling" -> "fall"), so "controlled" and
// "controller" reach "controll" while "control" stays "control". A final
// "ll" after more than one vowel-consonant run is undoubled in every
// word alike (Porter's step 5b): control/controlled -> "control",
// install/installing -> "instal", while fall/falling stay "fall".
if strings.HasSuffix(s, "ll") && measure(s) > 1 {
return s[:len(s)-1]
}
return s
}
func stripSuffixes(w string) string {
switch {
case strings.HasSuffix(w, "sses"):
w = w[:len(w)-2]
case strings.HasSuffix(w, "ies") && len(w) > 4:
w = w[:len(w)-3] + "y" // "studies" -> "study", "spies" -> "spy"
case strings.HasSuffix(w, "ies"):
w = w[:len(w)-1] // "ties" -> "tie": one letter before it, a word in "ie"
case strings.HasSuffix(w, "ss"):
// "process", "class": not a plural
case strings.HasSuffix(w, "s") && len(w) > 3:
w = w[:len(w)-1]
}
switch {
case strings.HasSuffix(w, "ied") && len(w) == 4:
return w[:3] // "tied" -> "tie", like "ties"
case strings.HasSuffix(w, "ied") && len(w) > 4:
w = w[:len(w)-3] + "y" // "studied" -> "study", "spied" -> "spy"
case strings.HasSuffix(w, "eed") && len(w) > 4:
return w[:len(w)-1]
case strings.HasSuffix(w, "ed") && len(w) > 4:
s := w[:len(w)-2]
return fixStem(s)
case strings.HasSuffix(w, "ing") && len(w) > 5:
s := w[:len(w)-3]
return fixStem(s)
}
switch {
case strings.HasSuffix(w, "ational") && len(w) > 8:
return w[:len(w)-5] // "operational" -> "operat", same as "operation"
case strings.HasSuffix(w, "ation") && len(w) > 6:
return w[:len(w)-3]
case strings.HasSuffix(w, "ence") && len(w) > 5:
return w[:len(w)-4]
case strings.HasSuffix(w, "ance") && len(w) > 5:
return w[:len(w)-4]
case strings.HasSuffix(w, "ment") && len(w) > 5:
return w[:len(w)-4]
case strings.HasSuffix(w, "ness") && len(w) > 5:
return w[:len(w)-4]
case strings.HasSuffix(w, "ity") && len(w) > 5:
return w[:len(w)-3]
case strings.HasSuffix(w, "ism") && len(w) > 5:
return w[:len(w)-3]
case strings.HasSuffix(w, "ist") && len(w) > 5:
return w[:len(w)-3]
case strings.HasSuffix(w, "ize") && len(w) > 5:
return w[:len(w)-3]
case strings.HasSuffix(w, "ise") && len(w) > 5:
return w[:len(w)-3]
case strings.HasSuffix(w, "ful") && len(w) > 5:
return w[:len(w)-3]
case strings.HasSuffix(w, "less") && len(w) > 5:
return w[:len(w)-4]
case strings.HasSuffix(w, "ly") && len(w) > 5 && w[len(w)-3] != 'p':
return w[:len(w)-2]
case strings.HasSuffix(w, "er") && len(w) > 4:
return w[:len(w)-2]
case strings.HasSuffix(w, "al") && len(w) > 4:
return w[:len(w)-2]
}
return w
}
// yToI turns a final consonant+y into i, so singular/plural pairs unify
// with the "ies" rule: "study" -> "studi", "studies" -> "studi";
// "agency" -> "agenci", "agencies" -> "agenci".
func yToI(w string) string {
if strings.HasSuffix(w, "y") && len(w) > 3 && !isVowel(rune(w[len(w)-2])) {
return w[:len(w)-1] + "i"
}
if strings.HasSuffix(w, "ie") && len(w) > 5 {
return w[:len(w)-1]
}
return w
}
// measure is Porter's m: the number of vowel-consonant runs in w, with y a
// vowel after a consonant ("controll" 2, "fall" 1, "tree" 0).
func measure(w string) int {
m := 0
prevVowel := false
for i := 0; i < len(w); i++ {
v := isVowel(rune(w[i])) || (w[i] == 'y' && i > 0 && !prevVowel)
if prevVowel && !v {
m++
}
prevVowel = v
}
return m
}
// fixStem handles consonant doubling after stripping -ed/-ing
// e.g. "hopping" -> "hopp" -> "hop".
func fixStem(s string) string {
if len(s) == 0 {
return s
}
if len(s) >= 2 {
last := s[len(s)-1]
secondLast := s[len(s)-2]
if last == secondLast && !isVowel(rune(last)) && last != 'l' && last != 's' && last != 'z' {
return s[:len(s)-1]
}
}
return s
}
func isVowel(r rune) bool {
return strings.ContainsRune("aeiou", r)
}
// foldTable maps accented Latin characters to their ASCII base form so
// "muller" finds "Müller-Maguhn" and "Sao Paulo" finds "São Paulo". The
// fold runs on BOTH the index and the query side, so it's symmetric.
var foldTable = map[rune]string{
'À': "A", 'Á': "A", 'Â': "A", 'Ã': "A", 'Ä': "A",
'Å': "A", 'Æ': "AE", 'Ç': "C", 'È': "E", 'É': "E", 'Ê': "E",
'Ë': "E", 'Ì': "I", 'Í': "I", 'Î': "I", 'Ï': "I", 'Ð': "D",
'Ñ': "N", 'Ò': "O", 'Ó': "O", 'Ô': "O", 'Õ': "O", 'Ö': "O",
'Ø': "O", 'Ù': "U", 'Ú': "U", 'Û': "U", 'Ü': "U", 'Ý': "Y",
'Þ': "TH", 'ß': "ss", 'à': "a", 'á': "a", 'â': "a", 'ã': "a",
'ä': "a", 'å': "a", 'æ': "ae", 'ç': "c", 'è': "e", 'é': "e",
'ê': "e", 'ë': "e", 'ì': "i", 'í': "i", 'î': "i", 'ï': "i",
'ð': "d", 'ñ': "n", 'ò': "o", 'ó': "o", 'ô': "o", 'õ': "o",
'ö': "o", 'ø': "o", 'ù': "u", 'ú': "u", 'û': "u", 'ü': "u",
'ý': "y", 'þ': "th", 'ÿ': "y", 'Ā': "A", 'ā': "a", 'Ă': "A",
'ă': "a", 'Ą': "A", 'ą': "a", 'Ć': "C", 'ć': "c", 'Ĉ': "C",
'ĉ': "c", 'Ċ': "C", 'ċ': "c", 'Č': "C", 'č': "c", 'Ď': "D",
'ď': "d", 'Đ': "D", 'đ': "d", 'Ē': "E", 'ē': "e", 'Ĕ': "E",
'ĕ': "e", 'Ė': "E", 'ė': "e", 'Ę': "E", 'ę': "e", 'Ě': "E",
'ě': "e", 'Ĝ': "G", 'ĝ': "g", 'Ğ': "G", 'ğ': "g", 'Ġ': "G",
'ġ': "g", 'Ģ': "G", 'ģ': "g", 'Ĥ': "H", 'ĥ': "h", 'Ħ': "H",
'ħ': "h", 'Ĩ': "I", 'ĩ': "i", 'Ī': "I", 'ī': "i", 'Ĭ': "I",
'ĭ': "i", 'Į': "I", 'į': "i", 'İ': "I", 'ı': "i", 'Ĵ': "J",
'ĵ': "j", 'Ķ': "K", 'ķ': "k", 'Ĺ': "L", 'ĺ': "l", 'Ļ': "L",
'ļ': "l", 'Ľ': "L", 'ľ': "l", 'Ŀ': "L", 'ŀ': "l", 'Ł': "L",
'ł': "l", 'Ń': "N", 'ń': "n", 'Ņ': "N", 'ņ': "n", 'Ň': "N",
'ň': "n", 'ʼn': "n", 'Ō': "O", 'ō': "o", 'Ŏ': "O", 'ŏ': "o",
'Ő': "O", 'ő': "o", 'Œ': "OE", 'œ': "oe", 'Ŕ': "R", 'ŕ': "r",
'Ŗ': "R", 'ŗ': "r", 'Ř': "R", 'ř': "r", 'Ś': "S", 'ś': "s",
'Ŝ': "S", 'ŝ': "s", 'Ş': "S", 'ş': "s", 'Š': "S", 'š': "s",
'Ţ': "T", 'ţ': "t", 'Ť': "T", 'ť': "t", 'Ŧ': "T", 'ŧ': "t",
'Ũ': "U", 'ũ': "u", 'Ū': "U", 'ū': "u", 'Ŭ': "U", 'ŭ': "u",
'Ů': "U", 'ů': "u", 'Ű': "U", 'ű': "u", 'Ų': "U", 'ų': "u",
'Ŵ': "W", 'ŵ': "w", 'Ŷ': "Y", 'ŷ': "y", 'Ÿ': "Y", 'Ź': "Z",
'ź': "z", 'Ż': "Z", 'ż': "z", 'Ž': "Z", 'ž': "z", 'Ș': "S",
'ș': "s", 'Ț': "T", 'ț': "t", 'ẞ': "SS",
}
// foldMin is the smallest rune foldTable maps, every rune below it folds
// to itself so the folds skip the table lookup for it (all of ASCII).
var foldMin = func() rune {
m := utf8.MaxRune
for r := range foldTable {
m = min(m, r)
}
return m
}()
// foldAccents replaces accented characters with their base form,
// preserving case for characters not in the table.
func foldAccents(s string) string {
needs := false
for _, r := range s {
if r < foldMin {
continue
}
if _, ok := foldTable[r]; ok {
needs = true
break
}
}
if !needs {
return s
}
var b strings.Builder
b.Grow(len(s))
for _, r := range s {
if r >= foldMin {
if f, ok := foldTable[r]; ok {
b.WriteString(f)
continue
}
}
b.WriteRune(r)
}
return b.String()
}
func normFoldString(s string) string {
return strings.ToLower(foldAccents(s))
}
// foldRune is normFoldString for one rune.
func foldRune(r rune) string {
if r < utf8.RuneSelf {
if 'A' <= r && r <= 'Z' {
r += 'a' - 'A'
}
return string(r)
}
if f, ok := foldTable[r]; ok {
return strings.ToLower(f)
}
return strings.ToLower(string(r))
}
func foldedLen(r rune) int {
if r < utf8.RuneSelf {
return 1
}
if f, ok := foldTable[r]; ok {
return len(f)
}
return utf8.RuneLen(unicode.ToLower(r))
}
// normFold returns the normal form of s plus a position map: posMap[i] is
// the byte offset in s of the rune that produced norm byte i (with a final
// entry of len(s)). Lets match highlighting (markTerms) mark on the normal
// form but slice the ORIGINAL text, so a "muller" query highlights "Müller"
// as written.
func normFold(s string) (string, []int) {
var b strings.Builder
b.Grow(len(s))
posMap := make([]int, 0, len(s)+1)
for i, r := range s {
if r < utf8.RuneSelf {
if 'A' <= r && r <= 'Z' {
r += 'a' - 'A'
}
b.WriteByte(byte(r))
posMap = append(posMap, i)
continue
}
f := foldRune(r)
for j := 0; j < len(f); j++ {
posMap = append(posMap, i)
}
b.WriteString(f)
}
posMap = append(posMap, len(s))
return b.String(), posMap
}
// splitWords splits s into alphanumeric runs (it does not change case).
func splitWords(s string) []string {
return strings.FieldsFunc(s, func(r rune) bool {
return !unicode.IsLetter(r) && !unicode.IsDigit(r)
})
}
func tokenize(s string) []string {
words := splitWords(normFoldString(s))
filtered := words[:0]
for _, w := range words {
if len(w) > 1 && !stopWords[w] {
filtered = append(filtered, stem(w))
}
}
return filtered
}
// eachToken calls f with every token of a stemmed stream (tokenize's
// output joined by single spaces) in order.
func eachToken(stream string, f func(tok string)) {
for i := 0; i < len(stream); {
j := strings.IndexByte(stream[i:], ' ')
if j < 0 {
j = len(stream) - i
}
if j > 0 {
f(stream[i : i+j])
}
i += j + 1
}
}
func buildTFIDF(docs []Document) TFIDFIndex {
idx := TFIDFIndex{
DocStats: make(map[string]map[string]float64, len(docs)),
DocFreq: make(map[string]int),
N: len(docs),
}
for i := range docs {
d := &docs[i]
// Title 3x, fulltext 2x, descriptions + metadata 1x. The metadata
// is every field the all-fields search reads (searchMeta), so a doc
// that matches only via metadata still gets a relevance score
// instead of always sorting last. Title, descriptions and body come
// from the stemmed forms loadData already built (tokenize() output
// joined by spaces).
tf := make(map[string]float64)
total := 0.0
add := func(weight float64) func(string) {
return func(w string) {
if w == valueSep {
return
}
tf[w] += weight
total += weight
}
}
eachToken(d.StemmedTitle, add(3))
eachToken(d.StemmedFull, add(2))
eachToken(d.StemmedDesc, add(1))
for _, w := range tokenize(searchMeta(d)) {
add(1)(w)
}
for w := range tf {
tf[w] /= total // total > 0 whenever tf has a key
idx.DocFreq[w]++
}
idx.DocStats[d.ID] = tf
}
log.Printf("TF-IDF index built: %d docs, %d unique terms", len(docs), len(idx.DocFreq))
return idx
}
// scorer reads one positive term of a query into its relevance score for a
// document. The term's words go through the same tokenize() pipeline as the
// index (stemming + stopword removal), and matching against stemmed fields
// is by whole token.
func (idx TFIDFIndex) scorer(query string) func(doc *Document) float64 {
// A proximity clause scores as its phrase - the window is a matcher
// concern, not a relevance signal (and the raw "~10" tail would
// otherwise leak a junk "10" term into the tokenizer).
if _, phrase, _, ok := parseProximity(query); ok {
query = phrase
}
// An exact-value clause scores as its phrase with the phrase bonuses on
// whole words; quoted text earns them wherever it occurs.
contains := strings.Contains
if v, ok := exactValue(query); ok {
query, contains = `"`+v+`"`, containsWord
}
// Folded so the phrase bonuses are accent-consistent with matching, and
// so the content check can use the pre-folded body (no per-doc fold).
clean := strings.Trim(normFoldString(strings.TrimSpace(query)), `"`)
// Wildcard words score by where they occur, not through their stems:
// +5 when the title has one, else +1 when the document does.
var wild []queryWord
if !isQuotedQuery(query) {
var kept []string
for _, w := range strings.Fields(clean) {
if _, _, _, isWild := wildcard(w); isWild {
wild = append(wild, literalWord(w))
} else {
kept = append(kept, w)
}
}
clean = strings.Join(kept, " ")
}
terms := tokenize(clean)
// A term's weight in a document is its share of the document times how
// rare it is in the corpus; a term no document holds weighs nothing.
idf := make([]float64, len(terms))
for i, w := range terms {
if df := idx.DocFreq[w]; df != 0 {
idf[i] = math.Log(float64(idx.N) / float64(df))
}
}
isPhrase := len(strings.Fields(clean)) > 1
return func(doc *Document) float64 {
stats, ok := idx.DocStats[doc.ID]
if !ok || (len(terms) == 0 && len(wild) == 0) {
return 0
}
title := "" // for the wildcard and phrase bonuses
if len(wild) > 0 || isPhrase {
title = doc.titleFold()
}
var score float64
for i := range wild {
switch {
case wild[i].kind == wordNever: // matches nothing (see wildcard), so earns nothing
case wild[i].in(title):
score += 5
case wild[i].in(doc.SearchText):
score++
}
}
if len(terms) == 0 {
return score
}
for i, w := range terms {
score += stats[w] * idf[i]
}
// bonus is whole when the field holds the exact phrase (or, for one
// word, the word), else all when it holds every term somewhere.
bonus := func(phraseIn string, holds func(term string) bool, whole, all float64) float64 {
if isPhrase && contains(phraseIn, clean) {
return whole
}
for _, w := range terms {
if !holds(w) {
return 0
}
}
if isPhrase {
return all
}
return whole
}
// Title +10 or +5; the whole document +3 or +1. The document's
// entry in the index holds exactly the terms of its all-fields
// stream.
inTitle := func(w string) bool { return containsTokens(doc.StemmedTitle, w) }
inDoc := func(w string) bool { _, ok := stats[w]; return ok }
return score + bonus(title, inTitle, 10, 5) + bonus(doc.FoldedFull, inDoc, 3, 1)
}
}
// similarity returns cosine similarity between two docs' TF-IDF vectors.
// Returns 0-1; used by relatedScore in data.go.
// TODO split this file or move this away from here?
func (idx TFIDFIndex) similarity(aID, bID string) float64 {
a, ok1 := idx.DocStats[aID]
b, ok2 := idx.DocStats[bID]
if !ok1 || !ok2 {
return 0
}
var dot, normA, normB float64
for w, tfA := range a {
df := idx.DocFreq[w]
if df == 0 {
continue
}
idf := math.Log(float64(idx.N) / float64(df))
tfidfA := tfA * idf
normA += tfidfA * tfidfA
if tfB, ok := b[w]; ok {
tfidfB := tfB * idf
dot += tfidfA * tfidfB
}
}
for w, tfB := range b {
df := idx.DocFreq[w]
if df == 0 {
continue
}
idf := math.Log(float64(idx.N) / float64(df))
tfidfB := tfB * idf
normB += tfidfB * tfidfB
}
if normA == 0 || normB == 0 {
return 0
}
return dot / (math.Sqrt(normA) * math.Sqrt(normB))
}
// --- Clause values ---
// isQuotedQuery returns true for "exact phrase" queries.
func isQuotedQuery(q string) bool {
q = strings.TrimSpace(q)
return len(q) >= 2 && strings.HasPrefix(q, `"`) && strings.HasSuffix(q, `"`)
}
// --- Exact values and word boundaries ---
//
// A clause value has four shapes: words (cable tapping, matched with their
// other forms), quoted text ("cable tapping", found anywhere, like grep -i),
// an exact value (="cable tapping", or =tap for one word) and a wildcard
// (tap*, *apping). An exact value means "exactly this, nothing attached", on
// a metadata field it is one whole value, so agency:=NSA does not match
// "NSA, GCHQ"; on the text fields (all fields, title, content, description)
// it is whole words exactly as typed, so =tap finds "tap" but not "taps" or
// "strap".
// exactValue reports whether q is an exact-value clause ("=") followed by
// a quoted value or a single bare word - and returns the value.
func exactValue(q string) (string, bool) {
q = strings.TrimSpace(q)
if !strings.HasPrefix(q, "=") {
return "", false
}
v := strings.TrimSpace(q[1:])
if isQuotedQuery(v) {
v = strings.TrimSpace(v[1 : len(v)-1])
} else if strings.ContainsAny(v, spaceBytes+`"`) {
return "", false
}
return v, v != ""
}
// exactClause builds the ="..." clause value the generated links use, so a
// count and its click compare whole values the same way.
func exactClause(v string) string { return `="` + v + `"` }
// isWordRune is the word-character test splitWords uses.
func isWordRune(r rune) bool { return unicode.IsLetter(r) || unicode.IsDigit(r) }
// indexWord returns the byte offset of the first occurrence of needle in s,
// at or after from, that stands on word boundaries (no letter or digit
// directly before or after it) or -1. A needle edge that is not a word
// character ("u.s.") needs no boundary on that side. s and needle must
// already share a normal form (normFoldString).
func indexWord(s, needle string, from int) int {
first, _ := utf8.DecodeRuneInString(needle)
last, _ := utf8.DecodeLastRuneInString(needle)
return indexBounded(s, needle, from, isWordRune(first), isWordRune(last))
}
// wildMinRunes is the fewest letters or digits a fragment behind a leading
// "*" must have: "*a" or "*ab" would match nearly every document.
const wildMinRunes = 3
// wildcard reports whether w is a wildcard word (a fragment with "*") at
// its end (tap*), its start (*apping) or both (*keys*), and returns the
// fragment in normal form plus which sides may run on into more of the
// word.
func wildcard(w string) (frag string, openStart, openEnd, ok bool) {
openStart, openEnd = strings.HasPrefix(w, "*"), strings.HasSuffix(w, "*")
if !openStart && !openEnd {
return "", false, false, false
}
frag = normFoldString(strings.Trim(w, "*"))
n := 0
for _, r := range frag {
if isWordRune(r) {
n++
}
}
if n == 0 || (openStart && n < wildMinRunes) || strings.Contains(frag, "*") {
return "", openStart, openEnd, true
}
return frag, openStart, openEnd, true
}
// indexWild finds a wildcard fragment in s, both in normal form. A closed
// side must sit on a word boundary and an open side may continue into the
// word, so tap* finds "tapping" but not "strap".
func indexWild(s, frag string, from int, openStart, openEnd bool) int {
first, _ := utf8.DecodeRuneInString(frag)
last, _ := utf8.DecodeLastRuneInString(frag)
return indexBounded(s, frag, from, !openStart && isWordRune(first), !openEnd && isWordRune(last))
}
// wordBounds widens [start, end) in s to the whole word around it.
func wordBounds(s string, start, end int) (int, int) {
for start > 0 {
r, size := utf8.DecodeLastRuneInString(s[:start])
if !isWordRune(r) {
break
}
start -= size
}
for end < len(s) {
r, size := utf8.DecodeRuneInString(s[end:])
if !isWordRune(r) {
break
}
end += size
}
return start, end
}
// indexBounded is indexWord with the boundary checks chosen by the caller:
// checkStart requires no letter or digit directly before the match,
// checkEnd none directly after it.
func indexBounded(s, needle string, from int, checkStart, checkEnd bool) int {
if from < 0 {
from = 0
}
if needle == "" {
if from <= len(s) {
return from
}
return -1
}
for from <= len(s)-len(needle) {
j := strings.Index(s[from:], needle)
if j < 0 {
return -1
}
i := from + j
end := i + len(needle)
ok := true
if checkStart && i > 0 {
if r, _ := utf8.DecodeLastRuneInString(s[:i]); isWordRune(r) {
ok = false
}
}
if ok && checkEnd && end < len(s) {
if r, _ := utf8.DecodeRuneInString(s[end:]); isWordRune(r) {
ok = false
}
}
if ok {
return i
}
_, size := utf8.DecodeRuneInString(s[i:])
from = i + size
}
return -1
}
// containsWord reports whether needle occurs in s on word boundaries.
func containsWord(s, needle string) bool { return indexWord(s, needle, 0) >= 0 }
// containsTokens reports whether phrase (one or more stemmed tokens joined
// by single spaces) occurs in a space-joined token stream as WHOLE tokens.
func containsTokens(stream, phrase string) bool {
if phrase == "" {
return true
}
for i := 0; i+len(phrase) <= len(stream); {
j := strings.Index(stream[i:], phrase)
if j < 0 {
return false
}
p := i + j
end := p + len(phrase)
if (p == 0 || stream[p-1] == ' ') && (end == len(stream) || stream[end] == ' ') {
return true
}
i = p + 1
}
return false
}
// valueSep joins a field's values for word and phrase matching. It is not a
// word character, so words still split on it, and nobody types it, so a
// quoted phrase cannot run from one value into the next.
const valueSep = "\x1f"
func valueKey(v string) string { return strings.ToLower(strings.TrimSpace(v)) }
func sameValue(a, b string) bool {
a, b = strings.TrimSpace(a), strings.TrimSpace(b)
if !isASCII(a) || !isASCII(b) {
return strings.ToLower(a) == strings.ToLower(b)
}
if len(a) != len(b) {
return false
}
for i := 0; i < len(a); i++ {
x, y := a[i], b[i]
if 'A' <= x && x <= 'Z' {
x += 'a' - 'A'
}
if 'A' <= y && y <= 'Z' {
y += 'a' - 'A'
}
if x != y {
return false
}
}
return true
}
func isASCII(s string) bool {
for i := 0; i < len(s); i++ {
if s[i] >= utf8.RuneSelf {
return false
}
}
return true
}
// --- Compiled clauses ---
//
// A search reads each clause once, into a test, and the scan runs the tests
// on every document.
// queryWord is one word of an unquoted clause.
type queryWord struct {
kind int
text string // the fragment (wordWild), the word (wordWhole, wordSub) or its stemmed tokens joined by spaces (wordStems)
openStart, openEnd bool // wordWild: which sides of the fragment had "*"
tokens []string // wordStems: the tokens of text
}
const (
wordNever = iota // a wildcard with nothing to search for (wildcard): it matches nothing
wordWild // tap*, *apping, *keys*: part of a word, in the folded text
wordWhole // a whole word as written, in the folded text
wordSub // a word with internal punctuation ("u.s."), as written, anywhere in the folded text
wordStems // the word's stemmed tokens in sequence, in the token stream
)
// literalWord reads a folded query word the way folded text can hold it: a
// wildcard, a whole word or, with internal punctuation, which word matching
// can't represent, a substring.
func literalWord(w string) queryWord {
if frag, openStart, openEnd, ok := wildcard(w); ok {
if frag == "" {
return queryWord{kind: wordNever}
}
return queryWord{kind: wordWild, text: frag, openStart: openStart, openEnd: openEnd}
}
if parts := splitWords(w); len(parts) == 1 {
return queryWord{kind: wordWhole, text: parts[0]}
}
return queryWord{kind: wordSub, text: w}
}
// in reports whether the word, read literally, occurs in folded text.
func (w *queryWord) in(folded string) bool {
switch w.kind {
case wordWild:
return indexWild(folded, w.text, 0, w.openStart, w.openEnd) >= 0
case wordWhole:
return containsWord(folded, w.text)
case wordSub:
return strings.Contains(folded, w.text)
}
return false
}
// literalTest is a clause read for folded text that is not tokenized (the
// values of a metadata field). Quoted text is found anywhere, like grep -i
// ("ard" inside "guardian"); otherwise every word must occur as literalWord
// reads it.
type literalTest struct {
quoted bool
needle string
words []queryWord
}
func compileLiteral(q string) literalTest {
q = normFoldString(strings.TrimSpace(q))
if isQuotedQuery(q) {
return literalTest{quoted: true, needle: strings.Trim(q, `"`)}
}
var t literalTest
for _, w := range strings.Fields(q) {
t.words = append(t.words, literalWord(w))
}
return t
}
func (t *literalTest) match(folded string) bool {
if t.quoted {
return strings.Contains(folded, t.needle)
}
for i := range t.words {
if !t.words[i].in(folded) {
return false
}
}
return true
}
// valuesTest is a clause read for a metadata field's values: ="..." is one
// whole value (the equality the browse pages and the refine tallies use
// the valueKey) and anything else the values' words and phrases
// (literalTest).
type valuesTest struct {
exact bool
key string // exact: the value's valueKey
text literalTest
}
func compileValues(q string) valuesTest {
if v, ok := exactValue(q); ok {
return valuesTest{exact: true, key: valueKey(v)}
}
return valuesTest{text: compileLiteral(q)}
}
// match tests the clause on a list of values.
func (t *valuesTest) match(vals []string) bool {
if t.exact {
for _, x := range vals {
if sameValue(x, t.key) {
return true
}
}
return false
}
return t.text.match(normFoldString(strings.Join(vals, valueSep)))
}
func (t *valuesTest) matchIndexed(keys []string, folded string) bool {
if t.exact {
return slices.Contains(keys, t.key)
}
return t.text.match(folded)
}
// compileWords reads an unquoted clause for a text field. Its words run
// through the same tokenize pipeline as the index, so stemming stays
// consistent with it.
func compileWords(q string) []queryWord {
var stems, literal []queryWord
for _, w := range strings.Fields(strings.ToLower(strings.TrimSpace(q))) {
if _, _, _, ok := wildcard(w); ok {
literal = append(literal, literalWord(w))
} else if toks := tokenize(w); len(toks) == 0 {
literal = append(literal, literalWord(normFoldString(w)))
} else {
stems = append(stems, queryWord{kind: wordStems, text: strings.Join(toks, " "), tokens: toks})
}
}
return append(stems, literal...)
}
// compile reads clause q (one negation-free value of a row) into the
// test of it on the field.
func (f *searchField) compile(q string) func(*Document) bool {
if terms, phrase, win, ok := parseProximity(q); ok {
// "..."~N - see the proximity block below.
if f.Stemmed != nil && len(terms) >= 2 {
return func(d *Document) bool {
return d.hasTerms(terms) && proximityMatch(f.Stemmed(d), terms, win)
}
}
// A metadata field, or a phrase that boiled down to fewer than two
// searchable words, so it adds nothing, match the plain
// quoted phrase on the requested field instead.
q = phrase
}
if f.Values != nil {
test := compileValues(q)
return func(d *Document) bool {
if d.fields == nil {
return test.match(f.read(d))
}
return test.matchIndexed(d.fields.keys[f.index], d.fields.folded[f.index])
}
}
// A text field: an exact value (=word) is a whole-word test and quoted
// text a substring test, both on the folded text; anything else runs on
// the stemmed tokens, and reads the folded text only for its wildcard
// and literal words.
if v, ok := exactValue(q); ok {
needle := normFoldString(v)
return func(d *Document) bool { return containsWord(f.Folded(d), needle) }
}
if isQuotedQuery(q) {
needle := strings.Trim(normFoldString(strings.TrimSpace(q)), `"`)
return func(d *Document) bool { return strings.Contains(f.Folded(d), needle) }
}
words := compileWords(q)
everyField := f.Code == "TE"
return func(d *Document) bool {
for i := range words {
w := &words[i]
switch {
case w.kind != wordStems:
if !w.in(f.Folded(d)) {
return false
}
case !d.hasTerms(w.tokens):
return false
case everyField && len(w.tokens) == 1 && d.terms != nil:
// stream needs no walking
case !containsTokens(f.Stemmed(d), w.text):
return false
}
}
return true
}
}
// --- Proximity operator ----------------------------------------------------
//
// "upstream collection"~10 matches documents where every quoted word occurs
// within a 10-word window, unordered and counted on the same stemmed,
// stopword-free token streams the bare-word matcher uses.
// parseProximity reports whether q has the "..."~N shape (the whole clause,
// nothing after the digits). terms are the tokenized, stemmed, deduplicated
// phrase words; phrase is the plain quoted phrase for callers that degrade;
// win is N, capped at 1000 (wider than any window that could still mean
// anything beyond plain AND).
func parseProximity(q string) (terms []string, phrase string, win int, ok bool) {
q = strings.TrimSpace(q)
if len(q) < 4 || q[0] != '"' {
return nil, "", 0, false
}
end := strings.LastIndexByte(q, '"')
if end == 0 || end+2 >= len(q) || q[end+1] != '~' {
return nil, "", 0, false
}
n, err := strconv.Atoi(q[end+2:])
if err != nil || n < 1 {
return nil, "", 0, false
}
if n > 1000 {
n = 1000
}
inner := q[1:end]
seen := map[string]bool{}
for _, t := range tokenize(inner) {
if !seen[t] {
seen[t] = true
terms = append(terms, t)
}
}
return terms, `"` + inner + `"`, n, true
}
// proximityMatch walks a space-joined stemmed stream once, tracking the most
// recent position of each term; whenever every term has been seen and the
// span from the oldest to the newest of those positions fits the window, a
// valid window exists. Last-seen positions minimize the span at every step,
// so no qualifying window is missed. One linear pass, no string splitting.
func proximityMatch(stream string, terms []string, win int) bool {
if len(terms) == 0 {
return false
}
last := make([]int, len(terms))
for i := range last {
last[i] = -1
}
need := len(terms)
pos := 0
i := 0
for i < len(stream) {
for i < len(stream) && stream[i] == ' ' {
i++
}
if i >= len(stream) {
break
}
j := i
for j < len(stream) && stream[j] != ' ' {
j++
}
tok := stream[i:j]
for t, term := range terms {
if tok != term {
continue
}
if last[t] < 0 {
need--
}
last[t] = pos
if need == 0 {
lo, hi := last[0], last[0]
for _, p := range last[1:] {
if p < lo {
lo = p
}
if p > hi {
hi = p
}
}
if hi-lo <= win {
return true
}
}
}
pos++
i = j
}
return false
}
// --- Snippet helpers ---
// getSnippets returns up to max non-overlapping +-60-char windows around
// query matches, in document order. The match runs on the normal form
// (folded + lowered) and each window's bounds are resolved back to the
// ORIGINAL text (one forward walk, see below), so accents and casing
// survive into the snippet.
func getSnippets(content, query string, max int) []string {
return snippetsIn(content, normFoldString(content), query, max)
}
// bodySnippets is getSnippets over the document's transcript, on the fold
// of it the loader already holds.
func (d *Document) bodySnippets(query string, max int) []string {
return snippetsIn(d.FullText, d.foldedBody(), query, max)
}
// anchorSource yields one needle's matches in a text, in document order and
// one at a time, so a search for three windows reads no further than them.
type anchorSource struct {
next func(from int) (pos, n, resume int) // pos < 0: no match from there on
from int
pos, n int // the match not yet taken; pos < 0 once there is none
}
func (a *anchorSource) advance() { a.pos, a.n, a.from = a.next(a.from) }
// snippetsIn is getSnippets for a caller that holds the normal form of
// content (normFoldString).
func snippetsIn(content, norm, query string, max int) []string {
if query == "" || content == "" || max <= 0 {
return nil
}
searchQuery := strings.TrimSpace(query)
bounded := false // an exact value anchors on whole words only
if v, ok := exactValue(searchQuery); ok {
searchQuery, bounded = `"`+v+`"`, true
}
isExact := isQuotedQuery(searchQuery)
cleanQuery := normFoldString(strings.Trim(searchQuery, `"`))
if cleanQuery == "" {
return nil
}
var sources []*anchorSource
add := func(next func(from int) (pos, n, resume int)) {
src := &anchorSource{next: next}
src.advance()
sources = append(sources, src)
}
addAll := func(needle string, wordsOnly bool) {
if needle == "" {
return
}
add(func(from int) (int, int, int) {
i := -1
if wordsOnly {
i = indexWord(norm, needle, from)
} else if j := strings.Index(norm[from:], needle); j >= 0 {
i = from + j
}
return i, len(needle), i + len(needle)
})
}
found := func() bool {
for _, src := range sources {
if src.pos >= 0 {
return true
}
}
return false
}
if isExact || strings.Contains(cleanQuery, " ") {
addAll(cleanQuery, !isExact || bounded)
}
if !isExact {
var literal []string // stopwords, single letters, "u.s.": anchors of last resort
for _, w := range strings.Fields(cleanQuery) {
if frag, openStart, openEnd, ok := wildcard(w); ok {
if frag != "" {
add(func(from int) (int, int, int) {
i := indexWild(norm, frag, from, openStart, openEnd)
return i, len(frag), i + len(frag)
})
}
continue
}
kept := 0
for _, p := range splitWords(w) {
if len(p) <= 1 || stopWords[p] {
continue
}
kept++
addAll(p, true)
s := stem(p)
n := 0
for n < len(s) && n < len(p) && s[n] == p[n] {
n++
}
if s == p || n < 2 {
continue
}
add(func(from int) (int, int, int) {
for {
i := indexWild(norm, p[:n], from, false, true)
if i < 0 {
return -1, 0, from
}
start, end := wordBounds(norm, i, i+n)
if word := norm[start:end]; word != p && stem(word) == s {
return start, end - start, end
}
from = end
}
})
}
if kept == 0 {
literal = append(literal, w)
}
}
if !found() {
for _, w := range literal {
parts := splitWords(w)
if len(parts) == 1 {
w = parts[0]
}
addAll(w, len(parts) == 1)
}
}
}
if !found() {
return nil
}
// First pass. Choose up to max non-overlapping +-60-byte windows, in
// NORM offsets, taking the anchors in document order.
type window struct{ start, end int }
var wins []window
lastEnd := -1
for len(wins) < max {
var first *anchorSource
for _, src := range sources {
if src.pos >= 0 && (first == nil || src.pos < first.pos) {
first = src
}
}
if first == nil {
break
}
pos, n := first.pos, first.n
first.advance()
if pos < lastEnd {
continue // already inside the previous window
}
start := pos - 60
if start < 0 {
start = 0
}
end := pos + n + 60
if end > len(norm) {
end = len(norm)
}
lastEnd = end
wins = append(wins, window{start, end})
}
// Second pass. Resolve those few norm offsets back to offsets in the
// original content with ONE forward walk. A norm offset resolves to the
// start of the rune whose folded form covers it; the offset just past
// the end resolves to len(content).
bounds := make([]int, 0, len(wins)*2)
for _, wnd := range wins {
bounds = append(bounds, wnd.start, wnd.end)
}
sort.Ints(bounds) // window paddings may overlap, so bounds aren't sorted yet
orig := make(map[int]int, len(bounds))
bi, normOff := 0, 0
for oi, r := range content {
if bi == len(bounds) {
break
}
l := foldedLen(r)
for bi < len(bounds) && bounds[bi] < normOff+l {
orig[bounds[bi]] = oi
bi++
}
normOff += l
}
for ; bi < len(bounds); bi++ {
orig[bounds[bi]] = len(content) // the past-the-end boundary
}
out := make([]string, 0, len(wins))
for _, wnd := range wins {
oStart, oEnd := orig[wnd.start], orig[wnd.end]
prefix := "... "
if oStart == 0 {
prefix = ""
}
suffix := " ..."
if oEnd == len(content) {
suffix = ""
}
out = append(out, prefix+content[oStart:oEnd]+suffix)
}
return out
}
// --- Search-result highlighting ---
// Highlight term kinds
const (
hlWord = iota // matches whole words
hlWild // marks the whole word a wildcard fragment matched in (tap*, *keys*)
hlPhrase // matches an exact value (=tap) at word boundaries
hlSub // matches raw substrings (quoted text, and punctuated terms)
)
// hlTerm is one positive query term prepared for markTerms.
type hlTerm struct {
kind int
needle string // in normal form (folded + lowered)
stemmed string // the needle's stem when word stemming applies
openStart, openEnd bool // which sides of the fragment had "*"
}
// buildHighlightTerms extracts the positive terms of the clause list in the
// shape markTerms consumes, mirroring how the matchers read a clause
// (literalWord, compileWords). A quoted value is one exact phrase, plain
// words match whole words (stem-unified, like the index), "word*" matches
// word prefixes, and words with internal punctuation fall back to substring
// matching. Negated clauses are skipped (their terms are what the results
// DON'T contain).
func buildHighlightTerms(queries, conjs []string) []hlTerm {
var out []hlTerm
seen := make(map[string]bool)
add := func(t hlTerm) {
k := strconv.Itoa(t.kind) + strconv.FormatBool(t.openStart) + strconv.FormatBool(t.openEnd) + "\x00" + t.needle
if t.needle == "" || seen[k] {
return
}
seen[k] = true
out = append(out, t)
}
for _, q := range positiveTerms(queries, conjs) {
if _, phrase, _, ok := parseProximity(q); ok {
// Proximity clause. The match is word-based, not an exact
// substring, so highlight its individual words (with the
// same stemming eligibility bare words get below).
q = strings.Trim(phrase, `"`)
}
if v, ok := exactValue(q); ok {
add(hlTerm{kind: hlPhrase, needle: normFoldString(v)})
continue
}
q = normFoldString(strings.TrimSpace(q))
if q == "" {
continue
}
if isQuotedQuery(q) {
add(hlTerm{kind: hlSub, needle: strings.Trim(q, `"`)})
continue
}
for _, w := range strings.Fields(q) {
if frag, openStart, openEnd, ok := wildcard(w); ok {
if frag != "" {
add(hlTerm{kind: hlWild, needle: frag, openStart: openStart, openEnd: openEnd})
}
continue
}
parts := splitWords(w)
if len(parts) != 1 {
// Internal punctuation. The matchers read these as
// substrings, so highlight them the same way.
add(hlTerm{kind: hlSub, needle: w})
continue
}
w = parts[0]
t := hlTerm{kind: hlWord, needle: w}
// Stem eligibility mirrors tokenize, so "taps" highlights
// "tapped" exactly when the index unified them.
if len(w) > 1 && !stopWords[w] {
t.stemmed = stem(w)
}
add(t)
}
}
return out
}
// markTerms HTML-escapes s and wraps every occurrence of the given terms in
// <mark> tags, for the search results page.
//
// Matching runs on the normal form (normFold) so casing and accents can't
// hide a match, and the marked ranges are resolved back to the ORIGINAL
// text through normFold's position map (e.g. a "muller" query highlights
// "Müller" as written). Word terms match whole words, or words whose stem
// equals the term's stem (the same unification the index does); prefix
// terms match the start of a word; substring terms match anywhere.
func markTerms(s string, terms []hlTerm) template.HTML {
if s == "" || len(terms) == 0 {
return template.HTML(template.HTMLEscapeString(s))
}
norm, posMap := normFold(s)
var wordTerms, subTerms []hlTerm
for _, t := range terms {
if t.kind != hlWord {
subTerms = append(subTerms, t)
} else {
wordTerms = append(wordTerms, t)
}
}
type span struct{ start, end int } // byte offsets in norm
var spans []span
if len(wordTerms) > 0 {
needStem := false
for _, t := range wordTerms {
if t.stemmed != "" {
needStem = true
break
}
}
ws := -1 // current word start; -1 = between words
flush := func(we int) {
if ws < 0 {
return
}
word := norm[ws:we]
wstem := ""
if needStem {
wstem = stem(word)
}
for _, t := range wordTerms {
if word == t.needle || (t.stemmed != "" && wstem == t.stemmed) {
spans = append(spans, span{ws, we})
break
}
}
ws = -1
}
for i, r := range norm {
if unicode.IsLetter(r) || unicode.IsDigit(r) {
if ws < 0 {
ws = i
}
} else {
flush(i)
}
}
flush(len(norm))
}
for _, t := range subTerms {
for from := 0; ; {
i := -1
switch t.kind {
case hlPhrase:
i = indexWord(norm, t.needle, from)
case hlWild:
i = indexWild(norm, t.needle, from, t.openStart, t.openEnd)
default:
if j := strings.Index(norm[from:], t.needle); j >= 0 {
i = from + j
}
}
if i < 0 {
break
}
start, end := i, i+len(t.needle)
from = end
if t.kind == hlWild {
start, end = wordBounds(norm, start, end) // mark the whole word
}
spans = append(spans, span{start, end})
}
}
if len(spans) == 0 {
return template.HTML(template.HTMLEscapeString(s))
}
// Merge overlapping and touching spans so nested or adjacent matches
// render as one contiguous highlight.
sort.Slice(spans, func(i, j int) bool { return spans[i].start < spans[j].start })
merged := spans[:1]
for _, sp := range spans[1:] {
if last := &merged[len(merged)-1]; sp.start <= last.end {
if sp.end > last.end {
last.end = sp.end
}
} else {
merged = append(merged, sp)
}
}
// Emit escaped gap / <mark> / escaped match / </mark> runs. Norm
// offsets resolve to original offsets via posMap; when a span boundary
// falls inside one rune's multi-byte fold (e.g. 'ẞ' -> "ss"), the whole
// rune is included.
var b strings.Builder
last := 0
for _, sp := range merged {
oStart := posMap[sp.start]
j := sp.end
for j < len(norm) && posMap[j] == posMap[sp.end-1] {
j++
}
oEnd := posMap[j]
if oStart < last {
oStart = last // rune snapping can nudge a span into the previous one
}
if oEnd <= oStart {
continue
}
b.WriteString(template.HTMLEscapeString(s[last:oStart]))
b.WriteString("<mark>")
b.WriteString(template.HTMLEscapeString(s[oStart:oEnd]))
b.WriteString("</mark>")
last = oEnd
}
b.WriteString(template.HTMLEscapeString(s[last:]))
return template.HTML(b.String())
}
// --- Search execution (shared by the HTML page and the JSON API) ---
// ScoredDoc is one match, with its relevance score. Doc points into archive.Docs, which is
// read-only while serving: the match set is walked, filtered and sorted as
// pointers, and the page a reader sees wraps them in searchResults, which
// carry the highlighting.
type ScoredDoc struct {
Doc *Document
Score float64
}
// searchResult is one row of the search page: a document and what the
// query marked in it.
type searchResult struct {
Doc *Document
Title, Desc template.HTML // escaped, query matches in <mark> (markTerms)
Snippets []template.HTML // up to 3 match-context windows, <mark>-highlighted
VersionSnippets []VersionSnippet // windows from versions' new lines, labelled with their copy
MatchesIn []*DocVersion // versions that matched when the main transcript didn't
}
const maxSnippetDocs = 100
const maxSearchClauses = 24
const maxSearchWords = 64
func searchWords(queries []string) int {
n := 0
for _, q := range queries {
n += len(strings.Fields(q))
}
return n
}
// normalizeClauses caps the clause count and pads/truncates fields and
// conjs to the length of queries.
func normalizeClauses(queries, fields, conjs []string) ([]string, []string, []string) {
if len(queries) > maxSearchClauses {
queries = queries[:maxSearchClauses]
}
trimmed := make([]string, len(queries))
for i, q := range queries {
trimmed[i] = strings.TrimSpace(q)
}
queries = trimmed
if len(fields) > len(queries) {
fields = fields[:len(queries)]
}
for len(fields) < len(queries) {
fields = append(fields, "TE")
}
if len(conjs) > len(queries) {
conjs = conjs[:len(queries)]
}
for len(conjs) < len(queries) {
conjs = append(conjs, "and")
}
return queries, fields, conjs
}
// searchField describes one searchable field: its wire code (the fqf
// value), UI label, simple-box operator aliases, and what it searches. This
// table is the single source of truth.
type searchField struct {
Code string
Label string
Aliases []string
Stemmed, Folded func(d *Document) string
Values func(d *Document) []string
read func(d *Document) []string
index int // the field's place in searchFields
}
type docFields struct {
values [][]string
keys [][]string
folded []string
pubYears []string // pubYearsOf
}
func (co *Corpus) buildDocFields() {
for i := range co.archive.Docs {
d := &co.archive.Docs[i]
d.fields = nil // the readers below work everything out
f := &docFields{
values: make([][]string, len(searchFields)),
keys: make([][]string, len(searchFields)),
folded: make([]string, len(searchFields)),
pubYears: pubYearsOf(d),
}
for fi := range searchFields {
sf := &searchFields[fi]
if sf.read == nil {
continue
}
vals := sf.read(d)
keys := make([]string, len(vals))
for vi, v := range vals {
keys[vi] = valueKey(v)
}
f.values[fi], f.keys[fi] = vals, keys
f.folded[fi] = normFoldString(strings.Join(vals, valueSep))
}
d.fields = f
}
}
// searchFields is in the form dropdown's display order.
var searchFields = []searchField{
// SearchText is stored pre-folded, so quoted text and exact words are
// direct tests against it.
{Code: "TE", Label: "All fields", Aliases: []string{"text"},
Stemmed: func(d *Document) string { return d.StemmedText },
Folded: func(d *Document) string { return d.SearchText }},
{Code: "TI", Label: "Title", Aliases: []string{"title"},
Stemmed: func(d *Document) string { return d.StemmedTitle },
Folded: (*Document).titleFold},
{Code: "FT", Label: "Content", Aliases: []string{"content"},
Stemmed: func(d *Document) string { return d.StemmedFull },
Folded: func(d *Document) string { return d.FoldedFull }},
{Code: "DE", Label: "Description", Aliases: []string{"description"},
Stemmed: func(d *Document) string { return d.StemmedDesc },
Folded: (*Document).descFold},
{Code: "CR", Label: "Agency", Aliases: []string{"agency"},
Values: func(d *Document) []string { return single(d.DocMeta.Creator) }},
{Code: "RE", Label: "Journalist", Aliases: []string{"journalist"},
Values: journalistsOf},
{Code: "PU", Label: "Publisher", Aliases: []string{"publisher"},
Values: publishersOf},
{Code: "CL", Label: "Classification", Aliases: []string{"classification"},
Values: func(d *Document) []string { return d.DocMeta.Classification }},
{Code: "PR", Label: "Program", Aliases: []string{"program"},
Values: func(d *Document) []string { return d.DocMeta.Programs }},
{Code: "DI", Label: "Dissemination", Aliases: []string{"dissemination"},
Values: func(d *Document) []string { return d.DocMeta.Dissemination }},
{Code: "BA", Label: "Banner", Aliases: []string{"banner"},
Values: func(d *Document) []string { return single(d.DocMeta.Banner) }},
{Code: "GE", Label: "Genre", Aliases: []string{"genre"},
Values: func(d *Document) []string { return single(d.DocMeta.Genre) }},
{Code: "SE", Label: "Series", Aliases: []string{"series"},
Values: func(d *Document) []string { return single(d.DocMeta.Series.Name) }},
{Code: "CO", Label: "Country", Aliases: []string{"country"},
Values: func(d *Document) []string { return d.DocMeta.Countries }},
{Code: "PE", Label: "People", Aliases: []string{"people"},
Values: func(d *Document) []string { return d.DocMeta.People }},
{Code: "TA", Label: "Tags", Aliases: []string{"tag"},
Values: func(d *Document) []string { return d.Tags }},
{Code: "TY", Label: "Type", Aliases: []string{"type"},
Values: func(d *Document) []string { return single(d.Type) }},
{Code: "PL", Label: "Portion level",
Values: func(d *Document) []string { return d.PortionLevels }},
{Code: "PM", Label: "Portion marking",
Values: func(d *Document) []string { return d.PortionMarkings }},
{Code: "PC", Label: "Portion control",
Values: func(d *Document) []string { return d.PortionControls }},
}
func single(v string) []string {
if v == "" {
return nil
}
return []string{v}
}
var (
fieldByCode = map[string]*searchField{}
simpleOps = map[string]string{}
)
func init() {
for i := range searchFields {
f := &searchFields[i]
f.index = i
if read := f.Values; read != nil {
f.read = read
f.Values = func(d *Document) []string {
if d.fields != nil {
return d.fields.values[i]
}
return read(d)
}
}
fieldByCode[f.Code] = f
for _, a := range f.Aliases {
simpleOps[a] = f.Code
}
}
}
type simpleDates struct {
DocStart, DocEnd, PubStart, PubEnd string
}
// padDateEnd widens a partial date for use as an inclusive range end:
// 2013 -> 2013-12-31, 2013-06 -> 2013-06-31 (string comparison makes the
// over-wide day harmless).
func padDateEnd(v string) string {
switch len(v) {
case 4:
return v + "-12-31"
case 7:
return v + "-31"
}
return v
}
// parseSimpleQuery turns a one-line query into fielded clauses plus date
// filters.
//
// program:XKEYSCORE country:Germany cable tapping
//
// becomes three AND clauses: PR=XKEYSCORE, CO=Germany, TE="cable tapping".
// Values can be double-quoted (journalist:"Poitras, Laura"); the quotes are
// KEPT on the clause value, so a quoted value has the same exact-match
// (substring/phrase) semantics it has when typed into the structured form,
// and a bare "quoted phrase" becomes its own exact all-fields clause rather
// than dissolving into the free-text words.
//
// A leading '-' negates: -term and -program:X become "and not" clauses
// (ordered after the positive ones, since the boolean evaluation is
// left-to-right). date:2013, date:2013-06, date:2013..2015 filter on the
// document date; published: does the same for publication dates.
func parseSimpleQuery(s string) (queries, fields, conjs []string, dates simpleDates) {
var freeText []string
type clause struct{ q, f string }
var negs []clause
for i, neg, ok := nextTerm(s, 0); ok; i, neg, ok = nextTerm(s, i) {
// try operator: word characters followed by ':'
opEnd := i
for opEnd < len(s) && (isWordByte(s[opEnd])) {
opEnd++
}
field := ""
dateOp := ""
valStart := i
if opEnd > i && opEnd < len(s) && s[opEnd] == ':' {
name := strings.ToLower(s[i:opEnd])
if code, ok := simpleOps[name]; ok {
field = code
valStart = opEnd + 1
} else if name == "date" || name == "published" {
dateOp = name
valStart = opEnd + 1
}
}
clauseVal, val, quoted, exact, next := readValue(s, valStart)
i = next
if val == "" {
continue
}
if dateOp != "" {
// date filters can't be negated; '-' before them is ignored.
start, end := val, val
if a, b, ok := strings.Cut(val, ".."); ok {
start, end = strings.TrimSpace(a), strings.TrimSpace(b)
}
end = padDateEnd(end)
if dateOp == "date" {
dates.DocStart, dates.DocEnd = start, end
} else {
dates.PubStart, dates.PubEnd = start, end
}
continue
}
val = clauseVal
switch {
case field != "" && neg:
negs = append(negs, clause{val, field})
case field != "":
queries = append(queries, val)
fields = append(fields, field)
case neg:
negs = append(negs, clause{val, "TE"})
case quoted || exact:
queries = append(queries, val)
fields = append(fields, "TE")
default:
freeText = append(freeText, val)
}
}
if len(freeText) > 0 {
queries = append(queries, strings.Join(freeText, " "))
fields = append(fields, "TE")
}
if len(queries) == 0 {
queries = []string{""}
fields = []string{"TE"}
}
conjs = make([]string, len(queries))
for i := range conjs {
conjs[i] = "and"
}
for _, n := range negs {
queries = append(queries, n.q)
fields = append(fields, n.f)
conjs = append(conjs, "not")
}
return queries, fields, conjs, dates
}
func isWordByte(b byte) bool {
return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') || b == '_'
}
const spaceBytes = " \t\n\v\f\r"
func isSpaceByte(b byte) bool { return strings.IndexByte(spaceBytes, b) >= 0 }
func proxSuffix(s string, i int) (string, int) {
if i >= len(s) || s[i] != '~' {
return "", i
}
j := i + 1
for j < len(s) && s[j] >= '0' && s[j] <= '9' {
j++
}
if j == i+1 || (j < len(s) && !isSpaceByte(s[j])) {
return "", i
}
return s[i:j], j
}
// readToken reads one search-value token starting at s[i]: either a
// "quoted span" returned with the quotes stripped and quoted=true, or
// a bare run up to the next whitespace. next is the index just past the
// token. This is THE tokenizer for search values: parseSimpleQuery and
// rowTerms both use it, behind nextTerm, so neither the quoting rule nor
// the negation rule can drift between them.
func readToken(s string, i int) (val string, quoted bool, next int) {
if i < len(s) && s[i] == '"' {
j := i + 1
if k := strings.IndexByte(s[j:], '"'); k >= 0 {
return s[j : j+k], true, j + k + 1
}
return s[i+1:], true, len(s)
}
j := i
for j < len(s) && !isSpaceByte(s[j]) {
j++
}
return s[i:j], false, j
}
// nextTerm finds the next term of a query from s[i] on: it skips the
// whitespace, and a leading '-' negates the term when it is attached to
// one. i is where the term's value (or its field: operator) starts; ok is
// false at the end of the string.
func nextTerm(s string, i int) (start int, neg, ok bool) {
for i < len(s) && isSpaceByte(s[i]) {
i++
}
if i >= len(s) {
return i, false, false
}
if s[i] == '-' && i+1 < len(s) && !isSpaceByte(s[i+1]) {
return i + 1, true, true
}
return i, false, true
}
// readValue reads one search value at s[i]
func readValue(s string, i int) (clause, inner string, quoted, exact bool, next int) {
if i < len(s) && s[i] == '=' {
exact = true
i++
}
inner, quoted, next = readToken(s, i)
prox := ""
if quoted {
prox, next = proxSuffix(s, next)
}
inner = strings.TrimSpace(inner)
switch {
case inner == "":
case exact && quoted:
clause = `="` + inner + `"`
case exact:
clause = "=" + inner
case quoted:
clause = `"` + inner + `"` + prox
default:
clause = inner
}
return clause, inner, quoted, exact, next
}
// rowTerms splits one search row into what it requires and what it
// excludes, the way parseSimpleQuery splits ?q=: each quoted phrase,
// proximity clause or exact value is a positive part of its own, the bare
// words together are one more (last), and google-style -term and -"phrase"
// tokens are exclusions on the row's own field.
//
// This is THE reading of a row: the matcher, the scorer, the snippet anchor
// and the highlighter all go through it, so "beta -gamma" is (beta AND NOT
// gamma) as one unit wherever the row sits.
func rowTerms(s string) (positives, negatives []string) {
var words []string
for i, neg, ok := nextTerm(s, 0); ok; i, neg, ok = nextTerm(s, i) {
tok, _, quoted, exact, next := readValue(s, i)
i = next
switch {
case tok == "" || tok == "-":
case neg:
negatives = append(negatives, tok)
case quoted || exact:
positives = append(positives, tok)
default:
words = append(words, tok)
}
}
if len(words) > 0 {
positives = append(positives, strings.Join(words, " "))
}
return positives, negatives
}
// executeSearch is the search pipeline the page and the API both run: over
// the whole archive, the collection filter, the dates and the boolean
// clause list, then the type filter, and the survivors ranked and sorted
// by p.Sort/p.Order, as a fresh slice the caller owns. The two filters are
// the browse pages' (inCollections, typeSelected). typeCounts tallies the
// types BEFORE the type filter. Only the documents that pass every filter
// are scored.
func (co *Corpus) executeSearch(p searchParams) (matches []ScoredDoc, typeCounts map[string]int) {
dateDocEnd := padDateEnd(p.DateDocEnd)
datePubEnd := padDateEnd(p.DatePubEnd)
query := compileQuery(p.Queries, p.Fields, p.Conjs)
byEntry := entryRows(p.Queries, p.Fields, p.Conjs)
var scorers []func(*Document) float64
for _, q := range positiveTerms(p.Queries, p.Conjs) {
scorers = append(scorers, co.tfidf.scorer(q))
}
typeCounts = make(map[string]int)
for i := range co.archive.Docs {
doc := &co.archive.Docs[i]
if !inCollections(doc, p.Coll) ||
!matchDates(doc, p.DateDocStart, dateDocEnd, p.DatePubStart, datePubEnd, byEntry) ||
!query.matches(doc) {
continue
}
typeCounts[doc.Type]++
if !typeSelected(doc, p.Types) {
continue
}
totalScore := 0.0
for _, score := range scorers {
totalScore += score(doc)
}
matches = append(matches, ScoredDoc{Doc: doc, Score: totalScore})
}
slices.SortStableFunc(matches, func(a, b ScoredDoc) int { return cmp.Compare(b.Score, a.Score) })
return sortScored(matches, p.Sort, p.Order), typeCounts
}
// --- Search handler (HTML) ---
// --- Faceted search refinement -----------------------------------------------
type searchFacet struct {
Label string
Values []facetEntry
}
func topFacetCounts(m map[string]int, total, n int, baseURL, field string) []facetEntry {
var out []facetEntry
for _, v := range rankCounts(m, -1) {
if len(out) == n {
break
}
if m[v] >= total {
continue
}
// ="value" -> a whole-value match on the field, the equality the
// tally counted, so the number is the number the click finds.
out = append(out, facetEntry{Name: v, Count: m[v], URL: appendClause(baseURL, field, exactClause(v))})
}
return out
}
func facetValues(d *Document, code string) []string {
if f := fieldByCode[code]; f != nil && f.Values != nil {
return f.Values(d)
}
return nil
}
type facetGroup struct {
Label string
Code string
}
func facetGroups(docs []*Document, groups []facetGroup, baseURL string, values func(*Document, string) []string) []searchFacet {
type tally struct{ count, first int }
tallies := make([]map[string]*tally, len(groups))
for i := range tallies {
tallies[i] = map[string]*tally{}
}
seq := 0
for i := range docs {
for gi, g := range groups {
for _, v := range values(docs[i], g.Code) {
t := tallies[gi][v]
if t == nil {
t = &tally{first: seq}
tallies[gi][v] = t
}
t.count++
seq++
}
}
}
out := make([]searchFacet, 0, len(groups))
for gi, g := range groups {
type folded struct {
name string
count, first int
}
byKey := make(map[string]*folded, len(tallies[gi]))
for v, t := range tallies[gi] {
k := valueKey(v)
f := byKey[k]
if f == nil {
byKey[k] = &folded{v, t.count, t.first}
continue
}
f.count += t.count
if t.first < f.first {
f.name, f.first = v, t.first
}
}
counts := make(map[string]int, len(byKey))
for _, f := range byKey {
counts[f.name] = f.count
}
if vals := topFacetCounts(counts, len(docs), 8, baseURL, g.Code); len(vals) > 0 {
out = append(out, searchFacet{Label: g.Label, Values: vals})
}
}
return out
}
func facetsWithinWords(facets []searchFacet, room int) []searchFacet {
out := facets[:0]
for _, f := range facets {
vals := f.Values[:0]
for _, v := range f.Values {
if searchWords([]string{exactClause(v.Name)}) <= room {
vals = append(vals, v)
}
}
if len(vals) > 0 {
f.Values = vals
out = append(out, f)
}
}
return out
}
func computeSearchFacets(docs []*Document, baseURL string, values func(*Document, string) []string) []searchFacet {
return facetGroups(docs, []facetGroup{
{"Programs", "PR"}, {"Agencies", "CR"}, {"Countries", "CO"}, {"Journalists", "RE"},
}, baseURL, values)
}
func collectionFilter(v url.Values) map[string]bool {
sel := map[string]bool{}
for _, c := range v["coll"] {
if validCollections[c] {
sel[c] = true
}
}
if len(sel) == 0 || len(sel) == len(validCollections) {
return nil
}
return sel
}
func snippetAnchor(queries, fields, conjs []string) (term, field string) {
for i, q := range queries {
if q == "" || conjs[i] == "not" {
continue
}
pos, _ := rowTerms(q)
if len(pos) == 0 {
continue // only exclusions: nothing in the results to anchor on
}
q = pos[0]
if _, phrase, _, ok := parseProximity(q); ok {
if ws := strings.Fields(strings.Trim(phrase, `"`)); len(ws) > 0 {
return ws[0], fields[i]
}
}
return q, fields[i]
}
return "", ""
}
// positiveTerms is what the query asks the results to contain: each
// non-empty row's positive part, skipping NOT-rows and every row's
// exclusions (rowTerms). The scorer and the highlighter read the query
// through it, so neither rewards or marks a term the results were selected
// for NOT containing.
func positiveTerms(queries, conjs []string) []string {
var out []string
for i, q := range queries {
if q == "" || (i < len(conjs) && conjs[i] == "not") {
continue
}
pos, _ := rowTerms(q)
out = append(out, pos...)
}
return out
}
// searchParams is one search's effective inputs as read by readSearchParams.
type searchParams struct {
Queries, Fields, Conjs []string
DateDocStart, DateDocEnd, DatePubStart, DatePubEnd string
FromQ bool // a ?q= shortcut replaced the structured params
Coll map[string]bool // collectionFilter: nil filters nothing
Types map[string]bool // selectedTypes: empty filters nothing
Sort, Order string // Sort defaults to "relevance"
OverClauses int // the requested clause count when it exceeds maxSearchClauses: the search must not run
OverWords int // the word count (searchWords) when it exceeds maxSearchWords: the search must not run
}
func (p searchParams) refused() bool { return p.OverClauses > 0 || p.OverWords > 0 }
// readSearchParams reads a search from URL params. this intake is shared by
// the HTML page and the JSON API.
//
// A non-empty ?q= replaces any structured clause params. Plain words become
// a single all-fields clause. Field operators like program:XKEYSCORE or
// journalist:"Poitras, Laura" become their own AND clauses (see
// parseSimpleQuery). Its parsed dates override the URL's when either end is
// set - date:..2015 has an empty start but is still a real filter.
func readSearchParams(v url.Values) searchParams {
p := searchParams{
Queries: v["fqv"],
Fields: v["fqf"],
Conjs: v["fqc"],
DateDocStart: v.Get("date_doc_start"),
DateDocEnd: v.Get("date_doc_end"),
DatePubStart: v.Get("date_pub_start"),
DatePubEnd: v.Get("date_pub_end"),
Coll: collectionFilter(v),
Sort: sortToken(v.Get("sort")),
Order: sortToken(v.Get("order")),
}
p.Types, _ = selectedTypes(v)
if docSortByKey[p.Sort] == nil {
p.Sort, p.Order = "relevance", ""
}
if s := v.Get("q"); s != "" {
var sd simpleDates
p.Queries, p.Fields, p.Conjs, sd = parseSimpleQuery(s)
if sd.DocStart != "" || sd.DocEnd != "" {
p.DateDocStart, p.DateDocEnd = sd.DocStart, sd.DocEnd
}
if sd.PubStart != "" || sd.PubEnd != "" {
p.DatePubStart, p.DatePubEnd = sd.PubStart, sd.PubEnd
}
p.FromQ = true
}
if n := len(p.Queries); n > maxSearchClauses {
p.OverClauses = n
} else if n := searchWords(p.Queries); n > maxSearchWords {
p.OverWords = n
}
p.Queries, p.Fields, p.Conjs = normalizeClauses(p.Queries, p.Fields, p.Conjs)
return p
}
func (p searchParams) dates() [4]string {
return [4]string{p.DateDocStart, p.DateDocEnd, p.DatePubStart, p.DatePubEnd}
}
// canonicalSearchQuery rebuilds the query params from the EFFECTIVE state
// of a search (the post-add/remove clause rows, the collection and type
// filters, the dates that a ?q= shortcut may have parsed out of the box)
// for every link the page generates (sorting, pagination, the histogram
// bars, the refine box, the export and the recent/pinned entries).
func canonicalSearchQuery(doSearch bool, p searchParams) url.Values {
q := url.Values{}
if doSearch {
// A ?q= entry has no do_search param, so the canonical form adds it
// explicitly.
q.Set("do_search", "1")
}
for i := range p.Queries {
q.Add("fqv", p.Queries[i])
q.Add("fqf", p.Fields[i])
q.Add("fqc", p.Conjs[i])
}
if colls := collectionParams(p.Coll); len(colls) > 0 {
q["coll"] = colls
}
if len(p.Types) > 0 {
types := make([]string, 0, len(p.Types))
for t := range p.Types {
types = append(types, t)
}
sort.Strings(types)
q["type"] = types
}
dates := p.dates()
for i, k := range []string{"date_doc_start", "date_doc_end", "date_pub_start", "date_pub_end"} {
if dates[i] != "" {
q.Set(k, dates[i])
}
}
return q
}
// searchDateHistogram is the year-bar strip under a result set.
func searchDateHistogram(results []*Document, q url.Values) []timelineBar {
counts := make(map[string]int)
undated := 0
for i := range results {
if y := docYear(results[i]); y != "" {
counts[y]++
} else {
undated++
}
}
_, maxCount := sortedYears(counts)
if undated > maxCount {
maxCount = undated // the undated bar shares the scale
}
start, end := q.Get("date_doc_start"), q.Get("date_doc_end")
bars := yearBars(counts, func(y string) string {
from, to := y, y
if start > y {
from = start
}
if end != "" && padDateEnd(end) < padDateEnd(y) {
to = end
}
yq := cloneValues(q)
yq.Set("date_doc_start", from)
yq.Set("date_doc_end", to)
return "/search?" + yq.Encode()
})
for i := range bars {
bars[i].Pct = barPct(bars[i].Count, maxCount, 1)
}
if undated > 0 {
bars = append(bars, timelineBar{
Year: "-",
Count: undated,
Pct: barPct(undated, maxCount, 1),
})
}
return bars
}
// decorateResults marks the query's terms in the titles, descriptions and
// body snippets of the results on the page.
func decorateResults(docs []*Document, queries, fields, conjs []string) (results []searchResult, snippetsCapped bool) {
snippetTerm, snippetField := snippetAnchor(queries, fields, conjs)
hl := buildHighlightTerms(queries, conjs)
results = make([]searchResult, len(docs))
for i, doc := range docs {
res := &results[i]
res.Doc = doc
res.Title = markTerms(doc.Title, hl)
res.Desc = markTerms(doc.Description, hl)
if snippetTerm == "" || i >= maxSnippetDocs {
continue
}
var raw []string
var hits []versionHit
mainHit := false
if snippetField == "TA" {
raw = getSnippets(strings.Join(doc.Tags, ", "), snippetTerm, 3)
} else {
raw = doc.bodySnippets(snippetTerm, 3)
mainHit = len(raw) > 0
if len(raw) < 3 {
hits = versionHits(doc, snippetTerm, 3)
}
if !mainHit && len(hits) == 0 {
raw = getSnippets(doc.Description, snippetTerm, 1)
}
}
for _, s := range raw {
res.Snippets = append(res.Snippets, markTerms(s, hl))
}
room := 3 - len(raw)
for _, h := range hits {
if !mainHit {
res.MatchesIn = append(res.MatchesIn, h.Version)
}
for _, s := range h.Snippets {
if room <= 0 {
break
}
res.VersionSnippets = append(res.VersionSnippets,
VersionSnippet{Slug: h.Version.Slug, Label: h.Version.Label, HTML: markTerms(s, hl)})
room--
}
}
}
return results, snippetTerm != "" && len(docs) > maxSnippetDocs
}
func (co *Corpus) handleSearch(w http.ResponseWriter, r *http.Request) {
params := r.URL.Query()
action := params.Get("action")
sp := readSearchParams(params)
prefs := readPrefs(r)
scope := pageScope(params, prefs, true)
sp.Coll = scope.filter
queries, fields, conjs := sp.Queries, sp.Fields, sp.Conjs
sortBy, order := sp.Sort, sp.Order
doSearch := (params.Get("do_search") == "1" || sp.FromQ) && !sp.refused()
if len(queries) == 0 {
queries = []string{""}
fields = []string{"TE"}
conjs = []string{"and"}
}
switch {
case strings.HasPrefix(action, "remove-"):
idxStr := strings.TrimPrefix(action, "remove-")
removeIdx, err := strconv.Atoi(idxStr)
if err == nil && removeIdx > 0 && removeIdx < len(queries) {
queries = append(queries[:removeIdx], queries[removeIdx+1:]...)
fields = append(fields[:removeIdx], fields[removeIdx+1:]...)
conjs = append(conjs[:removeIdx], conjs[removeIdx+1:]...)
}
case action == "add" && len(queries) < maxSearchClauses:
queries = append(queries, "")
fields = append(fields, "TE")
conjs = append(conjs, "and")
}
sp.Queries, sp.Fields, sp.Conjs = queries, fields, conjs
var scored []ScoredDoc
searchTypeCounts := map[string]int{}
if doSearch {
scored, searchTypeCounts = co.executeSearch(sp)
}
selColl := scope.shown
collFiltered := scope.explicit()
selTypeSet := sp.Types
results := make([]*Document, 0, len(scored))
for _, s := range scored {
results = append(results, s.Doc)
}
q := canonicalSearchQuery(doSearch, sp)
q["coll"] = scope.colls
canonicalQS := q.Encode()
searchBaseURL := "/search?" + canonicalQS + "&"
formParams := cloneValues(q)
if sortBy != "relevance" {
formParams.Set("sort", sortBy)
}
if order != "" {
formParams.Set("order", order)
}
page, _ := strconv.Atoi(params.Get("p"))
start, end, page, totalPages := paginate(len(results), page, prefs.cardsPerPage())
pageResults, snippetsCapped := decorateResults(results[start:end], queries, fields, conjs)
labelQ := q
if scope.defaulted {
labelQ = cloneValues(q)
delete(labelQ, "coll")
}
queryLabel, named := "", false // the page title's name for the search
if doSearch {
queryLabel, named = co.searchLabel(labelQ)
}
savable := named && len(canonicalQS) <= searchQSMax
var recorded []string
if prefs.TrackSearches && savable &&
(action == "" || action == "search") && !crossSiteFetch(r) {
recorded = recordRecentSearch(w, r, canonicalQS)
}
var recentSearches []recentSearch
if prefs.TrackSearches {
if recorded != nil {
recentSearches = co.parseSearchEntries(recorded)
} else {
recentSearches = co.readRecentSearches(r)
}
}
searchPinned := savable && isPinnedSearch(r, canonicalQS)
var dateHist []timelineBar
if doSearch && len(results) > 0 {
dateHist = searchDateHistogram(results, q)
}
var facets []searchFacet
if doSearch && len(results) >= 2 && len(queries) < maxSearchClauses {
values := facetValues
if !hasOrRow(queries, conjs) {
values = refineValues(sp.DatePubStart, sp.DatePubEnd)
}
facets = facetsWithinWords(computeSearchFacets(results, searchBaseURL, values),
maxSearchWords-searchWords(queries))
}
if len(recentSearches) > 0 {
noStore(w)
}
render(w, r, &searchPage{
FieldOptions: searchFields,
Facets: facets,
Queries: queries,
Fields: fields,
Conjs: conjs,
Results: pageResults,
SearchTypeCounts: searchTypeCounts,
SelTypeSet: selTypeSet,
FormParams: formParams,
Savable: savable,
ShowThumbs: prefs.thumbsOn(),
FOIAQS: prefs.foiaLinkQS(),
Base: pageWith(prefs),
Total: len(results),
Page: page,
TotalPages: totalPages,
RecentSearches: recentSearches,
SelColl: selColl,
CollFiltered: collFiltered,
CanonicalQS: canonicalQS,
SearchPinned: searchPinned,
Searched: doSearch,
QueryLabel: queryLabel,
DateDocStart: sp.DateDocStart,
DateDocEnd: sp.DateDocEnd,
DatePubStart: sp.DatePubStart,
DatePubEnd: sp.DatePubEnd,
Sort: sortBy,
Order: order,
SearchBaseURL: searchBaseURL,
DateHist: dateHist,
SnippetsCapped: snippetsCapped,
SnippetLimit: maxSnippetDocs,
OverClauses: sp.OverClauses,
ClauseLimit: maxSearchClauses,
OverWords: sp.OverWords,
WordLimit: maxSearchWords,
})
}
// --- Date helpers ---
func inDateRange(date, start, end string) bool {
if date == "" {
return start == "" && end == ""
}
if start != "" && date < start {
return false
}
if end != "" && date > end {
return false
}
return true
}
func matchDates(doc *Document, dateDocStart, dateDocEnd, datePubStart, datePubEnd string, byEntry []entryRow) bool {
if dateDocStart != "" || dateDocEnd != "" {
if !inDateRange(doc.DocMeta.DateCreated, dateDocStart, dateDocEnd) {
return false
}
}
if datePubStart == "" && datePubEnd == "" {
return true
}
inRange := func(i int) bool {
return inDateRange(doc.Publication[i].DatePublished, datePubStart, datePubEnd)
}
if len(byEntry) == 0 {
for i := range doc.Publication {
if inRange(i) {
return true
}
}
return false
}
for _, r := range byEntry {
found := false
for i := range doc.Publication {
if inRange(i) && r.matches(doc, i) {
found = true
break
}
}
if !found {
return false
}
}
return true
}
// entryRow is a journalist or publisher row a publication date is tied
// to, parsed once per search.
type entryRow struct {
field string // "RE" or "PU"
pos, negs []valuesTest
}
func entryRows(queries, fields, conjs []string) []entryRow {
if hasOrRow(queries, conjs) {
return nil
}
var rows []entryRow
for i, q := range queries {
if q == "" {
continue
}
if conjs[i] != "not" && (fields[i] == "RE" || fields[i] == "PU") {
pos, negs := rowTerms(q)
rows = append(rows, entryRow{fields[i], entryTests(pos), entryTests(negs)})
}
}
return rows
}
func hasOrRow(queries, conjs []string) bool {
for i, q := range queries {
if q != "" && conjs[i] == "or" {
return true
}
}
return false
}
func entryTests(terms []string) []valuesTest {
tests := make([]valuesTest, len(terms))
for i, t := range terms {
if _, phrase, _, ok := parseProximity(t); ok {
t = phrase
}
tests[i] = compileValues(t)
}
return tests
}
func entryValues(doc *Document, i int, field string) []string {
p := &doc.Publication[i]
switch {
case field == "RE":
return p.Journalists
case !p.ReportingOnly:
return []string{p.Publisher}
}
return nil
}
func refineValues(pubStart, pubEnd string) func(*Document, string) []string {
if pubStart == "" && pubEnd == "" {
return facetValues
}
pubEnd = padDateEnd(pubEnd)
return func(d *Document, code string) []string {
if code != "RE" && code != "PU" {
return facetValues(d, code)
}
var vals []string
for i := range d.Publication {
if inDateRange(d.Publication[i].DatePublished, pubStart, pubEnd) {
vals = append(vals, entryValues(d, i, code)...)
}
}
return normalizeList(vals)
}
}
// matches reports whether publication entry i matches the row on its own
// journalists or publisher (entryValues).
func (r entryRow) matches(doc *Document, i int) bool {
vals := entryValues(doc, i, r.field)
for t := range r.pos {
if !r.pos[t].match(vals) {
return false
}
}
for t := range r.negs {
if r.negs[t].match(vals) {
return false
}
}
return true
}
// --- The boolean clause list ---
// compiledQuery is a search's rows read once. The non-empty ones, in order,
// each with the tests of its positive values and of its exclusions on its
// field.
type compiledQuery []compiledRow
type compiledRow struct {
conj string
pos, negs []func(*Document) bool
}
func compileQuery(queries, fields, conjs []string) compiledQuery {
var rows compiledQuery
for i, q := range queries {
if q == "" {
continue
}
f := fieldByCode[fields[i]]
if f == nil {
f = fieldByCode["TE"]
}
row := compiledRow{conj: conjs[i]}
pos, negs := rowTerms(q)
for _, v := range pos {
row.pos = append(row.pos, f.compile(v))
}
for _, v := range negs {
row.negs = append(row.negs, f.compile(v))
}
rows = append(rows, row)
}
return rows
}
// matches evaluates the rows against a document, left to right, each row
// as one unit.
func (rows compiledQuery) matches(d *Document) bool {
overall := true
for i := range rows {
row := &rows[i]
if i == 0 {
overall = row.matches(d) != (row.conj == "not")
continue
}
switch row.conj {
case "or":
overall = overall || row.matches(d)
case "not":
overall = overall && !row.matches(d)
default: // "and"
overall = overall && row.matches(d)
}
}
return overall
}
// matches reports whether a document matches one row.
func (row *compiledRow) matches(d *Document) bool {
for _, test := range row.pos {
if !test(d) {
return false
}
}
for _, test := range row.negs {
if test(d) {
return false
}
}
return true
}
// versionHit is one of a document's versions whose new lines match a query,
// with snippet windows from those lines.
type versionHit struct {
Version *DocVersion
Snippets []string
}
// versionHits returns the versions of doc whose new lines (NovelText) match
// term in version order.
func versionHits(doc *Document, term string, max int) []versionHit {
var out []versionHit
for i := range doc.Versions {
v := &doc.Versions[i]
if s := getSnippets(v.NovelText, term, max); len(s) > 0 {
out = append(out, versionHit{Version: v, Snippets: s})
}
}
return out
}