package main
import (
"cmp"
"html/template"
"log"
"math"
"net/http"
"net/url"
"slices"
"sort"
"strconv"
"strings"
"unicode"
"unicode/utf8"
)
type TFIDFIndex struct {
DocStats map[string]map[string]float64
DocFreq map[string]int
N int
}
// stopWords are excluded from tokenization to avoid recreating the map on
// every tokenize() call.
var stopWords = map[string]bool{
"the": true, "and": true, "for": true, "that": true,
"this": true, "with": true, "from": true, "are": true,
"was": true, "were": true, "has": true, "have": true,
"not": true, "but": true, "they": true, "their": true,
"will": true, "been": true, "its": true, "also": true,
"into": true, "which": true, "such": true, "other": true,
}
// --- Stemmer ---
// Porter-lite: suffix stripping for English. Applied at both index time and
// query time so "surveillance" and "surveilling" map to the same token.
// Intentionally conservative - only strips high-confidence suffixes to avoid
// over-stemming domain-specific terms like "SIGINT" or "collection".
func stem(w string) string {
// The last two steps apply whichever suffix rule returned: "flying" and
// "flyer" leave "fly", which must meet "fly" itself.
s := yToI(stripSuffixes(w))
// fixStem keeps a doubled l ("falling" -> "fall"), so "controlled" and
// "controller" reach "controll" while "control" stays "control". A final
// "ll" after more than one vowel-consonant run is undoubled in every
// word alike (Porter's step 5b): control/controlled -> "control",
// install/installing -> "instal", while fall/falling stay "fall".
if strings.HasSuffix(s, "ll") && measure(s) > 1 {
return s[:len(s)-1]
}
return s
}
func stripSuffixes(w string) string {
switch {
case strings.HasSuffix(w, "sses"):
w = w[:len(w)-2]
case strings.HasSuffix(w, "ies") && len(w) > 4:
w = w[:len(w)-3] + "y" // "studies" -> "study", "spies" -> "spy"
case strings.HasSuffix(w, "ies"):
w = w[:len(w)-1] // "ties" -> "tie": one letter before it, a word in "ie"
case strings.HasSuffix(w, "ss"):
// "process", "class": not a plural
case strings.HasSuffix(w, "s") && len(w) > 3:
w = w[:len(w)-1]
}
switch {
case strings.HasSuffix(w, "ied") && len(w) == 4:
return w[:3] // "tied" -> "tie", like "ties"
case strings.HasSuffix(w, "ied") && len(w) > 4:
w = w[:len(w)-3] + "y" // "studied" -> "study", "spied" -> "spy"
case strings.HasSuffix(w, "eed") && len(w) > 4:
return w[:len(w)-1]
case strings.HasSuffix(w, "ed") && len(w) > 4:
s := w[:len(w)-2]
return fixStem(s)
case strings.HasSuffix(w, "ing") && len(w) > 5:
s := w[:len(w)-3]
return fixStem(s)
}
switch {
case strings.HasSuffix(w, "ational") && len(w) > 8:
return w[:len(w)-5] // "operational" -> "operat", same as "operation"
case strings.HasSuffix(w, "ation") && len(w) > 6:
return w[:len(w)-3]
case strings.HasSuffix(w, "ence") && len(w) > 5:
return w[:len(w)-4]
case strings.HasSuffix(w, "ance") && len(w) > 5:
return w[:len(w)-4]
case strings.HasSuffix(w, "ment") && len(w) > 5:
return w[:len(w)-4]
case strings.HasSuffix(w, "ness") && len(w) > 5:
return w[:len(w)-4]
case strings.HasSuffix(w, "ity") && len(w) > 5:
return w[:len(w)-3]
case strings.HasSuffix(w, "ism") && len(w) > 5:
return w[:len(w)-3]
case strings.HasSuffix(w, "ist") && len(w) > 5:
return w[:len(w)-3]
case strings.HasSuffix(w, "ize") && len(w) > 5:
return w[:len(w)-3]
case strings.HasSuffix(w, "ise") && len(w) > 5:
return w[:len(w)-3]
case strings.HasSuffix(w, "ful") && len(w) > 5:
return w[:len(w)-3]
case strings.HasSuffix(w, "less") && len(w) > 5:
return w[:len(w)-4]
case strings.HasSuffix(w, "ly") && len(w) > 5 && w[len(w)-3] != 'p':
return w[:len(w)-2]
case strings.HasSuffix(w, "er") && len(w) > 4:
return w[:len(w)-2]
case strings.HasSuffix(w, "al") && len(w) > 4:
return w[:len(w)-2]
}
return w
}
// yToI turns a final consonant+y into i, so singular/plural pairs unify
// with the "ies" rule: "study" -> "studi", "studies" -> "studi";
// "agency" -> "agenci", "agencies" -> "agenci".
func yToI(w string) string {
if strings.HasSuffix(w, "y") && len(w) > 3 && !isVowel(rune(w[len(w)-2])) {
return w[:len(w)-1] + "i"
}
if strings.HasSuffix(w, "ie") && len(w) > 5 {
return w[:len(w)-1]
}
return w
}
// measure is Porter's m: the number of vowel-consonant runs in w, with y a
// vowel after a consonant ("controll" 2, "fall" 1, "tree" 0).
func measure(w string) int {
m := 0
prevVowel := false
for i := 0; i < len(w); i++ {
v := isVowel(rune(w[i])) || (w[i] == 'y' && i > 0 && !prevVowel)
if prevVowel && !v {
m++
}
prevVowel = v
}
return m
}
// fixStem handles consonant doubling after stripping -ed/-ing
// e.g. "hopping" -> "hopp" -> "hop".
func fixStem(s string) string {
if len(s) == 0 {
return s
}
if len(s) >= 2 {
last := s[len(s)-1]
secondLast := s[len(s)-2]
if last == secondLast && !isVowel(rune(last)) && last != 'l' && last != 's' && last != 'z' {
return s[:len(s)-1]
}
}
return s
}
func isVowel(r rune) bool {
return strings.ContainsRune("aeiou", r)
}
// foldTable maps accented Latin characters to their ASCII base form so
// "muller" finds "Müller-Maguhn" and "Sao Paulo" finds "São Paulo". The
// fold runs on BOTH the index and the query side, so it's symmetric.
var foldTable = map[rune]string{
'À': "A", 'Á': "A", 'Â': "A", 'Ã': "A", 'Ä': "A",
'Å': "A", 'Æ': "AE", 'Ç': "C", 'È': "E", 'É': "E", 'Ê': "E",
'Ë': "E", 'Ì': "I", 'Í': "I", 'Î': "I", 'Ï': "I", 'Ð': "D",
'Ñ': "N", 'Ò': "O", 'Ó': "O", 'Ô': "O", 'Õ': "O", 'Ö': "O",
'Ø': "O", 'Ù': "U", 'Ú': "U", 'Û': "U", 'Ü': "U", 'Ý': "Y",
'Þ': "TH", 'ß': "ss", 'à': "a", 'á': "a", 'â': "a", 'ã': "a",
'ä': "a", 'å': "a", 'æ': "ae", 'ç': "c", 'è': "e", 'é': "e",
'ê': "e", 'ë': "e", 'ì': "i", 'í': "i", 'î': "i", 'ï': "i",
'ð': "d", 'ñ': "n", 'ò': "o", 'ó': "o", 'ô': "o", 'õ': "o",
'ö': "o", 'ø': "o", 'ù': "u", 'ú': "u", 'û': "u", 'ü': "u",
'ý': "y", 'þ': "th", 'ÿ': "y", 'Ā': "A", 'ā': "a", 'Ă': "A",
'ă': "a", 'Ą': "A", 'ą': "a", 'Ć': "C", 'ć': "c", 'Ĉ': "C",
'ĉ': "c", 'Ċ': "C", 'ċ': "c", 'Č': "C", 'č': "c", 'Ď': "D",
'ď': "d", 'Đ': "D", 'đ': "d", 'Ē': "E", 'ē': "e", 'Ĕ': "E",
'ĕ': "e", 'Ė': "E", 'ė': "e", 'Ę': "E", 'ę': "e", 'Ě': "E",
'ě': "e", 'Ĝ': "G", 'ĝ': "g", 'Ğ': "G", 'ğ': "g", 'Ġ': "G",
'ġ': "g", 'Ģ': "G", 'ģ': "g", 'Ĥ': "H", 'ĥ': "h", 'Ħ': "H",
'ħ': "h", 'Ĩ': "I", 'ĩ': "i", 'Ī': "I", 'ī': "i", 'Ĭ': "I",
'ĭ': "i", 'Į': "I", 'į': "i", 'İ': "I", 'ı': "i", 'Ĵ': "J",
'ĵ': "j", 'Ķ': "K", 'ķ': "k", 'Ĺ': "L", 'ĺ': "l", 'Ļ': "L",
'ļ': "l", 'Ľ': "L", 'ľ': "l", 'Ŀ': "L", 'ŀ': "l", 'Ł': "L",
'ł': "l", 'Ń': "N", 'ń': "n", 'Ņ': "N", 'ņ': "n", 'Ň': "N",
'ň': "n", 'ʼn': "n", 'Ō': "O", 'ō': "o", 'Ŏ': "O", 'ŏ': "o",
'Ő': "O", 'ő': "o", 'Œ': "OE", 'œ': "oe", 'Ŕ': "R", 'ŕ': "r",
'Ŗ': "R", 'ŗ': "r", 'Ř': "R", 'ř': "r", 'Ś': "S", 'ś': "s",
'Ŝ': "S", 'ŝ': "s", 'Ş': "S", 'ş': "s", 'Š': "S", 'š': "s",
'Ţ': "T", 'ţ': "t", 'Ť': "T", 'ť': "t", 'Ŧ': "T", 'ŧ': "t",
'Ũ': "U", 'ũ': "u", 'Ū': "U", 'ū': "u", 'Ŭ': "U", 'ŭ': "u",
'Ů': "U", 'ů': "u", 'Ű': "U", 'ű': "u", 'Ų': "U", 'ų': "u",
'Ŵ': "W", 'ŵ': "w", 'Ŷ': "Y", 'ŷ': "y", 'Ÿ': "Y", 'Ź': "Z",
'ź': "z", 'Ż': "Z", 'ż': "z", 'Ž': "Z", 'ž': "z", 'Ș': "S",
'ș': "s", 'Ț': "T", 'ț': "t", 'ẞ': "SS",
}
// foldMin is the smallest rune foldTable maps, every rune below it folds
// to itself so the folds skip the table lookup for it (all of ASCII).
var foldMin = func() rune {
m := utf8.MaxRune
for r := range foldTable {
m = min(m, r)
}
return m
}()
// foldAccents replaces accented characters with their base form,
// preserving case for characters not in the table.
func foldAccents(s string) string {
needs := false
for _, r := range s {
if r < foldMin {
continue
}
if _, ok := foldTable[r]; ok {
needs = true
break
}
}
if !needs {
return s
}
var b strings.Builder
b.Grow(len(s))
for _, r := range s {
if r >= foldMin {
if f, ok := foldTable[r]; ok {
b.WriteString(f)
continue
}
}
b.WriteRune(r)
}
return b.String()
}
func normFoldString(s string) string {
return strings.ToLower(foldAccents(s))
}
// foldRune is normFoldString for one rune.
func foldRune(r rune) string {
if r < utf8.RuneSelf {
if 'A' <= r && r <= 'Z' {
r += 'a' - 'A'
}
return string(r)
}
if f, ok := foldTable[r]; ok {
return strings.ToLower(f)
}
return strings.ToLower(string(r))
}
func foldedLen(r rune) int {
if r < utf8.RuneSelf {
return 1
}
if f, ok := foldTable[r]; ok {
return len(f)
}
return utf8.RuneLen(unicode.ToLower(r))
}
// normFold returns the normal form of s plus a position map: posMap[i] is
// the byte offset in s of the rune that produced norm byte i (with a final
// entry of len(s)). Lets match highlighting (markTerms) mark on the normal
// form but slice the ORIGINAL text, so a "muller" query highlights "Müller"
// as written.
func normFold(s string) (string, []int) {
var b strings.Builder
b.Grow(len(s))
posMap := make([]int, 0, len(s)+1)
for i, r := range s {
if r < utf8.RuneSelf {
if 'A' <= r && r <= 'Z' {
r += 'a' - 'A'
}
b.WriteByte(byte(r))
posMap = append(posMap, i)
continue
}
f := foldRune(r)
for j := 0; j < len(f); j++ {
posMap = append(posMap, i)
}
b.WriteString(f)
}
posMap = append(posMap, len(s))
return b.String(), posMap
}
// splitWords splits s into alphanumeric runs (it does not change case).
func splitWords(s string) []string {
return strings.FieldsFunc(s, func(r rune) bool {
return !unicode.IsLetter(r) && !unicode.IsDigit(r)
})
}
func tokenize(s string) []string {
words := splitWords(normFoldString(s))
filtered := words[:0]
for _, w := range words {
if len(w) > 1 && !stopWords[w] {
filtered = append(filtered, stem(w))
}
}
return filtered
}
// eachToken calls f with every token of a stemmed stream (tokenize's
// output joined by single spaces) in order.
func eachToken(stream string, f func(tok string)) {
for i := 0; i < len(stream); {
j := strings.IndexByte(stream[i:], ' ')
if j < 0 {
j = len(stream) - i
}
if j > 0 {
f(stream[i : i+j])
}
i += j + 1
}
}
func buildTFIDF(docs []Document) TFIDFIndex {
idx := TFIDFIndex{
DocStats: make(map[string]map[string]float64, len(docs)),
DocFreq: make(map[string]int),
N: len(docs),
}
for i := range docs {
d := &docs[i]
// Title 3x, fulltext 2x, descriptions + metadata 1x. The metadata
// is every field the all-fields search reads (searchMeta), so a doc
// that matches only via metadata still gets a relevance score
// instead of always sorting last. Title, descriptions and body come
// from the stemmed forms loadData already built (tokenize() output
// joined by spaces).
tf := make(map[string]float64)
total := 0.0
add := func(weight float64) func(string) {
return func(w string) {
if w == valueSep {
return
}
tf[w] += weight
total += weight
}
}
eachToken(d.StemmedTitle, add(3))
eachToken(d.StemmedFull, add(2))
eachToken(d.StemmedDesc, add(1))
for _, w := range tokenize(searchMeta(d)) {
add(1)(w)
}
for w := range tf {
tf[w] /= total // total > 0 whenever tf has a key
idx.DocFreq[w]++
}
idx.DocStats[d.ID] = tf
}
log.Printf("TF-IDF index built: %d docs, %d unique terms", len(docs), len(idx.DocFreq))
return idx
}
// scorer reads one positive term of a query into its relevance score for a
// document. The term's words go through the same tokenize() pipeline as the
// index (stemming + stopword removal), and matching against stemmed fields
// is by whole token.
func (idx TFIDFIndex) scorer(query string) func(doc *Document) float64 {
// A proximity clause scores as its phrase - the window is a matcher
// concern, not a relevance signal (and the raw "~10" tail would
// otherwise leak a junk "10" term into the tokenizer).
if _, phrase, _, ok := parseProximity(query); ok {
query = phrase
}
// An exact-value clause scores as its phrase with the phrase bonuses on
// whole words; quoted text earns them wherever it occurs.
contains := strings.Contains
if v, ok := exactValue(query); ok {
query, contains = `"`+v+`"`, containsWord
}
// Folded so the phrase bonuses are accent-consistent with matching, and
// so the content check can use the pre-folded body (no per-doc fold).
clean := strings.Trim(normFoldString(strings.TrimSpace(query)), `"`)
// Wildcard words score by where they occur, not through their stems:
// +5 when the title has one, else +1 when the document does.
var wild []queryWord
if !isQuotedQuery(query) {
var kept []string
for _, w := range strings.Fields(clean) {
if _, _, _, isWild := wildcard(w); isWild {
wild = append(wild, literalWord(w))
} else {
kept = append(kept, w)
}
}
clean = strings.Join(kept, " ")
}
terms := tokenize(clean)
// A term's weight in a document is its share of the document times how
// rare it is in the corpus; a term no document holds weighs nothing.
idf := make([]float64, len(terms))
for i, w := range terms {
if df := idx.DocFreq[w]; df != 0 {
idf[i] = math.Log(float64(idx.N) / float64(df))
}
}
isPhrase := len(strings.Fields(clean)) > 1
return func(doc *Document) float64 {
stats, ok := idx.DocStats[doc.ID]
if !ok || (len(terms) == 0 && len(wild) == 0) {
return 0
}
title := "" // for the wildcard and phrase bonuses
if len(wild) > 0 || isPhrase {
title = doc.titleFold()
}
var score float64
for i := range wild {
switch {
case wild[i].kind == wordNever: // matches nothing (see wildcard), so earns nothing
case wild[i].in(title):
score += 5
case wild[i].in(doc.SearchText):
score++
}
}
if len(terms) == 0 {
return score
}
for i, w := range terms {
score += stats[w] * idf[i]
}
// bonus is whole when the field holds the exact phrase (or, for one
// word, the word), else all when it holds every term somewhere.
bonus := func(phraseIn string, holds func(term string) bool, whole, all float64) float64 {
if isPhrase && contains(phraseIn, clean) {
return whole
}
for _, w := range terms {
if !holds(w) {
return 0
}
}
if isPhrase {
return all
}
return whole
}
// Title +10 or +5; the whole document +3 or +1. The document's
// entry in the index holds exactly the terms of its all-fields
// stream.
inTitle := func(w string) bool { return containsTokens(doc.StemmedTitle, w) }
inDoc := func(w string) bool { _, ok := stats[w]; return ok }
return score + bonus(title, inTitle, 10, 5) + bonus(doc.FoldedFull, inDoc, 3, 1)
}
}
// similarity returns cosine similarity between two docs' TF-IDF vectors.
// Returns 0-1; used by relatedScore in data.go.
// TODO split this file or move this away from here?
func (idx TFIDFIndex) similarity(aID, bID string) float64 {
a, ok1 := idx.DocStats[aID]
b, ok2 := idx.DocStats[bID]
if !ok1 || !ok2 {
return 0
}
var dot, normA, normB float64
for w, tfA := range a {
df := idx.DocFreq[w]
if df == 0 {
continue
}
idf := math.Log(float64(idx.N) / float64(df))
tfidfA := tfA * idf
normA += tfidfA * tfidfA
if tfB, ok := b[w]; ok {
tfidfB := tfB * idf
dot += tfidfA * tfidfB
}
}
for w, tfB := range b {
df := idx.DocFreq[w]
if df == 0 {
continue
}
idf := math.Log(float64(idx.N) / float64(df))
tfidfB := tfB * idf
normB += tfidfB * tfidfB
}
if normA == 0 || normB == 0 {
return 0
}
return dot / (math.Sqrt(normA) * math.Sqrt(normB))
}
// --- Clause values ---
// isQuotedQuery returns true for "exact phrase" queries.
func isQuotedQuery(q string) bool {
q = strings.TrimSpace(q)
return len(q) >= 2 && strings.HasPrefix(q, `"`) && strings.HasSuffix(q, `"`)
}
// --- Exact values and word boundaries ---
//
// A clause value has four shapes: words (cable tapping, matched with their
// other forms), quoted text ("cable tapping", found anywhere, like grep -i),
// an exact value (="cable tapping", or =tap for one word) and a wildcard
// (tap*, *apping). An exact value means "exactly this, nothing attached", on
// a metadata field it is one whole value, so agency:=NSA does not match
// "NSA, GCHQ"; on the text fields (all fields, title, content, description)
// it is whole words exactly as typed, so =tap finds "tap" but not "taps" or
// "strap".
// exactValue reports whether q is an exact-value clause ("=") followed by
// a quoted value or a single bare word - and returns the value.
func exactValue(q string) (string, bool) {
q = strings.TrimSpace(q)
if !strings.HasPrefix(q, "=") {
return "", false
}
v := strings.TrimSpace(q[1:])
if isQuotedQuery(v) {
v = strings.TrimSpace(v[1 : len(v)-1])
} else if strings.ContainsAny(v, spaceBytes+`"`) {
return "", false
}
return v, v != ""
}
// exactClause builds the ="..." clause value the generated links use, so a
// count and its click compare whole values the same way.
func exactClause(v string) string { return `="` + v + `"` }
// isWordRune is the word-character test splitWords uses.
func isWordRune(r rune) bool { return unicode.IsLetter(r) || unicode.IsDigit(r) }
// indexWord returns the byte offset of the first occurrence of needle in s,
// at or after from, that stands on word boundaries (no letter or digit
// directly before or after it) or -1. A needle edge that is not a word
// character ("u.s.") needs no boundary on that side. s and needle must
// already share a normal form (normFoldString).
func indexWord(s, needle string, from int) int {
first, _ := utf8.DecodeRuneInString(needle)
last, _ := utf8.DecodeLastRuneInString(needle)
return indexBounded(s, needle, from, isWordRune(first), isWordRune(last))
}
// wildMinRunes is the fewest letters or digits a fragment behind a leading
// "*" must have: "*a" or "*ab" would match nearly every document.
const wildMinRunes = 3
// wildcard reports whether w is a wildcard word (a fragment with "*") at
// its end (tap*), its start (*apping) or both (*keys*), and returns the
// fragment in normal form plus which sides may run on into more of the
// word.
func wildcard(w string) (frag string, openStart, openEnd, ok bool) {
openStart, openEnd = strings.HasPrefix(w, "*"), strings.HasSuffix(w, "*")
if !openStart && !openEnd {
return "", false, false, false
}
frag = normFoldString(strings.Trim(w, "*"))
n := 0
for _, r := range frag {
if isWordRune(r) {
n++
}
}
if n == 0 || (openStart && n < wildMinRunes) || strings.Contains(frag, "*") {
return "", openStart, openEnd, true
}
return frag, openStart, openEnd, true
}
// indexWild finds a wildcard fragment in s, both in normal form. A closed
// side must sit on a word boundary and an open side may continue into the
// word, so tap* finds "tapping" but not "strap".
func indexWild(s, frag string, from int, openStart, openEnd bool) int {
first, _ := utf8.DecodeRuneInString(frag)
last, _ := utf8.DecodeLastRuneInString(frag)
return indexBounded(s, frag, from, !openStart && isWordRune(first), !openEnd && isWordRune(last))
}
// wordBounds widens [start, end) in s to the whole word around it.
func wordBounds(s string, start, end int) (int, int) {
for start > 0 {
r, size := utf8.DecodeLastRuneInString(s[:start])
if !isWordRune(r) {
break
}
start -= size
}
for end < len(s) {
r, size := utf8.DecodeRuneInString(s[end:])
if !isWordRune(r) {
break
}
end += size
}
return start, end
}
// indexBounded is indexWord with the boundary checks chosen by the caller:
// checkStart requires no letter or digit directly before the match,
// checkEnd none directly after it.
func indexBounded(s, needle string, from int, checkStart, checkEnd bool) int {
if from < 0 {
from = 0
}
if needle == "" {
if from <= len(s) {
return from
}
return -1
}
for from <= len(s)-len(needle) {
j := strings.Index(s[from:], needle)
if j < 0 {
return -1
}
i := from + j
end := i + len(needle)
ok := true
if checkStart && i > 0 {
if r, _ := utf8.DecodeLastRuneInString(s[:i]); isWordRune(r) {
ok = false
}
}
if ok && checkEnd && end < len(s) {
if r, _ := utf8.DecodeRuneInString(s[end:]); isWordRune(r) {
ok = false
}
}
if ok {
return i
}
_, size := utf8.DecodeRuneInString(s[i:])
from = i + size
}
return -1
}
// containsWord reports whether needle occurs in s on word boundaries.
func containsWord(s, needle string) bool { return indexWord(s, needle, 0) >= 0 }
// containsTokens reports whether phrase (one or more stemmed tokens joined
// by single spaces) occurs in a space-joined token stream as WHOLE tokens.
func containsTokens(stream, phrase string) bool {
if phrase == "" {
return true
}
for i := 0; i+len(phrase) <= len(stream); {
j := strings.Index(stream[i:], phrase)
if j < 0 {
return false
}
p := i + j
end := p + len(phrase)
if (p == 0 || stream[p-1] == ' ') && (end == len(stream) || stream[end] == ' ') {
return true
}
i = p + 1
}
return false
}
// valueSep joins a field's values for word and phrase matching. It is not a
// word character, so words still split on it, and nobody types it, so a
// quoted phrase cannot run from one value into the next.
const valueSep = "\x1f"
func valueKey(v string) string { return strings.ToLower(strings.TrimSpace(v)) }
func sameValue(a, b string) bool {
a, b = strings.TrimSpace(a), strings.TrimSpace(b)
if !isASCII(a) || !isASCII(b) {
return strings.ToLower(a) == strings.ToLower(b)
}
if len(a) != len(b) {
return false
}
for i := 0; i < len(a); i++ {
x, y := a[i], b[i]
if 'A' <= x && x <= 'Z' {
x += 'a' - 'A'
}
if 'A' <= y && y <= 'Z' {
y += 'a' - 'A'
}
if x != y {
return false
}
}
return true
}
func isASCII(s string) bool {
for i := 0; i < len(s); i++ {
if s[i] >= utf8.RuneSelf {
return false
}
}
return true
}
// --- Compiled clauses ---
//
// A search reads each clause once, into a test, and the scan runs the tests
// on every document.
// queryWord is one word of an unquoted clause.
type queryWord struct {
kind int
text string // the fragment (wordWild), the word (wordWhole, wordSub) or its stemmed tokens joined by spaces (wordStems)
openStart, openEnd bool // wordWild: which sides of the fragment had "*"
tokens []string // wordStems: the tokens of text
}
const (
wordNever = iota // a wildcard with nothing to search for (wildcard): it matches nothing
wordWild // tap*, *apping, *keys*: part of a word, in the folded text
wordWhole // a whole word as written, in the folded text
wordSub // a word with internal punctuation ("u.s."), as written, anywhere in the folded text
wordStems // the word's stemmed tokens in sequence, in the token stream
)
// literalWord reads a folded query word the way folded text can hold it: a
// wildcard, a whole word or, with internal punctuation, which word matching
// can't represent, a substring.
func literalWord(w string) queryWord {
if frag, openStart, openEnd, ok := wildcard(w); ok {
if frag == "" {
return queryWord{kind: wordNever}
}
return queryWord{kind: wordWild, text: frag, openStart: openStart, openEnd: openEnd}
}
if parts := splitWords(w); len(parts) == 1 {
return queryWord{kind: wordWhole, text: parts[0]}
}
return queryWord{kind: wordSub, text: w}
}
// in reports whether the word, read literally, occurs in folded text.
func (w *queryWord) in(folded string) bool {
switch w.kind {
case wordWild:
return indexWild(folded, w.text, 0, w.openStart, w.openEnd) >= 0
case wordWhole:
return containsWord(folded, w.text)
case wordSub:
return strings.Contains(folded, w.text)
}
return false
}
// literalTest is a clause read for folded text that is not tokenized (the
// values of a metadata field). Quoted text is found anywhere, like grep -i
// ("ard" inside "guardian"); otherwise every word must occur as literalWord
// reads it.
type literalTest struct {
quoted bool
needle string
words []queryWord
}
func compileLiteral(q string) literalTest {
q = normFoldString(strings.TrimSpace(q))
if isQuotedQuery(q) {
return literalTest{quoted: true, needle: strings.Trim(q, `"`)}
}
var t literalTest
for _, w := range strings.Fields(q) {
t.words = append(t.words, literalWord(w))
}
return t
}
func (t *literalTest) match(folded string) bool {
if t.quoted {
return strings.Contains(folded, t.needle)
}
for i := range t.words {
if !t.words[i].in(folded) {
return false
}
}
return true
}
// valuesTest is a clause read for a metadata field's values: ="..." is one
// whole value (the equality the browse pages and the refine tallies use
// the valueKey) and anything else the values' words and phrases
// (literalTest).
type valuesTest struct {
exact bool
key string // exact: the value's valueKey
text literalTest
}
func compileValues(q string) valuesTest {
if v, ok := exactValue(q); ok {
return valuesTest{exact: true, key: valueKey(v)}
}
return valuesTest{text: compileLiteral(q)}
}
// match tests the clause on a list of values.
func (t *valuesTest) match(vals []string) bool {
if t.exact {
for _, x := range vals {
if sameValue(x, t.key) {
return true
}
}
return false
}
return t.text.match(normFoldString(strings.Join(vals, valueSep)))
}
func (t *valuesTest) matchIndexed(keys []string, folded string) bool {
if t.exact {
return slices.Contains(keys, t.key)
}
return t.text.match(folded)
}
// compileWords reads an unquoted clause for a text field. Its words run
// through the same tokenize pipeline as the index, so stemming stays
// consistent with it.
func compileWords(q string) []queryWord {
var stems, literal []queryWord
for _, w := range strings.Fields(strings.ToLower(strings.TrimSpace(q))) {
if _, _, _, ok := wildcard(w); ok {
literal = append(literal, literalWord(w))
} else if toks := tokenize(w); len(toks) == 0 {
literal = append(literal, literalWord(normFoldString(w)))
} else {
stems = append(stems, queryWord{kind: wordStems, text: strings.Join(toks, " "), tokens: toks})
}
}
return append(stems, literal...)
}
// compile reads clause q (one negation-free value of a row) into the
// test of it on the field.
func (f *searchField) compile(q string) func(*Document) bool {
if terms, phrase, win, ok := parseProximity(q); ok {
// "..."~N - see the proximity block below.
if f.Stemmed != nil && len(terms) >= 2 {
return func(d *Document) bool {
return d.hasTerms(terms) && proximityMatch(f.Stemmed(d), terms, win)
}
}
// A metadata field, or a phrase that boiled down to fewer than two
// searchable words, so it adds nothing, match the plain
// quoted phrase on the requested field instead.
q = phrase
}
if f.Values != nil {
test := compileValues(q)
return func(d *Document) bool {
if d.fields == nil {
return test.match(f.read(d))
}
return test.matchIndexed(d.fields.keys[f.index], d.fields.folded[f.index])
}
}
// A text field: an exact value (=word) is a whole-word test and quoted
// text a substring test, both on the folded text; anything else runs on
// the stemmed tokens, and reads the folded text only for its wildcard
// and literal words.
if v, ok := exactValue(q); ok {
needle := normFoldString(v)
return func(d *Document) bool { return containsWord(f.Folded(d), needle) }
}
if isQuotedQuery(q) {
needle := strings.Trim(normFoldString(strings.TrimSpace(q)), `"`)
return func(d *Document) bool { return strings.Contains(f.Folded(d), needle) }
}
words := compileWords(q)
everyField := f.Code == "TE"
return func(d *Document) bool {
for i := range words {
w := &words[i]
switch {
case w.kind != wordStems:
if !w.in(f.Folded(d)) {
return false
}
case !d.hasTerms(w.tokens):
return false
case everyField && len(w.tokens) == 1 && d.terms != nil:
// stream needs no walking
case !containsTokens(f.Stemmed(d), w.text):
return false
}
}
return true
}
}
// --- Proximity operator ----------------------------------------------------
//
// "upstream collection"~10 matches documents where every quoted word occurs
// within a 10-word window, unordered and counted on the same stemmed,
// stopword-free token streams the bare-word matcher uses.
// parseProximity reports whether q has the "..."~N shape (the whole clause,
// nothing after the digits). terms are the tokenized, stemmed, deduplicated
// phrase words; phrase is the plain quoted phrase for callers that degrade;
// win is N, capped at 1000 (wider than any window that could still mean
// anything beyond plain AND).
func parseProximity(q string) (terms []string, phrase string, win int, ok bool) {
q = strings.TrimSpace(q)
if len(q) < 4 || q[0] != '"' {
return nil, "", 0, false
}
end := strings.LastIndexByte(q, '"')
if end == 0 || end+2 >= len(q) || q[end+1] != '~' {
return nil, "", 0, false
}
n, err := strconv.Atoi(q[end+2:])
if err != nil || n < 1 {
return nil, "", 0, false
}
if n > 1000 {
n = 1000
}
inner := q[1:end]
seen := map[string]bool{}
for _, t := range tokenize(inner) {
if !seen[t] {
seen[t] = true
terms = append(terms, t)
}
}
return terms, `"` + inner + `"`, n, true
}
// proximityMatch walks a space-joined stemmed stream once, tracking the most
// recent position of each term; whenever every term has been seen and the
// span from the oldest to the newest of those positions fits the window, a
// valid window exists. Last-seen positions minimize the span at every step,
// so no qualifying window is missed. One linear pass, no string splitting.
func proximityMatch(stream string, terms []string, win int) bool {
if len(terms) == 0 {
return false
}
last := make([]int, len(terms))
for i := range last {
last[i] = -1
}
need := len(terms)
pos := 0
i := 0
for i < len(stream) {
for i < len(stream) && stream[i] == ' ' {
i++
}
if i >= len(stream) {
break
}
j := i
for j < len(stream) && stream[j] != ' ' {
j++
}
tok := stream[i:j]
for t, term := range terms {
if tok != term {
continue
}
if last[t] < 0 {
need--
}
last[t] = pos
if need == 0 {
lo, hi := last[0], last[0]
for _, p := range last[1:] {
if p < lo {
lo = p
}
if p > hi {
hi = p
}
}
if hi-lo <= win {
return true
}
}
}
pos++
i = j
}
return false
}
// --- Snippet helpers ---
// getSnippets returns up to max non-overlapping +-60-char windows around
// query matches, in document order. The match runs on the normal form
// (folded + lowered) and each window's bounds are resolved back to the
// ORIGINAL text (one forward walk, see below), so accents and casing
// survive into the snippet.
func getSnippets(content, query string, max int) []string {
return snippetsIn(content, normFoldString(content), query, max)
}
// bodySnippets is getSnippets over the document's transcript, on the fold
// of it the loader already holds.
func (d *Document) bodySnippets(query string, max int) []string {
return snippetsIn(d.FullText, d.foldedBody(), query, max)
}
// anchorSource yields one needle's matches in a text, in document order and
// one at a time, so a search for three windows reads no further than them.
type anchorSource struct {
next func(from int) (pos, n, resume int) // pos < 0: no match from there on
from int
pos, n int // the match not yet taken; pos < 0 once there is none
}
func (a *anchorSource) advance() { a.pos, a.n, a.from = a.next(a.from) }
// snippetsIn is getSnippets for a caller that holds the normal form of
// content (normFoldString).
func snippetsIn(content, norm, query string, max int) []string {
if query == "" || content == "" || max <= 0 {
return nil
}
searchQuery := strings.TrimSpace(query)
bounded := false // an exact value anchors on whole words only
if v, ok := exactValue(searchQuery); ok {
searchQuery, bounded = `"`+v+`"`, true
}
isExact := isQuotedQuery(searchQuery)
cleanQuery := normFoldString(strings.Trim(searchQuery, `"`))
if cleanQuery == "" {
return nil
}
var sources []*anchorSource
add := func(next func(from int) (pos, n, resume int)) {
src := &anchorSource{next: next}
src.advance()
sources = append(sources, src)
}
addAll := func(needle string, wordsOnly bool) {
if needle == "" {
return
}
add(func(from int) (int, int, int) {
i := -1
if wordsOnly {
i = indexWord(norm, needle, from)
} else if j := strings.Index(norm[from:], needle); j >= 0 {
i = from + j
}
return i, len(needle), i + len(needle)
})
}
found := func() bool {
for _, src := range sources {
if src.pos >= 0 {
return true
}
}
return false
}
if isExact || strings.Contains(cleanQuery, " ") {
addAll(cleanQuery, !isExact || bounded)
}
if !isExact {
var literal []string // stopwords, single letters, "u.s.": anchors of last resort
for _, w := range strings.Fields(cleanQuery) {
if frag, openStart, openEnd, ok := wildcard(w); ok {
if frag != "" {
add(func(from int) (int, int, int) {
i := indexWild(norm, frag, from, openStart, openEnd)
return i, len(frag), i + len(frag)
})
}
continue
}
kept := 0
for _, p := range splitWords(w) {
if len(p) <= 1 || stopWords[p] {
continue
}
kept++
addAll(p, true)
s := stem(p)
n := 0
for n < len(s) && n < len(p) && s[n] == p[n] {
n++
}
if s == p || n < 2 {
continue
}
add(func(from int) (int, int, int) {
for {
i := indexWild(norm, p[:n], from, false, true)
if i < 0 {
return -1, 0, from
}
start, end := wordBounds(norm, i, i+n)
if word := norm[start:end]; word != p && stem(word) == s {
return start, end - start, end
}
from = end
}
})
}
if kept == 0 {
literal = append(literal, w)
}
}
if !found() {
for _, w := range literal {
parts := splitWords(w)
if len(parts) == 1 {
w = parts[0]
}
addAll(w, len(parts) == 1)
}
}
}
if !found() {
return nil
}
// First pass. Choose up to max non-overlapping +-60-byte windows, in
// NORM offsets, taking the anchors in document order.
type window struct{ start, end int }
var wins []window
lastEnd := -1
for len(wins) < max {
var first *anchorSource
for _, src := range sources {
if src.pos >= 0 && (first == nil || src.pos < first.pos) {
first = src
}
}
if first == nil {
break
}
pos, n := first.pos, first.n
first.advance()
if pos < lastEnd {
continue // already inside the previous window
}
start := pos - 60
if start < 0 {
start = 0
}
end := pos + n + 60
if end > len(norm) {
end = len(norm)
}
lastEnd = end
wins = append(wins, window{start, end})
}
// Second pass. Resolve those few norm offsets back to offsets in the
// original content with ONE forward walk. A norm offset resolves to the
// start of the rune whose folded form covers it; the offset just past
// the end resolves to len(content).
bounds := make([]int, 0, len(wins)*2)
for _, wnd := range wins {
bounds = append(bounds, wnd.start, wnd.end)
}
sort.Ints(bounds) // window paddings may overlap, so bounds aren't sorted yet
orig := make(map[int]int, len(bounds))
bi, normOff := 0, 0
for oi, r := range content {
if bi == len(bounds) {
break
}
l := foldedLen(r)
for bi < len(bounds) && bounds[bi] < normOff+l {
orig[bounds[bi]] = oi
bi++
}
normOff += l
}
for ; bi < len(bounds); bi++ {
orig[bounds[bi]] = len(content) // the past-the-end boundary
}
out := make([]string, 0, len(wins))
for _, wnd := range wins {
oStart, oEnd := orig[wnd.start], orig[wnd.end]
prefix := "... "
if oStart == 0 {
prefix = ""
}
suffix := " ..."
if oEnd == len(content) {
suffix = ""
}
out = append(out, prefix+content[oStart:oEnd]+suffix)
}
return out
}
// --- Search-result highlighting ---
// Highlight term kinds
const (
hlWord = iota // matches whole words
hlWild // marks the whole word a wildcard fragment matched in (tap*, *keys*)
hlPhrase // matches an exact value (=tap) at word boundaries
hlSub // matches raw substrings (quoted text, and punctuated terms)
)
// hlTerm is one positive query term prepared for markTerms.
type hlTerm struct {
kind int
needle string // in normal form (folded + lowered)
stemmed string // the needle's stem when word stemming applies
openStart, openEnd bool // which sides of the fragment had "*"
}
// buildHighlightTerms extracts the positive terms of the clause list in the
// shape markTerms consumes, mirroring how the matchers read a clause
// (literalWord, compileWords). A quoted value is one exact phrase, plain
// words match whole words (stem-unified, like the index), "word*" matches
// word prefixes, and words with internal punctuation fall back to substring
// matching. Negated clauses are skipped (their terms are what the results
// DON'T contain).
func buildHighlightTerms(queries, conjs []string) []hlTerm {
var out []hlTerm
seen := make(map[string]bool)
add := func(t hlTerm) {
k := strconv.Itoa(t.kind) + strconv.FormatBool(t.openStart) + strconv.FormatBool(t.openEnd) + "\x00" + t.needle
if t.needle == "" || seen[k] {
return
}
seen[k] = true
out = append(out, t)
}
for _, q := range positiveTerms(queries, conjs) {
if _, phrase, _, ok := parseProximity(q); ok {
// Proximity clause. The match is word-based, not an exact
// substring, so highlight its individual words (with the
// same stemming eligibility bare words get below).
q = strings.Trim(phrase, `"`)
}
if v, ok := exactValue(q); ok {
add(hlTerm{kind: hlPhrase, needle: normFoldString(v)})
continue
}
q = normFoldString(strings.TrimSpace(q))
if q == "" {
continue
}
if isQuotedQuery(q) {
add(hlTerm{kind: hlSub, needle: strings.Trim(q, `"`)})
continue
}
for _, w := range strings.Fields(q) {
if frag, openStart, openEnd, ok := wildcard(w); ok {
if frag != "" {
add(hlTerm{kind: hlWild, needle: frag, openStart: openStart, openEnd: openEnd})
}
continue
}
parts := splitWords(w)
if len(parts) != 1 {
// Internal punctuation. The matchers read these as
// substrings, so highlight them the same way.
add(hlTerm{kind: hlSub, needle: w})
continue
}
w = parts[0]
t := hlTerm{kind: hlWord, needle: w}
// Stem eligibility mirrors tokenize, so "taps" highlights
// "tapped" exactly when the index unified them.
if len(w) > 1 && !stopWords[w] {
t.stemmed = stem(w)
}
add(t)
}
}
return out
}
// markTerms HTML-escapes s and wraps every occurrence of the given terms in
// tags, for the search results page.
//
// Matching runs on the normal form (normFold) so casing and accents can't
// hide a match, and the marked ranges are resolved back to the ORIGINAL
// text through normFold's position map (e.g. a "muller" query highlights
// "Müller" as written). Word terms match whole words, or words whose stem
// equals the term's stem (the same unification the index does); prefix
// terms match the start of a word; substring terms match anywhere.
func markTerms(s string, terms []hlTerm) template.HTML {
if s == "" || len(terms) == 0 {
return template.HTML(template.HTMLEscapeString(s))
}
norm, posMap := normFold(s)
var wordTerms, subTerms []hlTerm
for _, t := range terms {
if t.kind != hlWord {
subTerms = append(subTerms, t)
} else {
wordTerms = append(wordTerms, t)
}
}
type span struct{ start, end int } // byte offsets in norm
var spans []span
if len(wordTerms) > 0 {
needStem := false
for _, t := range wordTerms {
if t.stemmed != "" {
needStem = true
break
}
}
ws := -1 // current word start; -1 = between words
flush := func(we int) {
if ws < 0 {
return
}
word := norm[ws:we]
wstem := ""
if needStem {
wstem = stem(word)
}
for _, t := range wordTerms {
if word == t.needle || (t.stemmed != "" && wstem == t.stemmed) {
spans = append(spans, span{ws, we})
break
}
}
ws = -1
}
for i, r := range norm {
if unicode.IsLetter(r) || unicode.IsDigit(r) {
if ws < 0 {
ws = i
}
} else {
flush(i)
}
}
flush(len(norm))
}
for _, t := range subTerms {
for from := 0; ; {
i := -1
switch t.kind {
case hlPhrase:
i = indexWord(norm, t.needle, from)
case hlWild:
i = indexWild(norm, t.needle, from, t.openStart, t.openEnd)
default:
if j := strings.Index(norm[from:], t.needle); j >= 0 {
i = from + j
}
}
if i < 0 {
break
}
start, end := i, i+len(t.needle)
from = end
if t.kind == hlWild {
start, end = wordBounds(norm, start, end) // mark the whole word
}
spans = append(spans, span{start, end})
}
}
if len(spans) == 0 {
return template.HTML(template.HTMLEscapeString(s))
}
// Merge overlapping and touching spans so nested or adjacent matches
// render as one contiguous highlight.
sort.Slice(spans, func(i, j int) bool { return spans[i].start < spans[j].start })
merged := spans[:1]
for _, sp := range spans[1:] {
if last := &merged[len(merged)-1]; sp.start <= last.end {
if sp.end > last.end {
last.end = sp.end
}
} else {
merged = append(merged, sp)
}
}
// Emit escaped gap / / escaped match / runs. Norm
// offsets resolve to original offsets via posMap; when a span boundary
// falls inside one rune's multi-byte fold (e.g. 'ẞ' -> "ss"), the whole
// rune is included.
var b strings.Builder
last := 0
for _, sp := range merged {
oStart := posMap[sp.start]
j := sp.end
for j < len(norm) && posMap[j] == posMap[sp.end-1] {
j++
}
oEnd := posMap[j]
if oStart < last {
oStart = last // rune snapping can nudge a span into the previous one
}
if oEnd <= oStart {
continue
}
b.WriteString(template.HTMLEscapeString(s[last:oStart]))
b.WriteString("")
b.WriteString(template.HTMLEscapeString(s[oStart:oEnd]))
b.WriteString("")
last = oEnd
}
b.WriteString(template.HTMLEscapeString(s[last:]))
return template.HTML(b.String())
}
// --- Search execution (shared by the HTML page and the JSON API) ---
// ScoredDoc is one match, with its relevance score. Doc points into archive.Docs, which is
// read-only while serving: the match set is walked, filtered and sorted as
// pointers, and the page a reader sees wraps them in searchResults, which
// carry the highlighting.
type ScoredDoc struct {
Doc *Document
Score float64
}
// searchResult is one row of the search page: a document and what the
// query marked in it.
type searchResult struct {
Doc *Document
Title, Desc template.HTML // escaped, query matches in (markTerms)
Snippets []template.HTML // up to 3 match-context windows, -highlighted
VersionSnippets []VersionSnippet // windows from versions' new lines, labelled with their copy
MatchesIn []*DocVersion // versions that matched when the main transcript didn't
}
const maxSnippetDocs = 100
const maxSearchClauses = 24
const maxSearchWords = 64
func searchWords(queries []string) int {
n := 0
for _, q := range queries {
n += len(strings.Fields(q))
}
return n
}
// normalizeClauses caps the clause count and pads/truncates fields and
// conjs to the length of queries.
func normalizeClauses(queries, fields, conjs []string) ([]string, []string, []string) {
if len(queries) > maxSearchClauses {
queries = queries[:maxSearchClauses]
}
trimmed := make([]string, len(queries))
for i, q := range queries {
trimmed[i] = strings.TrimSpace(q)
}
queries = trimmed
if len(fields) > len(queries) {
fields = fields[:len(queries)]
}
for len(fields) < len(queries) {
fields = append(fields, "TE")
}
if len(conjs) > len(queries) {
conjs = conjs[:len(queries)]
}
for len(conjs) < len(queries) {
conjs = append(conjs, "and")
}
return queries, fields, conjs
}
// searchField describes one searchable field: its wire code (the fqf
// value), UI label, simple-box operator aliases, and what it searches. This
// table is the single source of truth.
type searchField struct {
Code string
Label string
Aliases []string
Stemmed, Folded func(d *Document) string
Values func(d *Document) []string
read func(d *Document) []string
index int // the field's place in searchFields
}
type docFields struct {
values [][]string
keys [][]string
folded []string
pubYears []string // pubYearsOf
}
func (co *Corpus) buildDocFields() {
for i := range co.archive.Docs {
d := &co.archive.Docs[i]
d.fields = nil // the readers below work everything out
f := &docFields{
values: make([][]string, len(searchFields)),
keys: make([][]string, len(searchFields)),
folded: make([]string, len(searchFields)),
pubYears: pubYearsOf(d),
}
for fi := range searchFields {
sf := &searchFields[fi]
if sf.read == nil {
continue
}
vals := sf.read(d)
keys := make([]string, len(vals))
for vi, v := range vals {
keys[vi] = valueKey(v)
}
f.values[fi], f.keys[fi] = vals, keys
f.folded[fi] = normFoldString(strings.Join(vals, valueSep))
}
d.fields = f
}
}
// searchFields is in the form dropdown's display order.
var searchFields = []searchField{
// SearchText is stored pre-folded, so quoted text and exact words are
// direct tests against it.
{Code: "TE", Label: "All fields", Aliases: []string{"text"},
Stemmed: func(d *Document) string { return d.StemmedText },
Folded: func(d *Document) string { return d.SearchText }},
{Code: "TI", Label: "Title", Aliases: []string{"title"},
Stemmed: func(d *Document) string { return d.StemmedTitle },
Folded: (*Document).titleFold},
{Code: "FT", Label: "Content", Aliases: []string{"content"},
Stemmed: func(d *Document) string { return d.StemmedFull },
Folded: func(d *Document) string { return d.FoldedFull }},
{Code: "DE", Label: "Description", Aliases: []string{"description"},
Stemmed: func(d *Document) string { return d.StemmedDesc },
Folded: (*Document).descFold},
{Code: "CR", Label: "Agency", Aliases: []string{"agency"},
Values: func(d *Document) []string { return single(d.DocMeta.Creator) }},
{Code: "RE", Label: "Journalist", Aliases: []string{"journalist"},
Values: journalistsOf},
{Code: "PU", Label: "Publisher", Aliases: []string{"publisher"},
Values: publishersOf},
{Code: "CL", Label: "Classification", Aliases: []string{"classification"},
Values: func(d *Document) []string { return d.DocMeta.Classification }},
{Code: "PR", Label: "Program", Aliases: []string{"program"},
Values: func(d *Document) []string { return d.DocMeta.Programs }},
{Code: "DI", Label: "Dissemination", Aliases: []string{"dissemination"},
Values: func(d *Document) []string { return d.DocMeta.Dissemination }},
{Code: "BA", Label: "Banner", Aliases: []string{"banner"},
Values: func(d *Document) []string { return single(d.DocMeta.Banner) }},
{Code: "GE", Label: "Genre", Aliases: []string{"genre"},
Values: func(d *Document) []string { return single(d.DocMeta.Genre) }},
{Code: "SE", Label: "Series", Aliases: []string{"series"},
Values: func(d *Document) []string { return single(d.DocMeta.Series.Name) }},
{Code: "CO", Label: "Country", Aliases: []string{"country"},
Values: func(d *Document) []string { return d.DocMeta.Countries }},
{Code: "PE", Label: "People", Aliases: []string{"people"},
Values: func(d *Document) []string { return d.DocMeta.People }},
{Code: "TA", Label: "Tags", Aliases: []string{"tag"},
Values: func(d *Document) []string { return d.Tags }},
{Code: "TY", Label: "Type", Aliases: []string{"type"},
Values: func(d *Document) []string { return single(d.Type) }},
{Code: "PL", Label: "Portion level",
Values: func(d *Document) []string { return d.PortionLevels }},
{Code: "PM", Label: "Portion marking",
Values: func(d *Document) []string { return d.PortionMarkings }},
{Code: "PC", Label: "Portion control",
Values: func(d *Document) []string { return d.PortionControls }},
}
func single(v string) []string {
if v == "" {
return nil
}
return []string{v}
}
var (
fieldByCode = map[string]*searchField{}
simpleOps = map[string]string{}
)
func init() {
for i := range searchFields {
f := &searchFields[i]
f.index = i
if read := f.Values; read != nil {
f.read = read
f.Values = func(d *Document) []string {
if d.fields != nil {
return d.fields.values[i]
}
return read(d)
}
}
fieldByCode[f.Code] = f
for _, a := range f.Aliases {
simpleOps[a] = f.Code
}
}
}
type simpleDates struct {
DocStart, DocEnd, PubStart, PubEnd string
}
// padDateEnd widens a partial date for use as an inclusive range end:
// 2013 -> 2013-12-31, 2013-06 -> 2013-06-31 (string comparison makes the
// over-wide day harmless).
func padDateEnd(v string) string {
switch len(v) {
case 4:
return v + "-12-31"
case 7:
return v + "-31"
}
return v
}
// parseSimpleQuery turns a one-line query into fielded clauses plus date
// filters.
//
// program:XKEYSCORE country:Germany cable tapping
//
// becomes three AND clauses: PR=XKEYSCORE, CO=Germany, TE="cable tapping".
// Values can be double-quoted (journalist:"Poitras, Laura"); the quotes are
// KEPT on the clause value, so a quoted value has the same exact-match
// (substring/phrase) semantics it has when typed into the structured form,
// and a bare "quoted phrase" becomes its own exact all-fields clause rather
// than dissolving into the free-text words.
//
// A leading '-' negates: -term and -program:X become "and not" clauses
// (ordered after the positive ones, since the boolean evaluation is
// left-to-right). date:2013, date:2013-06, date:2013..2015 filter on the
// document date; published: does the same for publication dates.
func parseSimpleQuery(s string) (queries, fields, conjs []string, dates simpleDates) {
var freeText []string
type clause struct{ q, f string }
var negs []clause
for i, neg, ok := nextTerm(s, 0); ok; i, neg, ok = nextTerm(s, i) {
// try operator: word characters followed by ':'
opEnd := i
for opEnd < len(s) && (isWordByte(s[opEnd])) {
opEnd++
}
field := ""
dateOp := ""
valStart := i
if opEnd > i && opEnd < len(s) && s[opEnd] == ':' {
name := strings.ToLower(s[i:opEnd])
if code, ok := simpleOps[name]; ok {
field = code
valStart = opEnd + 1
} else if name == "date" || name == "published" {
dateOp = name
valStart = opEnd + 1
}
}
clauseVal, val, quoted, exact, next := readValue(s, valStart)
i = next
if val == "" {
continue
}
if dateOp != "" {
// date filters can't be negated; '-' before them is ignored.
start, end := val, val
if a, b, ok := strings.Cut(val, ".."); ok {
start, end = strings.TrimSpace(a), strings.TrimSpace(b)
}
end = padDateEnd(end)
if dateOp == "date" {
dates.DocStart, dates.DocEnd = start, end
} else {
dates.PubStart, dates.PubEnd = start, end
}
continue
}
val = clauseVal
switch {
case field != "" && neg:
negs = append(negs, clause{val, field})
case field != "":
queries = append(queries, val)
fields = append(fields, field)
case neg:
negs = append(negs, clause{val, "TE"})
case quoted || exact:
queries = append(queries, val)
fields = append(fields, "TE")
default:
freeText = append(freeText, val)
}
}
if len(freeText) > 0 {
queries = append(queries, strings.Join(freeText, " "))
fields = append(fields, "TE")
}
if len(queries) == 0 {
queries = []string{""}
fields = []string{"TE"}
}
conjs = make([]string, len(queries))
for i := range conjs {
conjs[i] = "and"
}
for _, n := range negs {
queries = append(queries, n.q)
fields = append(fields, n.f)
conjs = append(conjs, "not")
}
return queries, fields, conjs, dates
}
func isWordByte(b byte) bool {
return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') || b == '_'
}
const spaceBytes = " \t\n\v\f\r"
func isSpaceByte(b byte) bool { return strings.IndexByte(spaceBytes, b) >= 0 }
func proxSuffix(s string, i int) (string, int) {
if i >= len(s) || s[i] != '~' {
return "", i
}
j := i + 1
for j < len(s) && s[j] >= '0' && s[j] <= '9' {
j++
}
if j == i+1 || (j < len(s) && !isSpaceByte(s[j])) {
return "", i
}
return s[i:j], j
}
// readToken reads one search-value token starting at s[i]: either a
// "quoted span" returned with the quotes stripped and quoted=true, or
// a bare run up to the next whitespace. next is the index just past the
// token. This is THE tokenizer for search values: parseSimpleQuery and
// rowTerms both use it, behind nextTerm, so neither the quoting rule nor
// the negation rule can drift between them.
func readToken(s string, i int) (val string, quoted bool, next int) {
if i < len(s) && s[i] == '"' {
j := i + 1
if k := strings.IndexByte(s[j:], '"'); k >= 0 {
return s[j : j+k], true, j + k + 1
}
return s[i+1:], true, len(s)
}
j := i
for j < len(s) && !isSpaceByte(s[j]) {
j++
}
return s[i:j], false, j
}
// nextTerm finds the next term of a query from s[i] on: it skips the
// whitespace, and a leading '-' negates the term when it is attached to
// one. i is where the term's value (or its field: operator) starts; ok is
// false at the end of the string.
func nextTerm(s string, i int) (start int, neg, ok bool) {
for i < len(s) && isSpaceByte(s[i]) {
i++
}
if i >= len(s) {
return i, false, false
}
if s[i] == '-' && i+1 < len(s) && !isSpaceByte(s[i+1]) {
return i + 1, true, true
}
return i, false, true
}
// readValue reads one search value at s[i]
func readValue(s string, i int) (clause, inner string, quoted, exact bool, next int) {
if i < len(s) && s[i] == '=' {
exact = true
i++
}
inner, quoted, next = readToken(s, i)
prox := ""
if quoted {
prox, next = proxSuffix(s, next)
}
inner = strings.TrimSpace(inner)
switch {
case inner == "":
case exact && quoted:
clause = `="` + inner + `"`
case exact:
clause = "=" + inner
case quoted:
clause = `"` + inner + `"` + prox
default:
clause = inner
}
return clause, inner, quoted, exact, next
}
// rowTerms splits one search row into what it requires and what it
// excludes, the way parseSimpleQuery splits ?q=: each quoted phrase,
// proximity clause or exact value is a positive part of its own, the bare
// words together are one more (last), and google-style -term and -"phrase"
// tokens are exclusions on the row's own field.
//
// This is THE reading of a row: the matcher, the scorer, the snippet anchor
// and the highlighter all go through it, so "beta -gamma" is (beta AND NOT
// gamma) as one unit wherever the row sits.
func rowTerms(s string) (positives, negatives []string) {
var words []string
for i, neg, ok := nextTerm(s, 0); ok; i, neg, ok = nextTerm(s, i) {
tok, _, quoted, exact, next := readValue(s, i)
i = next
switch {
case tok == "" || tok == "-":
case neg:
negatives = append(negatives, tok)
case quoted || exact:
positives = append(positives, tok)
default:
words = append(words, tok)
}
}
if len(words) > 0 {
positives = append(positives, strings.Join(words, " "))
}
return positives, negatives
}
// executeSearch is the search pipeline the page and the API both run: over
// the whole archive, the collection filter, the dates and the boolean
// clause list, then the type filter, and the survivors ranked and sorted
// by p.Sort/p.Order, as a fresh slice the caller owns. The two filters are
// the browse pages' (inCollections, typeSelected). typeCounts tallies the
// types BEFORE the type filter. Only the documents that pass every filter
// are scored.
func (co *Corpus) executeSearch(p searchParams) (matches []ScoredDoc, typeCounts map[string]int) {
dateDocEnd := padDateEnd(p.DateDocEnd)
datePubEnd := padDateEnd(p.DatePubEnd)
query := compileQuery(p.Queries, p.Fields, p.Conjs)
byEntry := entryRows(p.Queries, p.Fields, p.Conjs)
var scorers []func(*Document) float64
for _, q := range positiveTerms(p.Queries, p.Conjs) {
scorers = append(scorers, co.tfidf.scorer(q))
}
typeCounts = make(map[string]int)
for i := range co.archive.Docs {
doc := &co.archive.Docs[i]
if !inCollections(doc, p.Coll) ||
!matchDates(doc, p.DateDocStart, dateDocEnd, p.DatePubStart, datePubEnd, byEntry) ||
!query.matches(doc) {
continue
}
typeCounts[doc.Type]++
if !typeSelected(doc, p.Types) {
continue
}
totalScore := 0.0
for _, score := range scorers {
totalScore += score(doc)
}
matches = append(matches, ScoredDoc{Doc: doc, Score: totalScore})
}
slices.SortStableFunc(matches, func(a, b ScoredDoc) int { return cmp.Compare(b.Score, a.Score) })
return sortScored(matches, p.Sort, p.Order), typeCounts
}
// --- Search handler (HTML) ---
// --- Faceted search refinement -----------------------------------------------
type searchFacet struct {
Label string
Values []facetEntry
}
func topFacetCounts(m map[string]int, total, n int, baseURL, field string) []facetEntry {
var out []facetEntry
for _, v := range rankCounts(m, -1) {
if len(out) == n {
break
}
if m[v] >= total {
continue
}
// ="value" -> a whole-value match on the field, the equality the
// tally counted, so the number is the number the click finds.
out = append(out, facetEntry{Name: v, Count: m[v], URL: appendClause(baseURL, field, exactClause(v))})
}
return out
}
func facetValues(d *Document, code string) []string {
if f := fieldByCode[code]; f != nil && f.Values != nil {
return f.Values(d)
}
return nil
}
type facetGroup struct {
Label string
Code string
}
func facetGroups(docs []*Document, groups []facetGroup, baseURL string, values func(*Document, string) []string) []searchFacet {
type tally struct{ count, first int }
tallies := make([]map[string]*tally, len(groups))
for i := range tallies {
tallies[i] = map[string]*tally{}
}
seq := 0
for i := range docs {
for gi, g := range groups {
for _, v := range values(docs[i], g.Code) {
t := tallies[gi][v]
if t == nil {
t = &tally{first: seq}
tallies[gi][v] = t
}
t.count++
seq++
}
}
}
out := make([]searchFacet, 0, len(groups))
for gi, g := range groups {
type folded struct {
name string
count, first int
}
byKey := make(map[string]*folded, len(tallies[gi]))
for v, t := range tallies[gi] {
k := valueKey(v)
f := byKey[k]
if f == nil {
byKey[k] = &folded{v, t.count, t.first}
continue
}
f.count += t.count
if t.first < f.first {
f.name, f.first = v, t.first
}
}
counts := make(map[string]int, len(byKey))
for _, f := range byKey {
counts[f.name] = f.count
}
if vals := topFacetCounts(counts, len(docs), 8, baseURL, g.Code); len(vals) > 0 {
out = append(out, searchFacet{Label: g.Label, Values: vals})
}
}
return out
}
func facetsWithinWords(facets []searchFacet, room int) []searchFacet {
out := facets[:0]
for _, f := range facets {
vals := f.Values[:0]
for _, v := range f.Values {
if searchWords([]string{exactClause(v.Name)}) <= room {
vals = append(vals, v)
}
}
if len(vals) > 0 {
f.Values = vals
out = append(out, f)
}
}
return out
}
func computeSearchFacets(docs []*Document, baseURL string, values func(*Document, string) []string) []searchFacet {
return facetGroups(docs, []facetGroup{
{"Programs", "PR"}, {"Agencies", "CR"}, {"Countries", "CO"}, {"Journalists", "RE"},
}, baseURL, values)
}
func collectionFilter(v url.Values) map[string]bool {
sel := map[string]bool{}
for _, c := range v["coll"] {
if validCollections[c] {
sel[c] = true
}
}
if len(sel) == 0 || len(sel) == len(validCollections) {
return nil
}
return sel
}
func snippetAnchor(queries, fields, conjs []string) (term, field string) {
for i, q := range queries {
if q == "" || conjs[i] == "not" {
continue
}
pos, _ := rowTerms(q)
if len(pos) == 0 {
continue // only exclusions: nothing in the results to anchor on
}
q = pos[0]
if _, phrase, _, ok := parseProximity(q); ok {
if ws := strings.Fields(strings.Trim(phrase, `"`)); len(ws) > 0 {
return ws[0], fields[i]
}
}
return q, fields[i]
}
return "", ""
}
// positiveTerms is what the query asks the results to contain: each
// non-empty row's positive part, skipping NOT-rows and every row's
// exclusions (rowTerms). The scorer and the highlighter read the query
// through it, so neither rewards or marks a term the results were selected
// for NOT containing.
func positiveTerms(queries, conjs []string) []string {
var out []string
for i, q := range queries {
if q == "" || (i < len(conjs) && conjs[i] == "not") {
continue
}
pos, _ := rowTerms(q)
out = append(out, pos...)
}
return out
}
// searchParams is one search's effective inputs as read by readSearchParams.
type searchParams struct {
Queries, Fields, Conjs []string
DateDocStart, DateDocEnd, DatePubStart, DatePubEnd string
FromQ bool // a ?q= shortcut replaced the structured params
Coll map[string]bool // collectionFilter: nil filters nothing
Types map[string]bool // selectedTypes: empty filters nothing
Sort, Order string // Sort defaults to "relevance"
OverClauses int // the requested clause count when it exceeds maxSearchClauses: the search must not run
OverWords int // the word count (searchWords) when it exceeds maxSearchWords: the search must not run
}
func (p searchParams) refused() bool { return p.OverClauses > 0 || p.OverWords > 0 }
// readSearchParams reads a search from URL params. this intake is shared by
// the HTML page and the JSON API.
//
// A non-empty ?q= replaces any structured clause params. Plain words become
// a single all-fields clause. Field operators like program:XKEYSCORE or
// journalist:"Poitras, Laura" become their own AND clauses (see
// parseSimpleQuery). Its parsed dates override the URL's when either end is
// set - date:..2015 has an empty start but is still a real filter.
func readSearchParams(v url.Values) searchParams {
p := searchParams{
Queries: v["fqv"],
Fields: v["fqf"],
Conjs: v["fqc"],
DateDocStart: v.Get("date_doc_start"),
DateDocEnd: v.Get("date_doc_end"),
DatePubStart: v.Get("date_pub_start"),
DatePubEnd: v.Get("date_pub_end"),
Coll: collectionFilter(v),
Sort: sortToken(v.Get("sort")),
Order: sortToken(v.Get("order")),
}
p.Types, _ = selectedTypes(v)
if docSortByKey[p.Sort] == nil {
p.Sort, p.Order = "relevance", ""
}
if s := v.Get("q"); s != "" {
var sd simpleDates
p.Queries, p.Fields, p.Conjs, sd = parseSimpleQuery(s)
if sd.DocStart != "" || sd.DocEnd != "" {
p.DateDocStart, p.DateDocEnd = sd.DocStart, sd.DocEnd
}
if sd.PubStart != "" || sd.PubEnd != "" {
p.DatePubStart, p.DatePubEnd = sd.PubStart, sd.PubEnd
}
p.FromQ = true
}
if n := len(p.Queries); n > maxSearchClauses {
p.OverClauses = n
} else if n := searchWords(p.Queries); n > maxSearchWords {
p.OverWords = n
}
p.Queries, p.Fields, p.Conjs = normalizeClauses(p.Queries, p.Fields, p.Conjs)
return p
}
func (p searchParams) dates() [4]string {
return [4]string{p.DateDocStart, p.DateDocEnd, p.DatePubStart, p.DatePubEnd}
}
// canonicalSearchQuery rebuilds the query params from the EFFECTIVE state
// of a search (the post-add/remove clause rows, the collection and type
// filters, the dates that a ?q= shortcut may have parsed out of the box)
// for every link the page generates (sorting, pagination, the histogram
// bars, the refine box, the export and the recent/pinned entries).
func canonicalSearchQuery(doSearch bool, p searchParams) url.Values {
q := url.Values{}
if doSearch {
// A ?q= entry has no do_search param, so the canonical form adds it
// explicitly.
q.Set("do_search", "1")
}
for i := range p.Queries {
q.Add("fqv", p.Queries[i])
q.Add("fqf", p.Fields[i])
q.Add("fqc", p.Conjs[i])
}
if colls := collectionParams(p.Coll); len(colls) > 0 {
q["coll"] = colls
}
if len(p.Types) > 0 {
types := make([]string, 0, len(p.Types))
for t := range p.Types {
types = append(types, t)
}
sort.Strings(types)
q["type"] = types
}
dates := p.dates()
for i, k := range []string{"date_doc_start", "date_doc_end", "date_pub_start", "date_pub_end"} {
if dates[i] != "" {
q.Set(k, dates[i])
}
}
return q
}
// searchDateHistogram is the year-bar strip under a result set.
func searchDateHistogram(results []*Document, q url.Values) []timelineBar {
counts := make(map[string]int)
undated := 0
for i := range results {
if y := docYear(results[i]); y != "" {
counts[y]++
} else {
undated++
}
}
_, maxCount := sortedYears(counts)
if undated > maxCount {
maxCount = undated // the undated bar shares the scale
}
start, end := q.Get("date_doc_start"), q.Get("date_doc_end")
bars := yearBars(counts, func(y string) string {
from, to := y, y
if start > y {
from = start
}
if end != "" && padDateEnd(end) < padDateEnd(y) {
to = end
}
yq := cloneValues(q)
yq.Set("date_doc_start", from)
yq.Set("date_doc_end", to)
return "/search?" + yq.Encode()
})
for i := range bars {
bars[i].Pct = barPct(bars[i].Count, maxCount, 1)
}
if undated > 0 {
bars = append(bars, timelineBar{
Year: "-",
Count: undated,
Pct: barPct(undated, maxCount, 1),
})
}
return bars
}
// decorateResults marks the query's terms in the titles, descriptions and
// body snippets of the results on the page.
func decorateResults(docs []*Document, queries, fields, conjs []string) (results []searchResult, snippetsCapped bool) {
snippetTerm, snippetField := snippetAnchor(queries, fields, conjs)
hl := buildHighlightTerms(queries, conjs)
results = make([]searchResult, len(docs))
for i, doc := range docs {
res := &results[i]
res.Doc = doc
res.Title = markTerms(doc.Title, hl)
res.Desc = markTerms(doc.Description, hl)
if snippetTerm == "" || i >= maxSnippetDocs {
continue
}
var raw []string
var hits []versionHit
mainHit := false
if snippetField == "TA" {
raw = getSnippets(strings.Join(doc.Tags, ", "), snippetTerm, 3)
} else {
raw = doc.bodySnippets(snippetTerm, 3)
mainHit = len(raw) > 0
if len(raw) < 3 {
hits = versionHits(doc, snippetTerm, 3)
}
if !mainHit && len(hits) == 0 {
raw = getSnippets(doc.Description, snippetTerm, 1)
}
}
for _, s := range raw {
res.Snippets = append(res.Snippets, markTerms(s, hl))
}
room := 3 - len(raw)
for _, h := range hits {
if !mainHit {
res.MatchesIn = append(res.MatchesIn, h.Version)
}
for _, s := range h.Snippets {
if room <= 0 {
break
}
res.VersionSnippets = append(res.VersionSnippets,
VersionSnippet{Slug: h.Version.Slug, Label: h.Version.Label, HTML: markTerms(s, hl)})
room--
}
}
}
return results, snippetTerm != "" && len(docs) > maxSnippetDocs
}
func (co *Corpus) handleSearch(w http.ResponseWriter, r *http.Request) {
params := r.URL.Query()
action := params.Get("action")
sp := readSearchParams(params)
prefs := readPrefs(r)
scope := pageScope(params, prefs, true)
sp.Coll = scope.filter
queries, fields, conjs := sp.Queries, sp.Fields, sp.Conjs
sortBy, order := sp.Sort, sp.Order
doSearch := (params.Get("do_search") == "1" || sp.FromQ) && !sp.refused()
if len(queries) == 0 {
queries = []string{""}
fields = []string{"TE"}
conjs = []string{"and"}
}
switch {
case strings.HasPrefix(action, "remove-"):
idxStr := strings.TrimPrefix(action, "remove-")
removeIdx, err := strconv.Atoi(idxStr)
if err == nil && removeIdx > 0 && removeIdx < len(queries) {
queries = append(queries[:removeIdx], queries[removeIdx+1:]...)
fields = append(fields[:removeIdx], fields[removeIdx+1:]...)
conjs = append(conjs[:removeIdx], conjs[removeIdx+1:]...)
}
case action == "add" && len(queries) < maxSearchClauses:
queries = append(queries, "")
fields = append(fields, "TE")
conjs = append(conjs, "and")
}
sp.Queries, sp.Fields, sp.Conjs = queries, fields, conjs
var scored []ScoredDoc
searchTypeCounts := map[string]int{}
if doSearch {
scored, searchTypeCounts = co.executeSearch(sp)
}
selColl := scope.shown
collFiltered := scope.explicit()
selTypeSet := sp.Types
results := make([]*Document, 0, len(scored))
for _, s := range scored {
results = append(results, s.Doc)
}
q := canonicalSearchQuery(doSearch, sp)
q["coll"] = scope.colls
canonicalQS := q.Encode()
searchBaseURL := "/search?" + canonicalQS + "&"
formParams := cloneValues(q)
if sortBy != "relevance" {
formParams.Set("sort", sortBy)
}
if order != "" {
formParams.Set("order", order)
}
page, _ := strconv.Atoi(params.Get("p"))
start, end, page, totalPages := paginate(len(results), page, prefs.cardsPerPage())
pageResults, snippetsCapped := decorateResults(results[start:end], queries, fields, conjs)
labelQ := q
if scope.defaulted {
labelQ = cloneValues(q)
delete(labelQ, "coll")
}
queryLabel, named := "", false // the page title's name for the search
if doSearch {
queryLabel, named = co.searchLabel(labelQ)
}
savable := named && len(canonicalQS) <= searchQSMax
var recorded []string
if prefs.TrackSearches && savable &&
(action == "" || action == "search") && !crossSiteFetch(r) {
recorded = recordRecentSearch(w, r, canonicalQS)
}
var recentSearches []recentSearch
if prefs.TrackSearches {
if recorded != nil {
recentSearches = co.parseSearchEntries(recorded)
} else {
recentSearches = co.readRecentSearches(r)
}
}
searchPinned := savable && isPinnedSearch(r, canonicalQS)
var dateHist []timelineBar
if doSearch && len(results) > 0 {
dateHist = searchDateHistogram(results, q)
}
var facets []searchFacet
if doSearch && len(results) >= 2 && len(queries) < maxSearchClauses {
values := facetValues
if !hasOrRow(queries, conjs) {
values = refineValues(sp.DatePubStart, sp.DatePubEnd)
}
facets = facetsWithinWords(computeSearchFacets(results, searchBaseURL, values),
maxSearchWords-searchWords(queries))
}
if len(recentSearches) > 0 {
noStore(w)
}
render(w, r, &searchPage{
FieldOptions: searchFields,
Facets: facets,
Queries: queries,
Fields: fields,
Conjs: conjs,
Results: pageResults,
SearchTypeCounts: searchTypeCounts,
SelTypeSet: selTypeSet,
FormParams: formParams,
Savable: savable,
ShowThumbs: prefs.thumbsOn(),
FOIAQS: prefs.foiaLinkQS(),
Base: pageWith(prefs),
Total: len(results),
Page: page,
TotalPages: totalPages,
RecentSearches: recentSearches,
SelColl: selColl,
CollFiltered: collFiltered,
CanonicalQS: canonicalQS,
SearchPinned: searchPinned,
Searched: doSearch,
QueryLabel: queryLabel,
DateDocStart: sp.DateDocStart,
DateDocEnd: sp.DateDocEnd,
DatePubStart: sp.DatePubStart,
DatePubEnd: sp.DatePubEnd,
Sort: sortBy,
Order: order,
SearchBaseURL: searchBaseURL,
DateHist: dateHist,
SnippetsCapped: snippetsCapped,
SnippetLimit: maxSnippetDocs,
OverClauses: sp.OverClauses,
ClauseLimit: maxSearchClauses,
OverWords: sp.OverWords,
WordLimit: maxSearchWords,
})
}
// --- Date helpers ---
func inDateRange(date, start, end string) bool {
if date == "" {
return start == "" && end == ""
}
if start != "" && date < start {
return false
}
if end != "" && date > end {
return false
}
return true
}
func matchDates(doc *Document, dateDocStart, dateDocEnd, datePubStart, datePubEnd string, byEntry []entryRow) bool {
if dateDocStart != "" || dateDocEnd != "" {
if !inDateRange(doc.DocMeta.DateCreated, dateDocStart, dateDocEnd) {
return false
}
}
if datePubStart == "" && datePubEnd == "" {
return true
}
inRange := func(i int) bool {
return inDateRange(doc.Publication[i].DatePublished, datePubStart, datePubEnd)
}
if len(byEntry) == 0 {
for i := range doc.Publication {
if inRange(i) {
return true
}
}
return false
}
for _, r := range byEntry {
found := false
for i := range doc.Publication {
if inRange(i) && r.matches(doc, i) {
found = true
break
}
}
if !found {
return false
}
}
return true
}
// entryRow is a journalist or publisher row a publication date is tied
// to, parsed once per search.
type entryRow struct {
field string // "RE" or "PU"
pos, negs []valuesTest
}
func entryRows(queries, fields, conjs []string) []entryRow {
if hasOrRow(queries, conjs) {
return nil
}
var rows []entryRow
for i, q := range queries {
if q == "" {
continue
}
if conjs[i] != "not" && (fields[i] == "RE" || fields[i] == "PU") {
pos, negs := rowTerms(q)
rows = append(rows, entryRow{fields[i], entryTests(pos), entryTests(negs)})
}
}
return rows
}
func hasOrRow(queries, conjs []string) bool {
for i, q := range queries {
if q != "" && conjs[i] == "or" {
return true
}
}
return false
}
func entryTests(terms []string) []valuesTest {
tests := make([]valuesTest, len(terms))
for i, t := range terms {
if _, phrase, _, ok := parseProximity(t); ok {
t = phrase
}
tests[i] = compileValues(t)
}
return tests
}
func entryValues(doc *Document, i int, field string) []string {
p := &doc.Publication[i]
switch {
case field == "RE":
return p.Journalists
case !p.ReportingOnly:
return []string{p.Publisher}
}
return nil
}
func refineValues(pubStart, pubEnd string) func(*Document, string) []string {
if pubStart == "" && pubEnd == "" {
return facetValues
}
pubEnd = padDateEnd(pubEnd)
return func(d *Document, code string) []string {
if code != "RE" && code != "PU" {
return facetValues(d, code)
}
var vals []string
for i := range d.Publication {
if inDateRange(d.Publication[i].DatePublished, pubStart, pubEnd) {
vals = append(vals, entryValues(d, i, code)...)
}
}
return normalizeList(vals)
}
}
// matches reports whether publication entry i matches the row on its own
// journalists or publisher (entryValues).
func (r entryRow) matches(doc *Document, i int) bool {
vals := entryValues(doc, i, r.field)
for t := range r.pos {
if !r.pos[t].match(vals) {
return false
}
}
for t := range r.negs {
if r.negs[t].match(vals) {
return false
}
}
return true
}
// --- The boolean clause list ---
// compiledQuery is a search's rows read once. The non-empty ones, in order,
// each with the tests of its positive values and of its exclusions on its
// field.
type compiledQuery []compiledRow
type compiledRow struct {
conj string
pos, negs []func(*Document) bool
}
func compileQuery(queries, fields, conjs []string) compiledQuery {
var rows compiledQuery
for i, q := range queries {
if q == "" {
continue
}
f := fieldByCode[fields[i]]
if f == nil {
f = fieldByCode["TE"]
}
row := compiledRow{conj: conjs[i]}
pos, negs := rowTerms(q)
for _, v := range pos {
row.pos = append(row.pos, f.compile(v))
}
for _, v := range negs {
row.negs = append(row.negs, f.compile(v))
}
rows = append(rows, row)
}
return rows
}
// matches evaluates the rows against a document, left to right, each row
// as one unit.
func (rows compiledQuery) matches(d *Document) bool {
overall := true
for i := range rows {
row := &rows[i]
if i == 0 {
overall = row.matches(d) != (row.conj == "not")
continue
}
switch row.conj {
case "or":
overall = overall || row.matches(d)
case "not":
overall = overall && !row.matches(d)
default: // "and"
overall = overall && row.matches(d)
}
}
return overall
}
// matches reports whether a document matches one row.
func (row *compiledRow) matches(d *Document) bool {
for _, test := range row.pos {
if !test(d) {
return false
}
}
for _, test := range row.negs {
if test(d) {
return false
}
}
return true
}
// versionHit is one of a document's versions whose new lines match a query,
// with snippet windows from those lines.
type versionHit struct {
Version *DocVersion
Snippets []string
}
// versionHits returns the versions of doc whose new lines (NovelText) match
// term in version order.
func versionHits(doc *Document, term string, max int) []versionHit {
var out []versionHit
for i := range doc.Versions {
v := &doc.Versions[i]
if s := getSnippets(v.NovelText, term, max); len(s) > 0 {
out = append(out, versionHit{Version: v, Snippets: s})
}
}
return out
}