snowchive-server/data.go
2026-11-19 04:40:59 +02:00

1470 lines
39 KiB
Go

package main
import (
"encoding/json"
"fmt"
"html/template"
"image"
_ "image/gif"
_ "image/jpeg"
_ "image/png"
"log"
"math"
"os"
"path/filepath"
"slices"
"sort"
"strconv"
"strings"
"sync"
)
// Publication is one entry of a document's publication list.
type Publication struct {
Title string `json:"title,omitempty"`
Publisher string `json:"publisher"`
Journalists []string `json:"reporters"`
Contributors []string `json:"contributors"`
DatePublished string `json:"date_published"`
Link string `json:"link"`
Note string `json:"note,omitempty"`
Version string `json:"version,omitempty"`
DocSource string `json:"doc_source,omitempty"`
DocURL string `json:"doc_url,omitempty"`
ReportingOnly bool `json:"reporting_only,omitempty"`
}
type Document struct {
ID string `json:"id"`
AkaID []AkaEntry `json:"aka_id,omitempty"`
Title string `json:"title"`
Type string `json:"type"`
Collection string `json:"collection"`
Description string `json:"description"`
LongDescription string `json:"long_description"`
Tags []string `json:"tags"`
Pages int `json:"pages,omitempty"`
PDFSha256 string `json:"pdf_sha256,omitempty"`
ArchiveComment string `json:"archive_comment,omitempty"`
VersionMeta map[string]VersionInfo `json:"versions,omitempty"`
DocMeta struct {
Creator string `json:"creator"`
DateCreated string `json:"date_created"`
Programs []string `json:"programs"`
Classification []string `json:"classification"`
Dissemination []string `json:"dissemination"`
Banner string `json:"banner"`
Genre string `json:"genre"`
Series struct {
Name string `json:"name"`
Part int `json:"part"`
} `json:"series"`
Countries []string `json:"countries"`
People []string `json:"people"`
} `json:"document"`
Publication []Publication `json:"publication"`
Dir string `json:"-"`
FullText string `json:"-"`
SearchText string `json:"-"`
StemmedText string `json:"-"`
StemmedTitle string `json:"-"`
StemmedDesc string `json:"-"`
StemmedFull string `json:"-"` // OCR body only, for "Content" field searches
FoldedFull string `json:"-"` // accent-folded + lowercased OCR body, for fast quoted "Content" matching
bodyFoldLen int // how much of FoldedFull is the fold of FullText: the versions' lines follow it
foldedTitle string // normFoldString(Title)
foldedDesc string // normFoldString(Description + valueSep + LongDescription)
// terms is the document's entry in the TF-IDF index: every stemmed token
// it holds, in any field. fields is its metadata as the matchers and the
// tallies read it (buildDocFields). Both are nil until loadData sets them.
terms map[string]float64
fields *docFields
Images []string `json:"-"`
ImageSizes []imageSize `json:"-"`
PDFFile string `json:"-"`
PDFSize int64 `json:"-"`
TextFile string `json:"-"` // the .md OCR/transcript's file name, "" without one
Versions []DocVersion `json:"-"` // differing published copies from versions/<slug>/ (loadVersions)
PortionLevels []string `json:"-"`
PortionMarkings []string `json:"-"`
PortionControls []string `json:"-"`
HasThumb bool `json:"-"`
HasGifThumb bool `json:"-"`
HTMLContent template.HTML `json:"-"`
}
func (d *Document) foldedBody() string {
if d.bodyFoldLen > len(d.FoldedFull) || (d.bodyFoldLen == 0 && d.FullText != "") {
return normFoldString(d.FullText) // a document setBodySearchText never saw
}
return d.FoldedFull[:d.bodyFoldLen]
}
// ReportedIssue reports whether the document is a SIDtoday issue that was
// individually reported, and so is in the Reported collection as well.
func (d *Document) ReportedIssue() bool {
return d.Type == "SIDTODAY" && d.Collection == collectionNews
}
func (d *Document) JournalistsDeduped() []string { return facetValues(d, "RE") }
func (d *Document) Publishers() []string { return facetValues(d, "PU") }
func journalistsOf(d *Document) []string {
var all []string
for _, p := range d.Publication {
all = append(all, p.Journalists...)
}
return normalizeList(all)
}
func publishersOf(d *Document) []string {
var all []string
for _, p := range d.Publication {
if !p.ReportingOnly {
all = append(all, p.Publisher)
}
}
return normalizeList(all)
}
func (d *Document) titleFold() string {
if d.foldedTitle == "" && d.Title != "" {
return normFoldString(d.Title)
}
return d.foldedTitle
}
func (d *Document) descFold() string {
if d.foldedDesc == "" {
return normFoldString(d.Description + valueSep + d.LongDescription)
}
return d.foldedDesc
}
// hasTerms reports whether the document holds every one of the stemmed
// tokens, in any field.
func (d *Document) hasTerms(tokens []string) bool {
if d.terms == nil {
return true
}
for _, t := range tokens {
if _, ok := d.terms[t]; !ok {
return false
}
}
return true
}
// AkaEntry is another entry this document was published under.
type AkaEntry struct {
ID string `json:"id"` // usually a alternative PDF filename
DocSource string `json:"doc_source,omitempty"`
DocURL string `json:"doc_url,omitempty"`
}
type DocVersion struct {
Slug string
PDFFile string
PDFSize int64
TextFile string
FullText string
HTMLContent template.HTML
Images []string
ImageSizes []imageSize
PubIndex []int
SHA string
NovelText string
Label string
DocID string
ChangedPages []ChangedPage
}
type VersionInfo struct {
ChangedPages []int `json:"changed_pages,omitempty"`
PDFSha256 string `json:"pdf_sha256,omitempty"`
}
type ChangedPage struct {
Page int
Pos int
}
func (v *DocVersion) PageChanged(pos int) bool {
if pos < 1 {
return false
}
for _, c := range v.ChangedPages {
if c.Pos == pos {
return true
}
}
return false
}
func (d *Document) VersionFor(slug string) *DocVersion {
if slug == "" {
return nil
}
for i := range d.Versions {
if d.Versions[i].Slug == slug {
return &d.Versions[i]
}
}
return nil
}
// servedFile reports whether rel, a slash-separated path inside the
// document's directory, is one of the files the loader indexed: the
// metadata.json, the PDF, the transcript, the thumbnails and the page images,
// of the document or of one of its versions. /files/ serves these and nothing else a directory
// has.
func (d *Document) servedFile(rel string) bool {
if rest, ok := strings.CutPrefix(rel, "versions/"); ok {
slug, name, _ := strings.Cut(rest, "/")
v := d.VersionFor(slug)
return v != nil && contentFile(name, v.PDFFile, v.TextFile, v.Images)
}
switch rel {
case "metadata.json":
return true
case "thumb.jpg":
return d.HasThumb
case "thumb.gif":
return d.HasGifThumb
}
return contentFile(rel, d.PDFFile, d.TextFile, d.Images)
}
func contentFile(name, pdf, text string, images []string) bool {
if name == "" {
return false
}
if name == pdf || name == text {
return true
}
image, ok := strings.CutPrefix(name, "images/")
return ok && slices.Contains(images, image)
}
func (d *Document) UnlinkedVersions() []DocVersion {
var out []DocVersion
for _, v := range d.Versions {
if len(v.PubIndex) == 0 {
out = append(out, v)
}
}
return out
}
type collCounts struct {
News int `json:"reported"`
Archive int `json:"sidtoday_archive"`
FOIA int `json:"foia"`
}
func (c *collCounts) add(coll string) {
switch coll {
case collectionNews:
c.News++
case collectionArchive:
c.Archive++
case collectionFOIA:
c.FOIA++
}
}
func (c *collCounts) addType(typ string) {
switch typ {
case "SIDTODAY":
c.Archive++
case "FOIA":
c.FOIA++
default:
c.News++
}
}
var genreVocabList = []string{"slides", "report", "wiki", "manual", "guide", "newsletter", "other"}
var genreVocab = func() map[string]bool {
m := make(map[string]bool, len(genreVocabList))
for _, g := range genreVocabList {
m[g] = true
}
return m
}()
type Archive struct {
Docs []Document
Types map[string]int
NewsTypes map[string]int
CollectionCounts map[string]int
Facets map[string]map[string]int
FacetsByColl map[string]map[string]*collCounts
SIDTodayCount int
RelatedDocs map[string][]string
}
const (
collectionNews = "news"
collectionArchive = "sidtoday-archive"
collectionFOIA = "foia"
)
// dataDir is the corpus, the data/ directory of the archive root (-root).
var dataDir = "data"
// projectDir, the root's project/ directory, holds the files that describe
// the archive as a whole and are the same on every mirror: glossary.json,
// assemblies.json, changelog.json and related.json.
var projectDir = "project"
func setRoot(root string) {
dataDir, projectDir = filepath.Join(root, "data"), filepath.Join(root, "project")
}
var loadWarnings, loadErrors int
func warnf(format string, args ...interface{}) {
loadWarnings++
log.Printf("Warning: "+format, args...)
}
func errorf(format string, args ...interface{}) {
loadErrors++
log.Printf("Error: "+format, args...)
}
func projectPath(name string) string { return filepath.Join(projectDir, name) }
func relatedDocsPath() string { return projectPath("related.json") }
const defaultDocType = "REPORTED"
var pageImageExts = map[string]bool{
".jpg": true, ".jpeg": true, ".png": true, ".gif": true, ".webp": true,
}
func (co *Corpus) buildSerialNav() {
co.serialPrev = map[string]string{}
co.serialNext = map[string]string{}
var prev string
for _, yr := range co.archiveTimeline {
for _, mo := range yr.Months {
for _, d := range mo.Docs {
if prev != "" {
co.serialPrev[d.ID] = prev
co.serialNext[prev] = d.ID
}
prev = d.ID
}
}
}
}
var collectionDirs = map[string]string{
"foia": collectionFOIA,
"sidtoday-archive": collectionArchive,
}
type docLocation struct {
id string
dir string
collection string
}
func docLocations(dataDir string) ([]docLocation, error) {
entries, err := os.ReadDir(dataDir)
if err != nil {
return nil, err
}
var out []docLocation
for _, e := range entries {
if !e.IsDir() {
continue
}
if coll, ok := collectionDirs[strings.ToLower(e.Name())]; ok {
subs, err := os.ReadDir(filepath.Join(dataDir, e.Name()))
if err != nil {
warnf("cannot read %s: %v", filepath.Join(dataDir, e.Name()), err)
continue
}
for _, s := range subs {
if s.IsDir() {
out = append(out, docLocation{
id: s.Name(),
dir: filepath.Join(dataDir, e.Name(), s.Name()),
collection: coll,
})
}
}
continue
}
out = append(out, docLocation{
id: e.Name(),
dir: filepath.Join(dataDir, e.Name()),
})
}
return out, nil
}
func dirHasPDF(dir string) bool {
entries, err := os.ReadDir(dir)
if err != nil {
return false
}
for _, e := range entries {
if !e.IsDir() && isPDF(e.Name()) {
return true
}
}
return false
}
func isPDF(name string) bool { return strings.EqualFold(filepath.Ext(name), ".pdf") }
func isTranscript(name string) bool { return strings.EqualFold(filepath.Ext(name), ".md") }
func firstN(vals []string, n int) []string {
if len(vals) <= n {
return vals
}
return append(append([]string{}, vals[:n]...), fmt.Sprintf("and %d more", len(vals)-n))
}
func plural(n int, one, many string) string {
if n == 1 {
return one
}
return many
}
func normalizeList(vals []string) []string {
if len(vals) == 0 {
return vals
}
out := make([]string, 0, len(vals))
seen := make(map[string]bool)
for _, v := range vals {
v = strings.TrimSpace(v)
if v == "" {
continue
}
k := valueKey(v)
if seen[k] {
continue
}
seen[k] = true
out = append(out, v)
}
return out
}
type seriesGroup struct {
Name string
Docs []*Document
}
func (co *Corpus) buildSeriesIndex() {
co.seriesIndex = make(map[string]*seriesGroup)
for i := range co.archive.Docs {
dd := &co.archive.Docs[i]
name := dd.DocMeta.Series.Name
if name == "" {
continue
}
if dd.DocMeta.Series.Part <= 0 {
warnf("%s: series %q without a part number - order matters; sorting it last", dd.ID, name)
}
key := valueKey(name)
g := co.seriesIndex[key]
if g == nil {
g = &seriesGroup{Name: name}
co.seriesIndex[key] = g
} else if g.Name != name {
warnf("%s: series name casing %q differs from first-seen %q - merged, but copy-paste the exact name", dd.ID, name, g.Name)
}
g.Docs = append(g.Docs, dd)
}
for _, g := range co.seriesIndex {
sort.SliceStable(g.Docs, func(i, j int) bool {
a, b := g.Docs[i].DocMeta.Series.Part, g.Docs[j].DocMeta.Series.Part
if (a <= 0) != (b <= 0) {
return b <= 0 // numbered parts before unnumbered ones
}
if a != b {
return a < b
}
return g.Docs[i].Title < g.Docs[j].Title
})
seen := map[int]string{}
for _, m := range g.Docs {
p := m.DocMeta.Series.Part
if p <= 0 {
continue
}
if prev, dup := seen[p]; dup {
warnf("series %q: part %d claimed by both %s and %s", g.Name, p, prev, m.ID)
}
seen[p] = m.ID
}
}
}
func wrapRedactions(html string) string {
return strings.ReplaceAll(html, "[REDACTED]", `<span class="redacted">[REDACTED]</span>`)
}
// load reads the corpus under dataDir into co and builds every index the
// handlers serve from. Written here once, before the corpus goes live
// (loadData), and read-only from then on.
func (co *Corpus) load() {
loadWarnings, loadErrors = 0, 0
co.loadGlossary()
co.archive = Archive{
Docs: []Document{},
Types: make(map[string]int),
NewsTypes: make(map[string]int),
CollectionCounts: make(map[string]int),
RelatedDocs: make(map[string][]string),
}
locations, err := docLocations(dataDir)
if err != nil {
log.Fatalf("Critical: Could not read data directory: %v", err)
}
seen := make(map[string]string)
var skipped, skippedWithPDF []string
for _, loc := range locations {
metaPath := filepath.Join(loc.dir, "metadata.json")
b, err := os.ReadFile(metaPath)
if err != nil {
skipped = append(skipped, loc.id)
if dirHasPDF(loc.dir) {
skippedWithPDF = append(skippedWithPDF, loc.id)
}
continue
}
var d Document
if err := json.Unmarshal(b, &d); err != nil {
errorf("invalid JSON in %s: %v - skipping document", metaPath, err)
continue
}
d.Type = strings.ToUpper(strings.TrimSpace(d.Type))
if d.Type == "" {
switch loc.collection {
case collectionArchive:
d.Type = "SIDTODAY"
case collectionFOIA:
d.Type = "FOIA"
default:
d.Type = defaultDocType
}
}
d.ID = loc.id
d.Dir = loc.dir
if prev, dup := seen[d.ID]; dup {
errorf("duplicate document id %q in %s (already loaded from %s) - skipping",
d.ID, loc.dir, prev)
continue
}
if !docIDSafe(d.ID) {
warnf("document id %q contains characters outside [A-Za-z0-9._~-] - links, sitemap entries or pin/history cookies may misbehave; consider renaming the directory", d.ID)
}
d.Tags = normalizeList(d.Tags)
d.AkaID = normalizeAka(&d)
files, derr := scanContentDir(d.Dir)
if derr != nil {
errorf("cannot read %s: %v - skipping document", d.Dir, derr)
continue
}
if files.PDFFile == "" {
errorf("%s: no PDF in %s - skipping document", d.ID, d.Dir)
continue
}
seen[d.ID] = loc.dir
d.PDFFile, d.PDFSize, d.TextFile = files.PDFFile, files.PDFSize, files.TextFile
d.HasThumb, d.HasGifThumb = files.HasThumb, files.HasGifThumb
if files.Text != "" {
d.FullText = files.Text
d.HTMLContent = template.HTML(wrapRedactions(Render(d.FullText)))
}
d.Collection = strings.ToLower(strings.TrimSpace(d.Collection))
if loc.collection != "" {
if d.Collection != "" && d.Collection != loc.collection {
warnf("%s declares collection %q but lives in the %s directory - using %q",
d.ID, d.Collection, loc.collection, loc.collection)
}
d.Collection = loc.collection
} else {
switch d.Collection {
case collectionNews, collectionArchive, collectionFOIA:
// explicit, keep
case "":
if d.Type == "FOIA" {
d.Collection = collectionFOIA
} else {
d.Collection = collectionNews
}
default:
warnf("%s has unknown collection %q - treating as %q",
d.ID, d.Collection, collectionNews)
d.Collection = collectionNews
}
}
if d.Collection == collectionArchive && d.Type != "SIDTODAY" {
warnf("%s is in the SIDtoday archive but typed %q - the SIDtoday tab lists the type SIDTODAY, so no browse tab shows it", d.ID, d.Type)
}
co.archive.CollectionCounts[d.Collection]++
if d.Type == "SIDTODAY" {
co.archive.SIDTodayCount++
}
newsDoc := d.Collection == collectionNews
co.archive.Types[d.Type]++
if newsDoc {
co.archive.NewsTypes[d.Type]++
}
d.Images = loadPageImages(d.Dir, d.ID)
if d.Pages <= 0 {
d.Pages = len(d.Images)
}
d.DocMeta.Programs = normalizeList(d.DocMeta.Programs)
d.DocMeta.Countries = normalizeList(d.DocMeta.Countries)
d.DocMeta.People = normalizeList(d.DocMeta.People)
d.DocMeta.Classification = normalizeList(d.DocMeta.Classification)
for _, c := range d.DocMeta.Classification {
if full, ok := classAlias[strings.ToLower(c)]; ok {
warnf("%s: classification %q is an abbreviation - write %q; searches and counts treat the two as different values", d.ID, c, strings.ToUpper(full))
}
}
d.DocMeta.Dissemination = normalizeList(d.DocMeta.Dissemination)
for i := range d.Publication {
p := &d.Publication[i]
p.Publisher = strings.TrimSpace(p.Publisher)
p.Journalists = normalizeList(p.Journalists)
p.Contributors = normalizeList(p.Contributors)
p.Note = strings.TrimSpace(p.Note)
p.DocSource = strings.TrimSpace(p.DocSource)
p.DocURL = strings.TrimSpace(p.DocURL)
}
if v := d.DocMeta.DateCreated; v != "" && !validDate(v) {
warnf("%s: date_created %q is not YYYY, YYYY-MM or YYYY-MM-DD - treating the document as undated", d.ID, v)
d.DocMeta.DateCreated = ""
}
for i := range d.Publication {
if v := d.Publication[i].DatePublished; v != "" && !validDate(v) {
warnf("%s: publication %d date_published %q is not YYYY, YYYY-MM or YYYY-MM-DD - treating it as undated", d.ID, i+1, v)
d.Publication[i].DatePublished = ""
}
}
co.loadVersions(&d)
d.DocMeta.Creator = strings.TrimSpace(d.DocMeta.Creator)
d.DocMeta.Series.Name = strings.TrimSpace(d.DocMeta.Series.Name)
d.DocMeta.Banner = strings.TrimSpace(d.DocMeta.Banner)
d.DocMeta.Genre = strings.ToLower(strings.TrimSpace(d.DocMeta.Genre))
if d.DocMeta.Genre != "" && !genreVocab[d.DocMeta.Genre] {
warnf("%s: genre %q is outside the vocabulary (%s) - serving as-is", d.ID, d.DocMeta.Genre, strings.Join(genreVocabList, ", "))
}
for _, f := range searchFields {
if f.Values == nil {
continue
}
for _, v := range f.Values(&d) {
if strings.Contains(v, `"`) {
warnf("%s: %s %q contains a double quote - its search links cannot quote it; use ' or “ ” instead", d.ID, strings.ToLower(f.Label), v)
}
}
}
// The all-fields forms are the title, the descriptions and the rest
// of the metadata, then the body: each part is tokenized once, and
// tokenize of the parts joined is the parts' tokens in a row.
meta := searchMeta(&d)
metaText := strings.Join([]string{d.Title, d.Description, d.LongDescription, meta}, valueSep)
titleToks := tokenize(d.Title)
descToks := tokenize(d.Description + " " + d.LongDescription)
metaToks := tokenize(meta)
stems := make([]string, 0, len(titleToks)+len(descToks)+len(metaToks))
stems = append(append(append(stems, titleToks...), descToks...), metaToks...)
d.StemmedTitle = strings.Join(titleToks, " ")
d.StemmedDesc = strings.Join(descToks, " ")
d.foldedTitle = normFoldString(d.Title)
d.foldedDesc = normFoldString(d.Description + valueSep + d.LongDescription)
setBodySearchText(&d)
d.StemmedText = strings.TrimSpace(strings.Join(stems, " ") + " " + d.StemmedFull)
d.SearchText = normFoldString(metaText) + valueSep + d.FoldedFull
d.FoldedFull = d.SearchText[len(d.SearchText)-len(d.FoldedFull):]
d.StemmedFull = d.StemmedText[len(d.StemmedText)-len(d.StemmedFull):]
co.archive.Docs = append(co.archive.Docs, d)
}
if len(skipped) > 0 {
log.Printf("Note: %d directories have no metadata.json and were skipped: %s",
len(skipped), strings.Join(firstN(skipped, 8), ", "))
if len(skippedWithPDF) > 0 {
warnf("%d skipped %s a PDF but no metadata.json - a document nobody described: %s",
len(skippedWithPDF), plural(len(skippedWithPDF), "directory holds", "directories hold"),
strings.Join(firstN(skippedWithPDF, 8), ", "))
}
}
sizesRead := co.readImageSizes()
log.Printf("Building TF-IDF index...")
co.tfidf = buildTFIDF(co.archive.Docs)
log.Printf("TF-IDF done")
sizesRead()
co.docIndex = make(map[string]*Document, len(co.archive.Docs))
for i := range co.archive.Docs {
d := &co.archive.Docs[i]
co.docIndex[d.ID] = d
d.terms = co.tfidf.DocStats[d.ID]
}
co.buildSeriesIndex()
co.loadAssemblies()
log.Printf("Loading related docs index...")
co.archive.RelatedDocs = co.loadRelatedDocs(co.archive.Docs)
log.Printf("Related docs done")
co.buildArticles()
co.loadChangelog()
co.buildCollectionDocs()
co.buildCategoryDocs()
co.buildHomeStats()
co.buildCodewords()
co.buildCategoryLists()
co.buildExploreTables()
co.buildIDRows()
co.buildClassPatterns()
co.buildPortions()
co.buildDocFields()
co.buildOutletTables()
co.buildArchiveTimeline()
co.buildSerialNav()
co.buildFacetsJSON()
co.buildSitemapPaths()
co.buildCanonicalSitemap()
log.Printf("Loaded: %d documents (%d error%s, %d warning%s)",
len(co.archive.Docs), loadErrors, plural(loadErrors, "", "s"), loadWarnings, plural(loadWarnings, "", "s"))
}
// loadRelatedDocs reads related.json if it exists - run with --rebuild-related
// to regenerate the file.
func (co *Corpus) loadRelatedDocs(docs []Document) map[string][]string {
b, err := os.ReadFile(relatedDocsPath())
if err != nil {
warnf("related.json not found - related docs unavailable. Run with --rebuild-related to generate it.")
return make(map[string][]string)
}
var related map[string][]string
if err := json.Unmarshal(b, &related); err != nil {
warnf("related.json is malformed: %v - related docs unavailable.", err)
return make(map[string][]string)
}
staleKeys, staleLinks, staleLists := 0, 0, 0
for id, ids := range related {
if co.docIndex[id] == nil {
delete(related, id)
staleKeys++
continue
}
kept := slices.DeleteFunc(ids, co.unknownDoc)
if n := len(ids) - len(kept); n > 0 {
staleLinks += n
staleLists++
}
related[id] = kept
}
if staleKeys > 0 || staleLinks > 0 {
warnf("related.json: %d entries for documents no longer in the corpus, and %d links to such documents in %d lists - ignored; run --rebuild-related to recompute those lists",
staleKeys, staleLinks, staleLists)
}
missing := 0
for _, d := range docs {
if _, ok := related[d.ID]; !ok {
missing++
}
}
if missing > 0 {
warnf("%d/%d documents missing from related.json - run --rebuild-related to update.", missing, len(docs))
} else {
log.Printf("Related docs loaded: %d entries", len(related))
}
return related
}
func (co *Corpus) unknownDoc(id string) bool { return co.docIndex[id] == nil }
func saveRelatedDocs(related map[string][]string) error {
b, err := json.MarshalIndent(related, "", " ")
if err != nil {
return err
}
return os.WriteFile(relatedDocsPath(), b, 0644)
}
func (co *Corpus) buildRelatedDocs(targets, all []Document) map[string][]string {
type scored struct {
id string
score float64
collection string
}
programDocs := make(map[string][]int)
tagDocs := make(map[string][]int)
titleDocs := make(map[string][]int)
for i, d := range all {
for _, p := range d.DocMeta.Programs {
key := strings.ToLower(p)
programDocs[key] = append(programDocs[key], i)
}
for _, t := range d.Tags {
key := strings.ToLower(t)
tagDocs[key] = append(tagDocs[key], i)
}
for _, w := range uniqueStrings(tokenize(d.Title)) {
titleDocs[w] = append(titleDocs[w], i)
}
}
tw := titleWeights{df: make(map[string]int, len(titleDocs)), n: len(all)}
for w, docs := range titleDocs {
tw.df[w] = len(docs)
}
maxTitleDF := len(all) / 10
for w, docs := range titleDocs {
if len(docs) > maxTitleDF {
delete(titleDocs, w)
}
}
result := make(map[string][]string, len(targets))
for i := range targets {
d := &targets[i]
seen := make(map[int]bool)
for _, p := range d.DocMeta.Programs {
for _, j := range programDocs[strings.ToLower(p)] {
seen[j] = true
}
}
for _, t := range d.Tags {
for _, j := range tagDocs[strings.ToLower(t)] {
seen[j] = true
}
}
for _, w := range tokenize(d.Title) {
for _, j := range titleDocs[w] {
seen[j] = true
}
}
var candidates []scored
for j := range seen {
if all[j].ID == d.ID {
continue
}
s := co.relatedScore(d, &all[j], tw)
if s >= minRelatedScore {
candidates = append(candidates, scored{all[j].ID, s, all[j].Collection})
}
}
sort.Slice(candidates, func(a, b int) bool {
if candidates[a].score != candidates[b].score {
return candidates[a].score > candidates[b].score
}
return candidates[a].id < candidates[b].id
})
const topN = 5
maxBulk := topN
if d.Collection == collectionNews {
maxBulk = 2
}
var ids []string
var skipped []string
bulk := 0
for _, c := range candidates {
if len(ids) == topN {
break
}
if c.collection != collectionNews && bulk >= maxBulk {
skipped = append(skipped, c.id)
continue
}
if c.collection != collectionNews {
bulk++
}
ids = append(ids, c.id)
}
for _, id := range skipped {
if len(ids) == topN {
break
}
ids = append(ids, id)
}
result[d.ID] = ids
}
log.Printf("Related docs built: %d entries", len(result))
return result
}
const minRelatedScore = 1.5
type titleWeights struct {
df map[string]int
n int
}
func (t titleWeights) weight(w string) float64 {
if t.n < 2 {
return 1
}
return math.Log(float64(t.n+1)/float64(t.df[w]+1)) / math.Log(float64(t.n+1))
}
func uniqueStrings(in []string) []string {
seen := make(map[string]bool, len(in))
out := in[:0:0]
for _, s := range in {
if key := strings.ToLower(s); !seen[key] {
seen[key] = true
out = append(out, s)
}
}
return out
}
func sharedCount(a, b []string) int {
inA := make(map[string]bool, len(a))
for _, s := range a {
inA[strings.ToLower(s)] = true
}
n := 0
for _, s := range uniqueStrings(b) {
if inA[strings.ToLower(s)] {
n++
}
}
return n
}
func (co *Corpus) relatedScore(a, b *Document, tw titleWeights) float64 {
var score float64
aTitleWords := make(map[string]bool)
for _, w := range tokenize(a.Title) {
aTitleWords[w] = true
}
for _, w := range uniqueStrings(tokenize(b.Title)) {
if aTitleWords[w] {
score += 3.0 * tw.weight(w)
}
}
score += 4.0 * float64(sharedCount(a.DocMeta.Programs, b.DocMeta.Programs))
score += 2.0 * float64(sharedCount(a.Tags, b.Tags))
var ajournos, bjournos []string
for _, p := range a.Publication {
ajournos = append(ajournos, p.Journalists...)
}
for _, p := range b.Publication {
bjournos = append(bjournos, p.Journalists...)
}
score += 2.0 * float64(sharedCount(ajournos, bjournos))
if strings.EqualFold(a.DocMeta.Creator, b.DocMeta.Creator) && a.DocMeta.Creator != "" {
score += 1.0
}
score += 0.5 * float64(sharedCount(a.DocMeta.Countries, b.DocMeta.Countries))
score += co.tfidf.similarity(a.ID, b.ID) * 2.0
return score
}
func pageNumber(s string) (int, error) {
parts := strings.SplitN(s, "-", 2)
if len(parts) < 2 {
return 0, fmt.Errorf("unexpected image filename format: %q", s)
}
numPart := strings.SplitN(parts[1], ".", 2)[0]
n, err := strconv.Atoi(numPart)
if err != nil {
return 0, fmt.Errorf("could not parse page number from %q: %v", s, err)
}
return n, nil
}
func parseImageNum(s string) int {
n, err := pageNumber(s)
if err != nil {
warnf("%v", err)
return 0
}
return n
}
func docIDSafe(id string) bool {
for i := 0; i < len(id); i++ {
c := id[i]
switch {
case c >= 'a' && c <= 'z',
c >= 'A' && c <= 'Z',
c >= '0' && c <= '9',
c == '-', c == '.', c == '_', c == '~':
default:
return false
}
}
return len(id) > 0
}
func normalizeAka(d *Document) []AkaEntry {
var out []AkaEntry
seen := make(map[string]bool)
for i, a := range d.AkaID {
a.ID = strings.TrimSpace(a.ID)
a.DocSource = strings.TrimSpace(a.DocSource)
a.DocURL = strings.TrimSpace(a.DocURL)
if a.ID == "" {
warnf("%s: aka_id[%d] has no id - ignored", d.ID, i)
continue
}
key := strings.ToLower(a.ID)
if seen[key] {
warnf("%s: aka_id %q is listed twice - keeping the first", d.ID, a.ID)
continue
}
seen[key] = true
out = append(out, a)
}
return out
}
func searchMeta(d *Document) string {
var reporters, publishers, pubTitles, pubNotes, contributors []string
for _, p := range d.Publication {
if p.Title != "" {
pubTitles = append(pubTitles, p.Title)
}
if p.Note != "" {
pubNotes = append(pubNotes, p.Note)
}
if p.Publisher != "" {
publishers = append(publishers, p.Publisher)
}
contributors = append(contributors, p.Contributors...)
reporters = append(reporters, p.Journalists...)
}
return strings.Join([]string{
akaIDs(d.AkaID),
d.DocMeta.Creator,
strings.Join(reporters, valueSep),
strings.Join(contributors, valueSep),
strings.Join(publishers, valueSep),
strings.Join(pubTitles, valueSep),
strings.Join(pubNotes, valueSep),
d.ArchiveComment,
strings.Join(d.DocMeta.People, valueSep),
strings.Join(d.DocMeta.Countries, valueSep),
strings.Join(d.DocMeta.Programs, valueSep),
strings.Join(d.DocMeta.Classification, valueSep),
strings.Join(d.DocMeta.Dissemination, valueSep),
d.DocMeta.Banner,
d.DocMeta.Genre,
d.DocMeta.Series.Name,
d.ID,
strings.Join(d.Tags, valueSep),
}, valueSep)
}
func akaIDs(akas []AkaEntry) string {
ids := make([]string, len(akas))
for i, a := range akas {
ids[i] = a.ID
}
return strings.Join(ids, valueSep)
}
func loadPageImages(dir, label string) []string {
files, err := os.ReadDir(filepath.Join(dir, "images"))
if err != nil && !os.IsNotExist(err) {
warnf("%s: images directory unreadable: %v", label, err)
}
type pageImage struct {
name string
num int
}
var imgs []pageImage
for _, f := range files {
if !f.IsDir() && pageImageExts[strings.ToLower(filepath.Ext(f.Name()))] {
imgs = append(imgs, pageImage{f.Name(), parseImageNum(f.Name())})
}
}
sort.SliceStable(imgs, func(i, j int) bool { return imgs[i].num < imgs[j].num })
var out []string
for i, im := range imgs {
if i > 0 && im.num > 0 && im.num == imgs[i-1].num {
warnf("%s: images/%s and images/%s are both page %d - keep one", label, imgs[i-1].name, im.name, im.num)
}
out = append(out, im.name)
}
return out
}
type imageSize struct{ W, H int }
func (s imageSize) Attr() template.HTMLAttr {
if s.W <= 0 || s.H <= 0 {
return ""
}
return template.HTMLAttr(fmt.Sprintf(` width="%d" height="%d"`, s.W, s.H))
}
// readImageSizes reads every page image's pixel size from its header, so a
// page can reserve each image's height before it loads. It runs alongside the
// rest of the load; the returned func waits for it and reports the failures.
func (co *Corpus) readImageSizes() func() {
type job struct {
path string
size *imageSize
}
var jobs []job
add := func(dir string, names []string, sizes *[]imageSize) {
*sizes = make([]imageSize, len(names))
for i, name := range names {
if !strings.EqualFold(filepath.Ext(name), ".webp") {
jobs = append(jobs, job{filepath.Join(dir, "images", name), &(*sizes)[i]})
}
}
}
for i := range co.archive.Docs {
d := &co.archive.Docs[i]
add(d.Dir, d.Images, &d.ImageSizes)
for j := range d.Versions {
v := &d.Versions[j]
add(filepath.Join(d.Dir, "versions", v.Slug), v.Images, &v.ImageSizes)
}
}
ch := make(chan job)
var mu sync.Mutex
var wg sync.WaitGroup
var failed []string
for range 16 {
wg.Add(1)
go func() {
defer wg.Done()
for j := range ch {
f, err := os.Open(j.path)
if err == nil {
var c image.Config
c, _, err = image.DecodeConfig(f)
f.Close()
*j.size = imageSize{c.Width, c.Height}
}
if err != nil {
mu.Lock()
failed = append(failed, fmt.Sprintf("%s: %v", strings.TrimPrefix(j.path, dataDir+string(filepath.Separator)), err))
mu.Unlock()
}
}
}()
}
go func() {
for _, j := range jobs {
ch <- j
}
close(ch)
}()
return func() {
wg.Wait()
sort.Strings(failed)
for _, f := range failed {
warnf("page image unreadable, served without its size: %s", f)
}
}
}
func versionSlugSafe(s string) bool {
if s == "" || strings.Contains(s, "..") || strings.Contains(s, "~") || !docIDSafe(s) {
return false
}
c := s[0]
return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9'
}
func (co *Corpus) loadVersions(d *Document) {
vdir := filepath.Join(d.Dir, "versions")
entries, err := os.ReadDir(vdir)
if err != nil && !os.IsNotExist(err) {
warnf("%s: versions directory unreadable: %v", d.ID, err)
}
for _, e := range entries {
name := e.Name()
if !e.IsDir() {
warnf("%s: versions/%s is not a folder - each version is a folder holding that copy's PDF, .md and images/", d.ID, name)
continue
}
if !versionSlugSafe(name) {
warnf("%s: versions/%s: version names may only contain letters, digits, \".\", \"_\" and \"-\", starting with a letter or digit - skipping", d.ID, name)
continue
}
v, err := loadVersion(filepath.Join(vdir, name), d.ID+"/versions/"+name)
if err != nil {
errorf("%s: versions/%s: %v - skipping version", d.ID, name, err)
continue
}
v.Slug = name
v.DocID = d.ID
d.Versions = append(d.Versions, v)
}
for i := range d.Publication {
p := &d.Publication[i]
p.Version = strings.TrimSpace(p.Version)
if p.Version == "" {
continue
}
v := d.VersionFor(p.Version)
if v == nil {
warnf("%s: publication %d (%s) points at versions/%s, which is missing or failed to load - its copy isn't linked", d.ID, i+1, p.Publisher, p.Version)
continue
}
v.PubIndex = append(v.PubIndex, i)
}
for i := range d.Versions {
v := &d.Versions[i]
v.Label = versionLabel(d, v)
}
for slug, info := range d.VersionMeta {
v := d.VersionFor(slug)
if v == nil {
warnf("%s: \"versions\" has an entry for versions/%s, which is missing or failed to load", d.ID, slug)
continue
}
v.ChangedPages = changedPages(d.ID, v, info.ChangedPages)
v.SHA = strings.ToLower(strings.TrimSpace(info.PDFSha256))
}
for _, v := range d.Versions {
if v.SHA == "" {
warnf("%s: versions/%s has no pdf_sha256 in metadata.json - its checksum isn't listed", d.ID, v.Slug)
}
}
}
func changedPages(docID string, v *DocVersion, pages []int) []ChangedPage {
pos := make(map[int]int, len(v.Images))
for i, name := range v.Images {
if n, err := pageNumber(name); err == nil {
if _, seen := pos[n]; !seen {
pos[n] = i + 1
}
}
}
nums := append([]int(nil), pages...)
sort.Ints(nums)
var out []ChangedPage
for i, n := range nums {
if i > 0 && n == nums[i-1] {
continue
}
if n < 1 {
warnf("%s: versions/%s: changed page %d is not a page number - ignored", docID, v.Slug, n)
continue
}
p := pos[n]
if p == 0 {
warnf("%s: versions/%s: changed page %d has no page image with that number - listed without a link", docID, v.Slug, n)
}
out = append(out, ChangedPage{Page: n, Pos: p})
}
return out
}
type contentFiles struct {
Text string
TextFile string
PDFFile string
PDFSize int64
HasThumb bool
HasGifThumb bool
}
func scanContentDir(dir string) (contentFiles, error) {
var c contentFiles
entries, err := os.ReadDir(dir)
if err != nil {
return c, err
}
textDone := false
for _, f := range entries {
if f.IsDir() {
continue
}
name := f.Name()
if !textDone && isTranscript(name) {
textDone = true
c.TextFile = name
b, rerr := os.ReadFile(filepath.Join(dir, name))
if rerr != nil {
warnf("cannot read %s: %v", filepath.Join(dir, name), rerr)
} else {
c.Text = string(b)
}
}
if c.PDFFile == "" && isPDF(name) {
c.PDFFile = name
if info, ierr := f.Info(); ierr == nil {
c.PDFSize = info.Size()
}
}
if name == "thumb.jpg" {
c.HasThumb = true
}
if name == "thumb.gif" {
c.HasGifThumb = true
}
}
return c, nil
}
func loadVersion(dir, label string) (DocVersion, error) {
var v DocVersion
files, err := scanContentDir(dir)
if err != nil {
return v, err
}
if files.PDFFile == "" {
return v, fmt.Errorf("no PDF")
}
v.PDFFile, v.PDFSize, v.TextFile, v.FullText = files.PDFFile, files.PDFSize, files.TextFile, files.Text
if v.FullText != "" {
v.HTMLContent = template.HTML(wrapRedactions(Render(v.FullText)))
}
v.Images = loadPageImages(dir, label)
return v, nil
}
// VersionSnippet is a search-result snippet taken from one of the document's
// versions, with the copy it came from.
type VersionSnippet struct {
Slug string
Label string
HTML template.HTML
}
func versionLabel(d *Document, v *DocVersion) string {
var names []string
seen := make(map[string]bool)
date := ""
for _, i := range v.PubIndex {
p := d.Publication[i]
if key := valueKey(p.Publisher); key != "" && !seen[key] {
seen[key] = true
names = append(names, p.Publisher)
}
if p.DatePublished != "" && (date == "" || p.DatePublished < date) {
date = p.DatePublished
}
}
if len(names) == 0 {
return v.Slug
}
label := strings.Join(names, " / ")
if date != "" {
label += ", " + date
}
return label
}
// lineKey is the form transcript lines are compared in: folded, lowercased,
// whitespace collapsed. Lines without a letter or digit ("---", "|") key to
// "" and are never new.
func lineKey(line string) string {
k := strings.Join(strings.Fields(normFoldString(line)), " ")
if strings.IndexFunc(k, isWordRune) < 0 {
return ""
}
return k
}
func versionNovelText(d *Document) string {
if len(d.Versions) == 0 {
return ""
}
inMain := make(map[string]bool)
for _, line := range strings.Split(d.FullText, "\n") {
if k := lineKey(line); k != "" {
inMain[k] = true
}
}
added := make(map[string]bool)
var all []string
for i := range d.Versions {
v := &d.Versions[i]
var novel []string
for _, line := range strings.Split(v.FullText, "\n") {
k := lineKey(line)
if k == "" || inMain[k] {
continue
}
line = strings.TrimSpace(line)
novel = append(novel, line)
if !added[k] {
added[k] = true
all = append(all, line)
}
}
v.NovelText = strings.Join(novel, "\n")
}
return strings.Join(all, "\n")
}
func setBodySearchText(d *Document) {
d.StemmedFull = strings.Join(tokenize(d.FullText), " ")
d.FoldedFull = normFoldString(d.FullText)
d.bodyFoldLen = len(d.FoldedFull)
if novel := versionNovelText(d); novel != "" {
d.StemmedFull = strings.TrimSpace(d.StemmedFull + " " + valueSep + " " + strings.Join(tokenize(novel), " "))
d.FoldedFull += valueSep + normFoldString(novel)
}
}