1470 lines
39 KiB
Go
1470 lines
39 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"html/template"
|
|
"image"
|
|
_ "image/gif"
|
|
_ "image/jpeg"
|
|
_ "image/png"
|
|
"log"
|
|
"math"
|
|
"os"
|
|
"path/filepath"
|
|
"slices"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
)
|
|
|
|
// Publication is one entry of a document's publication list.
|
|
type Publication struct {
|
|
Title string `json:"title,omitempty"`
|
|
Publisher string `json:"publisher"`
|
|
Journalists []string `json:"reporters"`
|
|
Contributors []string `json:"contributors"`
|
|
DatePublished string `json:"date_published"`
|
|
Link string `json:"link"`
|
|
Note string `json:"note,omitempty"`
|
|
Version string `json:"version,omitempty"`
|
|
DocSource string `json:"doc_source,omitempty"`
|
|
DocURL string `json:"doc_url,omitempty"`
|
|
ReportingOnly bool `json:"reporting_only,omitempty"`
|
|
}
|
|
|
|
type Document struct {
|
|
ID string `json:"id"`
|
|
AkaID []AkaEntry `json:"aka_id,omitempty"`
|
|
Title string `json:"title"`
|
|
Type string `json:"type"`
|
|
Collection string `json:"collection"`
|
|
Description string `json:"description"`
|
|
LongDescription string `json:"long_description"`
|
|
Tags []string `json:"tags"`
|
|
Pages int `json:"pages,omitempty"`
|
|
PDFSha256 string `json:"pdf_sha256,omitempty"`
|
|
ArchiveComment string `json:"archive_comment,omitempty"`
|
|
VersionMeta map[string]VersionInfo `json:"versions,omitempty"`
|
|
DocMeta struct {
|
|
Creator string `json:"creator"`
|
|
DateCreated string `json:"date_created"`
|
|
Programs []string `json:"programs"`
|
|
Classification []string `json:"classification"`
|
|
Dissemination []string `json:"dissemination"`
|
|
Banner string `json:"banner"`
|
|
Genre string `json:"genre"`
|
|
Series struct {
|
|
Name string `json:"name"`
|
|
Part int `json:"part"`
|
|
} `json:"series"`
|
|
Countries []string `json:"countries"`
|
|
People []string `json:"people"`
|
|
} `json:"document"`
|
|
Publication []Publication `json:"publication"`
|
|
|
|
Dir string `json:"-"`
|
|
FullText string `json:"-"`
|
|
SearchText string `json:"-"`
|
|
StemmedText string `json:"-"`
|
|
StemmedTitle string `json:"-"`
|
|
StemmedDesc string `json:"-"`
|
|
StemmedFull string `json:"-"` // OCR body only, for "Content" field searches
|
|
FoldedFull string `json:"-"` // accent-folded + lowercased OCR body, for fast quoted "Content" matching
|
|
bodyFoldLen int // how much of FoldedFull is the fold of FullText: the versions' lines follow it
|
|
foldedTitle string // normFoldString(Title)
|
|
foldedDesc string // normFoldString(Description + valueSep + LongDescription)
|
|
// terms is the document's entry in the TF-IDF index: every stemmed token
|
|
// it holds, in any field. fields is its metadata as the matchers and the
|
|
// tallies read it (buildDocFields). Both are nil until loadData sets them.
|
|
terms map[string]float64
|
|
fields *docFields
|
|
|
|
Images []string `json:"-"`
|
|
ImageSizes []imageSize `json:"-"`
|
|
PDFFile string `json:"-"`
|
|
PDFSize int64 `json:"-"`
|
|
TextFile string `json:"-"` // the .md OCR/transcript's file name, "" without one
|
|
Versions []DocVersion `json:"-"` // differing published copies from versions/<slug>/ (loadVersions)
|
|
PortionLevels []string `json:"-"`
|
|
PortionMarkings []string `json:"-"`
|
|
PortionControls []string `json:"-"`
|
|
HasThumb bool `json:"-"`
|
|
HasGifThumb bool `json:"-"`
|
|
HTMLContent template.HTML `json:"-"`
|
|
}
|
|
|
|
func (d *Document) foldedBody() string {
|
|
if d.bodyFoldLen > len(d.FoldedFull) || (d.bodyFoldLen == 0 && d.FullText != "") {
|
|
return normFoldString(d.FullText) // a document setBodySearchText never saw
|
|
}
|
|
return d.FoldedFull[:d.bodyFoldLen]
|
|
}
|
|
|
|
// ReportedIssue reports whether the document is a SIDtoday issue that was
|
|
// individually reported, and so is in the Reported collection as well.
|
|
func (d *Document) ReportedIssue() bool {
|
|
return d.Type == "SIDTODAY" && d.Collection == collectionNews
|
|
}
|
|
|
|
func (d *Document) JournalistsDeduped() []string { return facetValues(d, "RE") }
|
|
|
|
func (d *Document) Publishers() []string { return facetValues(d, "PU") }
|
|
|
|
func journalistsOf(d *Document) []string {
|
|
var all []string
|
|
for _, p := range d.Publication {
|
|
all = append(all, p.Journalists...)
|
|
}
|
|
return normalizeList(all)
|
|
}
|
|
|
|
func publishersOf(d *Document) []string {
|
|
var all []string
|
|
for _, p := range d.Publication {
|
|
if !p.ReportingOnly {
|
|
all = append(all, p.Publisher)
|
|
}
|
|
}
|
|
return normalizeList(all)
|
|
}
|
|
|
|
func (d *Document) titleFold() string {
|
|
if d.foldedTitle == "" && d.Title != "" {
|
|
return normFoldString(d.Title)
|
|
}
|
|
return d.foldedTitle
|
|
}
|
|
|
|
func (d *Document) descFold() string {
|
|
if d.foldedDesc == "" {
|
|
return normFoldString(d.Description + valueSep + d.LongDescription)
|
|
}
|
|
return d.foldedDesc
|
|
}
|
|
|
|
// hasTerms reports whether the document holds every one of the stemmed
|
|
// tokens, in any field.
|
|
func (d *Document) hasTerms(tokens []string) bool {
|
|
if d.terms == nil {
|
|
return true
|
|
}
|
|
for _, t := range tokens {
|
|
if _, ok := d.terms[t]; !ok {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// AkaEntry is another entry this document was published under.
|
|
type AkaEntry struct {
|
|
ID string `json:"id"` // usually a alternative PDF filename
|
|
DocSource string `json:"doc_source,omitempty"`
|
|
DocURL string `json:"doc_url,omitempty"`
|
|
}
|
|
|
|
type DocVersion struct {
|
|
Slug string
|
|
PDFFile string
|
|
PDFSize int64
|
|
TextFile string
|
|
FullText string
|
|
HTMLContent template.HTML
|
|
Images []string
|
|
ImageSizes []imageSize
|
|
PubIndex []int
|
|
SHA string
|
|
NovelText string
|
|
Label string
|
|
DocID string
|
|
ChangedPages []ChangedPage
|
|
}
|
|
|
|
type VersionInfo struct {
|
|
ChangedPages []int `json:"changed_pages,omitempty"`
|
|
PDFSha256 string `json:"pdf_sha256,omitempty"`
|
|
}
|
|
|
|
type ChangedPage struct {
|
|
Page int
|
|
Pos int
|
|
}
|
|
|
|
func (v *DocVersion) PageChanged(pos int) bool {
|
|
if pos < 1 {
|
|
return false
|
|
}
|
|
for _, c := range v.ChangedPages {
|
|
if c.Pos == pos {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (d *Document) VersionFor(slug string) *DocVersion {
|
|
if slug == "" {
|
|
return nil
|
|
}
|
|
for i := range d.Versions {
|
|
if d.Versions[i].Slug == slug {
|
|
return &d.Versions[i]
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// servedFile reports whether rel, a slash-separated path inside the
|
|
// document's directory, is one of the files the loader indexed: the
|
|
// metadata.json, the PDF, the transcript, the thumbnails and the page images,
|
|
// of the document or of one of its versions. /files/ serves these and nothing else a directory
|
|
// has.
|
|
func (d *Document) servedFile(rel string) bool {
|
|
if rest, ok := strings.CutPrefix(rel, "versions/"); ok {
|
|
slug, name, _ := strings.Cut(rest, "/")
|
|
v := d.VersionFor(slug)
|
|
return v != nil && contentFile(name, v.PDFFile, v.TextFile, v.Images)
|
|
}
|
|
switch rel {
|
|
case "metadata.json":
|
|
return true
|
|
case "thumb.jpg":
|
|
return d.HasThumb
|
|
case "thumb.gif":
|
|
return d.HasGifThumb
|
|
}
|
|
return contentFile(rel, d.PDFFile, d.TextFile, d.Images)
|
|
}
|
|
|
|
func contentFile(name, pdf, text string, images []string) bool {
|
|
if name == "" {
|
|
return false
|
|
}
|
|
if name == pdf || name == text {
|
|
return true
|
|
}
|
|
image, ok := strings.CutPrefix(name, "images/")
|
|
return ok && slices.Contains(images, image)
|
|
}
|
|
|
|
func (d *Document) UnlinkedVersions() []DocVersion {
|
|
var out []DocVersion
|
|
for _, v := range d.Versions {
|
|
if len(v.PubIndex) == 0 {
|
|
out = append(out, v)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
type collCounts struct {
|
|
News int `json:"reported"`
|
|
Archive int `json:"sidtoday_archive"`
|
|
FOIA int `json:"foia"`
|
|
}
|
|
|
|
func (c *collCounts) add(coll string) {
|
|
switch coll {
|
|
case collectionNews:
|
|
c.News++
|
|
case collectionArchive:
|
|
c.Archive++
|
|
case collectionFOIA:
|
|
c.FOIA++
|
|
}
|
|
}
|
|
|
|
func (c *collCounts) addType(typ string) {
|
|
switch typ {
|
|
case "SIDTODAY":
|
|
c.Archive++
|
|
case "FOIA":
|
|
c.FOIA++
|
|
default:
|
|
c.News++
|
|
}
|
|
}
|
|
|
|
var genreVocabList = []string{"slides", "report", "wiki", "manual", "guide", "newsletter", "other"}
|
|
|
|
var genreVocab = func() map[string]bool {
|
|
m := make(map[string]bool, len(genreVocabList))
|
|
for _, g := range genreVocabList {
|
|
m[g] = true
|
|
}
|
|
return m
|
|
}()
|
|
|
|
type Archive struct {
|
|
Docs []Document
|
|
Types map[string]int
|
|
NewsTypes map[string]int
|
|
CollectionCounts map[string]int
|
|
Facets map[string]map[string]int
|
|
FacetsByColl map[string]map[string]*collCounts
|
|
SIDTodayCount int
|
|
RelatedDocs map[string][]string
|
|
}
|
|
|
|
const (
|
|
collectionNews = "news"
|
|
collectionArchive = "sidtoday-archive"
|
|
collectionFOIA = "foia"
|
|
)
|
|
|
|
// dataDir is the corpus, the data/ directory of the archive root (-root).
|
|
var dataDir = "data"
|
|
|
|
// projectDir, the root's project/ directory, holds the files that describe
|
|
// the archive as a whole and are the same on every mirror: glossary.json,
|
|
// assemblies.json, changelog.json and related.json.
|
|
var projectDir = "project"
|
|
|
|
func setRoot(root string) {
|
|
dataDir, projectDir = filepath.Join(root, "data"), filepath.Join(root, "project")
|
|
}
|
|
|
|
var loadWarnings, loadErrors int
|
|
|
|
func warnf(format string, args ...interface{}) {
|
|
loadWarnings++
|
|
log.Printf("Warning: "+format, args...)
|
|
}
|
|
|
|
func errorf(format string, args ...interface{}) {
|
|
loadErrors++
|
|
log.Printf("Error: "+format, args...)
|
|
}
|
|
|
|
func projectPath(name string) string { return filepath.Join(projectDir, name) }
|
|
|
|
func relatedDocsPath() string { return projectPath("related.json") }
|
|
|
|
const defaultDocType = "REPORTED"
|
|
|
|
var pageImageExts = map[string]bool{
|
|
".jpg": true, ".jpeg": true, ".png": true, ".gif": true, ".webp": true,
|
|
}
|
|
|
|
func (co *Corpus) buildSerialNav() {
|
|
co.serialPrev = map[string]string{}
|
|
co.serialNext = map[string]string{}
|
|
var prev string
|
|
for _, yr := range co.archiveTimeline {
|
|
for _, mo := range yr.Months {
|
|
for _, d := range mo.Docs {
|
|
if prev != "" {
|
|
co.serialPrev[d.ID] = prev
|
|
co.serialNext[prev] = d.ID
|
|
}
|
|
prev = d.ID
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
var collectionDirs = map[string]string{
|
|
"foia": collectionFOIA,
|
|
"sidtoday-archive": collectionArchive,
|
|
}
|
|
|
|
type docLocation struct {
|
|
id string
|
|
dir string
|
|
collection string
|
|
}
|
|
|
|
func docLocations(dataDir string) ([]docLocation, error) {
|
|
entries, err := os.ReadDir(dataDir)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var out []docLocation
|
|
for _, e := range entries {
|
|
if !e.IsDir() {
|
|
continue
|
|
}
|
|
if coll, ok := collectionDirs[strings.ToLower(e.Name())]; ok {
|
|
subs, err := os.ReadDir(filepath.Join(dataDir, e.Name()))
|
|
if err != nil {
|
|
warnf("cannot read %s: %v", filepath.Join(dataDir, e.Name()), err)
|
|
continue
|
|
}
|
|
for _, s := range subs {
|
|
if s.IsDir() {
|
|
out = append(out, docLocation{
|
|
id: s.Name(),
|
|
dir: filepath.Join(dataDir, e.Name(), s.Name()),
|
|
collection: coll,
|
|
})
|
|
}
|
|
}
|
|
continue
|
|
}
|
|
out = append(out, docLocation{
|
|
id: e.Name(),
|
|
dir: filepath.Join(dataDir, e.Name()),
|
|
})
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func dirHasPDF(dir string) bool {
|
|
entries, err := os.ReadDir(dir)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
for _, e := range entries {
|
|
if !e.IsDir() && isPDF(e.Name()) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func isPDF(name string) bool { return strings.EqualFold(filepath.Ext(name), ".pdf") }
|
|
|
|
func isTranscript(name string) bool { return strings.EqualFold(filepath.Ext(name), ".md") }
|
|
|
|
func firstN(vals []string, n int) []string {
|
|
if len(vals) <= n {
|
|
return vals
|
|
}
|
|
return append(append([]string{}, vals[:n]...), fmt.Sprintf("and %d more", len(vals)-n))
|
|
}
|
|
|
|
func plural(n int, one, many string) string {
|
|
if n == 1 {
|
|
return one
|
|
}
|
|
return many
|
|
}
|
|
|
|
func normalizeList(vals []string) []string {
|
|
if len(vals) == 0 {
|
|
return vals
|
|
}
|
|
out := make([]string, 0, len(vals))
|
|
seen := make(map[string]bool)
|
|
for _, v := range vals {
|
|
v = strings.TrimSpace(v)
|
|
if v == "" {
|
|
continue
|
|
}
|
|
k := valueKey(v)
|
|
if seen[k] {
|
|
continue
|
|
}
|
|
seen[k] = true
|
|
out = append(out, v)
|
|
}
|
|
return out
|
|
}
|
|
|
|
type seriesGroup struct {
|
|
Name string
|
|
Docs []*Document
|
|
}
|
|
|
|
func (co *Corpus) buildSeriesIndex() {
|
|
co.seriesIndex = make(map[string]*seriesGroup)
|
|
for i := range co.archive.Docs {
|
|
dd := &co.archive.Docs[i]
|
|
name := dd.DocMeta.Series.Name
|
|
if name == "" {
|
|
continue
|
|
}
|
|
if dd.DocMeta.Series.Part <= 0 {
|
|
warnf("%s: series %q without a part number - order matters; sorting it last", dd.ID, name)
|
|
}
|
|
key := valueKey(name)
|
|
g := co.seriesIndex[key]
|
|
if g == nil {
|
|
g = &seriesGroup{Name: name}
|
|
co.seriesIndex[key] = g
|
|
} else if g.Name != name {
|
|
warnf("%s: series name casing %q differs from first-seen %q - merged, but copy-paste the exact name", dd.ID, name, g.Name)
|
|
}
|
|
g.Docs = append(g.Docs, dd)
|
|
}
|
|
for _, g := range co.seriesIndex {
|
|
sort.SliceStable(g.Docs, func(i, j int) bool {
|
|
a, b := g.Docs[i].DocMeta.Series.Part, g.Docs[j].DocMeta.Series.Part
|
|
if (a <= 0) != (b <= 0) {
|
|
return b <= 0 // numbered parts before unnumbered ones
|
|
}
|
|
if a != b {
|
|
return a < b
|
|
}
|
|
return g.Docs[i].Title < g.Docs[j].Title
|
|
})
|
|
seen := map[int]string{}
|
|
for _, m := range g.Docs {
|
|
p := m.DocMeta.Series.Part
|
|
if p <= 0 {
|
|
continue
|
|
}
|
|
if prev, dup := seen[p]; dup {
|
|
warnf("series %q: part %d claimed by both %s and %s", g.Name, p, prev, m.ID)
|
|
}
|
|
seen[p] = m.ID
|
|
}
|
|
}
|
|
}
|
|
|
|
func wrapRedactions(html string) string {
|
|
return strings.ReplaceAll(html, "[REDACTED]", `<span class="redacted">[REDACTED]</span>`)
|
|
}
|
|
|
|
// load reads the corpus under dataDir into co and builds every index the
|
|
// handlers serve from. Written here once, before the corpus goes live
|
|
// (loadData), and read-only from then on.
|
|
func (co *Corpus) load() {
|
|
loadWarnings, loadErrors = 0, 0
|
|
|
|
co.loadGlossary()
|
|
|
|
co.archive = Archive{
|
|
Docs: []Document{},
|
|
Types: make(map[string]int),
|
|
NewsTypes: make(map[string]int),
|
|
CollectionCounts: make(map[string]int),
|
|
RelatedDocs: make(map[string][]string),
|
|
}
|
|
|
|
locations, err := docLocations(dataDir)
|
|
if err != nil {
|
|
log.Fatalf("Critical: Could not read data directory: %v", err)
|
|
}
|
|
|
|
seen := make(map[string]string)
|
|
var skipped, skippedWithPDF []string
|
|
for _, loc := range locations {
|
|
metaPath := filepath.Join(loc.dir, "metadata.json")
|
|
b, err := os.ReadFile(metaPath)
|
|
if err != nil {
|
|
skipped = append(skipped, loc.id)
|
|
if dirHasPDF(loc.dir) {
|
|
skippedWithPDF = append(skippedWithPDF, loc.id)
|
|
}
|
|
continue
|
|
}
|
|
|
|
var d Document
|
|
if err := json.Unmarshal(b, &d); err != nil {
|
|
errorf("invalid JSON in %s: %v - skipping document", metaPath, err)
|
|
continue
|
|
}
|
|
|
|
d.Type = strings.ToUpper(strings.TrimSpace(d.Type))
|
|
if d.Type == "" {
|
|
switch loc.collection {
|
|
case collectionArchive:
|
|
d.Type = "SIDTODAY"
|
|
case collectionFOIA:
|
|
d.Type = "FOIA"
|
|
default:
|
|
d.Type = defaultDocType
|
|
}
|
|
}
|
|
d.ID = loc.id
|
|
d.Dir = loc.dir
|
|
|
|
if prev, dup := seen[d.ID]; dup {
|
|
errorf("duplicate document id %q in %s (already loaded from %s) - skipping",
|
|
d.ID, loc.dir, prev)
|
|
continue
|
|
}
|
|
|
|
if !docIDSafe(d.ID) {
|
|
warnf("document id %q contains characters outside [A-Za-z0-9._~-] - links, sitemap entries or pin/history cookies may misbehave; consider renaming the directory", d.ID)
|
|
}
|
|
|
|
d.Tags = normalizeList(d.Tags)
|
|
d.AkaID = normalizeAka(&d)
|
|
|
|
files, derr := scanContentDir(d.Dir)
|
|
if derr != nil {
|
|
errorf("cannot read %s: %v - skipping document", d.Dir, derr)
|
|
continue
|
|
}
|
|
if files.PDFFile == "" {
|
|
errorf("%s: no PDF in %s - skipping document", d.ID, d.Dir)
|
|
continue
|
|
}
|
|
seen[d.ID] = loc.dir
|
|
d.PDFFile, d.PDFSize, d.TextFile = files.PDFFile, files.PDFSize, files.TextFile
|
|
d.HasThumb, d.HasGifThumb = files.HasThumb, files.HasGifThumb
|
|
if files.Text != "" {
|
|
d.FullText = files.Text
|
|
d.HTMLContent = template.HTML(wrapRedactions(Render(d.FullText)))
|
|
}
|
|
|
|
d.Collection = strings.ToLower(strings.TrimSpace(d.Collection))
|
|
if loc.collection != "" {
|
|
if d.Collection != "" && d.Collection != loc.collection {
|
|
warnf("%s declares collection %q but lives in the %s directory - using %q",
|
|
d.ID, d.Collection, loc.collection, loc.collection)
|
|
}
|
|
d.Collection = loc.collection
|
|
} else {
|
|
switch d.Collection {
|
|
case collectionNews, collectionArchive, collectionFOIA:
|
|
// explicit, keep
|
|
case "":
|
|
if d.Type == "FOIA" {
|
|
d.Collection = collectionFOIA
|
|
} else {
|
|
d.Collection = collectionNews
|
|
}
|
|
default:
|
|
warnf("%s has unknown collection %q - treating as %q",
|
|
d.ID, d.Collection, collectionNews)
|
|
d.Collection = collectionNews
|
|
}
|
|
}
|
|
if d.Collection == collectionArchive && d.Type != "SIDTODAY" {
|
|
warnf("%s is in the SIDtoday archive but typed %q - the SIDtoday tab lists the type SIDTODAY, so no browse tab shows it", d.ID, d.Type)
|
|
}
|
|
co.archive.CollectionCounts[d.Collection]++
|
|
if d.Type == "SIDTODAY" {
|
|
co.archive.SIDTodayCount++
|
|
}
|
|
|
|
newsDoc := d.Collection == collectionNews
|
|
co.archive.Types[d.Type]++
|
|
if newsDoc {
|
|
co.archive.NewsTypes[d.Type]++
|
|
}
|
|
|
|
d.Images = loadPageImages(d.Dir, d.ID)
|
|
|
|
if d.Pages <= 0 {
|
|
d.Pages = len(d.Images)
|
|
}
|
|
|
|
d.DocMeta.Programs = normalizeList(d.DocMeta.Programs)
|
|
d.DocMeta.Countries = normalizeList(d.DocMeta.Countries)
|
|
d.DocMeta.People = normalizeList(d.DocMeta.People)
|
|
d.DocMeta.Classification = normalizeList(d.DocMeta.Classification)
|
|
for _, c := range d.DocMeta.Classification {
|
|
if full, ok := classAlias[strings.ToLower(c)]; ok {
|
|
warnf("%s: classification %q is an abbreviation - write %q; searches and counts treat the two as different values", d.ID, c, strings.ToUpper(full))
|
|
}
|
|
}
|
|
d.DocMeta.Dissemination = normalizeList(d.DocMeta.Dissemination)
|
|
for i := range d.Publication {
|
|
p := &d.Publication[i]
|
|
p.Publisher = strings.TrimSpace(p.Publisher)
|
|
p.Journalists = normalizeList(p.Journalists)
|
|
p.Contributors = normalizeList(p.Contributors)
|
|
p.Note = strings.TrimSpace(p.Note)
|
|
p.DocSource = strings.TrimSpace(p.DocSource)
|
|
p.DocURL = strings.TrimSpace(p.DocURL)
|
|
}
|
|
|
|
if v := d.DocMeta.DateCreated; v != "" && !validDate(v) {
|
|
warnf("%s: date_created %q is not YYYY, YYYY-MM or YYYY-MM-DD - treating the document as undated", d.ID, v)
|
|
d.DocMeta.DateCreated = ""
|
|
}
|
|
for i := range d.Publication {
|
|
if v := d.Publication[i].DatePublished; v != "" && !validDate(v) {
|
|
warnf("%s: publication %d date_published %q is not YYYY, YYYY-MM or YYYY-MM-DD - treating it as undated", d.ID, i+1, v)
|
|
d.Publication[i].DatePublished = ""
|
|
}
|
|
}
|
|
|
|
co.loadVersions(&d)
|
|
|
|
d.DocMeta.Creator = strings.TrimSpace(d.DocMeta.Creator)
|
|
d.DocMeta.Series.Name = strings.TrimSpace(d.DocMeta.Series.Name)
|
|
d.DocMeta.Banner = strings.TrimSpace(d.DocMeta.Banner)
|
|
d.DocMeta.Genre = strings.ToLower(strings.TrimSpace(d.DocMeta.Genre))
|
|
if d.DocMeta.Genre != "" && !genreVocab[d.DocMeta.Genre] {
|
|
warnf("%s: genre %q is outside the vocabulary (%s) - serving as-is", d.ID, d.DocMeta.Genre, strings.Join(genreVocabList, ", "))
|
|
}
|
|
|
|
for _, f := range searchFields {
|
|
if f.Values == nil {
|
|
continue
|
|
}
|
|
for _, v := range f.Values(&d) {
|
|
if strings.Contains(v, `"`) {
|
|
warnf("%s: %s %q contains a double quote - its search links cannot quote it; use ' or “ ” instead", d.ID, strings.ToLower(f.Label), v)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The all-fields forms are the title, the descriptions and the rest
|
|
// of the metadata, then the body: each part is tokenized once, and
|
|
// tokenize of the parts joined is the parts' tokens in a row.
|
|
meta := searchMeta(&d)
|
|
metaText := strings.Join([]string{d.Title, d.Description, d.LongDescription, meta}, valueSep)
|
|
titleToks := tokenize(d.Title)
|
|
descToks := tokenize(d.Description + " " + d.LongDescription)
|
|
metaToks := tokenize(meta)
|
|
stems := make([]string, 0, len(titleToks)+len(descToks)+len(metaToks))
|
|
stems = append(append(append(stems, titleToks...), descToks...), metaToks...)
|
|
d.StemmedTitle = strings.Join(titleToks, " ")
|
|
d.StemmedDesc = strings.Join(descToks, " ")
|
|
d.foldedTitle = normFoldString(d.Title)
|
|
d.foldedDesc = normFoldString(d.Description + valueSep + d.LongDescription)
|
|
setBodySearchText(&d)
|
|
d.StemmedText = strings.TrimSpace(strings.Join(stems, " ") + " " + d.StemmedFull)
|
|
d.SearchText = normFoldString(metaText) + valueSep + d.FoldedFull
|
|
d.FoldedFull = d.SearchText[len(d.SearchText)-len(d.FoldedFull):]
|
|
d.StemmedFull = d.StemmedText[len(d.StemmedText)-len(d.StemmedFull):]
|
|
|
|
co.archive.Docs = append(co.archive.Docs, d)
|
|
}
|
|
|
|
if len(skipped) > 0 {
|
|
log.Printf("Note: %d directories have no metadata.json and were skipped: %s",
|
|
len(skipped), strings.Join(firstN(skipped, 8), ", "))
|
|
if len(skippedWithPDF) > 0 {
|
|
warnf("%d skipped %s a PDF but no metadata.json - a document nobody described: %s",
|
|
len(skippedWithPDF), plural(len(skippedWithPDF), "directory holds", "directories hold"),
|
|
strings.Join(firstN(skippedWithPDF, 8), ", "))
|
|
}
|
|
}
|
|
|
|
sizesRead := co.readImageSizes()
|
|
log.Printf("Building TF-IDF index...")
|
|
co.tfidf = buildTFIDF(co.archive.Docs)
|
|
log.Printf("TF-IDF done")
|
|
sizesRead()
|
|
|
|
co.docIndex = make(map[string]*Document, len(co.archive.Docs))
|
|
for i := range co.archive.Docs {
|
|
d := &co.archive.Docs[i]
|
|
co.docIndex[d.ID] = d
|
|
d.terms = co.tfidf.DocStats[d.ID]
|
|
}
|
|
|
|
co.buildSeriesIndex()
|
|
co.loadAssemblies()
|
|
|
|
log.Printf("Loading related docs index...")
|
|
co.archive.RelatedDocs = co.loadRelatedDocs(co.archive.Docs)
|
|
log.Printf("Related docs done")
|
|
|
|
co.buildArticles()
|
|
co.loadChangelog()
|
|
|
|
co.buildCollectionDocs()
|
|
co.buildCategoryDocs()
|
|
co.buildHomeStats()
|
|
co.buildCodewords()
|
|
co.buildCategoryLists()
|
|
co.buildExploreTables()
|
|
co.buildIDRows()
|
|
co.buildClassPatterns()
|
|
co.buildPortions()
|
|
co.buildDocFields()
|
|
co.buildOutletTables()
|
|
co.buildArchiveTimeline()
|
|
co.buildSerialNav()
|
|
co.buildFacetsJSON()
|
|
co.buildSitemapPaths()
|
|
co.buildCanonicalSitemap()
|
|
|
|
log.Printf("Loaded: %d documents (%d error%s, %d warning%s)",
|
|
len(co.archive.Docs), loadErrors, plural(loadErrors, "", "s"), loadWarnings, plural(loadWarnings, "", "s"))
|
|
}
|
|
|
|
// loadRelatedDocs reads related.json if it exists - run with --rebuild-related
|
|
// to regenerate the file.
|
|
func (co *Corpus) loadRelatedDocs(docs []Document) map[string][]string {
|
|
b, err := os.ReadFile(relatedDocsPath())
|
|
if err != nil {
|
|
warnf("related.json not found - related docs unavailable. Run with --rebuild-related to generate it.")
|
|
return make(map[string][]string)
|
|
}
|
|
|
|
var related map[string][]string
|
|
if err := json.Unmarshal(b, &related); err != nil {
|
|
warnf("related.json is malformed: %v - related docs unavailable.", err)
|
|
return make(map[string][]string)
|
|
}
|
|
|
|
staleKeys, staleLinks, staleLists := 0, 0, 0
|
|
for id, ids := range related {
|
|
if co.docIndex[id] == nil {
|
|
delete(related, id)
|
|
staleKeys++
|
|
continue
|
|
}
|
|
kept := slices.DeleteFunc(ids, co.unknownDoc)
|
|
if n := len(ids) - len(kept); n > 0 {
|
|
staleLinks += n
|
|
staleLists++
|
|
}
|
|
related[id] = kept
|
|
}
|
|
if staleKeys > 0 || staleLinks > 0 {
|
|
warnf("related.json: %d entries for documents no longer in the corpus, and %d links to such documents in %d lists - ignored; run --rebuild-related to recompute those lists",
|
|
staleKeys, staleLinks, staleLists)
|
|
}
|
|
|
|
missing := 0
|
|
for _, d := range docs {
|
|
if _, ok := related[d.ID]; !ok {
|
|
missing++
|
|
}
|
|
}
|
|
if missing > 0 {
|
|
warnf("%d/%d documents missing from related.json - run --rebuild-related to update.", missing, len(docs))
|
|
} else {
|
|
log.Printf("Related docs loaded: %d entries", len(related))
|
|
}
|
|
|
|
return related
|
|
}
|
|
|
|
func (co *Corpus) unknownDoc(id string) bool { return co.docIndex[id] == nil }
|
|
|
|
func saveRelatedDocs(related map[string][]string) error {
|
|
b, err := json.MarshalIndent(related, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return os.WriteFile(relatedDocsPath(), b, 0644)
|
|
}
|
|
|
|
func (co *Corpus) buildRelatedDocs(targets, all []Document) map[string][]string {
|
|
type scored struct {
|
|
id string
|
|
score float64
|
|
collection string
|
|
}
|
|
|
|
programDocs := make(map[string][]int)
|
|
tagDocs := make(map[string][]int)
|
|
titleDocs := make(map[string][]int)
|
|
|
|
for i, d := range all {
|
|
for _, p := range d.DocMeta.Programs {
|
|
key := strings.ToLower(p)
|
|
programDocs[key] = append(programDocs[key], i)
|
|
}
|
|
for _, t := range d.Tags {
|
|
key := strings.ToLower(t)
|
|
tagDocs[key] = append(tagDocs[key], i)
|
|
}
|
|
for _, w := range uniqueStrings(tokenize(d.Title)) {
|
|
titleDocs[w] = append(titleDocs[w], i)
|
|
}
|
|
}
|
|
|
|
tw := titleWeights{df: make(map[string]int, len(titleDocs)), n: len(all)}
|
|
for w, docs := range titleDocs {
|
|
tw.df[w] = len(docs)
|
|
}
|
|
maxTitleDF := len(all) / 10
|
|
for w, docs := range titleDocs {
|
|
if len(docs) > maxTitleDF {
|
|
delete(titleDocs, w)
|
|
}
|
|
}
|
|
|
|
result := make(map[string][]string, len(targets))
|
|
|
|
for i := range targets {
|
|
d := &targets[i]
|
|
|
|
seen := make(map[int]bool)
|
|
for _, p := range d.DocMeta.Programs {
|
|
for _, j := range programDocs[strings.ToLower(p)] {
|
|
seen[j] = true
|
|
}
|
|
}
|
|
for _, t := range d.Tags {
|
|
for _, j := range tagDocs[strings.ToLower(t)] {
|
|
seen[j] = true
|
|
}
|
|
}
|
|
for _, w := range tokenize(d.Title) {
|
|
for _, j := range titleDocs[w] {
|
|
seen[j] = true
|
|
}
|
|
}
|
|
|
|
var candidates []scored
|
|
for j := range seen {
|
|
if all[j].ID == d.ID {
|
|
continue
|
|
}
|
|
s := co.relatedScore(d, &all[j], tw)
|
|
if s >= minRelatedScore {
|
|
candidates = append(candidates, scored{all[j].ID, s, all[j].Collection})
|
|
}
|
|
}
|
|
|
|
sort.Slice(candidates, func(a, b int) bool {
|
|
if candidates[a].score != candidates[b].score {
|
|
return candidates[a].score > candidates[b].score
|
|
}
|
|
return candidates[a].id < candidates[b].id
|
|
})
|
|
|
|
const topN = 5
|
|
maxBulk := topN
|
|
if d.Collection == collectionNews {
|
|
maxBulk = 2
|
|
}
|
|
var ids []string
|
|
var skipped []string
|
|
bulk := 0
|
|
for _, c := range candidates {
|
|
if len(ids) == topN {
|
|
break
|
|
}
|
|
if c.collection != collectionNews && bulk >= maxBulk {
|
|
skipped = append(skipped, c.id)
|
|
continue
|
|
}
|
|
if c.collection != collectionNews {
|
|
bulk++
|
|
}
|
|
ids = append(ids, c.id)
|
|
}
|
|
for _, id := range skipped {
|
|
if len(ids) == topN {
|
|
break
|
|
}
|
|
ids = append(ids, id)
|
|
}
|
|
result[d.ID] = ids
|
|
}
|
|
|
|
log.Printf("Related docs built: %d entries", len(result))
|
|
return result
|
|
}
|
|
|
|
const minRelatedScore = 1.5
|
|
|
|
type titleWeights struct {
|
|
df map[string]int
|
|
n int
|
|
}
|
|
|
|
func (t titleWeights) weight(w string) float64 {
|
|
if t.n < 2 {
|
|
return 1
|
|
}
|
|
return math.Log(float64(t.n+1)/float64(t.df[w]+1)) / math.Log(float64(t.n+1))
|
|
}
|
|
|
|
func uniqueStrings(in []string) []string {
|
|
seen := make(map[string]bool, len(in))
|
|
out := in[:0:0]
|
|
for _, s := range in {
|
|
if key := strings.ToLower(s); !seen[key] {
|
|
seen[key] = true
|
|
out = append(out, s)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func sharedCount(a, b []string) int {
|
|
inA := make(map[string]bool, len(a))
|
|
for _, s := range a {
|
|
inA[strings.ToLower(s)] = true
|
|
}
|
|
n := 0
|
|
for _, s := range uniqueStrings(b) {
|
|
if inA[strings.ToLower(s)] {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
func (co *Corpus) relatedScore(a, b *Document, tw titleWeights) float64 {
|
|
var score float64
|
|
|
|
aTitleWords := make(map[string]bool)
|
|
for _, w := range tokenize(a.Title) {
|
|
aTitleWords[w] = true
|
|
}
|
|
for _, w := range uniqueStrings(tokenize(b.Title)) {
|
|
if aTitleWords[w] {
|
|
score += 3.0 * tw.weight(w)
|
|
}
|
|
}
|
|
|
|
score += 4.0 * float64(sharedCount(a.DocMeta.Programs, b.DocMeta.Programs))
|
|
score += 2.0 * float64(sharedCount(a.Tags, b.Tags))
|
|
|
|
var ajournos, bjournos []string
|
|
for _, p := range a.Publication {
|
|
ajournos = append(ajournos, p.Journalists...)
|
|
}
|
|
for _, p := range b.Publication {
|
|
bjournos = append(bjournos, p.Journalists...)
|
|
}
|
|
score += 2.0 * float64(sharedCount(ajournos, bjournos))
|
|
|
|
if strings.EqualFold(a.DocMeta.Creator, b.DocMeta.Creator) && a.DocMeta.Creator != "" {
|
|
score += 1.0
|
|
}
|
|
|
|
score += 0.5 * float64(sharedCount(a.DocMeta.Countries, b.DocMeta.Countries))
|
|
score += co.tfidf.similarity(a.ID, b.ID) * 2.0
|
|
|
|
return score
|
|
}
|
|
|
|
func pageNumber(s string) (int, error) {
|
|
parts := strings.SplitN(s, "-", 2)
|
|
if len(parts) < 2 {
|
|
return 0, fmt.Errorf("unexpected image filename format: %q", s)
|
|
}
|
|
numPart := strings.SplitN(parts[1], ".", 2)[0]
|
|
n, err := strconv.Atoi(numPart)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("could not parse page number from %q: %v", s, err)
|
|
}
|
|
return n, nil
|
|
}
|
|
|
|
func parseImageNum(s string) int {
|
|
n, err := pageNumber(s)
|
|
if err != nil {
|
|
warnf("%v", err)
|
|
return 0
|
|
}
|
|
return n
|
|
}
|
|
|
|
func docIDSafe(id string) bool {
|
|
for i := 0; i < len(id); i++ {
|
|
c := id[i]
|
|
switch {
|
|
case c >= 'a' && c <= 'z',
|
|
c >= 'A' && c <= 'Z',
|
|
c >= '0' && c <= '9',
|
|
c == '-', c == '.', c == '_', c == '~':
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
return len(id) > 0
|
|
}
|
|
|
|
func normalizeAka(d *Document) []AkaEntry {
|
|
var out []AkaEntry
|
|
seen := make(map[string]bool)
|
|
for i, a := range d.AkaID {
|
|
a.ID = strings.TrimSpace(a.ID)
|
|
a.DocSource = strings.TrimSpace(a.DocSource)
|
|
a.DocURL = strings.TrimSpace(a.DocURL)
|
|
if a.ID == "" {
|
|
warnf("%s: aka_id[%d] has no id - ignored", d.ID, i)
|
|
continue
|
|
}
|
|
key := strings.ToLower(a.ID)
|
|
if seen[key] {
|
|
warnf("%s: aka_id %q is listed twice - keeping the first", d.ID, a.ID)
|
|
continue
|
|
}
|
|
seen[key] = true
|
|
out = append(out, a)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func searchMeta(d *Document) string {
|
|
var reporters, publishers, pubTitles, pubNotes, contributors []string
|
|
for _, p := range d.Publication {
|
|
if p.Title != "" {
|
|
pubTitles = append(pubTitles, p.Title)
|
|
}
|
|
if p.Note != "" {
|
|
pubNotes = append(pubNotes, p.Note)
|
|
}
|
|
if p.Publisher != "" {
|
|
publishers = append(publishers, p.Publisher)
|
|
}
|
|
contributors = append(contributors, p.Contributors...)
|
|
reporters = append(reporters, p.Journalists...)
|
|
}
|
|
return strings.Join([]string{
|
|
akaIDs(d.AkaID),
|
|
d.DocMeta.Creator,
|
|
strings.Join(reporters, valueSep),
|
|
strings.Join(contributors, valueSep),
|
|
strings.Join(publishers, valueSep),
|
|
strings.Join(pubTitles, valueSep),
|
|
strings.Join(pubNotes, valueSep),
|
|
d.ArchiveComment,
|
|
strings.Join(d.DocMeta.People, valueSep),
|
|
strings.Join(d.DocMeta.Countries, valueSep),
|
|
strings.Join(d.DocMeta.Programs, valueSep),
|
|
strings.Join(d.DocMeta.Classification, valueSep),
|
|
strings.Join(d.DocMeta.Dissemination, valueSep),
|
|
d.DocMeta.Banner,
|
|
d.DocMeta.Genre,
|
|
d.DocMeta.Series.Name,
|
|
d.ID,
|
|
strings.Join(d.Tags, valueSep),
|
|
}, valueSep)
|
|
}
|
|
|
|
func akaIDs(akas []AkaEntry) string {
|
|
ids := make([]string, len(akas))
|
|
for i, a := range akas {
|
|
ids[i] = a.ID
|
|
}
|
|
return strings.Join(ids, valueSep)
|
|
}
|
|
|
|
func loadPageImages(dir, label string) []string {
|
|
files, err := os.ReadDir(filepath.Join(dir, "images"))
|
|
if err != nil && !os.IsNotExist(err) {
|
|
warnf("%s: images directory unreadable: %v", label, err)
|
|
}
|
|
type pageImage struct {
|
|
name string
|
|
num int
|
|
}
|
|
var imgs []pageImage
|
|
for _, f := range files {
|
|
if !f.IsDir() && pageImageExts[strings.ToLower(filepath.Ext(f.Name()))] {
|
|
imgs = append(imgs, pageImage{f.Name(), parseImageNum(f.Name())})
|
|
}
|
|
}
|
|
sort.SliceStable(imgs, func(i, j int) bool { return imgs[i].num < imgs[j].num })
|
|
var out []string
|
|
for i, im := range imgs {
|
|
if i > 0 && im.num > 0 && im.num == imgs[i-1].num {
|
|
warnf("%s: images/%s and images/%s are both page %d - keep one", label, imgs[i-1].name, im.name, im.num)
|
|
}
|
|
out = append(out, im.name)
|
|
}
|
|
return out
|
|
}
|
|
|
|
type imageSize struct{ W, H int }
|
|
|
|
func (s imageSize) Attr() template.HTMLAttr {
|
|
if s.W <= 0 || s.H <= 0 {
|
|
return ""
|
|
}
|
|
return template.HTMLAttr(fmt.Sprintf(` width="%d" height="%d"`, s.W, s.H))
|
|
}
|
|
|
|
// readImageSizes reads every page image's pixel size from its header, so a
|
|
// page can reserve each image's height before it loads. It runs alongside the
|
|
// rest of the load; the returned func waits for it and reports the failures.
|
|
func (co *Corpus) readImageSizes() func() {
|
|
type job struct {
|
|
path string
|
|
size *imageSize
|
|
}
|
|
var jobs []job
|
|
add := func(dir string, names []string, sizes *[]imageSize) {
|
|
*sizes = make([]imageSize, len(names))
|
|
for i, name := range names {
|
|
if !strings.EqualFold(filepath.Ext(name), ".webp") {
|
|
jobs = append(jobs, job{filepath.Join(dir, "images", name), &(*sizes)[i]})
|
|
}
|
|
}
|
|
}
|
|
for i := range co.archive.Docs {
|
|
d := &co.archive.Docs[i]
|
|
add(d.Dir, d.Images, &d.ImageSizes)
|
|
for j := range d.Versions {
|
|
v := &d.Versions[j]
|
|
add(filepath.Join(d.Dir, "versions", v.Slug), v.Images, &v.ImageSizes)
|
|
}
|
|
}
|
|
|
|
ch := make(chan job)
|
|
var mu sync.Mutex
|
|
var wg sync.WaitGroup
|
|
var failed []string
|
|
for range 16 {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
for j := range ch {
|
|
f, err := os.Open(j.path)
|
|
if err == nil {
|
|
var c image.Config
|
|
c, _, err = image.DecodeConfig(f)
|
|
f.Close()
|
|
*j.size = imageSize{c.Width, c.Height}
|
|
}
|
|
if err != nil {
|
|
mu.Lock()
|
|
failed = append(failed, fmt.Sprintf("%s: %v", strings.TrimPrefix(j.path, dataDir+string(filepath.Separator)), err))
|
|
mu.Unlock()
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
go func() {
|
|
for _, j := range jobs {
|
|
ch <- j
|
|
}
|
|
close(ch)
|
|
}()
|
|
return func() {
|
|
wg.Wait()
|
|
sort.Strings(failed)
|
|
for _, f := range failed {
|
|
warnf("page image unreadable, served without its size: %s", f)
|
|
}
|
|
}
|
|
}
|
|
|
|
func versionSlugSafe(s string) bool {
|
|
if s == "" || strings.Contains(s, "..") || strings.Contains(s, "~") || !docIDSafe(s) {
|
|
return false
|
|
}
|
|
c := s[0]
|
|
return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9'
|
|
}
|
|
|
|
func (co *Corpus) loadVersions(d *Document) {
|
|
vdir := filepath.Join(d.Dir, "versions")
|
|
entries, err := os.ReadDir(vdir)
|
|
if err != nil && !os.IsNotExist(err) {
|
|
warnf("%s: versions directory unreadable: %v", d.ID, err)
|
|
}
|
|
for _, e := range entries {
|
|
name := e.Name()
|
|
if !e.IsDir() {
|
|
warnf("%s: versions/%s is not a folder - each version is a folder holding that copy's PDF, .md and images/", d.ID, name)
|
|
continue
|
|
}
|
|
if !versionSlugSafe(name) {
|
|
warnf("%s: versions/%s: version names may only contain letters, digits, \".\", \"_\" and \"-\", starting with a letter or digit - skipping", d.ID, name)
|
|
continue
|
|
}
|
|
v, err := loadVersion(filepath.Join(vdir, name), d.ID+"/versions/"+name)
|
|
if err != nil {
|
|
errorf("%s: versions/%s: %v - skipping version", d.ID, name, err)
|
|
continue
|
|
}
|
|
v.Slug = name
|
|
v.DocID = d.ID
|
|
d.Versions = append(d.Versions, v)
|
|
}
|
|
|
|
for i := range d.Publication {
|
|
p := &d.Publication[i]
|
|
p.Version = strings.TrimSpace(p.Version)
|
|
if p.Version == "" {
|
|
continue
|
|
}
|
|
v := d.VersionFor(p.Version)
|
|
if v == nil {
|
|
warnf("%s: publication %d (%s) points at versions/%s, which is missing or failed to load - its copy isn't linked", d.ID, i+1, p.Publisher, p.Version)
|
|
continue
|
|
}
|
|
v.PubIndex = append(v.PubIndex, i)
|
|
}
|
|
for i := range d.Versions {
|
|
v := &d.Versions[i]
|
|
v.Label = versionLabel(d, v)
|
|
}
|
|
|
|
for slug, info := range d.VersionMeta {
|
|
v := d.VersionFor(slug)
|
|
if v == nil {
|
|
warnf("%s: \"versions\" has an entry for versions/%s, which is missing or failed to load", d.ID, slug)
|
|
continue
|
|
}
|
|
v.ChangedPages = changedPages(d.ID, v, info.ChangedPages)
|
|
v.SHA = strings.ToLower(strings.TrimSpace(info.PDFSha256))
|
|
}
|
|
|
|
for _, v := range d.Versions {
|
|
if v.SHA == "" {
|
|
warnf("%s: versions/%s has no pdf_sha256 in metadata.json - its checksum isn't listed", d.ID, v.Slug)
|
|
}
|
|
}
|
|
}
|
|
|
|
func changedPages(docID string, v *DocVersion, pages []int) []ChangedPage {
|
|
pos := make(map[int]int, len(v.Images))
|
|
for i, name := range v.Images {
|
|
if n, err := pageNumber(name); err == nil {
|
|
if _, seen := pos[n]; !seen {
|
|
pos[n] = i + 1
|
|
}
|
|
}
|
|
}
|
|
nums := append([]int(nil), pages...)
|
|
sort.Ints(nums)
|
|
var out []ChangedPage
|
|
for i, n := range nums {
|
|
if i > 0 && n == nums[i-1] {
|
|
continue
|
|
}
|
|
if n < 1 {
|
|
warnf("%s: versions/%s: changed page %d is not a page number - ignored", docID, v.Slug, n)
|
|
continue
|
|
}
|
|
p := pos[n]
|
|
if p == 0 {
|
|
warnf("%s: versions/%s: changed page %d has no page image with that number - listed without a link", docID, v.Slug, n)
|
|
}
|
|
out = append(out, ChangedPage{Page: n, Pos: p})
|
|
}
|
|
return out
|
|
}
|
|
|
|
type contentFiles struct {
|
|
Text string
|
|
TextFile string
|
|
PDFFile string
|
|
PDFSize int64
|
|
HasThumb bool
|
|
HasGifThumb bool
|
|
}
|
|
|
|
func scanContentDir(dir string) (contentFiles, error) {
|
|
var c contentFiles
|
|
entries, err := os.ReadDir(dir)
|
|
if err != nil {
|
|
return c, err
|
|
}
|
|
textDone := false
|
|
for _, f := range entries {
|
|
if f.IsDir() {
|
|
continue
|
|
}
|
|
name := f.Name()
|
|
if !textDone && isTranscript(name) {
|
|
textDone = true
|
|
c.TextFile = name
|
|
b, rerr := os.ReadFile(filepath.Join(dir, name))
|
|
if rerr != nil {
|
|
warnf("cannot read %s: %v", filepath.Join(dir, name), rerr)
|
|
} else {
|
|
c.Text = string(b)
|
|
}
|
|
}
|
|
if c.PDFFile == "" && isPDF(name) {
|
|
c.PDFFile = name
|
|
if info, ierr := f.Info(); ierr == nil {
|
|
c.PDFSize = info.Size()
|
|
}
|
|
}
|
|
if name == "thumb.jpg" {
|
|
c.HasThumb = true
|
|
}
|
|
if name == "thumb.gif" {
|
|
c.HasGifThumb = true
|
|
}
|
|
}
|
|
return c, nil
|
|
}
|
|
|
|
func loadVersion(dir, label string) (DocVersion, error) {
|
|
var v DocVersion
|
|
files, err := scanContentDir(dir)
|
|
if err != nil {
|
|
return v, err
|
|
}
|
|
if files.PDFFile == "" {
|
|
return v, fmt.Errorf("no PDF")
|
|
}
|
|
v.PDFFile, v.PDFSize, v.TextFile, v.FullText = files.PDFFile, files.PDFSize, files.TextFile, files.Text
|
|
if v.FullText != "" {
|
|
v.HTMLContent = template.HTML(wrapRedactions(Render(v.FullText)))
|
|
}
|
|
v.Images = loadPageImages(dir, label)
|
|
return v, nil
|
|
}
|
|
|
|
// VersionSnippet is a search-result snippet taken from one of the document's
|
|
// versions, with the copy it came from.
|
|
type VersionSnippet struct {
|
|
Slug string
|
|
Label string
|
|
HTML template.HTML
|
|
}
|
|
|
|
func versionLabel(d *Document, v *DocVersion) string {
|
|
var names []string
|
|
seen := make(map[string]bool)
|
|
date := ""
|
|
for _, i := range v.PubIndex {
|
|
p := d.Publication[i]
|
|
if key := valueKey(p.Publisher); key != "" && !seen[key] {
|
|
seen[key] = true
|
|
names = append(names, p.Publisher)
|
|
}
|
|
if p.DatePublished != "" && (date == "" || p.DatePublished < date) {
|
|
date = p.DatePublished
|
|
}
|
|
}
|
|
if len(names) == 0 {
|
|
return v.Slug
|
|
}
|
|
label := strings.Join(names, " / ")
|
|
if date != "" {
|
|
label += ", " + date
|
|
}
|
|
return label
|
|
}
|
|
|
|
// lineKey is the form transcript lines are compared in: folded, lowercased,
|
|
// whitespace collapsed. Lines without a letter or digit ("---", "|") key to
|
|
// "" and are never new.
|
|
func lineKey(line string) string {
|
|
k := strings.Join(strings.Fields(normFoldString(line)), " ")
|
|
if strings.IndexFunc(k, isWordRune) < 0 {
|
|
return ""
|
|
}
|
|
return k
|
|
}
|
|
|
|
func versionNovelText(d *Document) string {
|
|
if len(d.Versions) == 0 {
|
|
return ""
|
|
}
|
|
inMain := make(map[string]bool)
|
|
for _, line := range strings.Split(d.FullText, "\n") {
|
|
if k := lineKey(line); k != "" {
|
|
inMain[k] = true
|
|
}
|
|
}
|
|
added := make(map[string]bool)
|
|
var all []string
|
|
for i := range d.Versions {
|
|
v := &d.Versions[i]
|
|
var novel []string
|
|
for _, line := range strings.Split(v.FullText, "\n") {
|
|
k := lineKey(line)
|
|
if k == "" || inMain[k] {
|
|
continue
|
|
}
|
|
line = strings.TrimSpace(line)
|
|
novel = append(novel, line)
|
|
if !added[k] {
|
|
added[k] = true
|
|
all = append(all, line)
|
|
}
|
|
}
|
|
v.NovelText = strings.Join(novel, "\n")
|
|
}
|
|
return strings.Join(all, "\n")
|
|
}
|
|
|
|
func setBodySearchText(d *Document) {
|
|
d.StemmedFull = strings.Join(tokenize(d.FullText), " ")
|
|
d.FoldedFull = normFoldString(d.FullText)
|
|
d.bodyFoldLen = len(d.FoldedFull)
|
|
if novel := versionNovelText(d); novel != "" {
|
|
d.StemmedFull = strings.TrimSpace(d.StemmedFull + " " + valueSep + " " + strings.Join(tokenize(novel), " "))
|
|
d.FoldedFull += valueSep + normFoldString(novel)
|
|
}
|
|
}
|