package main import ( "encoding/json" "fmt" "html/template" "image" _ "image/gif" _ "image/jpeg" _ "image/png" "log" "math" "os" "path/filepath" "slices" "sort" "strconv" "strings" "sync" ) // Publication is one entry of a document's publication list. type Publication struct { Title string `json:"title,omitempty"` Publisher string `json:"publisher"` Journalists []string `json:"reporters"` Contributors []string `json:"contributors"` DatePublished string `json:"date_published"` Link string `json:"link"` Note string `json:"note,omitempty"` Version string `json:"version,omitempty"` DocSource string `json:"doc_source,omitempty"` DocURL string `json:"doc_url,omitempty"` ReportingOnly bool `json:"reporting_only,omitempty"` } type Document struct { ID string `json:"id"` AkaID []AkaEntry `json:"aka_id,omitempty"` Title string `json:"title"` Type string `json:"type"` Collection string `json:"collection"` Description string `json:"description"` LongDescription string `json:"long_description"` Tags []string `json:"tags"` Pages int `json:"pages,omitempty"` PDFSha256 string `json:"pdf_sha256,omitempty"` ArchiveComment string `json:"archive_comment,omitempty"` VersionMeta map[string]VersionInfo `json:"versions,omitempty"` DocMeta struct { Creator string `json:"creator"` DateCreated string `json:"date_created"` Programs []string `json:"programs"` Classification []string `json:"classification"` Dissemination []string `json:"dissemination"` Banner string `json:"banner"` Genre string `json:"genre"` Series struct { Name string `json:"name"` Part int `json:"part"` } `json:"series"` Countries []string `json:"countries"` People []string `json:"people"` } `json:"document"` Publication []Publication `json:"publication"` Dir string `json:"-"` FullText string `json:"-"` SearchText string `json:"-"` StemmedText string `json:"-"` StemmedTitle string `json:"-"` StemmedDesc string `json:"-"` StemmedFull string `json:"-"` // OCR body only, for "Content" field searches FoldedFull string `json:"-"` // accent-folded + lowercased OCR body, for fast quoted "Content" matching bodyFoldLen int // how much of FoldedFull is the fold of FullText: the versions' lines follow it foldedTitle string // normFoldString(Title) foldedDesc string // normFoldString(Description + valueSep + LongDescription) // terms is the document's entry in the TF-IDF index: every stemmed token // it holds, in any field. fields is its metadata as the matchers and the // tallies read it (buildDocFields). Both are nil until loadData sets them. terms map[string]float64 fields *docFields Images []string `json:"-"` ImageSizes []imageSize `json:"-"` PDFFile string `json:"-"` PDFSize int64 `json:"-"` TextFile string `json:"-"` // the .md OCR/transcript's file name, "" without one Versions []DocVersion `json:"-"` // differing published copies from versions// (loadVersions) PortionLevels []string `json:"-"` PortionMarkings []string `json:"-"` PortionControls []string `json:"-"` HasThumb bool `json:"-"` HasGifThumb bool `json:"-"` HTMLContent template.HTML `json:"-"` } func (d *Document) foldedBody() string { if d.bodyFoldLen > len(d.FoldedFull) || (d.bodyFoldLen == 0 && d.FullText != "") { return normFoldString(d.FullText) // a document setBodySearchText never saw } return d.FoldedFull[:d.bodyFoldLen] } // ReportedIssue reports whether the document is a SIDtoday issue that was // individually reported, and so is in the Reported collection as well. func (d *Document) ReportedIssue() bool { return d.Type == "SIDTODAY" && d.Collection == collectionNews } func (d *Document) JournalistsDeduped() []string { return facetValues(d, "RE") } func (d *Document) Publishers() []string { return facetValues(d, "PU") } func journalistsOf(d *Document) []string { var all []string for _, p := range d.Publication { all = append(all, p.Journalists...) } return normalizeList(all) } func publishersOf(d *Document) []string { var all []string for _, p := range d.Publication { if !p.ReportingOnly { all = append(all, p.Publisher) } } return normalizeList(all) } func (d *Document) titleFold() string { if d.foldedTitle == "" && d.Title != "" { return normFoldString(d.Title) } return d.foldedTitle } func (d *Document) descFold() string { if d.foldedDesc == "" { return normFoldString(d.Description + valueSep + d.LongDescription) } return d.foldedDesc } // hasTerms reports whether the document holds every one of the stemmed // tokens, in any field. func (d *Document) hasTerms(tokens []string) bool { if d.terms == nil { return true } for _, t := range tokens { if _, ok := d.terms[t]; !ok { return false } } return true } // AkaEntry is another entry this document was published under. type AkaEntry struct { ID string `json:"id"` // usually a alternative PDF filename DocSource string `json:"doc_source,omitempty"` DocURL string `json:"doc_url,omitempty"` } type DocVersion struct { Slug string PDFFile string PDFSize int64 TextFile string FullText string HTMLContent template.HTML Images []string ImageSizes []imageSize PubIndex []int SHA string NovelText string Label string DocID string ChangedPages []ChangedPage } type VersionInfo struct { ChangedPages []int `json:"changed_pages,omitempty"` PDFSha256 string `json:"pdf_sha256,omitempty"` } type ChangedPage struct { Page int Pos int } func (v *DocVersion) PageChanged(pos int) bool { if pos < 1 { return false } for _, c := range v.ChangedPages { if c.Pos == pos { return true } } return false } func (d *Document) VersionFor(slug string) *DocVersion { if slug == "" { return nil } for i := range d.Versions { if d.Versions[i].Slug == slug { return &d.Versions[i] } } return nil } // servedFile reports whether rel, a slash-separated path inside the // document's directory, is one of the files the loader indexed: the // metadata.json, the PDF, the transcript, the thumbnails and the page images, // of the document or of one of its versions. /files/ serves these and nothing else a directory // has. func (d *Document) servedFile(rel string) bool { if rest, ok := strings.CutPrefix(rel, "versions/"); ok { slug, name, _ := strings.Cut(rest, "/") v := d.VersionFor(slug) return v != nil && contentFile(name, v.PDFFile, v.TextFile, v.Images) } switch rel { case "metadata.json": return true case "thumb.jpg": return d.HasThumb case "thumb.gif": return d.HasGifThumb } return contentFile(rel, d.PDFFile, d.TextFile, d.Images) } func contentFile(name, pdf, text string, images []string) bool { if name == "" { return false } if name == pdf || name == text { return true } image, ok := strings.CutPrefix(name, "images/") return ok && slices.Contains(images, image) } func (d *Document) UnlinkedVersions() []DocVersion { var out []DocVersion for _, v := range d.Versions { if len(v.PubIndex) == 0 { out = append(out, v) } } return out } type collCounts struct { News int `json:"reported"` Archive int `json:"sidtoday_archive"` FOIA int `json:"foia"` } func (c *collCounts) add(coll string) { switch coll { case collectionNews: c.News++ case collectionArchive: c.Archive++ case collectionFOIA: c.FOIA++ } } func (c *collCounts) addType(typ string) { switch typ { case "SIDTODAY": c.Archive++ case "FOIA": c.FOIA++ default: c.News++ } } var genreVocabList = []string{"slides", "report", "wiki", "manual", "guide", "newsletter", "other"} var genreVocab = func() map[string]bool { m := make(map[string]bool, len(genreVocabList)) for _, g := range genreVocabList { m[g] = true } return m }() type Archive struct { Docs []Document Types map[string]int NewsTypes map[string]int CollectionCounts map[string]int Facets map[string]map[string]int FacetsByColl map[string]map[string]*collCounts SIDTodayCount int RelatedDocs map[string][]string } const ( collectionNews = "news" collectionArchive = "sidtoday-archive" collectionFOIA = "foia" ) // dataDir is the corpus, the data/ directory of the archive root (-root). var dataDir = "data" // projectDir, the root's project/ directory, holds the files that describe // the archive as a whole and are the same on every mirror: glossary.json, // assemblies.json, changelog.json and related.json. var projectDir = "project" func setRoot(root string) { dataDir, projectDir = filepath.Join(root, "data"), filepath.Join(root, "project") } var loadWarnings, loadErrors int func warnf(format string, args ...interface{}) { loadWarnings++ log.Printf("Warning: "+format, args...) } func errorf(format string, args ...interface{}) { loadErrors++ log.Printf("Error: "+format, args...) } func projectPath(name string) string { return filepath.Join(projectDir, name) } func relatedDocsPath() string { return projectPath("related.json") } const defaultDocType = "REPORTED" var pageImageExts = map[string]bool{ ".jpg": true, ".jpeg": true, ".png": true, ".gif": true, ".webp": true, } func (co *Corpus) buildSerialNav() { co.serialPrev = map[string]string{} co.serialNext = map[string]string{} var prev string for _, yr := range co.archiveTimeline { for _, mo := range yr.Months { for _, d := range mo.Docs { if prev != "" { co.serialPrev[d.ID] = prev co.serialNext[prev] = d.ID } prev = d.ID } } } } var collectionDirs = map[string]string{ "foia": collectionFOIA, "sidtoday-archive": collectionArchive, } type docLocation struct { id string dir string collection string } func docLocations(dataDir string) ([]docLocation, error) { entries, err := os.ReadDir(dataDir) if err != nil { return nil, err } var out []docLocation for _, e := range entries { if !e.IsDir() { continue } if coll, ok := collectionDirs[strings.ToLower(e.Name())]; ok { subs, err := os.ReadDir(filepath.Join(dataDir, e.Name())) if err != nil { warnf("cannot read %s: %v", filepath.Join(dataDir, e.Name()), err) continue } for _, s := range subs { if s.IsDir() { out = append(out, docLocation{ id: s.Name(), dir: filepath.Join(dataDir, e.Name(), s.Name()), collection: coll, }) } } continue } out = append(out, docLocation{ id: e.Name(), dir: filepath.Join(dataDir, e.Name()), }) } return out, nil } func dirHasPDF(dir string) bool { entries, err := os.ReadDir(dir) if err != nil { return false } for _, e := range entries { if !e.IsDir() && isPDF(e.Name()) { return true } } return false } func isPDF(name string) bool { return strings.EqualFold(filepath.Ext(name), ".pdf") } func isTranscript(name string) bool { return strings.EqualFold(filepath.Ext(name), ".md") } func firstN(vals []string, n int) []string { if len(vals) <= n { return vals } return append(append([]string{}, vals[:n]...), fmt.Sprintf("and %d more", len(vals)-n)) } func plural(n int, one, many string) string { if n == 1 { return one } return many } func normalizeList(vals []string) []string { if len(vals) == 0 { return vals } out := make([]string, 0, len(vals)) seen := make(map[string]bool) for _, v := range vals { v = strings.TrimSpace(v) if v == "" { continue } k := valueKey(v) if seen[k] { continue } seen[k] = true out = append(out, v) } return out } type seriesGroup struct { Name string Docs []*Document } func (co *Corpus) buildSeriesIndex() { co.seriesIndex = make(map[string]*seriesGroup) for i := range co.archive.Docs { dd := &co.archive.Docs[i] name := dd.DocMeta.Series.Name if name == "" { continue } if dd.DocMeta.Series.Part <= 0 { warnf("%s: series %q without a part number - order matters; sorting it last", dd.ID, name) } key := valueKey(name) g := co.seriesIndex[key] if g == nil { g = &seriesGroup{Name: name} co.seriesIndex[key] = g } else if g.Name != name { warnf("%s: series name casing %q differs from first-seen %q - merged, but copy-paste the exact name", dd.ID, name, g.Name) } g.Docs = append(g.Docs, dd) } for _, g := range co.seriesIndex { sort.SliceStable(g.Docs, func(i, j int) bool { a, b := g.Docs[i].DocMeta.Series.Part, g.Docs[j].DocMeta.Series.Part if (a <= 0) != (b <= 0) { return b <= 0 // numbered parts before unnumbered ones } if a != b { return a < b } return g.Docs[i].Title < g.Docs[j].Title }) seen := map[int]string{} for _, m := range g.Docs { p := m.DocMeta.Series.Part if p <= 0 { continue } if prev, dup := seen[p]; dup { warnf("series %q: part %d claimed by both %s and %s", g.Name, p, prev, m.ID) } seen[p] = m.ID } } } func wrapRedactions(html string) string { return strings.ReplaceAll(html, "[REDACTED]", `[REDACTED]`) } // load reads the corpus under dataDir into co and builds every index the // handlers serve from. Written here once, before the corpus goes live // (loadData), and read-only from then on. func (co *Corpus) load() { loadWarnings, loadErrors = 0, 0 co.loadGlossary() co.archive = Archive{ Docs: []Document{}, Types: make(map[string]int), NewsTypes: make(map[string]int), CollectionCounts: make(map[string]int), RelatedDocs: make(map[string][]string), } locations, err := docLocations(dataDir) if err != nil { log.Fatalf("Critical: Could not read data directory: %v", err) } seen := make(map[string]string) var skipped, skippedWithPDF []string for _, loc := range locations { metaPath := filepath.Join(loc.dir, "metadata.json") b, err := os.ReadFile(metaPath) if err != nil { skipped = append(skipped, loc.id) if dirHasPDF(loc.dir) { skippedWithPDF = append(skippedWithPDF, loc.id) } continue } var d Document if err := json.Unmarshal(b, &d); err != nil { errorf("invalid JSON in %s: %v - skipping document", metaPath, err) continue } d.Type = strings.ToUpper(strings.TrimSpace(d.Type)) if d.Type == "" { switch loc.collection { case collectionArchive: d.Type = "SIDTODAY" case collectionFOIA: d.Type = "FOIA" default: d.Type = defaultDocType } } d.ID = loc.id d.Dir = loc.dir if prev, dup := seen[d.ID]; dup { errorf("duplicate document id %q in %s (already loaded from %s) - skipping", d.ID, loc.dir, prev) continue } if !docIDSafe(d.ID) { warnf("document id %q contains characters outside [A-Za-z0-9._~-] - links, sitemap entries or pin/history cookies may misbehave; consider renaming the directory", d.ID) } d.Tags = normalizeList(d.Tags) d.AkaID = normalizeAka(&d) files, derr := scanContentDir(d.Dir) if derr != nil { errorf("cannot read %s: %v - skipping document", d.Dir, derr) continue } if files.PDFFile == "" { errorf("%s: no PDF in %s - skipping document", d.ID, d.Dir) continue } seen[d.ID] = loc.dir d.PDFFile, d.PDFSize, d.TextFile = files.PDFFile, files.PDFSize, files.TextFile d.HasThumb, d.HasGifThumb = files.HasThumb, files.HasGifThumb if files.Text != "" { d.FullText = files.Text d.HTMLContent = template.HTML(wrapRedactions(Render(d.FullText))) } d.Collection = strings.ToLower(strings.TrimSpace(d.Collection)) if loc.collection != "" { if d.Collection != "" && d.Collection != loc.collection { warnf("%s declares collection %q but lives in the %s directory - using %q", d.ID, d.Collection, loc.collection, loc.collection) } d.Collection = loc.collection } else { switch d.Collection { case collectionNews, collectionArchive, collectionFOIA: // explicit, keep case "": if d.Type == "FOIA" { d.Collection = collectionFOIA } else { d.Collection = collectionNews } default: warnf("%s has unknown collection %q - treating as %q", d.ID, d.Collection, collectionNews) d.Collection = collectionNews } } if d.Collection == collectionArchive && d.Type != "SIDTODAY" { warnf("%s is in the SIDtoday archive but typed %q - the SIDtoday tab lists the type SIDTODAY, so no browse tab shows it", d.ID, d.Type) } co.archive.CollectionCounts[d.Collection]++ if d.Type == "SIDTODAY" { co.archive.SIDTodayCount++ } newsDoc := d.Collection == collectionNews co.archive.Types[d.Type]++ if newsDoc { co.archive.NewsTypes[d.Type]++ } d.Images = loadPageImages(d.Dir, d.ID) if d.Pages <= 0 { d.Pages = len(d.Images) } d.DocMeta.Programs = normalizeList(d.DocMeta.Programs) d.DocMeta.Countries = normalizeList(d.DocMeta.Countries) d.DocMeta.People = normalizeList(d.DocMeta.People) d.DocMeta.Classification = normalizeList(d.DocMeta.Classification) for _, c := range d.DocMeta.Classification { if full, ok := classAlias[strings.ToLower(c)]; ok { warnf("%s: classification %q is an abbreviation - write %q; searches and counts treat the two as different values", d.ID, c, strings.ToUpper(full)) } } d.DocMeta.Dissemination = normalizeList(d.DocMeta.Dissemination) for i := range d.Publication { p := &d.Publication[i] p.Publisher = strings.TrimSpace(p.Publisher) p.Journalists = normalizeList(p.Journalists) p.Contributors = normalizeList(p.Contributors) p.Note = strings.TrimSpace(p.Note) p.DocSource = strings.TrimSpace(p.DocSource) p.DocURL = strings.TrimSpace(p.DocURL) } if v := d.DocMeta.DateCreated; v != "" && !validDate(v) { warnf("%s: date_created %q is not YYYY, YYYY-MM or YYYY-MM-DD - treating the document as undated", d.ID, v) d.DocMeta.DateCreated = "" } for i := range d.Publication { if v := d.Publication[i].DatePublished; v != "" && !validDate(v) { warnf("%s: publication %d date_published %q is not YYYY, YYYY-MM or YYYY-MM-DD - treating it as undated", d.ID, i+1, v) d.Publication[i].DatePublished = "" } } co.loadVersions(&d) d.DocMeta.Creator = strings.TrimSpace(d.DocMeta.Creator) d.DocMeta.Series.Name = strings.TrimSpace(d.DocMeta.Series.Name) d.DocMeta.Banner = strings.TrimSpace(d.DocMeta.Banner) d.DocMeta.Genre = strings.ToLower(strings.TrimSpace(d.DocMeta.Genre)) if d.DocMeta.Genre != "" && !genreVocab[d.DocMeta.Genre] { warnf("%s: genre %q is outside the vocabulary (%s) - serving as-is", d.ID, d.DocMeta.Genre, strings.Join(genreVocabList, ", ")) } for _, f := range searchFields { if f.Values == nil { continue } for _, v := range f.Values(&d) { if strings.Contains(v, `"`) { warnf("%s: %s %q contains a double quote - its search links cannot quote it; use ' or “ ” instead", d.ID, strings.ToLower(f.Label), v) } } } // The all-fields forms are the title, the descriptions and the rest // of the metadata, then the body: each part is tokenized once, and // tokenize of the parts joined is the parts' tokens in a row. meta := searchMeta(&d) metaText := strings.Join([]string{d.Title, d.Description, d.LongDescription, meta}, valueSep) titleToks := tokenize(d.Title) descToks := tokenize(d.Description + " " + d.LongDescription) metaToks := tokenize(meta) stems := make([]string, 0, len(titleToks)+len(descToks)+len(metaToks)) stems = append(append(append(stems, titleToks...), descToks...), metaToks...) d.StemmedTitle = strings.Join(titleToks, " ") d.StemmedDesc = strings.Join(descToks, " ") d.foldedTitle = normFoldString(d.Title) d.foldedDesc = normFoldString(d.Description + valueSep + d.LongDescription) setBodySearchText(&d) d.StemmedText = strings.TrimSpace(strings.Join(stems, " ") + " " + d.StemmedFull) d.SearchText = normFoldString(metaText) + valueSep + d.FoldedFull d.FoldedFull = d.SearchText[len(d.SearchText)-len(d.FoldedFull):] d.StemmedFull = d.StemmedText[len(d.StemmedText)-len(d.StemmedFull):] co.archive.Docs = append(co.archive.Docs, d) } if len(skipped) > 0 { log.Printf("Note: %d directories have no metadata.json and were skipped: %s", len(skipped), strings.Join(firstN(skipped, 8), ", ")) if len(skippedWithPDF) > 0 { warnf("%d skipped %s a PDF but no metadata.json - a document nobody described: %s", len(skippedWithPDF), plural(len(skippedWithPDF), "directory holds", "directories hold"), strings.Join(firstN(skippedWithPDF, 8), ", ")) } } sizesRead := co.readImageSizes() log.Printf("Building TF-IDF index...") co.tfidf = buildTFIDF(co.archive.Docs) log.Printf("TF-IDF done") sizesRead() co.docIndex = make(map[string]*Document, len(co.archive.Docs)) for i := range co.archive.Docs { d := &co.archive.Docs[i] co.docIndex[d.ID] = d d.terms = co.tfidf.DocStats[d.ID] } co.buildSeriesIndex() co.loadAssemblies() log.Printf("Loading related docs index...") co.archive.RelatedDocs = co.loadRelatedDocs(co.archive.Docs) log.Printf("Related docs done") co.buildArticles() co.loadChangelog() co.buildCollectionDocs() co.buildCategoryDocs() co.buildHomeStats() co.buildCodewords() co.buildCategoryLists() co.buildExploreTables() co.buildIDRows() co.buildClassPatterns() co.buildPortions() co.buildDocFields() co.buildOutletTables() co.buildArchiveTimeline() co.buildSerialNav() co.buildFacetsJSON() co.buildSitemapPaths() co.buildCanonicalSitemap() log.Printf("Loaded: %d documents (%d error%s, %d warning%s)", len(co.archive.Docs), loadErrors, plural(loadErrors, "", "s"), loadWarnings, plural(loadWarnings, "", "s")) } // loadRelatedDocs reads related.json if it exists - run with --rebuild-related // to regenerate the file. func (co *Corpus) loadRelatedDocs(docs []Document) map[string][]string { b, err := os.ReadFile(relatedDocsPath()) if err != nil { warnf("related.json not found - related docs unavailable. Run with --rebuild-related to generate it.") return make(map[string][]string) } var related map[string][]string if err := json.Unmarshal(b, &related); err != nil { warnf("related.json is malformed: %v - related docs unavailable.", err) return make(map[string][]string) } staleKeys, staleLinks, staleLists := 0, 0, 0 for id, ids := range related { if co.docIndex[id] == nil { delete(related, id) staleKeys++ continue } kept := slices.DeleteFunc(ids, co.unknownDoc) if n := len(ids) - len(kept); n > 0 { staleLinks += n staleLists++ } related[id] = kept } if staleKeys > 0 || staleLinks > 0 { warnf("related.json: %d entries for documents no longer in the corpus, and %d links to such documents in %d lists - ignored; run --rebuild-related to recompute those lists", staleKeys, staleLinks, staleLists) } missing := 0 for _, d := range docs { if _, ok := related[d.ID]; !ok { missing++ } } if missing > 0 { warnf("%d/%d documents missing from related.json - run --rebuild-related to update.", missing, len(docs)) } else { log.Printf("Related docs loaded: %d entries", len(related)) } return related } func (co *Corpus) unknownDoc(id string) bool { return co.docIndex[id] == nil } func saveRelatedDocs(related map[string][]string) error { b, err := json.MarshalIndent(related, "", " ") if err != nil { return err } return os.WriteFile(relatedDocsPath(), b, 0644) } func (co *Corpus) buildRelatedDocs(targets, all []Document) map[string][]string { type scored struct { id string score float64 collection string } programDocs := make(map[string][]int) tagDocs := make(map[string][]int) titleDocs := make(map[string][]int) for i, d := range all { for _, p := range d.DocMeta.Programs { key := strings.ToLower(p) programDocs[key] = append(programDocs[key], i) } for _, t := range d.Tags { key := strings.ToLower(t) tagDocs[key] = append(tagDocs[key], i) } for _, w := range uniqueStrings(tokenize(d.Title)) { titleDocs[w] = append(titleDocs[w], i) } } tw := titleWeights{df: make(map[string]int, len(titleDocs)), n: len(all)} for w, docs := range titleDocs { tw.df[w] = len(docs) } maxTitleDF := len(all) / 10 for w, docs := range titleDocs { if len(docs) > maxTitleDF { delete(titleDocs, w) } } result := make(map[string][]string, len(targets)) for i := range targets { d := &targets[i] seen := make(map[int]bool) for _, p := range d.DocMeta.Programs { for _, j := range programDocs[strings.ToLower(p)] { seen[j] = true } } for _, t := range d.Tags { for _, j := range tagDocs[strings.ToLower(t)] { seen[j] = true } } for _, w := range tokenize(d.Title) { for _, j := range titleDocs[w] { seen[j] = true } } var candidates []scored for j := range seen { if all[j].ID == d.ID { continue } s := co.relatedScore(d, &all[j], tw) if s >= minRelatedScore { candidates = append(candidates, scored{all[j].ID, s, all[j].Collection}) } } sort.Slice(candidates, func(a, b int) bool { if candidates[a].score != candidates[b].score { return candidates[a].score > candidates[b].score } return candidates[a].id < candidates[b].id }) const topN = 5 maxBulk := topN if d.Collection == collectionNews { maxBulk = 2 } var ids []string var skipped []string bulk := 0 for _, c := range candidates { if len(ids) == topN { break } if c.collection != collectionNews && bulk >= maxBulk { skipped = append(skipped, c.id) continue } if c.collection != collectionNews { bulk++ } ids = append(ids, c.id) } for _, id := range skipped { if len(ids) == topN { break } ids = append(ids, id) } result[d.ID] = ids } log.Printf("Related docs built: %d entries", len(result)) return result } const minRelatedScore = 1.5 type titleWeights struct { df map[string]int n int } func (t titleWeights) weight(w string) float64 { if t.n < 2 { return 1 } return math.Log(float64(t.n+1)/float64(t.df[w]+1)) / math.Log(float64(t.n+1)) } func uniqueStrings(in []string) []string { seen := make(map[string]bool, len(in)) out := in[:0:0] for _, s := range in { if key := strings.ToLower(s); !seen[key] { seen[key] = true out = append(out, s) } } return out } func sharedCount(a, b []string) int { inA := make(map[string]bool, len(a)) for _, s := range a { inA[strings.ToLower(s)] = true } n := 0 for _, s := range uniqueStrings(b) { if inA[strings.ToLower(s)] { n++ } } return n } func (co *Corpus) relatedScore(a, b *Document, tw titleWeights) float64 { var score float64 aTitleWords := make(map[string]bool) for _, w := range tokenize(a.Title) { aTitleWords[w] = true } for _, w := range uniqueStrings(tokenize(b.Title)) { if aTitleWords[w] { score += 3.0 * tw.weight(w) } } score += 4.0 * float64(sharedCount(a.DocMeta.Programs, b.DocMeta.Programs)) score += 2.0 * float64(sharedCount(a.Tags, b.Tags)) var ajournos, bjournos []string for _, p := range a.Publication { ajournos = append(ajournos, p.Journalists...) } for _, p := range b.Publication { bjournos = append(bjournos, p.Journalists...) } score += 2.0 * float64(sharedCount(ajournos, bjournos)) if strings.EqualFold(a.DocMeta.Creator, b.DocMeta.Creator) && a.DocMeta.Creator != "" { score += 1.0 } score += 0.5 * float64(sharedCount(a.DocMeta.Countries, b.DocMeta.Countries)) score += co.tfidf.similarity(a.ID, b.ID) * 2.0 return score } func pageNumber(s string) (int, error) { parts := strings.SplitN(s, "-", 2) if len(parts) < 2 { return 0, fmt.Errorf("unexpected image filename format: %q", s) } numPart := strings.SplitN(parts[1], ".", 2)[0] n, err := strconv.Atoi(numPart) if err != nil { return 0, fmt.Errorf("could not parse page number from %q: %v", s, err) } return n, nil } func parseImageNum(s string) int { n, err := pageNumber(s) if err != nil { warnf("%v", err) return 0 } return n } func docIDSafe(id string) bool { for i := 0; i < len(id); i++ { c := id[i] switch { case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9', c == '-', c == '.', c == '_', c == '~': default: return false } } return len(id) > 0 } func normalizeAka(d *Document) []AkaEntry { var out []AkaEntry seen := make(map[string]bool) for i, a := range d.AkaID { a.ID = strings.TrimSpace(a.ID) a.DocSource = strings.TrimSpace(a.DocSource) a.DocURL = strings.TrimSpace(a.DocURL) if a.ID == "" { warnf("%s: aka_id[%d] has no id - ignored", d.ID, i) continue } key := strings.ToLower(a.ID) if seen[key] { warnf("%s: aka_id %q is listed twice - keeping the first", d.ID, a.ID) continue } seen[key] = true out = append(out, a) } return out } func searchMeta(d *Document) string { var reporters, publishers, pubTitles, pubNotes, contributors []string for _, p := range d.Publication { if p.Title != "" { pubTitles = append(pubTitles, p.Title) } if p.Note != "" { pubNotes = append(pubNotes, p.Note) } if p.Publisher != "" { publishers = append(publishers, p.Publisher) } contributors = append(contributors, p.Contributors...) reporters = append(reporters, p.Journalists...) } return strings.Join([]string{ akaIDs(d.AkaID), d.DocMeta.Creator, strings.Join(reporters, valueSep), strings.Join(contributors, valueSep), strings.Join(publishers, valueSep), strings.Join(pubTitles, valueSep), strings.Join(pubNotes, valueSep), d.ArchiveComment, strings.Join(d.DocMeta.People, valueSep), strings.Join(d.DocMeta.Countries, valueSep), strings.Join(d.DocMeta.Programs, valueSep), strings.Join(d.DocMeta.Classification, valueSep), strings.Join(d.DocMeta.Dissemination, valueSep), d.DocMeta.Banner, d.DocMeta.Genre, d.DocMeta.Series.Name, d.ID, strings.Join(d.Tags, valueSep), }, valueSep) } func akaIDs(akas []AkaEntry) string { ids := make([]string, len(akas)) for i, a := range akas { ids[i] = a.ID } return strings.Join(ids, valueSep) } func loadPageImages(dir, label string) []string { files, err := os.ReadDir(filepath.Join(dir, "images")) if err != nil && !os.IsNotExist(err) { warnf("%s: images directory unreadable: %v", label, err) } type pageImage struct { name string num int } var imgs []pageImage for _, f := range files { if !f.IsDir() && pageImageExts[strings.ToLower(filepath.Ext(f.Name()))] { imgs = append(imgs, pageImage{f.Name(), parseImageNum(f.Name())}) } } sort.SliceStable(imgs, func(i, j int) bool { return imgs[i].num < imgs[j].num }) var out []string for i, im := range imgs { if i > 0 && im.num > 0 && im.num == imgs[i-1].num { warnf("%s: images/%s and images/%s are both page %d - keep one", label, imgs[i-1].name, im.name, im.num) } out = append(out, im.name) } return out } type imageSize struct{ W, H int } func (s imageSize) Attr() template.HTMLAttr { if s.W <= 0 || s.H <= 0 { return "" } return template.HTMLAttr(fmt.Sprintf(` width="%d" height="%d"`, s.W, s.H)) } // readImageSizes reads every page image's pixel size from its header, so a // page can reserve each image's height before it loads. It runs alongside the // rest of the load; the returned func waits for it and reports the failures. func (co *Corpus) readImageSizes() func() { type job struct { path string size *imageSize } var jobs []job add := func(dir string, names []string, sizes *[]imageSize) { *sizes = make([]imageSize, len(names)) for i, name := range names { if !strings.EqualFold(filepath.Ext(name), ".webp") { jobs = append(jobs, job{filepath.Join(dir, "images", name), &(*sizes)[i]}) } } } for i := range co.archive.Docs { d := &co.archive.Docs[i] add(d.Dir, d.Images, &d.ImageSizes) for j := range d.Versions { v := &d.Versions[j] add(filepath.Join(d.Dir, "versions", v.Slug), v.Images, &v.ImageSizes) } } ch := make(chan job) var mu sync.Mutex var wg sync.WaitGroup var failed []string for range 16 { wg.Add(1) go func() { defer wg.Done() for j := range ch { f, err := os.Open(j.path) if err == nil { var c image.Config c, _, err = image.DecodeConfig(f) f.Close() *j.size = imageSize{c.Width, c.Height} } if err != nil { mu.Lock() failed = append(failed, fmt.Sprintf("%s: %v", strings.TrimPrefix(j.path, dataDir+string(filepath.Separator)), err)) mu.Unlock() } } }() } go func() { for _, j := range jobs { ch <- j } close(ch) }() return func() { wg.Wait() sort.Strings(failed) for _, f := range failed { warnf("page image unreadable, served without its size: %s", f) } } } func versionSlugSafe(s string) bool { if s == "" || strings.Contains(s, "..") || strings.Contains(s, "~") || !docIDSafe(s) { return false } c := s[0] return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' } func (co *Corpus) loadVersions(d *Document) { vdir := filepath.Join(d.Dir, "versions") entries, err := os.ReadDir(vdir) if err != nil && !os.IsNotExist(err) { warnf("%s: versions directory unreadable: %v", d.ID, err) } for _, e := range entries { name := e.Name() if !e.IsDir() { warnf("%s: versions/%s is not a folder - each version is a folder holding that copy's PDF, .md and images/", d.ID, name) continue } if !versionSlugSafe(name) { warnf("%s: versions/%s: version names may only contain letters, digits, \".\", \"_\" and \"-\", starting with a letter or digit - skipping", d.ID, name) continue } v, err := loadVersion(filepath.Join(vdir, name), d.ID+"/versions/"+name) if err != nil { errorf("%s: versions/%s: %v - skipping version", d.ID, name, err) continue } v.Slug = name v.DocID = d.ID d.Versions = append(d.Versions, v) } for i := range d.Publication { p := &d.Publication[i] p.Version = strings.TrimSpace(p.Version) if p.Version == "" { continue } v := d.VersionFor(p.Version) if v == nil { warnf("%s: publication %d (%s) points at versions/%s, which is missing or failed to load - its copy isn't linked", d.ID, i+1, p.Publisher, p.Version) continue } v.PubIndex = append(v.PubIndex, i) } for i := range d.Versions { v := &d.Versions[i] v.Label = versionLabel(d, v) } for slug, info := range d.VersionMeta { v := d.VersionFor(slug) if v == nil { warnf("%s: \"versions\" has an entry for versions/%s, which is missing or failed to load", d.ID, slug) continue } v.ChangedPages = changedPages(d.ID, v, info.ChangedPages) v.SHA = strings.ToLower(strings.TrimSpace(info.PDFSha256)) } for _, v := range d.Versions { if v.SHA == "" { warnf("%s: versions/%s has no pdf_sha256 in metadata.json - its checksum isn't listed", d.ID, v.Slug) } } } func changedPages(docID string, v *DocVersion, pages []int) []ChangedPage { pos := make(map[int]int, len(v.Images)) for i, name := range v.Images { if n, err := pageNumber(name); err == nil { if _, seen := pos[n]; !seen { pos[n] = i + 1 } } } nums := append([]int(nil), pages...) sort.Ints(nums) var out []ChangedPage for i, n := range nums { if i > 0 && n == nums[i-1] { continue } if n < 1 { warnf("%s: versions/%s: changed page %d is not a page number - ignored", docID, v.Slug, n) continue } p := pos[n] if p == 0 { warnf("%s: versions/%s: changed page %d has no page image with that number - listed without a link", docID, v.Slug, n) } out = append(out, ChangedPage{Page: n, Pos: p}) } return out } type contentFiles struct { Text string TextFile string PDFFile string PDFSize int64 HasThumb bool HasGifThumb bool } func scanContentDir(dir string) (contentFiles, error) { var c contentFiles entries, err := os.ReadDir(dir) if err != nil { return c, err } textDone := false for _, f := range entries { if f.IsDir() { continue } name := f.Name() if !textDone && isTranscript(name) { textDone = true c.TextFile = name b, rerr := os.ReadFile(filepath.Join(dir, name)) if rerr != nil { warnf("cannot read %s: %v", filepath.Join(dir, name), rerr) } else { c.Text = string(b) } } if c.PDFFile == "" && isPDF(name) { c.PDFFile = name if info, ierr := f.Info(); ierr == nil { c.PDFSize = info.Size() } } if name == "thumb.jpg" { c.HasThumb = true } if name == "thumb.gif" { c.HasGifThumb = true } } return c, nil } func loadVersion(dir, label string) (DocVersion, error) { var v DocVersion files, err := scanContentDir(dir) if err != nil { return v, err } if files.PDFFile == "" { return v, fmt.Errorf("no PDF") } v.PDFFile, v.PDFSize, v.TextFile, v.FullText = files.PDFFile, files.PDFSize, files.TextFile, files.Text if v.FullText != "" { v.HTMLContent = template.HTML(wrapRedactions(Render(v.FullText))) } v.Images = loadPageImages(dir, label) return v, nil } // VersionSnippet is a search-result snippet taken from one of the document's // versions, with the copy it came from. type VersionSnippet struct { Slug string Label string HTML template.HTML } func versionLabel(d *Document, v *DocVersion) string { var names []string seen := make(map[string]bool) date := "" for _, i := range v.PubIndex { p := d.Publication[i] if key := valueKey(p.Publisher); key != "" && !seen[key] { seen[key] = true names = append(names, p.Publisher) } if p.DatePublished != "" && (date == "" || p.DatePublished < date) { date = p.DatePublished } } if len(names) == 0 { return v.Slug } label := strings.Join(names, " / ") if date != "" { label += ", " + date } return label } // lineKey is the form transcript lines are compared in: folded, lowercased, // whitespace collapsed. Lines without a letter or digit ("---", "|") key to // "" and are never new. func lineKey(line string) string { k := strings.Join(strings.Fields(normFoldString(line)), " ") if strings.IndexFunc(k, isWordRune) < 0 { return "" } return k } func versionNovelText(d *Document) string { if len(d.Versions) == 0 { return "" } inMain := make(map[string]bool) for _, line := range strings.Split(d.FullText, "\n") { if k := lineKey(line); k != "" { inMain[k] = true } } added := make(map[string]bool) var all []string for i := range d.Versions { v := &d.Versions[i] var novel []string for _, line := range strings.Split(v.FullText, "\n") { k := lineKey(line) if k == "" || inMain[k] { continue } line = strings.TrimSpace(line) novel = append(novel, line) if !added[k] { added[k] = true all = append(all, line) } } v.NovelText = strings.Join(novel, "\n") } return strings.Join(all, "\n") } func setBodySearchText(d *Document) { d.StemmedFull = strings.Join(tokenize(d.FullText), " ") d.FoldedFull = normFoldString(d.FullText) d.bodyFoldLen = len(d.FoldedFull) if novel := versionNovelText(d); novel != "" { d.StemmedFull = strings.TrimSpace(d.StemmedFull + " " + valueSep + " " + strings.Join(tokenize(novel), " ")) d.FoldedFull += valueSep + normFoldString(novel) } }