first commit

This commit is contained in:
libroot 2026-11-19 04:40:59 +02:00
commit 8c4dcae475
Signed by: libroot
GPG key ID: 2AC68C029EBE9ACE
89 changed files with 24623 additions and 0 deletions

211
articles.go Normal file
View file

@ -0,0 +1,211 @@
// articles.go - news articles as first-class objects.
//
// An article's identity is its publication LINK. Giving a publication entry
// a title is what promotes its article to a story.
package main
import (
"crypto/sha256"
"encoding/hex"
"net/http"
"net/url"
"slices"
"sort"
"strings"
)
type Article struct {
Key string // short stable hash of the link (the /article/{key} path)
Title string
Link string
Host string // display fallback when Title is empty
Publisher string
Date string // earliest date_published seen
Journalists []string // union across docs, first-seen order
DocIDs []string
Book bool // published by a book publisher, not a news outlet
}
// bookPublishers are the publication entries that are books, not news
// articles (Greenwald's No Place to Hide, Gellman's Dark Mirror, Poitras's
// Astro Noise).
var bookPublishers = map[string]bool{
"Metropolitan Books": true,
"Penguin Press": true,
"Yale University Press": true,
}
// articleKeyFor derives the stable URL key for a link
func articleKeyFor(link string) string {
h := sha256.Sum256([]byte(link))
return hex.EncodeToString(h[:])[:12]
}
func hostOf(link string) string {
u, err := url.Parse(link)
if err != nil || u.Host == "" {
return link
}
return strings.TrimPrefix(u.Host, "www.")
}
// buildArticles indexes every publication entry that has a link
func (co *Corpus) buildArticles() {
co.articleByLink = make(map[string]*Article)
co.articleIndex = make(map[string]*Article)
co.articles = nil
for _, d := range co.archive.Docs {
for _, p := range d.Publication {
if p.Link == "" {
continue
}
a := co.articleByLink[p.Link]
if a == nil {
a = &Article{
Key: articleKeyFor(p.Link),
Link: p.Link,
Host: hostOf(p.Link),
}
co.articleByLink[p.Link] = a
co.articleIndex[a.Key] = a
}
if a.Title == "" && p.Title != "" {
a.Title = p.Title
}
if a.Publisher == "" && p.Publisher != "" {
a.Publisher = p.Publisher
a.Book = bookPublishers[p.Publisher]
}
if p.DatePublished != "" && (a.Date == "" || p.DatePublished < a.Date) {
a.Date = p.DatePublished
}
for _, j := range p.Journalists {
if j == "" {
continue
}
if !slices.ContainsFunc(a.Journalists, func(x string) bool { return sameValue(x, j) }) {
a.Journalists = append(a.Journalists, j)
}
}
dup := false
for _, id := range a.DocIDs {
if id == d.ID {
dup = true
break
}
}
if !dup {
a.DocIDs = append(a.DocIDs, d.ID)
}
}
}
for _, a := range co.articleByLink {
co.articles = append(co.articles, *a)
}
// Newest first; undated articles sort last; ties by title then link so
// the order is deterministic.
sort.Slice(co.articles, func(i, j int) bool {
di, dj := co.articles[i].Date, co.articles[j].Date
if di == "" && dj != "" {
return false
}
if di != "" && dj == "" {
return true
}
if di != dj {
return di > dj
}
ti := strings.ToLower(displayTitle(co.articles[i]))
tj := strings.ToLower(displayTitle(co.articles[j]))
if ti != tj {
return ti < tj
}
return co.articles[i].Link < co.articles[j].Link
})
co.buildArticleGroups()
}
// inCollection reports whether one of the article's documents is in coll.
func (a *Article) inCollection(co *Corpus, coll string) bool {
for _, id := range a.DocIDs {
if d := co.docIndex[id]; d != nil && d.Collection == coll {
return true
}
}
return false
}
func displayTitle(a Article) string {
if a.Title != "" {
return a.Title
}
return a.Host
}
type articleYearGroup struct {
Year string
Articles []Article
}
func (co *Corpus) buildArticleGroups() {
co.articleGroups, co.articlesTitled, co.articlesBooks = nil, 0, 0
for _, a := range co.articles {
if a.Title == "" {
continue
}
co.articlesTitled++
if a.Book {
co.articlesBooks++
}
year := "Undated"
if len(a.Date) >= 4 {
year = a.Date[:4]
}
if len(co.articleGroups) == 0 || co.articleGroups[len(co.articleGroups)-1].Year != year {
co.articleGroups = append(co.articleGroups, articleYearGroup{Year: year})
}
co.articleGroups[len(co.articleGroups)-1].Articles = append(co.articleGroups[len(co.articleGroups)-1].Articles, a)
}
}
func (co *Corpus) handleArticles(w http.ResponseWriter, r *http.Request) {
render(w, r, &articlesPage{
Groups: co.articleGroups,
GroupsHTML: co.preHTML("articles"),
Total: co.articlesTitled - co.articlesBooks,
Books: co.articlesBooks,
})
}
func (co *Corpus) handleArticleView(w http.ResponseWriter, r *http.Request) {
a := co.articleIndex[r.PathValue("key")]
if a == nil {
notFound(w, r)
return
}
prefs := readPrefs(r)
scope := pageScope(nil, prefs, false)
docs := make([]*Document, 0, len(a.DocIDs))
for _, id := range a.DocIDs {
if d := co.docIndex[id]; d != nil {
docs = append(docs, d)
scope = scope.including(d.Collection)
}
}
data := &articlePage{
Base: pageWith(prefs),
Article: *a,
Total: len(docs),
BrowseQS: scope.suffix(),
Cards: newCardView(prefs, docs),
}
// A partner that only reported the story has no publisher page.
if scope.hasValue(co, "publisher", a.Publisher) {
data.PublisherURL = scope.link(categoryValueURL("publisher", a.Publisher))
}
render(w, r, data)
}