211 lines
5.2 KiB
Go
211 lines
5.2 KiB
Go
// articles.go - news articles as first-class objects.
|
|
//
|
|
// An article's identity is its publication LINK. Giving a publication entry
|
|
// a title is what promotes its article to a story.
|
|
|
|
package main
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"net/http"
|
|
"net/url"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
)
|
|
|
|
type Article struct {
|
|
Key string // short stable hash of the link (the /article/{key} path)
|
|
Title string
|
|
Link string
|
|
Host string // display fallback when Title is empty
|
|
Publisher string
|
|
Date string // earliest date_published seen
|
|
Journalists []string // union across docs, first-seen order
|
|
DocIDs []string
|
|
Book bool // published by a book publisher, not a news outlet
|
|
}
|
|
|
|
// bookPublishers are the publication entries that are books, not news
|
|
// articles (Greenwald's No Place to Hide, Gellman's Dark Mirror, Poitras's
|
|
// Astro Noise).
|
|
var bookPublishers = map[string]bool{
|
|
"Metropolitan Books": true,
|
|
"Penguin Press": true,
|
|
"Yale University Press": true,
|
|
}
|
|
|
|
// articleKeyFor derives the stable URL key for a link
|
|
func articleKeyFor(link string) string {
|
|
h := sha256.Sum256([]byte(link))
|
|
return hex.EncodeToString(h[:])[:12]
|
|
}
|
|
|
|
func hostOf(link string) string {
|
|
u, err := url.Parse(link)
|
|
if err != nil || u.Host == "" {
|
|
return link
|
|
}
|
|
return strings.TrimPrefix(u.Host, "www.")
|
|
}
|
|
|
|
// buildArticles indexes every publication entry that has a link
|
|
func (co *Corpus) buildArticles() {
|
|
co.articleByLink = make(map[string]*Article)
|
|
co.articleIndex = make(map[string]*Article)
|
|
co.articles = nil
|
|
|
|
for _, d := range co.archive.Docs {
|
|
for _, p := range d.Publication {
|
|
if p.Link == "" {
|
|
continue
|
|
}
|
|
a := co.articleByLink[p.Link]
|
|
if a == nil {
|
|
a = &Article{
|
|
Key: articleKeyFor(p.Link),
|
|
Link: p.Link,
|
|
Host: hostOf(p.Link),
|
|
}
|
|
co.articleByLink[p.Link] = a
|
|
co.articleIndex[a.Key] = a
|
|
}
|
|
if a.Title == "" && p.Title != "" {
|
|
a.Title = p.Title
|
|
}
|
|
if a.Publisher == "" && p.Publisher != "" {
|
|
a.Publisher = p.Publisher
|
|
a.Book = bookPublishers[p.Publisher]
|
|
}
|
|
if p.DatePublished != "" && (a.Date == "" || p.DatePublished < a.Date) {
|
|
a.Date = p.DatePublished
|
|
}
|
|
for _, j := range p.Journalists {
|
|
if j == "" {
|
|
continue
|
|
}
|
|
if !slices.ContainsFunc(a.Journalists, func(x string) bool { return sameValue(x, j) }) {
|
|
a.Journalists = append(a.Journalists, j)
|
|
}
|
|
}
|
|
dup := false
|
|
for _, id := range a.DocIDs {
|
|
if id == d.ID {
|
|
dup = true
|
|
break
|
|
}
|
|
}
|
|
if !dup {
|
|
a.DocIDs = append(a.DocIDs, d.ID)
|
|
}
|
|
}
|
|
}
|
|
|
|
for _, a := range co.articleByLink {
|
|
co.articles = append(co.articles, *a)
|
|
}
|
|
// Newest first; undated articles sort last; ties by title then link so
|
|
// the order is deterministic.
|
|
sort.Slice(co.articles, func(i, j int) bool {
|
|
di, dj := co.articles[i].Date, co.articles[j].Date
|
|
if di == "" && dj != "" {
|
|
return false
|
|
}
|
|
if di != "" && dj == "" {
|
|
return true
|
|
}
|
|
if di != dj {
|
|
return di > dj
|
|
}
|
|
ti := strings.ToLower(displayTitle(co.articles[i]))
|
|
tj := strings.ToLower(displayTitle(co.articles[j]))
|
|
if ti != tj {
|
|
return ti < tj
|
|
}
|
|
return co.articles[i].Link < co.articles[j].Link
|
|
})
|
|
|
|
co.buildArticleGroups()
|
|
}
|
|
|
|
// inCollection reports whether one of the article's documents is in coll.
|
|
func (a *Article) inCollection(co *Corpus, coll string) bool {
|
|
for _, id := range a.DocIDs {
|
|
if d := co.docIndex[id]; d != nil && d.Collection == coll {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func displayTitle(a Article) string {
|
|
if a.Title != "" {
|
|
return a.Title
|
|
}
|
|
return a.Host
|
|
}
|
|
|
|
type articleYearGroup struct {
|
|
Year string
|
|
Articles []Article
|
|
}
|
|
|
|
func (co *Corpus) buildArticleGroups() {
|
|
co.articleGroups, co.articlesTitled, co.articlesBooks = nil, 0, 0
|
|
for _, a := range co.articles {
|
|
if a.Title == "" {
|
|
continue
|
|
}
|
|
co.articlesTitled++
|
|
if a.Book {
|
|
co.articlesBooks++
|
|
}
|
|
year := "Undated"
|
|
if len(a.Date) >= 4 {
|
|
year = a.Date[:4]
|
|
}
|
|
if len(co.articleGroups) == 0 || co.articleGroups[len(co.articleGroups)-1].Year != year {
|
|
co.articleGroups = append(co.articleGroups, articleYearGroup{Year: year})
|
|
}
|
|
co.articleGroups[len(co.articleGroups)-1].Articles = append(co.articleGroups[len(co.articleGroups)-1].Articles, a)
|
|
}
|
|
}
|
|
|
|
func (co *Corpus) handleArticles(w http.ResponseWriter, r *http.Request) {
|
|
render(w, r, &articlesPage{
|
|
Groups: co.articleGroups,
|
|
GroupsHTML: co.preHTML("articles"),
|
|
Total: co.articlesTitled - co.articlesBooks,
|
|
Books: co.articlesBooks,
|
|
})
|
|
}
|
|
|
|
func (co *Corpus) handleArticleView(w http.ResponseWriter, r *http.Request) {
|
|
a := co.articleIndex[r.PathValue("key")]
|
|
if a == nil {
|
|
notFound(w, r)
|
|
return
|
|
}
|
|
prefs := readPrefs(r)
|
|
scope := pageScope(nil, prefs, false)
|
|
docs := make([]*Document, 0, len(a.DocIDs))
|
|
for _, id := range a.DocIDs {
|
|
if d := co.docIndex[id]; d != nil {
|
|
docs = append(docs, d)
|
|
scope = scope.including(d.Collection)
|
|
}
|
|
}
|
|
data := &articlePage{
|
|
Base: pageWith(prefs),
|
|
Article: *a,
|
|
Total: len(docs),
|
|
BrowseQS: scope.suffix(),
|
|
Cards: newCardView(prefs, docs),
|
|
}
|
|
// A partner that only reported the story has no publisher page.
|
|
if scope.hasValue(co, "publisher", a.Publisher) {
|
|
data.PublisherURL = scope.link(categoryValueURL("publisher", a.Publisher))
|
|
}
|
|
render(w, r, data)
|
|
}
|