first commit
This commit is contained in:
commit
8c4dcae475
89 changed files with 24623 additions and 0 deletions
211
articles.go
Normal file
211
articles.go
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
// articles.go - news articles as first-class objects.
|
||||
//
|
||||
// An article's identity is its publication LINK. Giving a publication entry
|
||||
// a title is what promotes its article to a story.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Article struct {
|
||||
Key string // short stable hash of the link (the /article/{key} path)
|
||||
Title string
|
||||
Link string
|
||||
Host string // display fallback when Title is empty
|
||||
Publisher string
|
||||
Date string // earliest date_published seen
|
||||
Journalists []string // union across docs, first-seen order
|
||||
DocIDs []string
|
||||
Book bool // published by a book publisher, not a news outlet
|
||||
}
|
||||
|
||||
// bookPublishers are the publication entries that are books, not news
|
||||
// articles (Greenwald's No Place to Hide, Gellman's Dark Mirror, Poitras's
|
||||
// Astro Noise).
|
||||
var bookPublishers = map[string]bool{
|
||||
"Metropolitan Books": true,
|
||||
"Penguin Press": true,
|
||||
"Yale University Press": true,
|
||||
}
|
||||
|
||||
// articleKeyFor derives the stable URL key for a link
|
||||
func articleKeyFor(link string) string {
|
||||
h := sha256.Sum256([]byte(link))
|
||||
return hex.EncodeToString(h[:])[:12]
|
||||
}
|
||||
|
||||
func hostOf(link string) string {
|
||||
u, err := url.Parse(link)
|
||||
if err != nil || u.Host == "" {
|
||||
return link
|
||||
}
|
||||
return strings.TrimPrefix(u.Host, "www.")
|
||||
}
|
||||
|
||||
// buildArticles indexes every publication entry that has a link
|
||||
func (co *Corpus) buildArticles() {
|
||||
co.articleByLink = make(map[string]*Article)
|
||||
co.articleIndex = make(map[string]*Article)
|
||||
co.articles = nil
|
||||
|
||||
for _, d := range co.archive.Docs {
|
||||
for _, p := range d.Publication {
|
||||
if p.Link == "" {
|
||||
continue
|
||||
}
|
||||
a := co.articleByLink[p.Link]
|
||||
if a == nil {
|
||||
a = &Article{
|
||||
Key: articleKeyFor(p.Link),
|
||||
Link: p.Link,
|
||||
Host: hostOf(p.Link),
|
||||
}
|
||||
co.articleByLink[p.Link] = a
|
||||
co.articleIndex[a.Key] = a
|
||||
}
|
||||
if a.Title == "" && p.Title != "" {
|
||||
a.Title = p.Title
|
||||
}
|
||||
if a.Publisher == "" && p.Publisher != "" {
|
||||
a.Publisher = p.Publisher
|
||||
a.Book = bookPublishers[p.Publisher]
|
||||
}
|
||||
if p.DatePublished != "" && (a.Date == "" || p.DatePublished < a.Date) {
|
||||
a.Date = p.DatePublished
|
||||
}
|
||||
for _, j := range p.Journalists {
|
||||
if j == "" {
|
||||
continue
|
||||
}
|
||||
if !slices.ContainsFunc(a.Journalists, func(x string) bool { return sameValue(x, j) }) {
|
||||
a.Journalists = append(a.Journalists, j)
|
||||
}
|
||||
}
|
||||
dup := false
|
||||
for _, id := range a.DocIDs {
|
||||
if id == d.ID {
|
||||
dup = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !dup {
|
||||
a.DocIDs = append(a.DocIDs, d.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, a := range co.articleByLink {
|
||||
co.articles = append(co.articles, *a)
|
||||
}
|
||||
// Newest first; undated articles sort last; ties by title then link so
|
||||
// the order is deterministic.
|
||||
sort.Slice(co.articles, func(i, j int) bool {
|
||||
di, dj := co.articles[i].Date, co.articles[j].Date
|
||||
if di == "" && dj != "" {
|
||||
return false
|
||||
}
|
||||
if di != "" && dj == "" {
|
||||
return true
|
||||
}
|
||||
if di != dj {
|
||||
return di > dj
|
||||
}
|
||||
ti := strings.ToLower(displayTitle(co.articles[i]))
|
||||
tj := strings.ToLower(displayTitle(co.articles[j]))
|
||||
if ti != tj {
|
||||
return ti < tj
|
||||
}
|
||||
return co.articles[i].Link < co.articles[j].Link
|
||||
})
|
||||
|
||||
co.buildArticleGroups()
|
||||
}
|
||||
|
||||
// inCollection reports whether one of the article's documents is in coll.
|
||||
func (a *Article) inCollection(co *Corpus, coll string) bool {
|
||||
for _, id := range a.DocIDs {
|
||||
if d := co.docIndex[id]; d != nil && d.Collection == coll {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func displayTitle(a Article) string {
|
||||
if a.Title != "" {
|
||||
return a.Title
|
||||
}
|
||||
return a.Host
|
||||
}
|
||||
|
||||
type articleYearGroup struct {
|
||||
Year string
|
||||
Articles []Article
|
||||
}
|
||||
|
||||
func (co *Corpus) buildArticleGroups() {
|
||||
co.articleGroups, co.articlesTitled, co.articlesBooks = nil, 0, 0
|
||||
for _, a := range co.articles {
|
||||
if a.Title == "" {
|
||||
continue
|
||||
}
|
||||
co.articlesTitled++
|
||||
if a.Book {
|
||||
co.articlesBooks++
|
||||
}
|
||||
year := "Undated"
|
||||
if len(a.Date) >= 4 {
|
||||
year = a.Date[:4]
|
||||
}
|
||||
if len(co.articleGroups) == 0 || co.articleGroups[len(co.articleGroups)-1].Year != year {
|
||||
co.articleGroups = append(co.articleGroups, articleYearGroup{Year: year})
|
||||
}
|
||||
co.articleGroups[len(co.articleGroups)-1].Articles = append(co.articleGroups[len(co.articleGroups)-1].Articles, a)
|
||||
}
|
||||
}
|
||||
|
||||
func (co *Corpus) handleArticles(w http.ResponseWriter, r *http.Request) {
|
||||
render(w, r, &articlesPage{
|
||||
Groups: co.articleGroups,
|
||||
GroupsHTML: co.preHTML("articles"),
|
||||
Total: co.articlesTitled - co.articlesBooks,
|
||||
Books: co.articlesBooks,
|
||||
})
|
||||
}
|
||||
|
||||
func (co *Corpus) handleArticleView(w http.ResponseWriter, r *http.Request) {
|
||||
a := co.articleIndex[r.PathValue("key")]
|
||||
if a == nil {
|
||||
notFound(w, r)
|
||||
return
|
||||
}
|
||||
prefs := readPrefs(r)
|
||||
scope := pageScope(nil, prefs, false)
|
||||
docs := make([]*Document, 0, len(a.DocIDs))
|
||||
for _, id := range a.DocIDs {
|
||||
if d := co.docIndex[id]; d != nil {
|
||||
docs = append(docs, d)
|
||||
scope = scope.including(d.Collection)
|
||||
}
|
||||
}
|
||||
data := &articlePage{
|
||||
Base: pageWith(prefs),
|
||||
Article: *a,
|
||||
Total: len(docs),
|
||||
BrowseQS: scope.suffix(),
|
||||
Cards: newCardView(prefs, docs),
|
||||
}
|
||||
// A partner that only reported the story has no publisher page.
|
||||
if scope.hasValue(co, "publisher", a.Publisher) {
|
||||
data.PublisherURL = scope.link(categoryValueURL("publisher", a.Publisher))
|
||||
}
|
||||
render(w, r, data)
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue