// articles.go - news articles as first-class objects. // // An article's identity is its publication LINK. Giving a publication entry // a title is what promotes its article to a story. package main import ( "crypto/sha256" "encoding/hex" "net/http" "net/url" "slices" "sort" "strings" ) type Article struct { Key string // short stable hash of the link (the /article/{key} path) Title string Link string Host string // display fallback when Title is empty Publisher string Date string // earliest date_published seen Journalists []string // union across docs, first-seen order DocIDs []string Book bool // published by a book publisher, not a news outlet } // bookPublishers are the publication entries that are books, not news // articles (Greenwald's No Place to Hide, Gellman's Dark Mirror, Poitras's // Astro Noise). var bookPublishers = map[string]bool{ "Metropolitan Books": true, "Penguin Press": true, "Yale University Press": true, } // articleKeyFor derives the stable URL key for a link func articleKeyFor(link string) string { h := sha256.Sum256([]byte(link)) return hex.EncodeToString(h[:])[:12] } func hostOf(link string) string { u, err := url.Parse(link) if err != nil || u.Host == "" { return link } return strings.TrimPrefix(u.Host, "www.") } // buildArticles indexes every publication entry that has a link func (co *Corpus) buildArticles() { co.articleByLink = make(map[string]*Article) co.articleIndex = make(map[string]*Article) co.articles = nil for _, d := range co.archive.Docs { for _, p := range d.Publication { if p.Link == "" { continue } a := co.articleByLink[p.Link] if a == nil { a = &Article{ Key: articleKeyFor(p.Link), Link: p.Link, Host: hostOf(p.Link), } co.articleByLink[p.Link] = a co.articleIndex[a.Key] = a } if a.Title == "" && p.Title != "" { a.Title = p.Title } if a.Publisher == "" && p.Publisher != "" { a.Publisher = p.Publisher a.Book = bookPublishers[p.Publisher] } if p.DatePublished != "" && (a.Date == "" || p.DatePublished < a.Date) { a.Date = p.DatePublished } for _, j := range p.Journalists { if j == "" { continue } if !slices.ContainsFunc(a.Journalists, func(x string) bool { return sameValue(x, j) }) { a.Journalists = append(a.Journalists, j) } } dup := false for _, id := range a.DocIDs { if id == d.ID { dup = true break } } if !dup { a.DocIDs = append(a.DocIDs, d.ID) } } } for _, a := range co.articleByLink { co.articles = append(co.articles, *a) } // Newest first; undated articles sort last; ties by title then link so // the order is deterministic. sort.Slice(co.articles, func(i, j int) bool { di, dj := co.articles[i].Date, co.articles[j].Date if di == "" && dj != "" { return false } if di != "" && dj == "" { return true } if di != dj { return di > dj } ti := strings.ToLower(displayTitle(co.articles[i])) tj := strings.ToLower(displayTitle(co.articles[j])) if ti != tj { return ti < tj } return co.articles[i].Link < co.articles[j].Link }) co.buildArticleGroups() } // inCollection reports whether one of the article's documents is in coll. func (a *Article) inCollection(co *Corpus, coll string) bool { for _, id := range a.DocIDs { if d := co.docIndex[id]; d != nil && d.Collection == coll { return true } } return false } func displayTitle(a Article) string { if a.Title != "" { return a.Title } return a.Host } type articleYearGroup struct { Year string Articles []Article } func (co *Corpus) buildArticleGroups() { co.articleGroups, co.articlesTitled, co.articlesBooks = nil, 0, 0 for _, a := range co.articles { if a.Title == "" { continue } co.articlesTitled++ if a.Book { co.articlesBooks++ } year := "Undated" if len(a.Date) >= 4 { year = a.Date[:4] } if len(co.articleGroups) == 0 || co.articleGroups[len(co.articleGroups)-1].Year != year { co.articleGroups = append(co.articleGroups, articleYearGroup{Year: year}) } co.articleGroups[len(co.articleGroups)-1].Articles = append(co.articleGroups[len(co.articleGroups)-1].Articles, a) } } func (co *Corpus) handleArticles(w http.ResponseWriter, r *http.Request) { render(w, r, &articlesPage{ Groups: co.articleGroups, GroupsHTML: co.preHTML("articles"), Total: co.articlesTitled - co.articlesBooks, Books: co.articlesBooks, }) } func (co *Corpus) handleArticleView(w http.ResponseWriter, r *http.Request) { a := co.articleIndex[r.PathValue("key")] if a == nil { notFound(w, r) return } prefs := readPrefs(r) scope := pageScope(nil, prefs, false) docs := make([]*Document, 0, len(a.DocIDs)) for _, id := range a.DocIDs { if d := co.docIndex[id]; d != nil { docs = append(docs, d) scope = scope.including(d.Collection) } } data := &articlePage{ Base: pageWith(prefs), Article: *a, Total: len(docs), BrowseQS: scope.suffix(), Cards: newCardView(prefs, docs), } // A partner that only reported the story has no publisher page. if scope.hasValue(co, "publisher", a.Publisher) { data.PublisherURL = scope.link(categoryValueURL("publisher", a.Publisher)) } render(w, r, data) }