233 lines
6.3 KiB
Go
233 lines
6.3 KiB
Go
// seo.go - robots.txt, sitemap.xml, and the per-document metadata embeds.
|
|
|
|
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"encoding/xml"
|
|
"fmt"
|
|
"html/template"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
)
|
|
|
|
var canonicalBase string
|
|
|
|
func requestBase(r *http.Request) string {
|
|
if canonicalBase != "" {
|
|
return canonicalBase
|
|
}
|
|
scheme := "http"
|
|
if r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https" {
|
|
scheme = "https"
|
|
}
|
|
return scheme + "://" + r.Host
|
|
}
|
|
|
|
func handleRobots(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Cache-Control", "public, max-age=86400")
|
|
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
|
fmt.Fprintf(w, `User-agent: *
|
|
Allow: /
|
|
Disallow: /search
|
|
Disallow: /api/
|
|
Disallow: /random
|
|
Disallow: /pinned
|
|
Disallow: /preferences
|
|
|
|
Sitemap: %s/sitemap.xml
|
|
`, requestBase(r))
|
|
}
|
|
|
|
type sitemapURL struct {
|
|
Loc string `xml:"loc"`
|
|
}
|
|
|
|
type sitemapSet struct {
|
|
XMLName xml.Name `xml:"urlset"`
|
|
Xmlns string `xml:"xmlns,attr"`
|
|
URLs []sitemapURL `xml:"url"`
|
|
}
|
|
|
|
func (co *Corpus) buildSitemapPaths() {
|
|
paths := []string{
|
|
"/", "/browse", "/articles", "/glossary", "/changelog", "/about",
|
|
}
|
|
for _, c := range categories {
|
|
paths = append(paths, "/browse/"+c.Slug)
|
|
}
|
|
paths = append(paths, "/explore")
|
|
for _, m := range exploreMetrics {
|
|
paths = append(paths, "/explore/"+m.Slug)
|
|
}
|
|
paths = append(paths, "/explore/terms")
|
|
paths = append(paths, "/explore/classification")
|
|
paths = append(paths, "/explore/portions")
|
|
paths = append(paths, "/explore/outlets")
|
|
if len(co.assemblies) > 0 {
|
|
paths = append(paths, assembliesPath)
|
|
for _, a := range co.assemblies {
|
|
paths = append(paths, assemblyPath(a.ID))
|
|
}
|
|
}
|
|
for _, d := range co.archive.Docs {
|
|
paths = append(paths, docPath(d.ID))
|
|
}
|
|
for _, a := range co.articles {
|
|
if a.Title != "" {
|
|
paths = append(paths, "/article/"+a.Key)
|
|
}
|
|
}
|
|
|
|
co.sitemapPaths = paths
|
|
}
|
|
|
|
func (co *Corpus) marshalSitemap(base string) ([]byte, error) {
|
|
set := sitemapSet{Xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9"}
|
|
set.URLs = make([]sitemapURL, len(co.sitemapPaths))
|
|
for i, p := range co.sitemapPaths {
|
|
set.URLs[i] = sitemapURL{Loc: base + p}
|
|
}
|
|
out, err := xml.MarshalIndent(set, "", " ")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return append([]byte(xml.Header), out...), nil
|
|
}
|
|
|
|
func parseCanonicalBase(base string) (string, error) {
|
|
u, err := url.Parse(base)
|
|
if err != nil {
|
|
return "", fmt.Errorf("invalid base URL %q: %w", base, err)
|
|
}
|
|
if (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil ||
|
|
(u.Path != "" && u.Path != "/") || u.RawQuery != "" || u.Fragment != "" {
|
|
return "", fmt.Errorf("base must be scheme://host with no path/query/fragment/userinfo, e.g. https://archive.example.org (got %q)", base)
|
|
}
|
|
return u.Scheme + "://" + u.Host, nil
|
|
}
|
|
|
|
const sitemapBaseMark = "{base url}"
|
|
|
|
func (co *Corpus) buildCanonicalSitemap() {
|
|
co.sitemapParts = nil
|
|
if canonicalBase == "" {
|
|
if b, err := co.marshalSitemap(sitemapBaseMark); err == nil {
|
|
co.sitemapParts = bytes.Split(b, []byte(sitemapBaseMark))
|
|
}
|
|
return
|
|
}
|
|
b, err := co.marshalSitemap(canonicalBase)
|
|
if err != nil {
|
|
errorf("sitemap: %v - building it per request instead", err)
|
|
return
|
|
}
|
|
co.canonicalSitemap = b
|
|
}
|
|
|
|
func (co *Corpus) handleSitemap(w http.ResponseWriter, r *http.Request) {
|
|
body := co.canonicalSitemap
|
|
var base bytes.Buffer
|
|
if body == nil && co.sitemapParts != nil {
|
|
xml.EscapeText(&base, []byte(requestBase(r)))
|
|
} else if body == nil {
|
|
var err error
|
|
body, err = co.marshalSitemap(requestBase(r))
|
|
if err != nil {
|
|
serverError(w, err)
|
|
return
|
|
}
|
|
}
|
|
w.Header().Set("Cache-Control", "public, max-age=3600")
|
|
w.Header().Set("Content-Type", "application/xml; charset=utf-8")
|
|
if body != nil {
|
|
w.Write(body)
|
|
return
|
|
}
|
|
for i, part := range co.sitemapParts {
|
|
if i > 0 {
|
|
w.Write(base.Bytes())
|
|
}
|
|
w.Write(part)
|
|
}
|
|
}
|
|
|
|
// docEmbeds is handed to doc.html as data["Embeds"].
|
|
type docEmbeds struct {
|
|
JSONLDTag template.HTML
|
|
COinS string
|
|
}
|
|
|
|
func buildDocEmbeds(doc *Document, absURL, pdfURL string) docEmbeds {
|
|
type entity struct {
|
|
Type string `json:"@type"`
|
|
Name string `json:"name"`
|
|
}
|
|
type media struct {
|
|
Type string `json:"@type"`
|
|
ContentURL string `json:"contentUrl"`
|
|
EncodingFormat string `json:"encodingFormat"`
|
|
}
|
|
keywords := strings.Join(append(append([]string{}, doc.DocMeta.Programs...), doc.Tags...), ", ")
|
|
ld := struct {
|
|
Context string `json:"@context"`
|
|
Type string `json:"@type"`
|
|
Name string `json:"name"`
|
|
URL string `json:"url"`
|
|
Identifier string `json:"identifier"`
|
|
Description string `json:"description,omitempty"`
|
|
DateCreated string `json:"dateCreated,omitempty"`
|
|
DatePublished string `json:"datePublished,omitempty"`
|
|
InLanguage string `json:"inLanguage"`
|
|
Keywords string `json:"keywords,omitempty"`
|
|
Author *entity `json:"author,omitempty"`
|
|
Publisher entity `json:"publisher"`
|
|
AssociatedMedia media `json:"associatedMedia"`
|
|
}{
|
|
Context: "https://schema.org",
|
|
Type: "DigitalDocument",
|
|
Name: doc.Title,
|
|
URL: absURL,
|
|
Identifier: absURL,
|
|
Description: doc.Description,
|
|
DateCreated: doc.DocMeta.DateCreated,
|
|
DatePublished: oldestPubDate(doc),
|
|
InLanguage: "en",
|
|
Keywords: keywords,
|
|
Publisher: entity{Type: "Organization", Name: "Snowden Archive"},
|
|
AssociatedMedia: media{
|
|
Type: "MediaObject",
|
|
ContentURL: pdfURL,
|
|
EncodingFormat: "application/pdf",
|
|
},
|
|
}
|
|
if doc.DocMeta.Creator != "" {
|
|
ld.Author = &entity{Type: "Organization", Name: doc.DocMeta.Creator}
|
|
}
|
|
|
|
var tag template.HTML
|
|
if b, err := json.Marshal(ld); err == nil {
|
|
tag = template.HTML(`<script type="application/ld+json">` + string(b) + `</script>`)
|
|
}
|
|
|
|
v := url.Values{}
|
|
v.Set("ctx_ver", "Z39.88-2004")
|
|
v.Set("rft_val_fmt", "info:ofi/fmt:kev:mtx:dc")
|
|
v.Set("rft.title", doc.Title)
|
|
if doc.DocMeta.Creator != "" {
|
|
v.Set("rft.creator", doc.DocMeta.Creator)
|
|
}
|
|
if doc.DocMeta.DateCreated != "" {
|
|
v.Set("rft.date", doc.DocMeta.DateCreated)
|
|
}
|
|
v.Set("rft.type", "Text")
|
|
v.Set("rft.format", "application/pdf")
|
|
v.Set("rft.identifier", absURL)
|
|
v.Set("rft.publisher", "Snowden Archive")
|
|
for _, p := range doc.DocMeta.Programs {
|
|
v.Add("rft.subject", p)
|
|
}
|
|
return docEmbeds{JSONLDTag: tag, COinS: v.Encode()}
|
|
}
|