first commit

This commit is contained in:
libroot 2026-11-19 04:40:59 +02:00
commit 8c4dcae475
Signed by: libroot
GPG key ID: 2AC68C029EBE9ACE
89 changed files with 24623 additions and 0 deletions

233
seo.go Normal file
View file

@ -0,0 +1,233 @@
// seo.go - robots.txt, sitemap.xml, and the per-document metadata embeds.
package main
import (
"bytes"
"encoding/json"
"encoding/xml"
"fmt"
"html/template"
"net/http"
"net/url"
"strings"
)
var canonicalBase string
func requestBase(r *http.Request) string {
if canonicalBase != "" {
return canonicalBase
}
scheme := "http"
if r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https" {
scheme = "https"
}
return scheme + "://" + r.Host
}
func handleRobots(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Cache-Control", "public, max-age=86400")
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
fmt.Fprintf(w, `User-agent: *
Allow: /
Disallow: /search
Disallow: /api/
Disallow: /random
Disallow: /pinned
Disallow: /preferences
Sitemap: %s/sitemap.xml
`, requestBase(r))
}
type sitemapURL struct {
Loc string `xml:"loc"`
}
type sitemapSet struct {
XMLName xml.Name `xml:"urlset"`
Xmlns string `xml:"xmlns,attr"`
URLs []sitemapURL `xml:"url"`
}
func (co *Corpus) buildSitemapPaths() {
paths := []string{
"/", "/browse", "/articles", "/glossary", "/changelog", "/about",
}
for _, c := range categories {
paths = append(paths, "/browse/"+c.Slug)
}
paths = append(paths, "/explore")
for _, m := range exploreMetrics {
paths = append(paths, "/explore/"+m.Slug)
}
paths = append(paths, "/explore/terms")
paths = append(paths, "/explore/classification")
paths = append(paths, "/explore/portions")
paths = append(paths, "/explore/outlets")
if len(co.assemblies) > 0 {
paths = append(paths, assembliesPath)
for _, a := range co.assemblies {
paths = append(paths, assemblyPath(a.ID))
}
}
for _, d := range co.archive.Docs {
paths = append(paths, docPath(d.ID))
}
for _, a := range co.articles {
if a.Title != "" {
paths = append(paths, "/article/"+a.Key)
}
}
co.sitemapPaths = paths
}
func (co *Corpus) marshalSitemap(base string) ([]byte, error) {
set := sitemapSet{Xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9"}
set.URLs = make([]sitemapURL, len(co.sitemapPaths))
for i, p := range co.sitemapPaths {
set.URLs[i] = sitemapURL{Loc: base + p}
}
out, err := xml.MarshalIndent(set, "", " ")
if err != nil {
return nil, err
}
return append([]byte(xml.Header), out...), nil
}
func parseCanonicalBase(base string) (string, error) {
u, err := url.Parse(base)
if err != nil {
return "", fmt.Errorf("invalid base URL %q: %w", base, err)
}
if (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil ||
(u.Path != "" && u.Path != "/") || u.RawQuery != "" || u.Fragment != "" {
return "", fmt.Errorf("base must be scheme://host with no path/query/fragment/userinfo, e.g. https://archive.example.org (got %q)", base)
}
return u.Scheme + "://" + u.Host, nil
}
const sitemapBaseMark = "{base url}"
func (co *Corpus) buildCanonicalSitemap() {
co.sitemapParts = nil
if canonicalBase == "" {
if b, err := co.marshalSitemap(sitemapBaseMark); err == nil {
co.sitemapParts = bytes.Split(b, []byte(sitemapBaseMark))
}
return
}
b, err := co.marshalSitemap(canonicalBase)
if err != nil {
errorf("sitemap: %v - building it per request instead", err)
return
}
co.canonicalSitemap = b
}
func (co *Corpus) handleSitemap(w http.ResponseWriter, r *http.Request) {
body := co.canonicalSitemap
var base bytes.Buffer
if body == nil && co.sitemapParts != nil {
xml.EscapeText(&base, []byte(requestBase(r)))
} else if body == nil {
var err error
body, err = co.marshalSitemap(requestBase(r))
if err != nil {
serverError(w, err)
return
}
}
w.Header().Set("Cache-Control", "public, max-age=3600")
w.Header().Set("Content-Type", "application/xml; charset=utf-8")
if body != nil {
w.Write(body)
return
}
for i, part := range co.sitemapParts {
if i > 0 {
w.Write(base.Bytes())
}
w.Write(part)
}
}
// docEmbeds is handed to doc.html as data["Embeds"].
type docEmbeds struct {
JSONLDTag template.HTML
COinS string
}
func buildDocEmbeds(doc *Document, absURL, pdfURL string) docEmbeds {
type entity struct {
Type string `json:"@type"`
Name string `json:"name"`
}
type media struct {
Type string `json:"@type"`
ContentURL string `json:"contentUrl"`
EncodingFormat string `json:"encodingFormat"`
}
keywords := strings.Join(append(append([]string{}, doc.DocMeta.Programs...), doc.Tags...), ", ")
ld := struct {
Context string `json:"@context"`
Type string `json:"@type"`
Name string `json:"name"`
URL string `json:"url"`
Identifier string `json:"identifier"`
Description string `json:"description,omitempty"`
DateCreated string `json:"dateCreated,omitempty"`
DatePublished string `json:"datePublished,omitempty"`
InLanguage string `json:"inLanguage"`
Keywords string `json:"keywords,omitempty"`
Author *entity `json:"author,omitempty"`
Publisher entity `json:"publisher"`
AssociatedMedia media `json:"associatedMedia"`
}{
Context: "https://schema.org",
Type: "DigitalDocument",
Name: doc.Title,
URL: absURL,
Identifier: absURL,
Description: doc.Description,
DateCreated: doc.DocMeta.DateCreated,
DatePublished: oldestPubDate(doc),
InLanguage: "en",
Keywords: keywords,
Publisher: entity{Type: "Organization", Name: "Snowden Archive"},
AssociatedMedia: media{
Type: "MediaObject",
ContentURL: pdfURL,
EncodingFormat: "application/pdf",
},
}
if doc.DocMeta.Creator != "" {
ld.Author = &entity{Type: "Organization", Name: doc.DocMeta.Creator}
}
var tag template.HTML
if b, err := json.Marshal(ld); err == nil {
tag = template.HTML(`<script type="application/ld+json">` + string(b) + `</script>`)
}
v := url.Values{}
v.Set("ctx_ver", "Z39.88-2004")
v.Set("rft_val_fmt", "info:ofi/fmt:kev:mtx:dc")
v.Set("rft.title", doc.Title)
if doc.DocMeta.Creator != "" {
v.Set("rft.creator", doc.DocMeta.Creator)
}
if doc.DocMeta.DateCreated != "" {
v.Set("rft.date", doc.DocMeta.DateCreated)
}
v.Set("rft.type", "Text")
v.Set("rft.format", "application/pdf")
v.Set("rft.identifier", absURL)
v.Set("rft.publisher", "Snowden Archive")
for _, p := range doc.DocMeta.Programs {
v.Add("rft.subject", p)
}
return docEmbeds{JSONLDTag: tag, COinS: v.Encode()}
}