114 lines
2.7 KiB
Go
114 lines
2.7 KiB
Go
// explore_ids.go - /explore/ids: every document's identifiers in one table.
|
|
//
|
|
// /explore/ids.txt is the same checksums in sha256sum(1) format, so a mirror
|
|
// or a download verifies with "sha256sum -c" in the corpus root.
|
|
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"net/http"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
)
|
|
|
|
type idVersionRow struct {
|
|
Slug string
|
|
PDFFile string
|
|
PDFURL string
|
|
Size string
|
|
Pages int
|
|
SHA string
|
|
}
|
|
|
|
type idRow struct {
|
|
ID string
|
|
Collection string
|
|
Pages int
|
|
PDFFile string
|
|
PDFURL string
|
|
Size string
|
|
SHA string
|
|
Aka []AkaEntry
|
|
Versions []idVersionRow
|
|
}
|
|
|
|
func humanSize(n int64) string {
|
|
switch {
|
|
case n <= 0:
|
|
return ""
|
|
case n < 1<<10:
|
|
return fmt.Sprintf("%d B", n)
|
|
case n < 1<<20:
|
|
return fmt.Sprintf("%.0f KB", float64(n)/(1<<10))
|
|
case n < 1<<30:
|
|
return fmt.Sprintf("%.1f MB", float64(n)/(1<<20))
|
|
}
|
|
return fmt.Sprintf("%.2f GB", float64(n)/(1<<30))
|
|
}
|
|
|
|
func (co *Corpus) buildIDRows() {
|
|
co.idRows = make([]idRow, 0, len(co.archive.Docs))
|
|
var sums strings.Builder
|
|
|
|
for i := range co.archive.Docs {
|
|
d := &co.archive.Docs[i]
|
|
row := idRow{
|
|
ID: d.ID,
|
|
Collection: d.Collection,
|
|
Pages: d.Pages,
|
|
PDFFile: d.PDFFile,
|
|
PDFURL: filePath(d.ID, d.PDFFile),
|
|
Size: humanSize(d.PDFSize),
|
|
SHA: strings.TrimSpace(d.PDFSha256),
|
|
Aka: d.AkaID,
|
|
}
|
|
for _, v := range d.Versions {
|
|
row.Versions = append(row.Versions, idVersionRow{
|
|
Slug: v.Slug,
|
|
PDFFile: v.PDFFile,
|
|
PDFURL: versionFilePath(d.ID, v.Slug, v.PDFFile),
|
|
Size: humanSize(v.PDFSize),
|
|
Pages: len(v.Images),
|
|
SHA: v.SHA,
|
|
})
|
|
}
|
|
co.idRows = append(co.idRows, row)
|
|
|
|
rel, err := filepath.Rel(dataDir, d.Dir)
|
|
if err != nil {
|
|
rel = d.ID
|
|
}
|
|
if row.SHA != "" {
|
|
sums.WriteString(row.SHA + " " + filepath.ToSlash(filepath.Join(rel, d.PDFFile)) + "\n")
|
|
}
|
|
for _, v := range d.Versions {
|
|
if v.SHA != "" {
|
|
sums.WriteString(v.SHA + " " + filepath.ToSlash(filepath.Join(rel, "versions", v.Slug, v.PDFFile)) + "\n")
|
|
}
|
|
}
|
|
}
|
|
|
|
sort.Slice(co.idRows, func(i, j int) bool {
|
|
a, b := strings.ToLower(co.idRows[i].ID), strings.ToLower(co.idRows[j].ID)
|
|
if a != b {
|
|
return a < b
|
|
}
|
|
return co.idRows[i].ID < co.idRows[j].ID
|
|
})
|
|
co.idChecksums = sums.String()
|
|
}
|
|
|
|
func (co *Corpus) handleExploreIDs(w http.ResponseWriter, r *http.Request) {
|
|
render(w, r, &idsPage{
|
|
Rows: co.idRows,
|
|
RowsHTML: co.preHTML("ids"),
|
|
Shown: len(co.idRows),
|
|
})
|
|
}
|
|
|
|
func (co *Corpus) handleExploreIDsText(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
|
w.Header().Set("Cache-Control", "public, max-age=3600")
|
|
w.Write([]byte(co.idChecksums))
|
|
}
|