first commit

This commit is contained in:
libroot 2026-11-19 04:40:59 +02:00
commit 8c4dcae475
Signed by: libroot
GPG key ID: 2AC68C029EBE9ACE
89 changed files with 24623 additions and 0 deletions

591
mdparser.go Normal file
View file

@ -0,0 +1,591 @@
// mdparser.go - minimal dependency-free Markdown parser.
//
// Input trust: everything rendered through this parser is the archive's own
// curator-produced OCR never user-supplied input.
package main
import (
"fmt"
"html"
"regexp"
"strconv"
"strings"
"unicode"
"unicode/utf8"
)
// -- Inline patterns ----------------------------------------------------
var (
reInlineCode = regexp.MustCompile("`(.+?)`")
reStrikethrough = regexp.MustCompile(`~~(.+?)~~`)
reHardBreak = regexp.MustCompile(`(?i)<br\s*/?>`)
reEscape = regexp.MustCompile("\\\\([!-/:-@\\[-`|{}~])")
reOrderedItem = regexp.MustCompile(`^(\s*)(\d{1,9})\.(?:\s+(.*))?\s*$`)
reUnorderedItem = regexp.MustCompile(`^(\s*)[-*+](?:\s+(.*))?\s*$`)
reTableSep = regexp.MustCompile(`^\|[-| :]+\|$`)
)
// parseInline converts inline Markdown to HTML.
func parseInline(s string) string {
s = strings.ReplaceAll(s, "\x00", "")
s = strings.ReplaceAll(s, "\x01", "")
s = html.EscapeString(s)
// Backslash escapes (CommonMark): "\*" is a literal "*", not emphasis
// syntax. Pull escaped ASCII punctuation out to placeholders BEFORE the
// emphasis passes so the character can't be interpreted as a marker,
// then restore it at the end.
var escaped []string
if strings.IndexByte(s, '\\') >= 0 {
s = reEscape.ReplaceAllStringFunc(s, func(m string) string {
// m is "\X": EscapeString leaves the backslash alone. Keep
// the char after it.
ch := m[len(m)-1:]
escaped = append(escaped, ch)
return "\x01" + strconv.Itoa(len(escaped)-1) + "\x01"
})
}
// Extract inline code spans next and replace them with placeholders so
// the strikethrough/bold/italic passes can't reformat code content
// (e.g. `*x*` must render as literal *x*, not <em>x</em>). Escape
// placeholders that landed INSIDE a code span are put back as their
// original "\X" source text first: backslash escapes are not escapes
// within code spans (CommonMark), so `\*` renders as \*, not *. The
// escape pass still runs before this one on purpose - that ordering is
// what lets \` be a literal backtick that cannot OPEN a code span.
var codes []string
if strings.IndexByte(s, '`') >= 0 {
s = reInlineCode.ReplaceAllStringFunc(s, func(m string) string {
inner := reInlineCode.FindStringSubmatch(m)[1]
for i, ch := range escaped {
inner = strings.ReplaceAll(inner, "\x01"+strconv.Itoa(i)+"\x01", "\\"+ch)
}
codes = append(codes, "<code>"+inner+"</code>")
return "\x00" + strconv.Itoa(len(codes)-1) + "\x00"
})
}
// After the code-span pass, so `<br>` inside backticks stays literal.
if strings.Contains(s, "&lt;") {
s = reHardBreak.ReplaceAllString(s, "<br>")
}
if strings.Contains(s, "~~") {
s = reStrikethrough.ReplaceAllString(s, `<del>$1</del>`)
}
// ***x*** first, so it nests as <em><strong> rather than crossing.
s = emphasize(s, '*', 3, "<em><strong>", "</strong></em>")
s = emphasize(s, '*', 2, "<strong>", "</strong>")
s = emphasize(s, '_', 2, "<strong>", "</strong>")
s = emphasize(s, '*', 1, "<em>", "</em>")
s = emphasize(s, '_', 1, "<em>", "</em>")
// Restore code spans.
for i, code := range codes {
s = strings.Replace(s, "\x00"+strconv.Itoa(i)+"\x00", code, 1)
}
// Restore backslash-escaped characters as their literal selves.
// (Placeholders that sat inside a code span were already restored to
// their "\X" source form up in the code-extraction pass and no longer
// occur here; Replace on an absent placeholder is a no-op.)
for i, ch := range escaped {
s = strings.Replace(s, "\x01"+strconv.Itoa(i)+"\x01", ch, 1)
}
return s
}
// emphasize wraps the text between two runs of k delimiters c ('*' or '_')
// in open/close. It pairs them as CommonMark does: left to right, a run
// that can close gives its first k delimiters to the nearest unpaired
// opener before it, and a run that can open offers its last k to a later
// closer. Which runs can open or close follows CommonMark's flanking rules
// (delimiterRun).
func emphasize(s string, c byte, k int, open, close string) string {
var openers []int // offsets of unpaired openers, nearest last
tags := map[int]string{} // offset of k paired delimiters -> their tag
for i := 0; i < len(s); {
if s[i] != c {
i++
continue
}
end := i
for end < len(s) && s[end] == c {
end++
}
canOpen, canClose := delimiterRun(s, i, end, c)
n := end - i
if n >= k && canClose && len(openers) > 0 {
tags[openers[len(openers)-1]], tags[i] = open, close
openers = openers[:len(openers)-1]
n -= k
}
if n >= k && canOpen {
openers = append(openers, end-k)
}
i = end
}
if len(tags) == 0 {
return s
}
var b strings.Builder
for i := 0; i < len(s); i++ {
if t, ok := tags[i]; ok {
b.WriteString(t)
i += k - 1
} else {
b.WriteByte(s[i])
}
}
return b.String()
}
// delimiterRun reports whether the run of delimiters c at s[start:end]
// can open and can close emphasis, by CommonMark's flanking rules, which
// look at the whole run.
func delimiterRun(s string, start, end int, c byte) (canOpen, canClose bool) {
prev, next := ' ', ' '
if start > 0 {
prev, _ = utf8.DecodeLastRuneInString(s[:start])
}
if end < len(s) {
next, _ = utf8.DecodeRuneInString(s[end:])
}
left := !unicode.IsSpace(next) &&
(!isPunct(next) || unicode.IsSpace(prev) || isPunct(prev))
right := !unicode.IsSpace(prev) &&
(!isPunct(prev) || unicode.IsSpace(next) || isPunct(next))
if c == '_' {
return left && (!right || isPunct(prev)), right && (!left || isPunct(next))
}
return left, right
}
// isPunct is CommonMark's punctuation (Unicode P and S). The \x00/\x01
// placeholders count too: they stand for a code span (opened by a
// backtick) or a backslash-escaped punctuation character.
func isPunct(r rune) bool {
return r == 0 || r == 1 || unicode.IsPunct(r) || unicode.IsSymbol(r)
}
// -- Block-level parser --------------------------------------------
type parser struct {
lines []string
pos int
out strings.Builder
open []listLevel // the lists parseList is inside, outermost first
// numbered holds the last number of the ordered list at each indent,
// for as long as list lines keep coming (see endsItem).
numbered map[int]int
// top is the source's highest heading level and last the level of the
// heading emitted last (see parseHeading).
top, last int
}
// listLevel is one open list: the indent of its items and their kind.
type listLevel struct {
indent int
ordered bool
}
func newParser(src string) *parser {
src = strings.ReplaceAll(src, "\r\n", "\n")
src = strings.ReplaceAll(src, "\r", "\n")
p := &parser{lines: strings.Split(src, "\n"), top: 7, last: 1}
fence := ""
for _, line := range p.lines {
switch {
case fence != "":
if strings.HasPrefix(line, fence) {
fence = ""
}
case isFencedCodeStart(line):
fence = line[:3]
case isHeading(line):
p.top = min(p.top, headingLevel(line))
}
}
return p
}
func (p *parser) peek() (string, bool) {
if p.pos >= len(p.lines) {
return "", false
}
return p.lines[p.pos], true
}
func (p *parser) next() (string, bool) {
l, ok := p.peek()
if ok {
p.pos++
}
return l, ok
}
func (p *parser) emit(s string) { p.out.WriteString(s) }
// parse processes all lines and returns HTML.
func (p *parser) parse() string {
for p.pos < len(p.lines) {
line, _ := p.peek()
if !isListItem(line) {
p.numbered = nil // the run of list lines is over
}
switch {
case isFencedCodeStart(line):
p.parseFencedCode()
case isHR(line):
p.next()
p.emit("<hr>\n")
case isHeading(line):
p.parseHeading()
case isTableRow(line):
p.parseTable()
case isListItem(line):
p.parseList(reOrderedItem.MatchString(line), indentOf(line))
case strings.TrimSpace(line) == "":
p.next()
default:
p.parseParagraph()
}
}
return p.out.String()
}
// -- Heading --------------------------------------------------
// headingLevel returns the heading level of an ATX heading line - one to
// six '#' followed by whitespace or the end of the line - or 0 when the
// line is not a heading.
func headingLevel(s string) int {
n := 0
for n < len(s) && s[n] == '#' {
n++
}
if n == 0 || n > 6 || (n < len(s) && s[n] != ' ' && s[n] != '\t') {
return 0
}
return n
}
func isHeading(s string) bool { return headingLevel(s) > 0 }
// parseHeading emits a heading under the page's own <h1>: the source's top
// level becomes <h2> and no heading skips a level. The md-hN class keeps the
// size of the level the source wrote.
func (p *parser) parseHeading() {
line, _ := p.next()
level := headingLevel(line)
content := strings.TrimSpace(line[level:])
tag := min(level-p.top+2, p.last+1, 6)
p.last = tag
p.emit(fmt.Sprintf("<h%d class=\"md-h%d\">%s</h%d>\n", tag, level, parseInline(content), tag))
}
// -- Horizontal rule -------------------------------------------------
// isHR reports whether s is a thematic break: three or more of the same
// '-', '*' or '_', optionally with spaces or tabs between them.
func isHR(s string) bool {
t := strings.TrimSpace(s)
if t == "" || (t[0] != '-' && t[0] != '*' && t[0] != '_') {
return false
}
n := 0
for i := 0; i < len(t); i++ {
switch t[i] {
case t[0]:
n++
case ' ', '\t':
default:
return false
}
}
return n >= 3
}
// Fenced code block ---------------------------------------------------------
func isFencedCodeStart(s string) bool {
return strings.HasPrefix(s, "```") || strings.HasPrefix(s, "~~~")
}
func (p *parser) parseFencedCode() {
opener, _ := p.next()
fence := opener[:3]
var body strings.Builder
for {
line, ok := p.next()
if !ok || strings.HasPrefix(line, fence) {
break
}
body.WriteString(html.EscapeString(line))
body.WriteByte('\n')
}
p.emit("<pre><code>" + body.String() + "</code></pre>\n")
}
// -- Lists (unified, handles nesting) -----------------------------------------
// indentOf returns the number of leading whitespace characters in s.
func indentOf(s string) int {
return len(s) - len(strings.TrimLeft(s, " \t"))
}
// isListItem returns true if the line is a ul or ol item at any indent.
// A thematic break wins over a bullet ("* * *" is a rule, not an item).
func isListItem(s string) bool {
// A list marker is the first non-space byte: rule out the rest (most
// lines) before any regex runs.
t := strings.TrimLeft(s, "\t\n\f\r ")
if t == "" || (t[0] != '-' && t[0] != '*' && t[0] != '+' && (t[0] < '0' || t[0] > '9')) {
return false
}
return !isHR(s) && (reUnorderedItem.MatchString(s) || reOrderedItem.MatchString(s))
}
// itemContent strips the list marker and returns the item text (possibly
// empty for a bare marker like "6.").
func itemContent(s string) string {
if sub := reUnorderedItem.FindStringSubmatch(s); sub != nil {
return sub[2]
}
if sub := reOrderedItem.FindStringSubmatch(s); sub != nil {
return sub[3]
}
return strings.TrimSpace(s)
}
// orderedNum returns the source number of an ordered-list line ("7." -> 7)
// and true, or false if the line isn't an ordered item. "0." is a real
// number, so it cannot double as the "not ordered" answer.
func orderedNum(s string) (int, bool) {
if sub := reOrderedItem.FindStringSubmatch(s); sub != nil {
n, _ := strconv.Atoi(sub[2]) // at most nine digits: cannot fail
return n, true
}
return 0, false
}
// startsList reports whether list-item line s may open a new list on the
// line after text, where it would otherwise continue that text.
func startsList(s string) bool {
if itemContent(s) == "" {
return false
}
n, ok := orderedNum(s)
return !ok || n == 1
}
// endsItem reports whether line s, met while collecting the lazily
// continued text of a list item, ends that item rather than continuing it.
func (p *parser) endsItem(s string) bool {
if isHR(s) || isHeading(s) || isFencedCodeStart(s) || isTableRow(s) {
return true
}
if !isListItem(s) {
return false
}
if startsList(s) {
return true
}
indent, ordered := indentOf(s), reOrderedItem.MatchString(s)
for _, l := range p.open {
if l.indent == indent && l.ordered == ordered {
return true
}
}
if n, ok := orderedNum(s); ok {
for _, last := range p.numbered {
if n > last && n <= last+2 {
return true
}
}
}
return false
}
// parseList recursively parses a ul or ol at the given indent level.
func (p *parser) parseList(ordered bool, minIndent int) {
tag := "ul"
if ordered {
tag = "ol"
}
p.emit("<" + tag + ">\n")
p.open = append(p.open, listLevel{minIndent, ordered})
defer func() { p.open = p.open[:len(p.open)-1] }()
expected := 0 // running ordered-list counter, for value="N" decisions
for {
line, ok := p.peek()
if !ok || strings.TrimSpace(line) == "" || !isListItem(line) {
break
}
indent := indentOf(line)
if indent < minIndent {
// Belongs to a parent list - stop.
break
}
if indent == minIndent && reOrderedItem.MatchString(line) != ordered {
// Bullets after numbers, or numbers after bullets: a new
// list (as in CommonMark). Kept in this one, a number would
// lose itself to a bullet and a bullet would gain a number.
break
}
if indent > minIndent {
// Deeper indent mid-list without a parent <li> - treat as nested.
p.parseList(reOrderedItem.MatchString(line), indent)
continue
}
p.next()
// Item body: the marker line plus any lazily-continued lines after it.
var body []string
if c := itemContent(line); c != "" {
body = append(body, c)
}
for {
cont, ok := p.peek()
if !ok || strings.TrimSpace(cont) == "" || p.endsItem(cont) {
break
}
p.next()
body = append(body, strings.TrimSpace(cont))
}
liOpen := "<li>"
if ordered {
n, _ := orderedNum(line)
expected++
if n != expected {
liOpen = "<li value=\"" + strconv.Itoa(n) + "\">"
expected = n
}
if p.numbered == nil {
p.numbered = map[int]int{}
}
p.numbered[minIndent] = n
}
var parts []string
for _, l := range body {
parts = append(parts, parseInline(l))
}
text := strings.Join(parts, "<br>\n")
// Peek: if next line is deeper, nest a list inside this <li> -
// as many as follow, since bullets turning into numbers down
// there start a second nested list, which belongs here too.
next, hasNext := p.peek()
if hasNext && isListItem(next) && indentOf(next) > indent {
p.emit(liOpen + text + "\n")
for hasNext && isListItem(next) && indentOf(next) > indent {
p.parseList(reOrderedItem.MatchString(next), indentOf(next))
next, hasNext = p.peek()
}
p.emit("</li>\n")
} else {
p.emit(liOpen + text + "</li>\n")
}
}
p.emit("</" + tag + ">\n")
}
// -- Table ---------------------------------------------------------------------
func isTableRow(s string) bool {
t := strings.TrimSpace(s)
return len(t) >= 3 && t[0] == '|' && t[len(t)-1] == '|' && !strings.Contains(t, "\n")
}
// splitCells splits a pipe-delimited row into trimmed cell strings.
func splitCells(row string) []string {
row = strings.TrimPrefix(strings.TrimSpace(row), "|")
if strings.HasSuffix(row, "|") && !strings.HasSuffix(row, `\|`) {
row = row[:len(row)-1]
}
var cells []string
var cell strings.Builder
for i := 0; i < len(row); i++ {
switch {
case row[i] == '\\' && i+1 < len(row) && row[i+1] == '|':
cell.WriteByte('|')
i++
case row[i] == '|':
cells = append(cells, strings.TrimSpace(cell.String()))
cell.Reset()
default:
cell.WriteByte(row[i])
}
}
return append(cells, strings.TrimSpace(cell.String()))
}
func (p *parser) parseTable() {
headerLine, _ := p.next()
headers := splitCells(headerLine)
// Consume separator row (|---|---|) if present.
if sep, ok := p.peek(); ok && reTableSep.MatchString(strings.TrimSpace(sep)) {
p.next()
}
p.emit("<div class=\"table-scroll\">\n<table>\n<thead>\n<tr>\n")
for _, h := range headers {
p.emit("<th>" + parseInline(h) + "</th>\n")
}
p.emit("</tr>\n</thead>\n<tbody>\n")
for {
line, ok := p.peek()
if !ok || !isTableRow(line) {
break
}
p.next()
p.emit("<tr>\n")
for _, cell := range splitCells(line) {
p.emit("<td>" + parseInline(cell) + "</td>\n")
}
p.emit("</tr>\n")
}
p.emit("</tbody>\n</table>\n</div>\n")
}
// -- Paragraph -----------------------------------------------------------------
func (p *parser) parseParagraph() {
var lines []string
for {
line, ok := p.peek()
if !ok || strings.TrimSpace(line) == "" {
break
}
if isHeading(line) || isFencedCodeStart(line) || isHR(line) ||
(isListItem(line) && startsList(line)) || isTableRow(line) {
break
}
p.next()
lines = append(lines, strings.TrimSpace(line))
}
if len(lines) > 0 {
var parts []string
for _, l := range lines {
parts = append(parts, parseInline(l))
}
p.emit("<p>" + strings.Join(parts, "<br>\n") + "</p>\n")
}
}
// Render converts Markdown src to an HTML string.
func Render(src string) string {
return newParser(src).parse()
}