rdb/tools/handbuch_md.rb

#!/usr/bin/env ruby
# frozen_string_literal: true

# Turns doc/handbuch.txt (OCR, one run of text per page) into doc/handbuch.md:
# one section per book section, running page headers dropped, wrapped lines
# joined into paragraphs, commands and table output in code blocks, and a
# marker with PDF and book page before each page's text.
#
#   ruby tools/handbuch_md.rb [in.txt [out.md]]
#
# The OCR is not corrected. The conversion is heuristic (see prose?/heading?),
# so doc/handbuch.txt stays the source and the scan is the final authority.

SRC = ARGV[0] || File.expand_path('../doc/handbuch.txt', __dir__)
DST = ARGV[1] || File.expand_path('../doc/handbuch.md', __dir__)

PAGE = /\A==== PDF-Seite (\d+)(?: \/ Buchseite (\d+))? ====\z/
SHELLISH = %r{[|<>$=\[\]{}\\~`]|^\s*[#/]|\b(?:echo|cat|cd|sh|awk|sed|sort|ls|exec|read|set|if|then|fi|done|do|for|while|case|esac)\b\s}
SECTION = /\ASection (\d+|[A-Z]):\z/

# A book page starts with a running header that carries the book page number:
# "90 Shell Programming" on even pages, "Shell Programming 91" on odd ones.
def header?(line, book)
  book && line.size <= 70 && (line.match?(/\A#{book}[ _]+\S/) || line.match?(/\S[ _]+#{book}\z/))
end

# Page number and tab label ("TUTORIAL", "LEXICON") at the foot of a page.
def footer?(block, book)
  block.size == 1 && (block[0] == book.to_s || %w[TUTORIAL LEXICON].include?(block[0]))
end

# Prose goes through Markdown: keep its * _ < > from being read as markup.
def escape(text)
  text.gsub(/([*_<>`\\])/, '\\\\\1')
end

def prose?(lines)
  return false if lines.size == 1

  longest = lines.map(&:size).max
  return true if longest >= 80 && lines.count { |l| l.match?(SHELLISH) } * 2 < lines.size

  lines.first.size >= 70 && lines.none? { |l| l.match?(/\A\s*[$#]|\|\s*\w+/) } && lines.last.match?(/[a-z.,;:)]\z/)
end

# One line that looks like a command or a name from a listing, not a sentence.
def command?(line)
  (line.match?(SHELLISH) && line.split.size < 6) || (line.match?(/\A[a-z]/) && line.size <= 40 && line !~ /[.!?:]\z/)
end

def heading?(lines, following)
  return false unless lines.size == 1

  l = lines[0]
  return false if l.size > 70 || l.size < 4 || l.match?(/[.,;]\z/) || l !~ /\A[A-Z0-9]/
  return false if l.match?(SHELLISH) || l.count(' ') < 1 || l.count(' ') > 9
  return false unless following && prose?(following)

  l.scan(/\b[A-Za-z][a-z]*\b/).first(6).count { |w| w.match?(/\A[A-Z]/) || %w[and or of the to in for a an with on from by is as at].include?(w) } >= [l.split.size - 1, 1].max
end

def join(lines)
  lines.each_with_object(+'') do |l, out|
    if out.empty? then out << l
    elsif out.match?(/[a-z]-\z/) && l.match?(/\A[a-z]/) then out.chop!; out << l
    else out << ' ' << l
    end
  end
end

pages = []
cur = nil
File.readlines(SRC, chomp: true).each do |line|
  if (m = line.match(PAGE))
    cur = { pdf: m[1].to_i, book: m[2]&.to_i, lines: [] }
    pages << cur
  elsif cur
    cur[:lines] << line.rstrip
  end
end

out = +"# COHERENT /rdb Relational Database Manager\n\n"
out << "*Mark Williams Company, 1992.* Text per OCR (tesseract) aus dem Scan, nicht korrigiert: Tabulatoren fehlen, Tabellen und Zeichen können falsch sein. ".dup
out << "Im Zweifel gilt der Scan. Dieses Dokument entsteht aus `doc/handbuch.txt` mit `tools/handbuch_md.rb`; Änderungen gehören dorthin oder in den Konverter.\n\n"

pages.each do |pg|
  lines = pg[:lines].dup
  label = pg[:book] ? "PDF-Seite #{pg[:pdf]} · Buchseite #{pg[:book]}" : "PDF-Seite #{pg[:pdf]}"
  out << %(<a id="s#{pg[:pdf]}"></a>\n*— #{label} —*\n\n)

  # Front matter (cover, copyright, contents): keep as it is, in a block.
  if pg[:book].nil?
    text = lines.join("\n").strip
    out << "```text\n#{text}\n```\n\n" unless text.empty?
    next
  end

  lines.shift while lines.first&.strip&.empty?
  lines.shift if lines.first && header?(lines.first, pg[:book])
  blocks = lines.slice_when { |a, b| a.strip.empty? != b.strip.empty? }.reject { |b| b.first.strip.empty? }
  blocks.reject! { |b| footer?(b.map(&:strip), pg[:book]) }
  blocks.each_with_index do |b, i|
    b = b.map(&:strip)
    nxt = blocks[i + 1]&.map(&:strip)
    if b[0].match?(SECTION) && b.size <= 2
      title = b[1] || (nxt&.size == 1 ? nxt.first : nil)
      blocks[i + 1] = [''] if b.size == 1 && title # consumed
      out << "# #{b[0].sub(/:\z/, '')}#{" — #{title}" if title}\n\n"
    elsif b == ['']
      next
    elsif heading?(b, nxt)
      out << "## #{b[0]}\n\n"
    elsif prose?(b) || (b.size == 1 && !command?(b[0]))
      # list items inside a paragraph: "1. ..." or an OCR bullet (* ° •)
      item = /\A(\d+\.|[*°•-])\s+/
      b.slice_before { |l| l.match?(item) }.each do |part|
        if part.first.match?(item)
          out << part.then { |l| join(l) }.sub(item) { Regexp.last_match(1).match?(/\d/) ? Regexp.last_match(0) : '- ' }.then { |t| escape(t) } << "\n"
        else
          out << "\n" << escape(join(part)) << "\n\n"
        end
      end
      out << "\n"
    else
      out << "```text\n#{b.join("\n")}\n```\n\n"
    end
  end
end

File.write(DST, out.gsub(/\n{3,}/, "\n\n"))
puts "#{DST}: #{pages.size} Seiten, #{out.lines.size} Zeilen"