Handbuch als Markdown: doc/handbuch.md aus dem OCR-Text, mit Konverter

tools/handbuch_md.rb

@@ -0,0 +1,131 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Turns doc/handbuch.txt (OCR, one run of text per page) into doc/handbuch.md:
+# one section per book section, running page headers dropped, wrapped lines
+# joined into paragraphs, commands and table output in code blocks, and a
+# marker with PDF and book page before each page's text.
+#
+#   ruby tools/handbuch_md.rb [in.txt [out.md]]
+#
+# The OCR is not corrected. The conversion is heuristic (see prose?/heading?),
+# so doc/handbuch.txt stays the source and the scan is the final authority.
+
+SRC = ARGV[0] || File.expand_path('../doc/handbuch.txt', __dir__)
+DST = ARGV[1] || File.expand_path('../doc/handbuch.md', __dir__)
+
+PAGE = /\A==== PDF-Seite (\d+)(?: \/ Buchseite (\d+))? ====\z/
+SHELLISH = %r{[|<>$=\[\]{}\\~`]|^\s*[#/]|\b(?:echo|cat|cd|sh|awk|sed|sort|ls|exec|read|set|if|then|fi|done|do|for|while|case|esac)\b\s}
+SECTION = /\ASection (\d+|[A-Z]):\z/
+
+# A book page starts with a running header that carries the book page number:
+# "90 Shell Programming" on even pages, "Shell Programming 91" on odd ones.
+def header?(line, book)
+  book && line.size <= 70 && (line.match?(/\A#{book}[ _]+\S/) || line.match?(/\S[ _]+#{book}\z/))
+end
+
+# Page number and tab label ("TUTORIAL", "LEXICON") at the foot of a page.
+def footer?(block, book)
+  block.size == 1 && (block[0] == book.to_s || %w[TUTORIAL LEXICON].include?(block[0]))
+end
+
+# Prose goes through Markdown: keep its * _ < > from being read as markup.
+def escape(text)
+  text.gsub(/([*_<>`\\])/, '\\\\\1')
+end
+
+def prose?(lines)
+  return false if lines.size == 1
+
+  longest = lines.map(&:size).max
+  return true if longest >= 80 && lines.count { |l| l.match?(SHELLISH) } * 2 < lines.size
+
+  lines.first.size >= 70 && lines.none? { |l| l.match?(/\A\s*[$#]|\|\s*\w+/) } && lines.last.match?(/[a-z.,;:)]\z/)
+end
+
+# One line that looks like a command or a name from a listing, not a sentence.
+def command?(line)
+  (line.match?(SHELLISH) && line.split.size < 6) || (line.match?(/\A[a-z]/) && line.size <= 40 && line !~ /[.!?:]\z/)
+end
+
+def heading?(lines, following)
+  return false unless lines.size == 1
+
+  l = lines[0]
+  return false if l.size > 70 || l.size < 4 || l.match?(/[.,;]\z/) || l !~ /\A[A-Z0-9]/
+  return false if l.match?(SHELLISH) || l.count(' ') < 1 || l.count(' ') > 9
+  return false unless following && prose?(following)
+
+  l.scan(/\b[A-Za-z][a-z]*\b/).first(6).count { |w| w.match?(/\A[A-Z]/) || %w[and or of the to in for a an with on from by is as at].include?(w) } >= [l.split.size - 1, 1].max
+end
+
+def join(lines)
+  lines.each_with_object(+'') do |l, out|
+    if out.empty? then out << l
+    elsif out.match?(/[a-z]-\z/) && l.match?(/\A[a-z]/) then out.chop!; out << l
+    else out << ' ' << l
+    end
+  end
+end
+
+pages = []
+cur = nil
+File.readlines(SRC, chomp: true).each do |line|
+  if (m = line.match(PAGE))
+    cur = { pdf: m[1].to_i, book: m[2]&.to_i, lines: [] }
+    pages << cur
+  elsif cur
+    cur[:lines] << line.rstrip
+  end
+end
+
+out = +"# COHERENT /rdb Relational Database Manager\n\n"
+out << "*Mark Williams Company, 1992.* Text per OCR (tesseract) aus dem Scan, nicht korrigiert: Tabulatoren fehlen, Tabellen und Zeichen können falsch sein. ".dup
+out << "Im Zweifel gilt der Scan. Dieses Dokument entsteht aus `doc/handbuch.txt` mit `tools/handbuch_md.rb`; Änderungen gehören dorthin oder in den Konverter.\n\n"
+
+pages.each do |pg|
+  lines = pg[:lines].dup
+  label = pg[:book] ? "PDF-Seite #{pg[:pdf]} · Buchseite #{pg[:book]}" : "PDF-Seite #{pg[:pdf]}"
+  out << %(<a id="s#{pg[:pdf]}"></a>\n*— #{label} —*\n\n)
+
+  # Front matter (cover, copyright, contents): keep as it is, in a block.
+  if pg[:book].nil?
+    text = lines.join("\n").strip
+    out << "```text\n#{text}\n```\n\n" unless text.empty?
+    next
+  end
+
+  lines.shift while lines.first&.strip&.empty?
+  lines.shift if lines.first && header?(lines.first, pg[:book])
+  blocks = lines.slice_when { |a, b| a.strip.empty? != b.strip.empty? }.reject { |b| b.first.strip.empty? }
+  blocks.reject! { |b| footer?(b.map(&:strip), pg[:book]) }
+  blocks.each_with_index do |b, i|
+    b = b.map(&:strip)
+    nxt = blocks[i + 1]&.map(&:strip)
+    if b[0].match?(SECTION) && b.size <= 2
+      title = b[1] || (nxt&.size == 1 ? nxt.first : nil)
+      blocks[i + 1] = [''] if b.size == 1 && title # consumed
+      out << "# #{b[0].sub(/:\z/, '')}#{" — #{title}" if title}\n\n"
+    elsif b == ['']
+      next
+    elsif heading?(b, nxt)
+      out << "## #{b[0]}\n\n"
+    elsif prose?(b) || (b.size == 1 && !command?(b[0]))
+      # list items inside a paragraph: "1. ..." or an OCR bullet (* ° •)
+      item = /\A(\d+\.|[*°•-])\s+/
+      b.slice_before { |l| l.match?(item) }.each do |part|
+        if part.first.match?(item)
+          out << part.then { |l| join(l) }.sub(item) { Regexp.last_match(1).match?(/\d/) ? Regexp.last_match(0) : '- ' }.then { |t| escape(t) } << "\n"
+        else
+          out << "\n" << escape(join(part)) << "\n\n"
+        end
+      end
+      out << "\n"
+    else
+      out << "```text\n#{b.join("\n")}\n```\n\n"
+    end
+  end
+end
+
+File.write(DST, out.gsub(/\n{3,}/, "\n\n"))
+puts "#{DST}: #{pages.size} Seiten, #{out.lines.size} Zeilen"