wkh/datapipeline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Layout: Formatieren, Festbreiten-Plugin, Lesen von Satzarten-Dateien

Browse Source
This commit is contained in:
wkh
2026-10-09 21:42:33 +02:00
parent 48c900558b
commit f82fe50655
8 changed files with 414 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/datapipeline.rb
+1
View File
@@ -17,6 +17,7 @@ class DataPipeline
autoload :SqliteStorage, File.expand_path('datapipeline/storage/sqlite', __dir__)
autoload :ExcelStorage, File.expand_path('datapipeline/storage/excel', __dir__)
autoload :RestStorage, File.expand_path('datapipeline/storage/rest', __dir__)
autoload :FixedStorage, File.expand_path('datapipeline/storage/fixed', __dir__)
autoload :Slack, File.expand_path('datapipeline/notification/slack', __dir__)
# passed to header, trailer, map and filter
lib/datapipeline/layout.rb
+157 -8
View File
@@ -18,8 +18,15 @@ class DataPipeline
# keys of a row are matched without regard to case, string or symbol.
#
# write container(:csv, file: ..., layout: layout) -> values in field order, unknown keys raise
# write container(:fixed, file: ..., layout: layout) -> fixed width lines (see FixedStorage)
# read container(:csv, ..., layout: layout) -> rows with the field names as keys
# validate layout -> required, length, format per field
#
# format: true converts values to the text format of the field type, before checking and writing:
# N numbers with the decimals of the length (13.2 -> 2), rounded half up; numeric texts too
# DAT Date, Time or 'YYYY-MM-DD' -> YYYYMMDD
# TIM Time or 'HH:MM:SS' -> HHMMSS
#
class Layout
Field = Struct.new(:name, :type, :length, :required) do
def key = name.downcase
@@ -30,13 +37,30 @@ class DataPipeline
after ||= 0
[before - after, after]
end
# characters in a fixed width file: N 13.2 -> 14 (11 digits, point, 2 decimals), DAT 8, TIM 6
def width
case type
when 'DAT' then 8
when 'TIM' then 6
when 'N'
return unless length
before, after = digits
before + after + (after.positive? ? 1 : 0)
else length&.to_i
end
end
end
NUMBER = /\A[+-]?\d+(\.\d+)?\z/
attr_reader :records
def initialize type_field: nil, strict: true, &block
def initialize type_field: nil, strict: true, format: false, &block
@type_field = type_field&.to_s&.downcase
@strict = strict
@format = format
@records = {}
instance_eval(&block) if block
end
@@ -56,23 +80,148 @@ class DataPipeline
def values row
fields = fields_for(row)
normalized = normalize(row)
if @strict
unknown = normalized.keys - fields.map(&:key)
raise LayoutError, "#{record_type(row)}: fields not in layout: #{unknown.join(', ')}" if unknown.any?
end
fields.map { |field| normalized[field.key] }
unknown!(row, fields, normalized)
fields.map { |field| output(field, normalized[field.key]) }
end
def errors row
fields = fields_for(row)
normalized = normalize(row)
fields.flat_map { |field| check(field, normalized[field.key]) }
fields.flat_map { |field| check(field, output(field, normalized[field.key])) }
rescue LayoutError => e
[e.message]
end
# value as text in the format of the field type (see format: true)
def format_value field, value
case value
when nil then nil
when Numeric then field.type == 'N' ? format_number(field, value) : value.to_s
when Date, Time then format_time(field, value)
else format_text(field, value.to_s)
end
end
# one line of a fixed width file: text left aligned, numbers right aligned with zeros, empty fields blank
def line row
fields = fields_for(row)
normalized = normalize(row)
unknown!(row, fields, normalized)
fields.map { |field| fixed(field, format_value(field, normalized[field.key])) }.join
end
# values of a line in field order (csv without header) -> row with the field names as keys
def parse values
type, fields = find_record { |fields| values[type_index(fields)].to_s.strip }
raise LayoutError, "record type #{values[0].inspect} not in layout (#{@records.keys.join(', ')})" unless fields
raise LayoutError, "#{type}: #{values.size} values, layout has #{fields.size} fields" if @strict && values.size > fields.size
fields.each_with_index.to_h { |field, i| [field.key, values[i]] }
end
# a line of a fixed width file -> row with the field names as keys, values without padding, blank -> nil
def parse_line line
type, fields = find_record { |fields| slices(line, fields)[type_index(fields)].to_s.strip }
raise LayoutError, "record type of #{line[0, 20].inspect} not in layout (#{@records.keys.join(', ')})" unless fields
fields.zip(slices(line, fields)).to_h do |field, text|
text = field.type == 'AN' ? text.to_s.rstrip : text.to_s.strip
[field.key, text.empty? ? nil : text]
end
end
# fixed width needs a length for every AN and N field
def check_widths!
missing = @records.flat_map { |type, fields| fields.reject(&:width).map { |f| "#{type}.#{f.name}" } }
raise LayoutError, "fixed width needs a length for: #{missing.join(', ')}" if missing.any?
self
end
private
def output field, value
@format ? format_value(field, value) : value
end
def unknown! row, fields, normalized
return unless @strict
unknown = normalized.keys - fields.map(&:key)
raise LayoutError, "#{record_type(row)}: fields not in layout: #{unknown.join(', ')}" if unknown.any?
end
def find_record
@records.find { |type, fields| yield(fields) == type }
end
def type_index fields
@type_field ? fields.index { |f| f.key == @type_field } || 0 : 0
end
def slices line, fields
offset = 0
fields.map do |field|
text = line[offset, field.width]
offset += field.width
text
end
end
# 12.345 -> '12.35' for N 11.2; floats by their shortest text, so 12.345 is not 12.3449999...
def format_number field, value
_before, after = field.length ? field.digits : [nil, 0]
number = value.is_a?(Float) ? value.to_s.to_r : value.to_r
number = number.round(after, half: :up)
sign = number.negative? ? '-' : ''
int, frac = number.abs.divmod(1)
return "#{sign}#{int}" if after.zero?
"#{sign}#{int}.#{(frac * 10**after).round.to_s.rjust(after, '0')}"
end
def format_time field, value
case field.type
when 'DAT' then value.strftime('%Y%m%d')
when 'TIM' then value.respond_to?(:hour) ? value.strftime('%H%M%S') : value.to_s
else value.respond_to?(:hour) ? value.iso8601 : value.to_s
end
end
# numeric texts get the decimals of the field, ISO dates and times lose their separators
def format_text field, text
case field.type
when 'N'
_before, after = field.length ? field.digits : [nil, 0]
return text unless text.match?(NUMBER) && after.positive?
decimals = text.split('.')[1].to_s
decimals.length > after ? text : "#{text.split('.')[0]}.#{decimals.ljust(after, '0')}"
when 'DAT' then text.match?(/\A\d{4}-\d{2}-\d{2}\z/) ? text.delete('-') : text
when 'TIM' then text.match?(/\A\d{2}:\d{2}:\d{2}\z/) ? text.delete(':') : text
else text
end
end
def fixed field, text
width = field.width
return ' ' * width if text.nil? || text.empty?
raise LayoutError, "#{field.name}: #{text.inspect} longer than #{width}" if text.length > width
case field.type
when 'N' then raise LayoutError, "#{field.name}: #{text.inspect} is not a number" unless text.match?(NUMBER)
when 'DAT', 'TIM' then raise LayoutError, "#{field.name}: #{text.inspect} needs #{width} digits" unless text.match?(/\A\d{#{width}}\z/)
else return text.ljust(width)
end
return text if field.type != 'N'
sign = text.start_with?('-') ? '-' : ''
digits = text.delete('+-')
raise LayoutError, "#{field.name}: #{text.inspect} longer than #{width}" if sign.length + digits.length > width
sign + digits.rjust(width - sign.length, '0')
end
def record_type row
@type_field ? normalize(row)[@type_field] : row.values.first
end
@@ -95,7 +244,7 @@ class DataPipeline
end
def check_number field, text
return ["#{field.name}: not a number"] unless text.match?(/\A[+-]?\d+(\.\d+)?\z/)
return ["#{field.name}: not a number"] unless text.match?(NUMBER)
return [] unless field.length
before, after = field.digits
lib/datapipeline/storage/csv.rb
+3 -1
View File
@@ -10,7 +10,8 @@ class DataPipeline
# columns: fixed column order for writing (and keys for reading without headers)
# without columns the values are written in hash order
# layout: DataPipeline::Layout, field order per record type (several record types in one file),
# unknown fields raise; no header line
# unknown fields raise; no header line. reading with headers: false: keys from the fields
# of the record type
# encoding: encoding of the file, e.g. 'ISO-8859-1' or 'Windows-1252' (default UTF-8);
# read as UTF-8 strings, written transcoded from UTF-8; a UTF-8 BOM is skipped when reading
# replace: '?' -> characters the encoding cannot hold are replaced when writing (default: error)
@@ -131,6 +132,7 @@ class DataPipeline
# rows with headers are CSV::Row, without headers plain arrays
def to_hash row
return row.to_h if row.is_a?(CSV::Row)
return @options[:layout].parse(row) if @options[:layout]
keys = columns&.map(&:to_s) || (0...row.length).map(&:to_s)
keys.zip(row).to_h
lib/datapipeline/storage/fixed.rb
+77
View File
@@ -0,0 +1,77 @@
require_relative 'storage_plugin'
class DataPipeline
#
# fixed width file, one line per row, field widths from a DataPipeline::Layout
#
# options
# file: path of the file
# layout: DataPipeline::Layout, every AN and N field needs a length
# encoding: encoding of the file (default UTF-8), read as UTF-8, written transcoded
# replace: '?' -> characters the encoding cannot hold are replaced when writing (default: error)
# line_end: "\n" (default) or "\r\n"
#
# writing: values formatted as with Layout format: true, text left aligned with blanks, numbers right
# aligned with zeros (sign in front), empty fields blank; a value longer than its field raises LayoutError
# reading: record type from the type field, keys are the field names in lower case, values without padding,
# blank fields nil
#
class FixedStorage < StoragePlugin
def initialize **options, &block
@options = options
@file_name = options.fetch(:file) { raise ArgumentError, 'FixedStorage: option file missing' }
@layout = options.fetch(:layout) { raise ArgumentError, 'FixedStorage: option layout missing' }.check_widths!
@line_end = options.fetch(:line_end, "\n")
yield if block_given?
end
def read(&block)
return each_line.map { |line| @layout.parse_line(line) } unless block_given?
each_line { |line| yield @layout.parse_line(line) }
nil
end
def write data
data.each { |row| write_row row }
ensure
close
end
def write_row row
@file ||= File.open(@file_name, 'w', **write_options)
@file.write(@layout.line(row), @line_end)
end
def close
@file&.close
@file = nil
end
private
def utf8?
@options[:encoding].nil? || Encoding.find(@options[:encoding]) == Encoding::UTF_8
end
# empty lines are skipped
def each_line
return enum_for(:each_line) unless block_given?
encoding = utf8? ? 'bom|utf-8' : "#{@options[:encoding]}:utf-8"
File.foreach(@file_name, encoding:) do |line|
line = line.chomp
yield line unless line.strip.empty?
end
end
def write_options
return { encoding: 'utf-8' } if utf8?
options = { encoding: @options[:encoding] }
options.merge!(undef: :replace, invalid: :replace, replace: @options[:replace]) if @options[:replace]
options
end
end
end