263 lines
9.4 KiB
Ruby
263 lines
9.4 KiB
Ruby
require 'date'
|
|
|
|
class DataPipeline
|
|
class LayoutError < StandardError; end
|
|
|
|
#
|
|
# fixed field list per record type, for files with several record types (e.g. SAP CUDO)
|
|
#
|
|
# layout = DataPipeline::Layout.new do
|
|
# record 'DOHD', [['SATYP', 'AN', 4, true], ['LFDNR', 'N', 10, true], ['BELNR', 'AN', 10], ...]
|
|
# record 'LARC', [['SATYP', 'AN', 4, true], ['SUMME', 'N', 10, true]]
|
|
# end
|
|
#
|
|
# field: [name, type, length, required]
|
|
# type AN (text), N (number, length 10 or 13.2 = digits.decimals), DAT (YYYYMMDD), TIM (HHMMSS)
|
|
#
|
|
# the record type of a row is the value of type_field (default: the first value of the row).
|
|
# keys of a row are matched without regard to case, string or symbol.
|
|
#
|
|
# write container(:csv, file: ..., layout: layout) -> values in field order, unknown keys raise
|
|
# write container(:fixed, file: ..., layout: layout) -> fixed width lines (see FixedStorage)
|
|
# read container(:csv, ..., layout: layout) -> rows with the field names as keys
|
|
# validate layout -> required, length, format per field
|
|
#
|
|
# format: true converts values to the text format of the field type, before checking and writing:
|
|
# N numbers with the decimals of the length (13.2 -> 2), rounded half up; numeric texts too
|
|
# DAT Date, Time or 'YYYY-MM-DD' -> YYYYMMDD
|
|
# TIM Time or 'HH:MM:SS' -> HHMMSS
|
|
#
|
|
class Layout
|
|
Field = Struct.new(:name, :type, :length, :required) do
|
|
def key = name.downcase
|
|
|
|
# 13.2 -> [11, 2] digits before and after the point, 10 -> [10, 0]
|
|
def digits
|
|
before, after = length.to_s.split('.').map(&:to_i)
|
|
after ||= 0
|
|
[before - after, after]
|
|
end
|
|
|
|
# characters in a fixed width file: N 13.2 -> 14 (11 digits, point, 2 decimals), DAT 8, TIM 6
|
|
def width
|
|
case type
|
|
when 'DAT' then 8
|
|
when 'TIM' then 6
|
|
when 'N'
|
|
return unless length
|
|
|
|
before, after = digits
|
|
before + after + (after.positive? ? 1 : 0)
|
|
else length&.to_i
|
|
end
|
|
end
|
|
end
|
|
|
|
NUMBER = /\A[+-]?\d+(\.\d+)?\z/
|
|
|
|
attr_reader :records
|
|
|
|
def initialize type_field: nil, strict: true, format: false, &block
|
|
@type_field = type_field&.to_s&.downcase
|
|
@strict = strict
|
|
@format = format
|
|
@records = {}
|
|
instance_eval(&block) if block
|
|
end
|
|
|
|
def record type, fields
|
|
@records[type.to_s] = fields.map do |name, kind, length, required|
|
|
Field.new(name.to_s, (kind || 'AN').to_s.upcase, length, required ? true : false)
|
|
end
|
|
end
|
|
|
|
def fields_for row
|
|
type = record_type(row)
|
|
@records.fetch(type.to_s) { raise LayoutError, "record type #{type.inspect} not in layout (#{@records.keys.join(', ')})" }
|
|
end
|
|
|
|
# values in the order of the layout; unknown keys raise (strict), missing keys are nil
|
|
def values row
|
|
fields = fields_for(row)
|
|
normalized = normalize(row)
|
|
unknown!(row, fields, normalized)
|
|
fields.map { |field| output(field, normalized[field.key]) }
|
|
end
|
|
|
|
def errors row
|
|
fields = fields_for(row)
|
|
normalized = normalize(row)
|
|
fields.flat_map { |field| check(field, output(field, normalized[field.key])) }
|
|
rescue LayoutError => e
|
|
[e.message]
|
|
end
|
|
|
|
# value as text in the format of the field type (see format: true)
|
|
def format_value field, value
|
|
case value
|
|
when nil then nil
|
|
when Numeric then field.type == 'N' ? format_number(field, value) : value.to_s
|
|
when Date, Time then format_time(field, value)
|
|
else format_text(field, value.to_s)
|
|
end
|
|
end
|
|
|
|
# one line of a fixed width file: text left aligned, numbers right aligned with zeros, empty fields blank
|
|
def line row
|
|
fields = fields_for(row)
|
|
normalized = normalize(row)
|
|
unknown!(row, fields, normalized)
|
|
fields.map { |field| fixed(field, format_value(field, normalized[field.key])) }.join
|
|
end
|
|
|
|
# values of a line in field order (csv without header) -> row with the field names as keys
|
|
def parse values
|
|
type, fields = find_record { |fields| values[type_index(fields)].to_s.strip }
|
|
raise LayoutError, "record type #{values[0].inspect} not in layout (#{@records.keys.join(', ')})" unless fields
|
|
raise LayoutError, "#{type}: #{values.size} values, layout has #{fields.size} fields" if @strict && values.size > fields.size
|
|
|
|
fields.each_with_index.to_h { |field, i| [field.key, values[i]] }
|
|
end
|
|
|
|
# a line of a fixed width file -> row with the field names as keys, values without padding, blank -> nil
|
|
def parse_line line
|
|
type, fields = find_record { |fields| slices(line, fields)[type_index(fields)].to_s.strip }
|
|
raise LayoutError, "record type of #{line[0, 20].inspect} not in layout (#{@records.keys.join(', ')})" unless fields
|
|
|
|
fields.zip(slices(line, fields)).to_h do |field, text|
|
|
text = field.type == 'AN' ? text.to_s.rstrip : text.to_s.strip
|
|
[field.key, text.empty? ? nil : text]
|
|
end
|
|
end
|
|
|
|
# fixed width needs a length for every AN and N field
|
|
def check_widths!
|
|
missing = @records.flat_map { |type, fields| fields.reject(&:width).map { |f| "#{type}.#{f.name}" } }
|
|
raise LayoutError, "fixed width needs a length for: #{missing.join(', ')}" if missing.any?
|
|
|
|
self
|
|
end
|
|
|
|
private
|
|
|
|
def output field, value
|
|
@format ? format_value(field, value) : value
|
|
end
|
|
|
|
def unknown! row, fields, normalized
|
|
return unless @strict
|
|
|
|
unknown = normalized.keys - fields.map(&:key)
|
|
raise LayoutError, "#{record_type(row)}: fields not in layout: #{unknown.join(', ')}" if unknown.any?
|
|
end
|
|
|
|
def find_record
|
|
@records.find { |type, fields| yield(fields) == type }
|
|
end
|
|
|
|
def type_index fields
|
|
@type_field ? fields.index { |f| f.key == @type_field } || 0 : 0
|
|
end
|
|
|
|
def slices line, fields
|
|
offset = 0
|
|
fields.map do |field|
|
|
text = line[offset, field.width]
|
|
offset += field.width
|
|
text
|
|
end
|
|
end
|
|
|
|
# 12.345 -> '12.35' for N 11.2; floats by their shortest text, so 12.345 is not 12.3449999...
|
|
def format_number field, value
|
|
_before, after = field.length ? field.digits : [nil, 0]
|
|
number = value.is_a?(Float) ? value.to_s.to_r : value.to_r
|
|
number = number.round(after, half: :up)
|
|
sign = number.negative? ? '-' : ''
|
|
int, frac = number.abs.divmod(1)
|
|
return "#{sign}#{int}" if after.zero?
|
|
|
|
"#{sign}#{int}.#{(frac * 10**after).round.to_s.rjust(after, '0')}"
|
|
end
|
|
|
|
def format_time field, value
|
|
case field.type
|
|
when 'DAT' then value.strftime('%Y%m%d')
|
|
when 'TIM' then value.respond_to?(:hour) ? value.strftime('%H%M%S') : value.to_s
|
|
else value.respond_to?(:hour) ? value.iso8601 : value.to_s
|
|
end
|
|
end
|
|
|
|
# numeric texts get the decimals of the field, ISO dates and times lose their separators
|
|
def format_text field, text
|
|
case field.type
|
|
when 'N'
|
|
_before, after = field.length ? field.digits : [nil, 0]
|
|
return text unless text.match?(NUMBER) && after.positive?
|
|
|
|
decimals = text.split('.')[1].to_s
|
|
decimals.length > after ? text : "#{text.split('.')[0]}.#{decimals.ljust(after, '0')}"
|
|
when 'DAT' then text.match?(/\A\d{4}-\d{2}-\d{2}\z/) ? text.delete('-') : text
|
|
when 'TIM' then text.match?(/\A\d{2}:\d{2}:\d{2}\z/) ? text.delete(':') : text
|
|
else text
|
|
end
|
|
end
|
|
|
|
def fixed field, text
|
|
width = field.width
|
|
return ' ' * width if text.nil? || text.empty?
|
|
raise LayoutError, "#{field.name}: #{text.inspect} longer than #{width}" if text.length > width
|
|
|
|
case field.type
|
|
when 'N' then raise LayoutError, "#{field.name}: #{text.inspect} is not a number" unless text.match?(NUMBER)
|
|
when 'DAT', 'TIM' then raise LayoutError, "#{field.name}: #{text.inspect} needs #{width} digits" unless text.match?(/\A\d{#{width}}\z/)
|
|
else return text.ljust(width)
|
|
end
|
|
return text if field.type != 'N'
|
|
|
|
sign = text.start_with?('-') ? '-' : ''
|
|
digits = text.delete('+-')
|
|
raise LayoutError, "#{field.name}: #{text.inspect} longer than #{width}" if sign.length + digits.length > width
|
|
|
|
sign + digits.rjust(width - sign.length, '0')
|
|
end
|
|
|
|
def record_type row
|
|
@type_field ? normalize(row)[@type_field] : row.values.first
|
|
end
|
|
|
|
def normalize row
|
|
row.to_h { |key, value| [key.to_s.downcase, value] }
|
|
end
|
|
|
|
def check field, value
|
|
text = value.to_s
|
|
return(field.required ? ["#{field.name}: missing"] : []) if text.strip.empty?
|
|
|
|
case field.type
|
|
when 'N' then check_number(field, text)
|
|
when 'DAT' then valid_date?(text) ? [] : ["#{field.name}: not a date YYYYMMDD"]
|
|
when 'TIM' then text.match?(/\A([01]\d|2[0-3])[0-5]\d[0-5]\d\z/) ? [] : ["#{field.name}: not a time HHMMSS"]
|
|
else
|
|
field.length && text.length > field.length.to_i ? ["#{field.name}: longer than #{field.length}"] : []
|
|
end
|
|
end
|
|
|
|
def check_number field, text
|
|
return ["#{field.name}: not a number"] unless text.match?(NUMBER)
|
|
return [] unless field.length
|
|
|
|
before, after = field.digits
|
|
int, dec = text.delete('+-').split('.')
|
|
return ["#{field.name}: more than #{before} digits"] if int.length > before
|
|
return ["#{field.name}: more than #{after} decimals"] if dec && dec.length > after
|
|
|
|
[]
|
|
end
|
|
|
|
def valid_date? text
|
|
text.match?(/\A\d{8}\z/) && Date.valid_date?(text[0, 4].to_i, text[4, 2].to_i, text[6, 2].to_i)
|
|
end
|
|
end
|
|
end
|