125 lines
7.0 KiB
Ruby
125 lines
7.0 KiB
Ruby
require_relative '../test_helper'
|
|
|
|
class LayoutTest < PipelineTest
|
|
LAYOUT = DataPipeline::Layout.new do
|
|
record 'HEAD', [['SATYP', 'AN', 4, true], ['DATUM', 'DAT', 8, true], ['ZEIT', 'TIM', 6]]
|
|
record 'POS', [['SATYP', 'AN', 4, true], ['NR', 'N', 3, true], ['TEXT', 'AN', 5], ['BETRAG', 'N', '13.2']]
|
|
end
|
|
|
|
def test_values_in_field_order_any_key_case
|
|
assert_equal ['POS', '1', 'abc', nil], LAYOUT.values({ SATYP: 'POS', betrag: nil, 'text' => 'abc', 'NR' => '1' })
|
|
assert_equal ['HEAD', '20251103', nil], LAYOUT.values({ 'satyp' => 'HEAD', 'datum' => '20251103' })
|
|
end
|
|
|
|
def test_unknown_field_and_record_type
|
|
assert_includes assert_raises(DataPipeline::LayoutError) { LAYOUT.values({ 'satyp' => 'POS', 'extra' => 1 }) }.message, 'extra'
|
|
assert_includes assert_raises(DataPipeline::LayoutError) { LAYOUT.values({ 'satyp' => 'XXXX' }) }.message, 'XXXX'
|
|
lenient = DataPipeline::Layout.new(strict: false) { record 'A', [['SATYP']] }
|
|
assert_equal ['A'], lenient.values({ 'satyp' => 'A', 'extra' => 1 })
|
|
end
|
|
|
|
def test_type_field
|
|
layout = DataPipeline::Layout.new(type_field: 'art') { record 'X', [['ID'], ['ART']] }
|
|
assert_equal %w[7 X], layout.values({ 'id' => '7', 'art' => 'X' })
|
|
end
|
|
|
|
def test_errors
|
|
assert_empty LAYOUT.errors({ 'satyp' => 'POS', 'nr' => '001', 'text' => 'abcde', 'betrag' => '-123.45' })
|
|
assert_equal ['NR: missing', 'TEXT: longer than 5', 'BETRAG: more than 2 decimals'],
|
|
LAYOUT.errors({ 'satyp' => 'POS', 'text' => 'abcdef', 'betrag' => '1.234' })
|
|
assert_equal ['NR: more than 3 digits', 'BETRAG: not a number'], LAYOUT.errors({ 'satyp' => 'POS', 'nr' => '1000', 'betrag' => '1,5' })
|
|
assert_equal ['DATUM: not a date YYYYMMDD', 'ZEIT: not a time HHMMSS'], LAYOUT.errors({ 'satyp' => 'HEAD', 'datum' => '20251331', 'zeit' => '250000' })
|
|
assert_equal ['record type "?" not in layout (HEAD, POS)'], LAYOUT.errors({ 'satyp' => '?' })
|
|
end
|
|
|
|
def test_csv_with_layout
|
|
with_tmpdir do |dir|
|
|
rows = [{ SATYP: 'HEAD', DATUM: '20251103' }, { 'satyp' => 'POS', 'betrag' => '1.50', 'nr' => '1' }]
|
|
DataPipeline::CsvStorage.new(file: "#{dir}/l.ok", layout: LAYOUT, csv: { col_sep: '|', quote_char: '' }).write(rows)
|
|
assert_equal "HEAD|20251103|\nPOS|1||1.50\n", File.read("#{dir}/l.ok")
|
|
end
|
|
end
|
|
|
|
def test_validate_layout_abort_warn_and_ignore
|
|
rows = [{ 'satyp' => 'POS', 'nr' => '1' }, { 'satyp' => 'POS', 'text' => 'zu lang' }]
|
|
assert_raises(DataPipeline::ValidationError) { run_pipeline(rows) { validate LAYOUT } }
|
|
|
|
target, stats = run_pipeline(rows) { validate LAYOUT, warn: true }
|
|
assert_equal 2, target.output.size
|
|
assert_equal 1, stats.warnings
|
|
assert_match(/warnings=1/, stats.to_s)
|
|
|
|
_target, stats = run_pipeline(rows) { validate LAYOUT, warn: true, ignore: %w[NR TEXT] }
|
|
assert_equal 0, stats.warnings
|
|
end
|
|
|
|
def test_warning_log_is_capped
|
|
log = StringIO.new
|
|
stats = DataPipeline.new do
|
|
logger Logger.new(log)
|
|
read MemoryStorage.new(rows: Array.new(150) { { 'satyp' => 'POS' } })
|
|
write MemoryStorage.new
|
|
validate LAYOUT, warn: true
|
|
end.run
|
|
assert_equal 150, stats.warnings
|
|
assert_equal 101, log.string.lines.count { |l| l.include?('WARN') }
|
|
assert_includes log.string, 'more than 100 warnings'
|
|
end
|
|
|
|
def test_validate_hash_without_braces
|
|
target, stats = run_pipeline([{ 'a' => 'x' }, { 'a' => '1' }]) { on_error :skip; validate 'a' => { format: /\A\d\z/ } }
|
|
assert_equal [{ 'a' => '1' }], target.output
|
|
assert_equal 1, stats.rejected
|
|
end
|
|
|
|
FORMATTED = DataPipeline::Layout.new(format: true) do
|
|
record 'K', [['SATYP', 'AN', 2, true], ['DATUM', 'DAT'], ['ZEIT', 'TIM'], ['BETRAG', 'N', '11.2'], ['NR', 'N', 5], ['TEXT', 'AN', 6]]
|
|
end
|
|
|
|
def test_format_values_by_type
|
|
row = { 'satyp' => 'K', 'datum' => Date.new(2026, 10, 9), 'zeit' => Time.new(2026, 10, 9, 7, 5, 3), 'betrag' => 12.345, 'nr' => 7, 'text' => 'abc' }
|
|
assert_equal ['K', '20261009', '070503', '12.35', '7', 'abc'], FORMATTED.values(row)
|
|
assert_equal ['K', '20261009', '070503', '3.50', nil, nil], FORMATTED.values({ 'satyp' => 'K', 'datum' => '2026-10-09', 'zeit' => '07:05:03', 'betrag' => '3.5' })
|
|
assert_equal '-0.01', FORMATTED.values({ 'satyp' => 'K', 'betrag' => -0.005 })[3]
|
|
assert_equal '0.33', FORMATTED.values({ 'satyp' => 'K', 'betrag' => Rational(1, 3) })[3]
|
|
assert_equal '1.234', FORMATTED.values({ 'satyp' => 'K', 'betrag' => '1.234' })[3] # texts are not rounded
|
|
assert_equal ['K', nil, nil, nil, nil, nil], FORMATTED.values({ 'satyp' => 'K' })
|
|
assert_equal [nil, 12.345], [LAYOUT.values({ 'satyp' => 'HEAD' })[1], DataPipeline::Layout.new { record 'K', [['S'], ['B', 'N', '5.2']] }.values({ 's' => 'K', 'b' => 12.345 })[1]]
|
|
end
|
|
|
|
def test_errors_check_the_formatted_values
|
|
assert_empty FORMATTED.errors({ 'satyp' => 'K', 'datum' => Date.today, 'betrag' => 0.1 + 0.2 })
|
|
assert_equal ['BETRAG: more than 2 decimals'], DataPipeline::Layout.new { record 'K', [['S'], ['BETRAG', 'N', '11.2']] }.errors({ 's' => 'K', 'betrag' => 0.1 + 0.2 })
|
|
end
|
|
|
|
def test_fixed_line_and_parse_line
|
|
row = { 'satyp' => 'K', 'datum' => Date.new(2026, 10, 9), 'betrag' => -12.5, 'nr' => 7, 'text' => 'Müller' }
|
|
line = FORMATTED.line(row)
|
|
assert_equal 'K 20261009 -00000012.5000007Müller', line
|
|
assert_equal 2 + 8 + 6 + 12 + 5 + 6, line.length
|
|
assert_equal({ 'satyp' => 'K', 'datum' => '20261009', 'zeit' => nil, 'betrag' => '-00000012.50', 'nr' => '00007', 'text' => 'Müller' },
|
|
FORMATTED.parse_line(line))
|
|
assert_includes assert_raises(DataPipeline::LayoutError) { FORMATTED.line(row.merge('text' => 'zu lang')) }.message, 'longer than 6'
|
|
assert_includes assert_raises(DataPipeline::LayoutError) { FORMATTED.line(row.merge('nr' => 'x')) }.message, 'not a number'
|
|
assert_includes assert_raises(DataPipeline::LayoutError) { FORMATTED.line(row.merge('extra' => 1)) }.message, 'extra'
|
|
assert_includes assert_raises(DataPipeline::LayoutError) { FORMATTED.parse_line('XX') }.message, 'not in layout'
|
|
end
|
|
|
|
def test_check_widths
|
|
layout = DataPipeline::Layout.new { record 'A', [['SATYP', 'AN', 1], ['TEXT'], ['NR', 'N'], ['D', 'DAT']] }
|
|
assert_equal 'fixed width needs a length for: A.TEXT, A.NR', assert_raises(DataPipeline::LayoutError) { layout.check_widths! }.message
|
|
assert_same FORMATTED, FORMATTED.check_widths!
|
|
end
|
|
|
|
def test_parse_values_and_csv_read_with_layout
|
|
assert_equal({ 'satyp' => 'POS', 'nr' => '1', 'text' => nil, 'betrag' => nil }, LAYOUT.parse(%w[POS 1]))
|
|
assert_includes assert_raises(DataPipeline::LayoutError) { LAYOUT.parse(%w[HEAD 1 2 3]) }.message, '4 values'
|
|
with_tmpdir do |dir|
|
|
File.write("#{dir}/in.txt", "HEAD|20251103|\nPOS|1|abc|1.50\n")
|
|
rows = DataPipeline::CsvStorage.new(file: "#{dir}/in.txt", layout: LAYOUT, csv: { headers: false, col_sep: '|' }).read
|
|
assert_equal [{ 'satyp' => 'HEAD', 'datum' => '20251103', 'zeit' => nil },
|
|
{ 'satyp' => 'POS', 'nr' => '1', 'text' => 'abc', 'betrag' => '1.50' }], rows
|
|
end
|
|
end
|
|
end
|