wkh/datapipeline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Layout: Formatieren, Festbreiten-Plugin, Lesen von Satzarten-Dateien

Browse Source
This commit is contained in:
wkh
2026-10-09 21:42:33 +02:00
parent 48c900558b
commit f82fe50655
8 changed files with 414 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/core/layout_test.rb
+50
View File
@@ -71,4 +71,54 @@ class LayoutTest < PipelineTest
assert_equal [{ 'a' => '1' }], target.output
assert_equal 1, stats.rejected
end
FORMATTED = DataPipeline::Layout.new(format: true) do
record 'K', [['SATYP', 'AN', 2, true], ['DATUM', 'DAT'], ['ZEIT', 'TIM'], ['BETRAG', 'N', '11.2'], ['NR', 'N', 5], ['TEXT', 'AN', 6]]
end
def test_format_values_by_type
row = { 'satyp' => 'K', 'datum' => Date.new(2026, 10, 9), 'zeit' => Time.new(2026, 10, 9, 7, 5, 3), 'betrag' => 12.345, 'nr' => 7, 'text' => 'abc' }
assert_equal ['K', '20261009', '070503', '12.35', '7', 'abc'], FORMATTED.values(row)
assert_equal ['K', '20261009', '070503', '3.50', nil, nil], FORMATTED.values({ 'satyp' => 'K', 'datum' => '2026-10-09', 'zeit' => '07:05:03', 'betrag' => '3.5' })
assert_equal '-0.01', FORMATTED.values({ 'satyp' => 'K', 'betrag' => -0.005 })[3]
assert_equal '0.33', FORMATTED.values({ 'satyp' => 'K', 'betrag' => Rational(1, 3) })[3]
assert_equal '1.234', FORMATTED.values({ 'satyp' => 'K', 'betrag' => '1.234' })[3] # texts are not rounded
assert_equal ['K', nil, nil, nil, nil, nil], FORMATTED.values({ 'satyp' => 'K' })
assert_equal [nil, 12.345], [LAYOUT.values({ 'satyp' => 'HEAD' })[1], DataPipeline::Layout.new { record 'K', [['S'], ['B', 'N', '5.2']] }.values({ 's' => 'K', 'b' => 12.345 })[1]]
end
def test_errors_check_the_formatted_values
assert_empty FORMATTED.errors({ 'satyp' => 'K', 'datum' => Date.today, 'betrag' => 0.1 + 0.2 })
assert_equal ['BETRAG: more than 2 decimals'], DataPipeline::Layout.new { record 'K', [['S'], ['BETRAG', 'N', '11.2']] }.errors({ 's' => 'K', 'betrag' => 0.1 + 0.2 })
end
def test_fixed_line_and_parse_line
row = { 'satyp' => 'K', 'datum' => Date.new(2026, 10, 9), 'betrag' => -12.5, 'nr' => 7, 'text' => 'Müller' }
line = FORMATTED.line(row)
assert_equal 'K 20261009 -00000012.5000007Müller', line
assert_equal 2 + 8 + 6 + 12 + 5 + 6, line.length
assert_equal({ 'satyp' => 'K', 'datum' => '20261009', 'zeit' => nil, 'betrag' => '-00000012.50', 'nr' => '00007', 'text' => 'Müller' },
FORMATTED.parse_line(line))
assert_includes assert_raises(DataPipeline::LayoutError) { FORMATTED.line(row.merge('text' => 'zu lang')) }.message, 'longer than 6'
assert_includes assert_raises(DataPipeline::LayoutError) { FORMATTED.line(row.merge('nr' => 'x')) }.message, 'not a number'
assert_includes assert_raises(DataPipeline::LayoutError) { FORMATTED.line(row.merge('extra' => 1)) }.message, 'extra'
assert_includes assert_raises(DataPipeline::LayoutError) { FORMATTED.parse_line('XX') }.message, 'not in layout'
end
def test_check_widths
layout = DataPipeline::Layout.new { record 'A', [['SATYP', 'AN', 1], ['TEXT'], ['NR', 'N'], ['D', 'DAT']] }
assert_equal 'fixed width needs a length for: A.TEXT, A.NR', assert_raises(DataPipeline::LayoutError) { layout.check_widths! }.message
assert_same FORMATTED, FORMATTED.check_widths!
end
def test_parse_values_and_csv_read_with_layout
assert_equal({ 'satyp' => 'POS', 'nr' => '1', 'text' => nil, 'betrag' => nil }, LAYOUT.parse(%w[POS 1]))
assert_includes assert_raises(DataPipeline::LayoutError) { LAYOUT.parse(%w[HEAD 1 2 3]) }.message, '4 values'
with_tmpdir do |dir|
File.write("#{dir}/in.txt", "HEAD|20251103|\nPOS|1|abc|1.50\n")
rows = DataPipeline::CsvStorage.new(file: "#{dir}/in.txt", layout: LAYOUT, csv: { headers: false, col_sep: '|' }).read
assert_equal [{ 'satyp' => 'HEAD', 'datum' => '20251103', 'zeit' => nil },
{ 'satyp' => 'POS', 'nr' => '1', 'text' => 'abc', 'betrag' => '1.50' }], rows
end
end
end
test/plugins/fixed_storage_test.rb
+63
View File
@@ -0,0 +1,63 @@
require_relative '../test_helper'
class FixedStorageTest < PipelineTest
LAYOUT = DataPipeline::Layout.new do
record 'KOPF', [['SATYP', 'AN', 4, true], ['BELNR', 'AN', 6, true], ['DATUM', 'DAT', 8]]
record 'POS', [['SATYP', 'AN', 4, true], ['TEXT', 'AN', 10], ['BETRAG', 'N', '7.2']]
end
ROWS = [{ 'satyp' => 'KOPF', 'belnr' => 'R1', 'datum' => Date.new(2026, 10, 9) },
{ 'satyp' => 'POS', 'text' => 'Müller', 'betrag' => 12.5 },
{ 'satyp' => 'POS', 'betrag' => -3 }].freeze
def test_write_and_read_back
with_tmpdir do |dir|
DataPipeline::FixedStorage.new(file: "#{dir}/out.txt", layout: LAYOUT).write(ROWS)
assert_equal "KOPFR1 20261009\nPOS Müller 00012.50\nPOS -0003.00\n", File.read("#{dir}/out.txt")
assert_equal [{ 'satyp' => 'KOPF', 'belnr' => 'R1', 'datum' => '20261009' },
{ 'satyp' => 'POS', 'text' => 'Müller', 'betrag' => '00012.50' },
{ 'satyp' => 'POS', 'text' => nil, 'betrag' => '-0003.00' }],
DataPipeline::FixedStorage.new(file: "#{dir}/out.txt", layout: LAYOUT).read
end
end
def test_encoding_line_end_and_streaming_read
with_tmpdir do |dir|
DataPipeline::FixedStorage.new(file: "#{dir}/out.txt", layout: LAYOUT, encoding: 'ISO-8859-1', line_end: "\r\n").write(ROWS.first(2))
assert_equal "KOPFR1 20261009\r\nPOS M\xFCller 00012.50\r\n".b, File.binread("#{dir}/out.txt")
rows = []
DataPipeline::FixedStorage.new(file: "#{dir}/out.txt", layout: LAYOUT, encoding: 'ISO-8859-1').read { |row| rows << row }
assert_equal 'Müller', rows.last['text']
end
end
def test_pipeline_with_container_and_dry_run
with_tmpdir do |dir|
rows = ROWS.map(&:dup)
out = capture_stdout do
DataPipeline.new do
logger PipelineHelper::QUIET
dry_run
read MemoryStorage.new(rows:)
write container(:fixed, file: "#{dir}/dry.txt", layout: LAYOUT)
end.run
end
refute File.exist?("#{dir}/dry.txt")
assert_includes out, 'DRY RUN'
DataPipeline.new do
logger PipelineHelper::QUIET
read MemoryStorage.new(rows:)
write container(:fixed, file: "#{dir}/out.txt", layout: LAYOUT)
end.run
assert_equal 3, File.readlines("#{dir}/out.txt").size
end
end
def test_options_and_lengths_required
assert_raises(ArgumentError) { DataPipeline::FixedStorage.new(layout: LAYOUT) }
assert_raises(ArgumentError) { DataPipeline::FixedStorage.new(file: 'x') }
layout = DataPipeline::Layout.new { record 'A', [['SATYP', 'AN']] }
assert_raises(DataPipeline::LayoutError) { DataPipeline::FixedStorage.new(file: 'x', layout:) }
end
end