require_relative '../test_helper' class LayoutTest < PipelineTest LAYOUT = DataPipeline::Layout.new do record 'HEAD', [['SATYP', 'AN', 4, true], ['DATUM', 'DAT', 8, true], ['ZEIT', 'TIM', 6]] record 'POS', [['SATYP', 'AN', 4, true], ['NR', 'N', 3, true], ['TEXT', 'AN', 5], ['BETRAG', 'N', '13.2']] end def test_values_in_field_order_any_key_case assert_equal ['POS', '1', 'abc', nil], LAYOUT.values({ SATYP: 'POS', betrag: nil, 'text' => 'abc', 'NR' => '1' }) assert_equal ['HEAD', '20251103', nil], LAYOUT.values({ 'satyp' => 'HEAD', 'datum' => '20251103' }) end def test_unknown_field_and_record_type assert_includes assert_raises(DataPipeline::LayoutError) { LAYOUT.values({ 'satyp' => 'POS', 'extra' => 1 }) }.message, 'extra' assert_includes assert_raises(DataPipeline::LayoutError) { LAYOUT.values({ 'satyp' => 'XXXX' }) }.message, 'XXXX' lenient = DataPipeline::Layout.new(strict: false) { record 'A', [['SATYP']] } assert_equal ['A'], lenient.values({ 'satyp' => 'A', 'extra' => 1 }) end def test_type_field layout = DataPipeline::Layout.new(type_field: 'art') { record 'X', [['ID'], ['ART']] } assert_equal %w[7 X], layout.values({ 'id' => '7', 'art' => 'X' }) end def test_errors assert_empty LAYOUT.errors({ 'satyp' => 'POS', 'nr' => '001', 'text' => 'abcde', 'betrag' => '-123.45' }) assert_equal ['NR: missing', 'TEXT: longer than 5', 'BETRAG: more than 2 decimals'], LAYOUT.errors({ 'satyp' => 'POS', 'text' => 'abcdef', 'betrag' => '1.234' }) assert_equal ['NR: more than 3 digits', 'BETRAG: not a number'], LAYOUT.errors({ 'satyp' => 'POS', 'nr' => '1000', 'betrag' => '1,5' }) assert_equal ['DATUM: not a date YYYYMMDD', 'ZEIT: not a time HHMMSS'], LAYOUT.errors({ 'satyp' => 'HEAD', 'datum' => '20251331', 'zeit' => '250000' }) assert_equal ['record type "?" not in layout (HEAD, POS)'], LAYOUT.errors({ 'satyp' => '?' }) end def test_csv_with_layout with_tmpdir do |dir| rows = [{ SATYP: 'HEAD', DATUM: '20251103' }, { 'satyp' => 'POS', 'betrag' => '1.50', 'nr' => '1' }] DataPipeline::CsvStorage.new(file: "#{dir}/l.ok", layout: LAYOUT, csv: { col_sep: '|', quote_char: '' }).write(rows) assert_equal "HEAD|20251103|\nPOS|1||1.50\n", File.read("#{dir}/l.ok") end end def test_validate_layout_abort_warn_and_ignore rows = [{ 'satyp' => 'POS', 'nr' => '1' }, { 'satyp' => 'POS', 'text' => 'zu lang' }] assert_raises(DataPipeline::ValidationError) { run_pipeline(rows) { validate LAYOUT } } target, stats = run_pipeline(rows) { validate LAYOUT, warn: true } assert_equal 2, target.output.size assert_equal 1, stats.warnings assert_match(/warnings=1/, stats.to_s) _target, stats = run_pipeline(rows) { validate LAYOUT, warn: true, ignore: %w[NR TEXT] } assert_equal 0, stats.warnings end def test_warning_log_is_capped log = StringIO.new stats = DataPipeline.new do logger Logger.new(log) read MemoryStorage.new(rows: Array.new(150) { { 'satyp' => 'POS' } }) write MemoryStorage.new validate LAYOUT, warn: true end.run assert_equal 150, stats.warnings assert_equal 101, log.string.lines.count { |l| l.include?('WARN') } assert_includes log.string, 'more than 100 warnings' end def test_validate_hash_without_braces target, stats = run_pipeline([{ 'a' => 'x' }, { 'a' => '1' }]) { on_error :skip; validate 'a' => { format: /\A\d\z/ } } assert_equal [{ 'a' => '1' }], target.output assert_equal 1, stats.rejected end end