DSL: Kurzschritte rename, select, cast, default; transform, Parallelmodus und String-Bedingungen veraltet

This commit is contained in:
wkh
2026-10-09 21:44:13 +02:00
parent f82fe50655
commit fffb2e4961
5 changed files with 192 additions and 2 deletions
+76
View File
@@ -0,0 +1,76 @@
require_relative '../test_helper'
# rename, select, cast, default and the deprecation warnings
class StepsTest < PipelineTest
ROWS = [{ 'kunnr' => '1', 'betrag' => '12.50', 'datum' => '2026-10-09', 'land' => nil },
{ 'kunnr' => '2', 'betrag' => '', 'datum' => ' ', 'land' => 'DE' }].freeze
def test_rename_keeps_the_position
assert_equal [%w[kundennr betrag datum land]] * 2, output(ROWS) { rename 'kunnr' => 'kundennr' }.map(&:keys)
assert_equal [{ 'b' => 1 }], output([{ a: 1 }]) { rename a: 'b' }
end
def test_select_order_and_missing_keys
assert_equal [{ 'betrag' => '12.50', 'kunnr' => '1', 'x' => nil }], output(ROWS.first(1)) { select %w[betrag kunnr x] }
assert_equal [{ 'kunnr' => '1' }], output([{ kunnr: '1', b: 2 }]) { select 'kunnr' }
end
def test_cast_types_and_empty_values
rows = output(ROWS) { cast 'betrag' => :decimal, 'datum' => :date, 'kunnr' => :integer }
assert_equal [1, 12.5, Date.new(2026, 10, 9)], rows[0].values_at('kunnr', 'betrag', 'datum')
assert_equal [2, nil, nil], rows[1].values_at('kunnr', 'betrag', 'datum')
assert_equal [{ 'a' => '5' }], output([{ 'a' => 5 }]) { cast 'a' => :string }
assert_includes assert_raises(ArgumentError) { output(ROWS) { cast 'a' => :money } }.message, 'money unknown'
end
def test_cast_error_goes_to_on_error
target, stats = run_pipeline([{ 'n' => 'x' }, { 'n' => '3' }]) { on_error :skip; cast 'n' => :integer }
assert_equal [{ 'n' => 3 }], target.output
assert_equal [{ step: 'cast#1', error: 'n: "x" is not integer', row: { 'n' => 'x' } }], stats.errors
end
def test_default_value_and_lambda
rows = output(ROWS) { default 'land' => 'CH', 'datum' => ->(row) { "neu #{row['kunnr']}" }, 'neu' => 0 }
assert_equal [['CH', '2026-10-09', 0], ['DE', 'neu 2', 0]], rows.map { |r| r.values_at('land', 'datum', 'neu') }
end
def test_steps_chain_with_map_and_validate
rows = output(ROWS) do
rename 'kunnr' => 'id'
default 'betrag' => '0'
cast 'betrag' => :decimal
validate 'id' => { required: true }
select %w[id betrag]
end
assert_equal [{ 'id' => '1', 'betrag' => 12.5 }, { 'id' => '2', 'betrag' => 0.0 }], rows
end
def deprecation_log(&block)
log = StringIO.new
DataPipeline.new do
logger Logger.new(log)
read MemoryStorage.new(rows: [{ 'a' => 1 }])
write MemoryStorage.new
instance_eval(&block)
end.run
log.string.lines.grep(/deprecated/)
end
def test_deprecations_once_per_process
lines = deprecation_log do
transform(condition: 'true') { |r| r }
transform { |r| r }
end
assert_equal 3, lines.size
assert lines.any? { |l| l.include?('transform is deprecated') }
assert lines.any? { |l| l.include?('parallel mode') }
assert lines.any? { |l| l.include?('condition: as string') }
assert_empty deprecation_log { transform { |r| r } }
end
def test_no_deprecation_for_map_filter_and_single_transform_in_chained_mode
assert_empty deprecation_log { map { |r| r }; filter(condition: ->(_r) { true }) { true } }
lines = deprecation_log { chained; transform { |r| r }; transform { |r| r } }
assert_equal 1, lines.size # only transform, no parallel mode
end
end
+1
View File
@@ -55,6 +55,7 @@ module PipelineHelper
DataPipeline.reset_settings!
DataPipeline.runs.clear
DataPipeline.params.clear
DataPipeline.reset_deprecations!
end
# runs a pipeline from rows into a MemoryStorage, returns [target, stats]