wkh/datapipeline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

datapipeline 3.0.0: transform, Parallelmodus, chained und String-Bedingungen entfernt

Browse Source
This commit is contained in:
wkh
2026-10-09 22:04:12 +02:00
parent 0b72635e08
commit a8b8381293
13 changed files with 117 additions and 347 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/cli/etl_cli_test.rb
+4 -4
View File
@@ -14,8 +14,8 @@ class EtlCliTest < PipelineTest
"#{dir}/#{name}"
end
# like import.*.rb: own OptionParser for --from/--till, transform api
def test_legacy_script_gets_from_till_and_limit
# scripts with an own OptionParser for --from/--till
def test_script_with_own_option_parser_gets_from_till_and_limit
with_tmpdir do |dir|
path = script(dir, 'legacy.rb', <<~RUBY)
require 'optparse/date'
@@ -24,8 +24,8 @@ class EtlCliTest < PipelineTest
File.write('args.txt', "\#{opts[:from]} \#{opts[:till]}")
DataPipeline.new do
read container(:csv, file: 'in.csv')
transform(processing: :prepend) { |_, _| { 'h' => 'HEAD' } }
transform { |r, i| r.merge('i' => i) }
header { { 'h' => 'HEAD' } }
map { |r, ctx| r.merge('i' => ctx.index) }
write container(:csv, file: 'out.csv', csv: { headers: false, col_sep: '|' })
end.run
RUBY
test/core/chained_test.rb
+21 -15
View File
@@ -4,16 +4,22 @@ require_relative '../test_helper'
class ChainedTest < PipelineTest
ROWS = [{ 'a' => 1 }, { 'a' => 2 }, { 'a' => 3 }].freeze
def test_chained_transform_gets_previous_output
out = output(ROWS) { chained; transform { |r| r.merge('b' => 1) }; transform { |r| r.merge('c' => 2) } }
def test_each_step_gets_the_previous_output
out = output(ROWS) { map { |r| r.merge('b' => 1) }; map { |r| r.merge('c' => 2) } }
assert_equal ROWS.map { |r| r.merge('b' => 1, 'c' => 2) }, out
end
def test_one_to_many_then_index
out = output(ROWS) { chained; transform { |r| [r, r] }; transform { |r, i| r.merge('i' => i) } }
out = output(ROWS) { map { |r| [r, r] }; map { |r, ctx| r.merge('i' => ctx.index) } }
assert_equal (0...6).to_a, out.map { |r| r['i'] }
end
def test_nil_and_empty_array_drop_rows
assert_equal [{ 'a' => 1 }], output(ROWS) { map { |r| r['a'] == 1 ? r : nil } }
assert_equal ROWS, output(ROWS) { map { |r| [r, nil] } }
assert_equal [], output(ROWS) { map { [] } }
end
def test_map_and_filter
out = output(ROWS) do
map { |r| r.merge('d' => r['a'] * 2) }
@@ -23,12 +29,6 @@ class ChainedTest < PipelineTest
assert_equal [{ 'a' => 2, 'd' => 4, 'i' => 0 }, { 'a' => 3, 'd' => 6, 'i' => 1 }], out
end
def test_map_turns_chaining_on_for_transform
out = output(ROWS) { transform { |r| r.merge('b' => 1) }; map { |r| r.merge('c' => 1) } }
assert_equal 3, out.size
assert(out.all? { |r| r['b'] && r['c'] })
end
def test_header_and_trailer_with_context
out = output(ROWS) do
header { |c| { 'H' => c.read, 'W' => c.written } }
@@ -41,9 +41,14 @@ class ChainedTest < PipelineTest
assert_equal 6, out.size
end
def test_edge_order_follows_registration
out = output(ROWS) { header { { 'h' => 1 } }; transform(processing: :prepend) { { 'p' => 1 } }; map { |r| r } }
assert_equal [{ 'h' => 1 }, { 'p' => 1 }], out.first(2)
def test_several_headers_and_trailers_in_registration_order
out = output(ROWS) { header { { 'h' => 1 } }; trailer { { 't' => 1 } }; header { { 'h' => 2 } }; trailer { { 't' => 2 } } }
assert_equal [{ 'h' => 1 }, { 'h' => 2 }, *ROWS, { 't' => 1 }, { 't' => 2 }], out
end
def test_stable_order_of_many_steps
out = output(ROWS) { 20.times { |k| map { |r| r.merge("k#{k}" => k) } } }
assert_equal (0...20).to_a, out.first.values.drop(1)
end
def test_header_and_trailer_nil_and_arrays
@@ -51,12 +56,13 @@ class ChainedTest < PipelineTest
assert_equal ROWS + [{ 'x' => 1 }, { 'y' => 2 }], out
end
def test_condition_on_map
def test_condition_on_map_and_filter
assert_equal [{ 'a' => 1 }], output(ROWS) { map(condition: ->(r) { r['a'] == 1 }) { |r| r } }
assert_equal [{ 'a' => 3 }], output(ROWS) { filter(condition: ->(r) { r['a'] > 1 }) { |r| r['a'] == 3 } }
end
def test_chained_false
assert_equal 6, output(ROWS) { chained false; transform { |r| r }; transform { |r| r } }.size
def test_condition_must_be_a_lambda
assert_includes assert_raises(ArgumentError) { output(ROWS) { map(condition: "row['a'] == 1") { |r| r } } }.message, 'lambda'
end
def test_csv_end_to_end
test/core/compat_test.rb
-81
View File
@@ -1,81 +0,0 @@
require_relative '../test_helper'
#
# parallel mode (transform without chained/map/filter) must keep the output of v1.2.1,
# the version the CUDO importers were built on. expected values were recorded with v1.2.1.
#
class CompatTest < PipelineTest
ROWS = (1..6).map { |i| { 'id' => i, 'land' => i.even? ? 'DE' : 'AT' } }
def assert_output expected_json, &block
assert_equal expected_json, JSON.generate(output(ROWS, &block))
end
# pattern of import.customer.rb
def test_customer_pattern
assert_output '[{"SATYP":"FIRC","LFDNR":0},{"id":1,"land":"AT","lfdnr":"0000000001"},' \
'{"id":2,"land":"DE","lfdnr":"0000000002"},{"id":3,"land":"AT","lfdnr":"0000000003"},' \
'{"id":4,"land":"DE","lfdnr":"0000000004"},{"id":5,"land":"AT","lfdnr":"0000000005"},' \
'{"id":6,"land":"DE","lfdnr":"0000000006"},{"topic":"LARC","counter":"0000000006"}]' do
transform(processing: :prepend) { |_row, n| { 'SATYP': 'FIRC', 'LFDNR': n } }
transform(processing: :append) { |_row, n| { 'topic': 'LARC', 'counter': format('%010d', n) } }
transform { |row, _i| row.merge('lfdnr' => format('%010d', row['id'])) }
end
end
# pattern of import.posting.rb: closure counter, 1:n
def test_posting_pattern
expected = [{ SATYP: 'FIRC', LFDNR: 0 }]
lfdnr = 0
ROWS.each do |row|
expected << row.merge('lfdnr' => lfdnr += 1)
row['id'].times { |k| expected << { 'pos' => k, 'lfdnr' => lfdnr += 1 } }
end
expected << { SATYPE: 'LARC', SUMME: '0000000027' }
assert_output JSON.generate(expected) do
cnt = 1
transform(processing: :prepend) { |_row, _n| { 'SATYP': 'FIRC', 'LFDNR': 0 } }
transform do |row|
out = [row.merge('lfdnr' => cnt)]
cnt += 1
row['id'].times do |k|
out << { 'pos' => k, 'lfdnr' => cnt }
cnt += 1
end
out
end
transform(processing: :append) { |_row| { 'SATYPE': 'LARC', 'SUMME': format('%010d', cnt - 1) } }
end
end
def test_append_registered_before_prepend
assert_output '[{"h":0},{"id":1,"land":"AT"},{"id":2,"land":"DE"},{"id":3,"land":"AT"},' \
'{"id":4,"land":"DE"},{"id":5,"land":"AT"},{"id":6,"land":"DE"},{"t":6}]' do
transform(processing: :append) { |_, n| { 't' => n } }
transform { |r| r }
transform(processing: :prepend) { |_, n| { 'h' => n } }
end
end
def test_condition_string
assert_output '[{"id":2,"land":"DE","i":1},{"id":4,"land":"DE","i":3},{"id":6,"land":"DE","i":5}]' do
transform(condition: "row['land'] == 'DE'") { |r, i| r.merge('i' => i) }
end
end
def test_two_parallel_transforms
expected = ROWS.map { |r| r.merge('a' => 1) } + ROWS.map { |r| r.merge('b' => 1) }
assert_output JSON.generate(expected) do
transform { |r| r.merge('a' => 1) }
transform { |r| [r.merge('b' => 1), nil] }
end
end
def test_nil_and_empty_array
assert_output '[{"id":1,"land":"AT"}]' do
transform { |r| r['id'] > 3 ? nil : [] }
transform { |r| r['id'] == 1 ? r : nil }
end
end
end
test/core/operations_test.rb
+4 -4
View File
@@ -41,8 +41,8 @@ class OperationsTest < PipelineTest
assert_raises(RuntimeError) { run_pipeline(ROWS) { on_error :skip; header { raise 'h' }; map { |r| r } } }
end
def test_skip_in_parallel_mode
target, stats = run_pipeline(ROWS) { on_error :skip; transform { |r| r['id'] == '1' ? raise('x') : r } }
def test_skip_in_map
target, stats = run_pipeline(ROWS) { on_error :skip; map { |r| r['id'] == '1' ? raise('x') : r } }
assert_equal 4, target.output.size
assert_equal 1, stats.rejected
end
@@ -67,8 +67,8 @@ class OperationsTest < PipelineTest
assert_empty DataPipeline::Validation.errors({ 'a' => '' }, 'a' => { max: 1, type: :integer })
end
def test_validate_turns_chaining_on
target, = run_pipeline(ROWS) { on_error :skip; transform { |r| r.merge('x' => 1) }; validate RULES }
def test_validate_gets_the_output_of_the_previous_step
target, = run_pipeline(ROWS) { on_error :skip; map { |r| r.merge('x' => 1) }; validate RULES }
assert_equal 3, target.output.size
assert(target.output.all? { |r| r['x'] })
end
test/core/pipeline_test.rb
+3 -40
View File
@@ -1,17 +1,11 @@
require_relative '../test_helper'
# dsl, settings, plugin lookup, parallel mode
# dsl, settings, plugin lookup
class PipelineCoreTest < PipelineTest
ROWS = [{ 'a' => 1 }, { 'a' => 2 }, { 'a' => 3 }].freeze
def settings_of(pipeline) = pipeline.instance_variable_get(:@settings)
def test_nil_filters_rows
assert_equal [{ 'a' => 1 }], output(ROWS) { transform { |r| r['a'] == 1 ? r : nil } }
assert_equal ROWS, output(ROWS) { transform { |r| [r, nil] } }
assert_equal ROWS, output(ROWS) { transform(processing: :prepend) { nil }; transform { |r| r }; transform(processing: :append) { nil } }
end
def test_dsl_switches
assert_equal false, settings_of(DataPipeline.new { verbose false })[:verbose]
assert_equal true, settings_of(DataPipeline.new { verbose })[:verbose]
@@ -73,39 +67,8 @@ class PipelineCoreTest < PipelineTest
assert_equal Array.new(8) { |i| (i % 3) + 1 }, reads
end
def test_parallel_mode_is_side_by_side
out = output(ROWS) { transform { |r| r.merge('b' => 1) }; transform { |r| r.merge('c' => 2) } }
assert_equal 6, out.size
end
def test_parallel_mode_copies_rows_per_step
out = output(ROWS) { transform { |r| r['x'] = 1; r }; transform { |r| r.key?('x') ? nil : r } }
assert_equal 6, out.size
assert_equal [{ 'a' => 10 }, { 'a' => 20 }, { 'a' => 30 }], output(ROWS) { transform { |r| [r.transform_values! { |v| v * 10 }] } }
end
def test_stable_order_and_all_prepends_appends
out = output(ROWS) { 20.times { |k| transform { |r| r['a'] == 1 ? { 'k' => k } : nil } } }
assert_equal (0...20).to_a, out.map { |r| r['k'] }
out = output(ROWS) do
transform(processing: :prepend) { { 'h' => 1 } }
transform(processing: :prepend) { { 'h' => 2 } }
transform { |r| r }
transform(processing: :append) { { 't' => 1 } }
transform(processing: :append) { { 't' => 2 } }
end
assert_equal [{ 'h' => 1 }, { 'h' => 2 }, *ROWS, { 't' => 1 }, { 't' => 2 }], out
end
def test_append_count_parallel_vs_chained
assert_equal({ 'n' => 3 }, output(ROWS) { transform { |r| [r, r] }; transform(processing: :append) { |_, n| { 'n' => n } } }.last)
assert_equal({ 'n' => 6 }, output(ROWS) { chained; transform { |r| [r, r] }; transform(processing: :append) { |_, n| { 'n' => n } } }.last)
end
def test_conditions
assert_equal [{ 'a' => 1 }, { 'a' => 3 }], output(ROWS) { transform(condition: ->(r) { r['a'] != 2 }) { |r| r } }
assert_equal [{ 'a' => 2 }], output(ROWS) { transform(condition: "row['a'] == 2") { |r| r } }
def test_removed_dsl_of_version_2
%i[transform chained].each { |name| assert_raises(NoMethodError) { DataPipeline.new { send(name) { |r| r } } } }
end
def test_copy_without_steps
test/core/steps_test.rb
+1 -30
View File
@@ -1,6 +1,6 @@
require_relative '../test_helper'
# rename, select, cast, default and the deprecation warnings
# rename, select, cast, default
class StepsTest < PipelineTest
ROWS = [{ 'kunnr' => '1', 'betrag' => '12.50', 'datum' => '2026-10-09', 'land' => nil },
{ 'kunnr' => '2', 'betrag' => '', 'datum' => ' ', 'land' => 'DE' }].freeze
@@ -44,33 +44,4 @@ class StepsTest < PipelineTest
end
assert_equal [{ 'id' => '1', 'betrag' => 12.5 }, { 'id' => '2', 'betrag' => 0.0 }], rows
end
def deprecation_log(&block)
log = StringIO.new
DataPipeline.new do
logger Logger.new(log)
read MemoryStorage.new(rows: [{ 'a' => 1 }])
write MemoryStorage.new
instance_eval(&block)
end.run
log.string.lines.grep(/deprecated/)
end
def test_deprecations_once_per_process
lines = deprecation_log do
transform(condition: 'true') { |r| r }
transform { |r| r }
end
assert_equal 3, lines.size
assert lines.any? { |l| l.include?('transform is deprecated') }
assert lines.any? { |l| l.include?('parallel mode') }
assert lines.any? { |l| l.include?('condition: as string') }
assert_empty deprecation_log { transform { |r| r } }
end
def test_no_deprecation_for_map_filter_and_single_transform_in_chained_mode
assert_empty deprecation_log { map { |r| r }; filter(condition: ->(_r) { true }) { true } }
lines = deprecation_log { chained; transform { |r| r }; transform { |r| r } }
assert_equal 1, lines.size # only transform, no parallel mode
end
end
test/core/streaming_test.rb
+5 -14
View File
@@ -21,23 +21,14 @@ class StreamingTest < PipelineTest
assert_equal({ 'T' => 4, 'R' => 3 }, stream.last)
end
def test_stream_prepend_append
def test_stream_several_steps_one_to_many
out = output(ROWS, streamed: true) do
chained
transform(processing: :prepend) { |_, n| { 'p' => n } }
transform { |r| [r, r] }
transform(processing: :append) { |_, n| { 'n' => n } }
map { |r| [r, r] }
map { |r, ctx| r.merge('i' => ctx.index) }
trailer { |ctx| { 'n' => ctx.written } }
end
assert_equal({ 'p' => 0 }, out.first)
assert_equal({ 'n' => 6 }, out.last)
assert_equal 8, out.size
end
def test_stream_parallel_single_step_and_error_for_several
out = output(ROWS, streamed: true) { transform { |r| [r, r] }; transform(processing: :append) { |_, n| { 'n' => n } } }
assert_equal({ 'n' => 3 }, out.last)
error = assert_raises(ArgumentError) { output(ROWS, streamed: true) { transform { |r| r }; transform { |r| r } } }
assert_includes error.message, 'chained'
assert_equal (0...6).to_a, out.first(6).map { |r| r['i'] }
end
def test_stream_condition_and_copy
test/test_helper.rb
-1
View File
@@ -55,7 +55,6 @@ module PipelineHelper
DataPipeline.reset_settings!
DataPipeline.runs.clear
DataPipeline.params.clear
DataPipeline.reset_deprecations!
end
# runs a pipeline from rows into a MemoryStorage, returns [target, stats]