wkh/datapipeline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

datapipeline 2.0.0: Gem aus dem Projekt etl herausgelöst

Browse Source
This commit is contained in:
wkh
2026-10-08 15:30:02 +02:00
commit cce7b03ca9
56 changed files with 5205 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/core/chained_test.rb
+75
View File
@@ -0,0 +1,75 @@
require_relative '../test_helper'
# chained steps: map, filter, header, trailer, context
class ChainedTest < PipelineTest
ROWS = [{ 'a' => 1 }, { 'a' => 2 }, { 'a' => 3 }].freeze
def test_chained_transform_gets_previous_output
out = output(ROWS) { chained; transform { |r| r.merge('b' => 1) }; transform { |r| r.merge('c' => 2) } }
assert_equal ROWS.map { |r| r.merge('b' => 1, 'c' => 2) }, out
end
def test_one_to_many_then_index
out = output(ROWS) { chained; transform { |r| [r, r] }; transform { |r, i| r.merge('i' => i) } }
assert_equal (0...6).to_a, out.map { |r| r['i'] }
end
def test_map_and_filter
out = output(ROWS) do
map { |r| r.merge('d' => r['a'] * 2) }
filter { |r| r['d'] > 2 }
map { |r, ctx| r.merge('i' => ctx.index) }
end
assert_equal [{ 'a' => 2, 'd' => 4, 'i' => 0 }, { 'a' => 3, 'd' => 6, 'i' => 1 }], out
end
def test_map_turns_chaining_on_for_transform
out = output(ROWS) { transform { |r| r.merge('b' => 1) }; map { |r| r.merge('c' => 1) } }
assert_equal 3, out.size
assert(out.all? { |r| r['b'] && r['c'] })
end
def test_header_and_trailer_with_context
out = output(ROWS) do
header { |c| { 'H' => c.read, 'W' => c.written } }
filter { |r| r['a'] > 1 }
map { |r| [r, r] }
trailer { |c| { 'T' => c.written } }
end
assert_equal({ 'H' => 3, 'W' => 4 }, out.first)
assert_equal({ 'T' => 4 }, out.last)
assert_equal 6, out.size
end
def test_edge_order_follows_registration
out = output(ROWS) { header { { 'h' => 1 } }; transform(processing: :prepend) { { 'p' => 1 } }; map { |r| r } }
assert_equal [{ 'h' => 1 }, { 'p' => 1 }], out.first(2)
end
def test_header_and_trailer_nil_and_arrays
out = output(ROWS) { header { nil }; trailer { [{ 'x' => 1 }, nil, { 'y' => 2 }] }; map { |r| r } }
assert_equal ROWS + [{ 'x' => 1 }, { 'y' => 2 }], out
end
def test_condition_on_map
assert_equal [{ 'a' => 1 }], output(ROWS) { map(condition: ->(r) { r['a'] == 1 }) { |r| r } }
end
def test_chained_false
assert_equal 6, output(ROWS) { chained false; transform { |r| r }; transform { |r| r } }.size
end
def test_csv_end_to_end
with_tmpdir do |dir|
DataPipeline.new do
logger PipelineHelper::QUIET
read MemoryStorage.new(rows: ROWS)
filter { |r| r['a'].odd? }
map { |r| { 'id' => r['a'], 'x' => 'y' } }
trailer { |c| { 'id' => 'TOTAL', 'x' => c.written } }
write container(:csv, file: "#{dir}/c.csv", columns: %w[id x])
end.run
assert_equal "id,x\n1,y\n3,y\nTOTAL,2\n", File.read("#{dir}/c.csv")
end
end
end
test/core/compat_test.rb
+81
View File
@@ -0,0 +1,81 @@
require_relative '../test_helper'
#
# parallel mode (transform without chained/map/filter) must keep the output of v1.2.1,
# the version the CUDO importers were built on. expected values were recorded with v1.2.1.
#
class CompatTest < PipelineTest
ROWS = (1..6).map { |i| { 'id' => i, 'land' => i.even? ? 'DE' : 'AT' } }
def assert_output expected_json, &block
assert_equal expected_json, JSON.generate(output(ROWS, &block))
end
# pattern of import.customer.rb
def test_customer_pattern
assert_output '[{"SATYP":"FIRC","LFDNR":0},{"id":1,"land":"AT","lfdnr":"0000000001"},' \
'{"id":2,"land":"DE","lfdnr":"0000000002"},{"id":3,"land":"AT","lfdnr":"0000000003"},' \
'{"id":4,"land":"DE","lfdnr":"0000000004"},{"id":5,"land":"AT","lfdnr":"0000000005"},' \
'{"id":6,"land":"DE","lfdnr":"0000000006"},{"topic":"LARC","counter":"0000000006"}]' do
transform(processing: :prepend) { |_row, n| { 'SATYP': 'FIRC', 'LFDNR': n } }
transform(processing: :append) { |_row, n| { 'topic': 'LARC', 'counter': format('%010d', n) } }
transform { |row, _i| row.merge('lfdnr' => format('%010d', row['id'])) }
end
end
# pattern of import.posting.rb: closure counter, 1:n
def test_posting_pattern
expected = [{ SATYP: 'FIRC', LFDNR: 0 }]
lfdnr = 0
ROWS.each do |row|
expected << row.merge('lfdnr' => lfdnr += 1)
row['id'].times { |k| expected << { 'pos' => k, 'lfdnr' => lfdnr += 1 } }
end
expected << { SATYPE: 'LARC', SUMME: '0000000027' }
assert_output JSON.generate(expected) do
cnt = 1
transform(processing: :prepend) { |_row, _n| { 'SATYP': 'FIRC', 'LFDNR': 0 } }
transform do |row|
out = [row.merge('lfdnr' => cnt)]
cnt += 1
row['id'].times do |k|
out << { 'pos' => k, 'lfdnr' => cnt }
cnt += 1
end
out
end
transform(processing: :append) { |_row| { 'SATYPE': 'LARC', 'SUMME': format('%010d', cnt - 1) } }
end
end
def test_append_registered_before_prepend
assert_output '[{"h":0},{"id":1,"land":"AT"},{"id":2,"land":"DE"},{"id":3,"land":"AT"},' \
'{"id":4,"land":"DE"},{"id":5,"land":"AT"},{"id":6,"land":"DE"},{"t":6}]' do
transform(processing: :append) { |_, n| { 't' => n } }
transform { |r| r }
transform(processing: :prepend) { |_, n| { 'h' => n } }
end
end
def test_condition_string
assert_output '[{"id":2,"land":"DE","i":1},{"id":4,"land":"DE","i":3},{"id":6,"land":"DE","i":5}]' do
transform(condition: "row['land'] == 'DE'") { |r, i| r.merge('i' => i) }
end
end
def test_two_parallel_transforms
expected = ROWS.map { |r| r.merge('a' => 1) } + ROWS.map { |r| r.merge('b' => 1) }
assert_output JSON.generate(expected) do
transform { |r| r.merge('a' => 1) }
transform { |r| [r.merge('b' => 1), nil] }
end
end
def test_nil_and_empty_array
assert_output '[{"id":1,"land":"AT"}]' do
transform { |r| r['id'] > 3 ? nil : [] }
transform { |r| r['id'] == 1 ? r : nil }
end
end
end
test/core/layout_test.rb
+74
View File
@@ -0,0 +1,74 @@
require_relative '../test_helper'
class LayoutTest < PipelineTest
LAYOUT = DataPipeline::Layout.new do
record 'HEAD', [['SATYP', 'AN', 4, true], ['DATUM', 'DAT', 8, true], ['ZEIT', 'TIM', 6]]
record 'POS', [['SATYP', 'AN', 4, true], ['NR', 'N', 3, true], ['TEXT', 'AN', 5], ['BETRAG', 'N', '13.2']]
end
def test_values_in_field_order_any_key_case
assert_equal ['POS', '1', 'abc', nil], LAYOUT.values({ SATYP: 'POS', betrag: nil, 'text' => 'abc', 'NR' => '1' })
assert_equal ['HEAD', '20251103', nil], LAYOUT.values({ 'satyp' => 'HEAD', 'datum' => '20251103' })
end
def test_unknown_field_and_record_type
assert_includes assert_raises(DataPipeline::LayoutError) { LAYOUT.values({ 'satyp' => 'POS', 'extra' => 1 }) }.message, 'extra'
assert_includes assert_raises(DataPipeline::LayoutError) { LAYOUT.values({ 'satyp' => 'XXXX' }) }.message, 'XXXX'
lenient = DataPipeline::Layout.new(strict: false) { record 'A', [['SATYP']] }
assert_equal ['A'], lenient.values({ 'satyp' => 'A', 'extra' => 1 })
end
def test_type_field
layout = DataPipeline::Layout.new(type_field: 'art') { record 'X', [['ID'], ['ART']] }
assert_equal %w[7 X], layout.values({ 'id' => '7', 'art' => 'X' })
end
def test_errors
assert_empty LAYOUT.errors({ 'satyp' => 'POS', 'nr' => '001', 'text' => 'abcde', 'betrag' => '-123.45' })
assert_equal ['NR: missing', 'TEXT: longer than 5', 'BETRAG: more than 2 decimals'],
LAYOUT.errors({ 'satyp' => 'POS', 'text' => 'abcdef', 'betrag' => '1.234' })
assert_equal ['NR: more than 3 digits', 'BETRAG: not a number'], LAYOUT.errors({ 'satyp' => 'POS', 'nr' => '1000', 'betrag' => '1,5' })
assert_equal ['DATUM: not a date YYYYMMDD', 'ZEIT: not a time HHMMSS'], LAYOUT.errors({ 'satyp' => 'HEAD', 'datum' => '20251331', 'zeit' => '250000' })
assert_equal ['record type "?" not in layout (HEAD, POS)'], LAYOUT.errors({ 'satyp' => '?' })
end
def test_csv_with_layout
with_tmpdir do |dir|
rows = [{ SATYP: 'HEAD', DATUM: '20251103' }, { 'satyp' => 'POS', 'betrag' => '1.50', 'nr' => '1' }]
DataPipeline::CsvStorage.new(file: "#{dir}/l.ok", layout: LAYOUT, csv: { col_sep: '|', quote_char: '' }).write(rows)
assert_equal "HEAD|20251103|\nPOS|1||1.50\n", File.read("#{dir}/l.ok")
end
end
def test_validate_layout_abort_warn_and_ignore
rows = [{ 'satyp' => 'POS', 'nr' => '1' }, { 'satyp' => 'POS', 'text' => 'zu lang' }]
assert_raises(DataPipeline::ValidationError) { run_pipeline(rows) { validate LAYOUT } }
target, stats = run_pipeline(rows) { validate LAYOUT, warn: true }
assert_equal 2, target.output.size
assert_equal 1, stats.warnings
assert_match(/warnings=1/, stats.to_s)
_target, stats = run_pipeline(rows) { validate LAYOUT, warn: true, ignore: %w[NR TEXT] }
assert_equal 0, stats.warnings
end
def test_warning_log_is_capped
log = StringIO.new
stats = DataPipeline.new do
logger Logger.new(log)
read MemoryStorage.new(rows: Array.new(150) { { 'satyp' => 'POS' } })
write MemoryStorage.new
validate LAYOUT, warn: true
end.run
assert_equal 150, stats.warnings
assert_equal 101, log.string.lines.count { |l| l.include?('WARN') }
assert_includes log.string, 'more than 100 warnings'
end
def test_validate_hash_without_braces
target, stats = run_pipeline([{ 'a' => 'x' }, { 'a' => '1' }]) { on_error :skip; validate 'a' => { format: /\A\d\z/ } }
assert_equal [{ 'a' => '1' }], target.output
assert_equal 1, stats.rejected
end
end
test/core/operations_test.rb
+178
View File
@@ -0,0 +1,178 @@
require_relative '../test_helper'
# on_error, validate, stats, hooks, watermark, dry run, limit
class OperationsTest < PipelineTest
ROWS = (1..5).map { |i| { 'id' => i.to_s, 'shkzg' => i == 3 ? 'X' : 'S', 'betrag' => i == 4 ? 'abc' : "#{i}.50" } }.freeze
RULES = { 'id' => { required: true, format: /\A\d+\z/ }, 'shkzg' => { in: %w[S H] }, 'betrag' => { type: :decimal } }.freeze
def failing_map = proc { map { |r| r['id'] == '2' ? raise('kaputt') : r } }
def test_abort_is_default
error = assert_raises(RuntimeError) { run_pipeline(ROWS, &failing_map) }
assert_equal 'kaputt', error.message
assert_equal 'kaputt', DataPipeline.runs.last.error.message
end
def test_skip
map = failing_map
target, stats = run_pipeline(ROWS) { on_error :skip; instance_eval(&map) }
assert_equal 4, target.output.size
assert_equal 1, stats.rejected
assert_equal 'map#1', stats.errors.first[:step]
refute stats.ok?
end
def test_collect_batch_and_stream
[false, true].each do |streamed|
rejects = MemoryStorage.new
map = failing_map
target, = run_pipeline(ROWS, streamed:) { on_error :collect, to: rejects; instance_eval(&map) }
assert_equal [ROWS[1].merge('_step' => 'map#1', '_error' => 'kaputt')], rejects.streamed_rows
assert_equal 4, target.output.size
end
end
def test_on_error_arguments
assert_raises(ArgumentError) { DataPipeline.new { on_error :collect } }
assert_raises(ArgumentError) { DataPipeline.new { on_error :ignore } }
end
def test_edge_errors_always_abort
assert_raises(RuntimeError) { run_pipeline(ROWS) { on_error :skip; header { raise 'h' }; map { |r| r } } }
end
def test_skip_in_parallel_mode
target, stats = run_pipeline(ROWS) { on_error :skip; transform { |r| r['id'] == '1' ? raise('x') : r } }
assert_equal 4, target.output.size
assert_equal 1, stats.rejected
end
def test_validate_rules
assert_raises(DataPipeline::ValidationError) { run_pipeline(ROWS) { validate RULES } }
target, stats = run_pipeline(ROWS) { on_error :skip; validate RULES }
assert_equal %w[1 2 5], target.output.map { |r| r['id'] }
assert_equal 2, stats.rejected
end
def test_validate_block
target, stats = run_pipeline(ROWS) { on_error :skip; validate { |r| 'zu gross' if r['id'].to_i > 3 } }
assert_equal 3, target.output.size
assert_equal 'zu gross', stats.errors.first[:error]
end
def test_validation_messages
errors = DataPipeline::Validation.errors({ 'a' => '12345', 'd' => '2025-13-01' },
'a' => { max: 3, format: /x/ }, 'd' => { type: :date }, 'b' => { required: true })
assert_equal ['a: longer than 3', 'a: format /x/', 'd: not date', 'b: missing'], errors
assert_empty DataPipeline::Validation.errors({ 'a' => '' }, 'a' => { max: 1, type: :integer })
end
def test_validate_turns_chaining_on
target, = run_pipeline(ROWS) { on_error :skip; transform { |r| r.merge('x' => 1) }; validate RULES }
assert_equal 3, target.output.size
assert(target.output.all? { |r| r['x'] })
end
def test_stats
_target, stats = run_pipeline(ROWS) { map { |r| [r, r] } }
assert_equal [5, 10, 10], [stats.read, stats.written, stats.output]
assert stats.ok?
assert_kind_of Float, stats.duration
map = failing_map
_target, stats = run_pipeline(ROWS) { on_error :skip; instance_eval(&map) }
assert_match(/\Aread=5 written=4 rejected=1/, stats.to_s)
end
def test_on_finish_after_success_and_abort
seen = []
run_pipeline(ROWS) { on_finish { |s| seen << s.error } }
assert_raises(RuntimeError) { run_pipeline(ROWS) { on_finish { |s| seen << s.error.class }; map { raise 'x' } } }
assert_equal [nil, RuntimeError], seen
end
def test_hook_error_is_only_logged
log = StringIO.new
stats = DataPipeline.new do
logger Logger.new(log)
read MemoryStorage.new(rows: ROWS)
write MemoryStorage.new
on_finish { raise 'hook' }
end.run
assert stats.ok?
assert_includes log.string, 'on_finish: RuntimeError: hook'
end
def test_logger_messages
log = StringIO.new
DataPipeline.new do
logger Logger.new(log)
on_error :skip
read MemoryStorage.new(rows: ROWS)
write MemoryStorage.new
map { |r| r['id'] == '1' ? raise('weg') : r }
end.run
assert_includes log.string, 'rejected by map#1: weg'
assert_includes log.string, 'finish read=5 written=4 rejected=1'
end
def test_watermark
with_tmpdir do |dir|
file = "#{dir}/state.json"
wm = nil
run_pipeline(ROWS) { wm = watermark :t, column: 'id', file:, initial: 0 }
assert_equal '5', wm.value
run_pipeline([{ 'id' => '9' }, { 'id' => '10' }]) { watermark :t, column: 'id', file: }
assert_equal '10', JSON.parse(File.read(file))['t'], 'numeric texts compared as numbers'
assert_raises(RuntimeError) { run_pipeline([{ 'id' => '99' }]) { watermark :t, column: 'id', file:; map { raise 'x' } } }
assert_equal '10', JSON.parse(File.read(file))['t'], 'not saved after abort'
run_pipeline(ROWS, streamed: true) { watermark :s, column: 'id', file: }
assert_equal({ 't' => '10', 's' => '5' }, JSON.parse(File.read(file)))
end
end
def test_dry_run
with_tmpdir do |dir|
rejects = MemoryStorage.new
target = nil
text = capture_stdout do
target, = run_pipeline(ROWS, dry_run: true, preview: 2) do
watermark :d, column: 'id', file: "#{dir}/state.json"
on_error :collect, to: rejects
map { |r| r['id'] == '1' ? raise('x') : r }
end
end
assert_nil target.written
assert_empty target.streamed_rows
assert_empty rejects.streamed_rows
refute File.exist?("#{dir}/state.json")
assert text.start_with?('DRY RUN')
assert_equal 3, text.lines.size
assert DataPipeline.runs.last.dry_run
end
end
def test_dry_run_creates_no_csv_file
with_tmpdir do |dir|
capture_stdout do
DataPipeline.new(dry_run: true) do
logger PipelineHelper::QUIET
read MemoryStorage.new(rows: ROWS)
write container(:csv, file: "#{dir}/dry.csv")
end.run
end
refute File.exist?("#{dir}/dry.csv")
end
end
def test_limit
target, stats = run_pipeline(ROWS, limit: 2) { map { |r| r } }
assert_equal [2, 2], [target.output.size, stats.read]
target, stats = run_pipeline(ROWS, streamed: true, limit: 3) { trailer { |c| { 'n' => c.read } } }
assert_equal({ 'n' => 3 }, target.output.last)
assert_equal 3, stats.read
end
end
test/core/pipeline_test.rb
+131
View File
@@ -0,0 +1,131 @@
require_relative '../test_helper'
# dsl, settings, plugin lookup, parallel mode
class PipelineCoreTest < PipelineTest
ROWS = [{ 'a' => 1 }, { 'a' => 2 }, { 'a' => 3 }].freeze
def settings_of(pipeline) = pipeline.instance_variable_get(:@settings)
def test_nil_filters_rows
assert_equal [{ 'a' => 1 }], output(ROWS) { transform { |r| r['a'] == 1 ? r : nil } }
assert_equal ROWS, output(ROWS) { transform { |r| [r, nil] } }
assert_equal ROWS, output(ROWS) { transform(processing: :prepend) { nil }; transform { |r| r }; transform(processing: :append) { nil } }
end
def test_dsl_switches
assert_equal false, settings_of(DataPipeline.new { verbose false })[:verbose]
assert_equal true, settings_of(DataPipeline.new { verbose })[:verbose]
assert_equal false, settings_of(DataPipeline.new { streamed false })[:streamed]
assert_equal '1.0', settings_of(DataPipeline.new { version '1.0' })[:version]
assert_equal true, settings_of(DataPipeline.new { configure verbose: true })[:verbose]
end
def test_typo_raises
assert_raises(NoMethodError) { DataPipeline.new { verbsoe true } }
end
def test_unknown_plugin
error = assert_raises(RuntimeError) { DataPipeline.new { container :gibtsnicht } }
assert_includes error.message, 'GibtsnichtStorage'
end
def test_plugin_errors_pass_through
assert_raises(ArgumentError) { DataPipeline.new { container :csv } }
broken = Class.new(DataPipeline::StoragePlugin) { def initialize(**) = undefined_thing }
DataPipeline.register :broken, broken
assert_includes assert_raises(NameError) { DataPipeline.new { container :broken } }.message, 'undefined_thing'
end
def test_register_lazy_path
with_tmpdir do |dir|
File.write("#{dir}/lazy_storage.rb", "class LazyStorage < MemoryStorage; end\n")
DataPipeline.register :lazy, 'LazyStorage', path: "#{dir}/lazy_storage.rb"
assert_empty $LOADED_FEATURES.grep(/lazy_storage/)
storage = nil
DataPipeline.new { storage = container :lazy }
assert_equal 'LazyStorage', storage.class.name
refute_empty $LOADED_FEATURES.grep(/lazy_storage/)
end
end
def test_csv_plugin_by_convention
assert_equal DataPipeline::CsvStorage, DataPipeline.plugin(:csv)
end
def test_settings_are_frozen_and_replaced
assert DataPipeline.settings.frozen?
pipeline = DataPipeline.new { logger PipelineHelper::QUIET }
DataPipeline.settings(verbose: true)
assert_equal false, settings_of(pipeline)[:verbose]
assert_equal true, DataPipeline.settings[:verbose]
DataPipeline.reset_settings!
assert_equal DataPipeline::DEFAULT_SETTINGS, DataPipeline.settings
end
def test_settings_per_pipeline
_target, stats = run_pipeline(ROWS, limit: 1)
assert_equal 1, stats.read
assert_nil DataPipeline.settings[:limit]
end
def test_parallel_threads
reads = Array.new(8) { |i| Thread.new { run_pipeline(ROWS, limit: (i % 3) + 1).last.read } }.map(&:value)
assert_equal Array.new(8) { |i| (i % 3) + 1 }, reads
end
def test_parallel_mode_is_side_by_side
out = output(ROWS) { transform { |r| r.merge('b' => 1) }; transform { |r| r.merge('c' => 2) } }
assert_equal 6, out.size
end
def test_parallel_mode_copies_rows_per_step
out = output(ROWS) { transform { |r| r['x'] = 1; r }; transform { |r| r.key?('x') ? nil : r } }
assert_equal 6, out.size
assert_equal [{ 'a' => 10 }, { 'a' => 20 }, { 'a' => 30 }], output(ROWS) { transform { |r| [r.transform_values! { |v| v * 10 }] } }
end
def test_stable_order_and_all_prepends_appends
out = output(ROWS) { 20.times { |k| transform { |r| r['a'] == 1 ? { 'k' => k } : nil } } }
assert_equal (0...20).to_a, out.map { |r| r['k'] }
out = output(ROWS) do
transform(processing: :prepend) { { 'h' => 1 } }
transform(processing: :prepend) { { 'h' => 2 } }
transform { |r| r }
transform(processing: :append) { { 't' => 1 } }
transform(processing: :append) { { 't' => 2 } }
end
assert_equal [{ 'h' => 1 }, { 'h' => 2 }, *ROWS, { 't' => 1 }, { 't' => 2 }], out
end
def test_append_count_parallel_vs_chained
assert_equal({ 'n' => 3 }, output(ROWS) { transform { |r| [r, r] }; transform(processing: :append) { |_, n| { 'n' => n } } }.last)
assert_equal({ 'n' => 6 }, output(ROWS) { chained; transform { |r| [r, r] }; transform(processing: :append) { |_, n| { 'n' => n } } }.last)
end
def test_conditions
assert_equal [{ 'a' => 1 }, { 'a' => 3 }], output(ROWS) { transform(condition: ->(r) { r['a'] != 2 }) { |r| r } }
assert_equal [{ 'a' => 2 }], output(ROWS) { transform(condition: "row['a'] == 2") { |r| r } }
end
def test_copy_without_steps
assert_equal ROWS, output(ROWS)
end
def test_closes_source_and_target
source = MemoryStorage.new(rows: ROWS)
target = MemoryStorage.new
DataPipeline.new { logger PipelineHelper::QUIET; read source; write target }.run
assert source.closed?
assert target.closed?
end
def test_read_and_write_required
assert_includes assert_raises(ArgumentError) { DataPipeline.new { logger PipelineHelper::QUIET; read MemoryStorage.new }.run }.message, 'write'
assert_includes assert_raises(ArgumentError) { DataPipeline.new { logger PipelineHelper::QUIET; write MemoryStorage.new }.run }.message, 'read'
end
def test_block_required
assert_raises(ArgumentError) { DataPipeline.new { map } }
end
end
test/core/streaming_test.rb
+153
View File
@@ -0,0 +1,153 @@
require_relative '../test_helper'
# streamed mode, lookups, several targets
class StreamingTest < PipelineTest
ROWS = [{ 'a' => 1 }, { 'a' => 2 }, { 'a' => 3 }].freeze
CHAIN = proc do
header { |c| { 'H' => c.written } }
filter { |r| r['a'] > 1 }
map { |r, c| [r, r.merge('k' => c.index)] }
map { |r, c| r.merge('i' => c.index) }
trailer { |c| { 'T' => c.written, 'R' => c.read } }
end
def test_batch_and_stream_same_body
batch = output(ROWS, &CHAIN)
stream = output(ROWS, streamed: true, &CHAIN)
assert_equal batch[1..-2], stream[1..-2]
assert_equal({ 'H' => 4 }, batch.first)
assert_equal({ 'H' => nil }, stream.first)
assert_equal({ 'T' => 4, 'R' => 3 }, stream.last)
end
def test_stream_prepend_append
out = output(ROWS, streamed: true) do
chained
transform(processing: :prepend) { |_, n| { 'p' => n } }
transform { |r| [r, r] }
transform(processing: :append) { |_, n| { 'n' => n } }
end
assert_equal({ 'p' => 0 }, out.first)
assert_equal({ 'n' => 6 }, out.last)
assert_equal 8, out.size
end
def test_stream_parallel_single_step_and_error_for_several
out = output(ROWS, streamed: true) { transform { |r| [r, r] }; transform(processing: :append) { |_, n| { 'n' => n } } }
assert_equal({ 'n' => 3 }, out.last)
error = assert_raises(ArgumentError) { output(ROWS, streamed: true) { transform { |r| r }; transform { |r| r } } }
assert_includes error.message, 'chained'
end
def test_stream_condition_and_copy
assert_equal [{ 'a' => 1 }, { 'a' => 3 }], output(ROWS, streamed: true) { map(condition: ->(r) { r['a'] != 2 }) { |r| r } }
assert_equal ROWS, output(ROWS, streamed: true)
end
def test_stream_csv_to_csv
with_tmpdir do |dir|
File.write("#{dir}/in.csv", "a,b\n1,x\n2,y\n3,z\n")
DataPipeline.new do
logger PipelineHelper::QUIET
streamed
read container(:csv, file: "#{dir}/in.csv")
filter { |r| r['a'] != '2' }
trailer { |c| { 'a' => 'T', 'b' => c.written } }
write container(:csv, file: "#{dir}/out.csv")
end.run
assert_equal "a,b\n1,x\n3,z\nT,2\n", File.read("#{dir}/out.csv")
end
end
def test_stream_postgres_with_query_on_same_connection
with_pg_database do |db|
pg_exec db, "create table k as select i as id, case when i % 2 = 0 then 'DE' else 'AT' end as land from generate_series(1, 5) i;
create table l (land text, name text); insert into l values ('DE','Deutschland'),('AT','Oesterreich')"
with_tmpdir do |dir|
DataPipeline.new do
logger PipelineHelper::QUIET
streamed
pg = container :postgres, **db
read pg, query: 'select id, land from k where id <= $1 order by id', params: [5], fetch_size: 2
map { |r| r.merge('name' => query(pg, query: 'select name from l where land = $1', params: [r['land']]).first['name']) }
write container(:csv, file: "#{dir}/o.csv", columns: %w[id name])
end.run
assert_equal 6, File.readlines("#{dir}/o.csv").size
assert_includes File.read("#{dir}/o.csv"), '2,Deutschland'
end
end
end
def test_lookup_grouped_unique_and_lambda
with_tmpdir do |dir|
File.write("#{dir}/lk.csv", "id,grp\n1,a\n2,b\n3,a\n")
grouped = unique = by_lambda = nil
DataPipeline.new do
csv = container(:csv, file: "#{dir}/lk.csv")
grouped = lookup csv, key: 'grp'
unique = lookup csv, key: 'id', unique: true
by_lambda = lookup csv, key: ->(r) { r['grp'].upcase }
end
assert_equal 2, grouped['a'].size
assert_equal 'b', unique['2']['grp']
assert_equal %w[A B], by_lambda.keys.sort
end
end
def test_lookup_postgres_join_without_n_plus_one
with_pg_database do |db|
pg_exec db, "create table k as select i as id, case when i % 2 = 0 then 'DE' else 'AT' end as land from generate_series(1, 3) i;
create table l (land text, name text); insert into l values ('DE','Deutschland'),('AT','Oesterreich')"
out = nil
DataPipeline.new do
logger PipelineHelper::QUIET
pg = container :postgres, **db
names = lookup pg, query: 'select * from l where land like $1', params: ['%'], key: 'land', unique: true
read pg, query: 'select id, land from k order by id'
map { |r| r.merge('name' => names.dig(r['land'], 'name')) }
write(out = MemoryStorage.new)
end.run
assert_equal %w[Oesterreich Deutschland Oesterreich], out.output.map { |r| r['name'] }
end
end
def test_several_targets_with_only
rows = [{ 'satyp' => 'DOHD', 'n' => 1 }, { 'satyp' => 'DOPS', 'n' => 2 }, { 'satyp' => 'DOPS', 'n' => 3 }]
[false, true].each do |streamed|
all = MemoryStorage.new
heads = MemoryStorage.new
positions = MemoryStorage.new
trailers = MemoryStorage.new
DataPipeline.new(streamed:) do
logger PipelineHelper::QUIET
read MemoryStorage.new(rows:)
header { { 'satyp' => 'FIRC' } }
map { |r| r }
trailer { |c| { 'satyp' => 'LARC', 'n' => c.written } }
write all
write heads, only: ->(r) { r['satyp'] == 'DOHD' }
write positions, only: ->(r) { r['satyp'] == 'DOPS' }
write trailers, only: ->(r) { r['satyp'] == 'LARC' }
end.run
assert_equal 5, all.output.size, "streamed=#{streamed}"
assert_equal [1], heads.output.map { |r| r['n'] }
assert_equal 2, positions.output.size
assert_equal [{ 'satyp' => 'LARC', 'n' => 3 }], trailers.output
end
end
def test_memory_stays_flat_when_streaming
with_tmpdir do |dir|
File.open("#{dir}/big.csv", 'w') { |f| f.puts 'id,text'; 50_000.times { |i| f.puts "#{i},#{'x' * 80}" } }
stats = DataPipeline.new(streamed: true) do
logger PipelineHelper::QUIET
read container(:csv, file: "#{dir}/big.csv")
map { |r| r.merge('len' => r['text'].size) }
write container(:csv, file: "#{dir}/out.csv", columns: %w[id len])
end.run
assert_equal 50_000, stats.written
assert_equal 50_001, File.foreach("#{dir}/out.csv").count
end
end
end