wkh/datapipeline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

datapipeline 2.0.0: Gem aus dem Projekt etl herausgelöst

Browse Source
This commit is contained in:
wkh
2026-10-08 15:30:02 +02:00
commit cce7b03ca9
56 changed files with 5205 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/cli/etl_cli_test.rb
+89
View File
@@ -0,0 +1,89 @@
require_relative '../test_helper'
require 'open3'
# bin/etl as a subprocess: arguments, params, dry run, exit codes
class EtlCliTest < PipelineTest
def etl dir, *args
env = { 'BUNDLE_GEMFILE' => File.join(ROOT, 'Gemfile') }
Open3.capture3(env, RbConfig.ruby, File.join(ROOT, 'bin/etl'), *args, chdir: dir)
end
def script dir, name, body
File.write("#{dir}/in.csv", "a,b\n1,x\n2,y\nz,q\n")
File.write("#{dir}/#{name}", "require #{File.join(ROOT, 'lib/datapipeline').inspect}\n#{body}")
"#{dir}/#{name}"
end
# like import.*.rb: own OptionParser for --from/--till, transform api
def test_legacy_script_gets_from_till_and_limit
with_tmpdir do |dir|
path = script(dir, 'legacy.rb', <<~RUBY)
require 'optparse/date'
opts = {}
OptionParser.new { |p| p.on('--from=DATE', Date) { |v| opts[:from] = v }; p.on('--till=DATE', Date) { |v| opts[:till] = v } }.parse!
File.write('args.txt', "\#{opts[:from]} \#{opts[:till]}")
DataPipeline.new do
read container(:csv, file: 'in.csv')
transform(processing: :prepend) { |_, _| { 'h' => 'HEAD' } }
transform { |r, i| r.merge('i' => i) }
write container(:csv, file: 'out.csv', csv: { headers: false, col_sep: '|' })
end.run
RUBY
_out, err, status = etl(dir, 'run', path, '--from=2025-11-01', '--till=2025-11-30', '--limit=2')
assert_equal 0, status.exitstatus, err
assert_equal '2025-11-01 2025-11-30', File.read("#{dir}/args.txt")
assert_equal "HEAD\n1|x|0\n2|y|1\n", File.read("#{dir}/out.csv")
assert_includes err, 'INFO finish read=2 written=2 rejected=0'
end
end
def test_params_and_exit_code_2_for_rejected_rows
with_tmpdir do |dir|
path = script(dir, 'neu.rb', <<~RUBY)
DataPipeline.new do
on_error :skip
read container(:csv, file: 'in.csv')
validate 'a' => { format: /\\A\\d+\\z/ }
map { |r| r.merge('land' => params[:land]) }
write container(:csv, file: 'out.csv')
end.run
RUBY
_out, err, status = etl(dir, 'run', path, '--param', 'land=DE')
assert_equal 2, status.exitstatus, err
assert_equal "a,b,land\n1,x,DE\n2,y,DE\n", File.read("#{dir}/out.csv")
File.delete("#{dir}/out.csv")
out, _err, status = etl(dir, 'run', path, '--param', 'land=AT', '--dry-run', '--quiet')
assert_equal 2, status.exitstatus
refute File.exist?("#{dir}/out.csv")
assert out.start_with?('DRY RUN')
assert_includes out, '"land":"AT"'
end
end
def test_run_is_optional
with_tmpdir do |dir|
path = script(dir, 'kopie.rb', "DataPipeline.new { read container(:csv, file: 'in.csv'); write container(:csv, file: 'out.csv') }.run\n")
_out, err, status = etl(dir, path, '--quiet')
assert_equal 0, status.exitstatus, err
assert_equal File.read("#{dir}/in.csv"), File.read("#{dir}/out.csv")
end
end
def test_exit_code_1_on_abort_and_usage
with_tmpdir do |dir|
path = script(dir, 'kaputt.rb', "DataPipeline.new { read container(:csv, file: 'in.csv'); map { raise 'boom' }; write container(:csv, file: 'k.csv') }.run\n")
_out, err, status = etl(dir, 'run', path, '--quiet')
assert_equal 1, status.exitstatus
assert_includes err, 'RuntimeError: boom'
_out, err, status = etl(dir, 'gibtsnicht.rb')
assert_equal 1, status.exitstatus
assert_includes err, 'gibtsnicht.rb not found'
_out, err, status = etl(dir)
assert_equal 1, status.exitstatus
assert_includes err, 'Usage: etl [run] PIPELINE.rb'
end
end
end
test/core/chained_test.rb
+75
View File
@@ -0,0 +1,75 @@
require_relative '../test_helper'
# chained steps: map, filter, header, trailer, context
class ChainedTest < PipelineTest
ROWS = [{ 'a' => 1 }, { 'a' => 2 }, { 'a' => 3 }].freeze
def test_chained_transform_gets_previous_output
out = output(ROWS) { chained; transform { |r| r.merge('b' => 1) }; transform { |r| r.merge('c' => 2) } }
assert_equal ROWS.map { |r| r.merge('b' => 1, 'c' => 2) }, out
end
def test_one_to_many_then_index
out = output(ROWS) { chained; transform { |r| [r, r] }; transform { |r, i| r.merge('i' => i) } }
assert_equal (0...6).to_a, out.map { |r| r['i'] }
end
def test_map_and_filter
out = output(ROWS) do
map { |r| r.merge('d' => r['a'] * 2) }
filter { |r| r['d'] > 2 }
map { |r, ctx| r.merge('i' => ctx.index) }
end
assert_equal [{ 'a' => 2, 'd' => 4, 'i' => 0 }, { 'a' => 3, 'd' => 6, 'i' => 1 }], out
end
def test_map_turns_chaining_on_for_transform
out = output(ROWS) { transform { |r| r.merge('b' => 1) }; map { |r| r.merge('c' => 1) } }
assert_equal 3, out.size
assert(out.all? { |r| r['b'] && r['c'] })
end
def test_header_and_trailer_with_context
out = output(ROWS) do
header { |c| { 'H' => c.read, 'W' => c.written } }
filter { |r| r['a'] > 1 }
map { |r| [r, r] }
trailer { |c| { 'T' => c.written } }
end
assert_equal({ 'H' => 3, 'W' => 4 }, out.first)
assert_equal({ 'T' => 4 }, out.last)
assert_equal 6, out.size
end
def test_edge_order_follows_registration
out = output(ROWS) { header { { 'h' => 1 } }; transform(processing: :prepend) { { 'p' => 1 } }; map { |r| r } }
assert_equal [{ 'h' => 1 }, { 'p' => 1 }], out.first(2)
end
def test_header_and_trailer_nil_and_arrays
out = output(ROWS) { header { nil }; trailer { [{ 'x' => 1 }, nil, { 'y' => 2 }] }; map { |r| r } }
assert_equal ROWS + [{ 'x' => 1 }, { 'y' => 2 }], out
end
def test_condition_on_map
assert_equal [{ 'a' => 1 }], output(ROWS) { map(condition: ->(r) { r['a'] == 1 }) { |r| r } }
end
def test_chained_false
assert_equal 6, output(ROWS) { chained false; transform { |r| r }; transform { |r| r } }.size
end
def test_csv_end_to_end
with_tmpdir do |dir|
DataPipeline.new do
logger PipelineHelper::QUIET
read MemoryStorage.new(rows: ROWS)
filter { |r| r['a'].odd? }
map { |r| { 'id' => r['a'], 'x' => 'y' } }
trailer { |c| { 'id' => 'TOTAL', 'x' => c.written } }
write container(:csv, file: "#{dir}/c.csv", columns: %w[id x])
end.run
assert_equal "id,x\n1,y\n3,y\nTOTAL,2\n", File.read("#{dir}/c.csv")
end
end
end
test/core/compat_test.rb
+81
View File
@@ -0,0 +1,81 @@
require_relative '../test_helper'
#
# parallel mode (transform without chained/map/filter) must keep the output of v1.2.1,
# the version the CUDO importers were built on. expected values were recorded with v1.2.1.
#
class CompatTest < PipelineTest
ROWS = (1..6).map { |i| { 'id' => i, 'land' => i.even? ? 'DE' : 'AT' } }
def assert_output expected_json, &block
assert_equal expected_json, JSON.generate(output(ROWS, &block))
end
# pattern of import.customer.rb
def test_customer_pattern
assert_output '[{"SATYP":"FIRC","LFDNR":0},{"id":1,"land":"AT","lfdnr":"0000000001"},' \
'{"id":2,"land":"DE","lfdnr":"0000000002"},{"id":3,"land":"AT","lfdnr":"0000000003"},' \
'{"id":4,"land":"DE","lfdnr":"0000000004"},{"id":5,"land":"AT","lfdnr":"0000000005"},' \
'{"id":6,"land":"DE","lfdnr":"0000000006"},{"topic":"LARC","counter":"0000000006"}]' do
transform(processing: :prepend) { |_row, n| { 'SATYP': 'FIRC', 'LFDNR': n } }
transform(processing: :append) { |_row, n| { 'topic': 'LARC', 'counter': format('%010d', n) } }
transform { |row, _i| row.merge('lfdnr' => format('%010d', row['id'])) }
end
end
# pattern of import.posting.rb: closure counter, 1:n
def test_posting_pattern
expected = [{ SATYP: 'FIRC', LFDNR: 0 }]
lfdnr = 0
ROWS.each do |row|
expected << row.merge('lfdnr' => lfdnr += 1)
row['id'].times { |k| expected << { 'pos' => k, 'lfdnr' => lfdnr += 1 } }
end
expected << { SATYPE: 'LARC', SUMME: '0000000027' }
assert_output JSON.generate(expected) do
cnt = 1
transform(processing: :prepend) { |_row, _n| { 'SATYP': 'FIRC', 'LFDNR': 0 } }
transform do |row|
out = [row.merge('lfdnr' => cnt)]
cnt += 1
row['id'].times do |k|
out << { 'pos' => k, 'lfdnr' => cnt }
cnt += 1
end
out
end
transform(processing: :append) { |_row| { 'SATYPE': 'LARC', 'SUMME': format('%010d', cnt - 1) } }
end
end
def test_append_registered_before_prepend
assert_output '[{"h":0},{"id":1,"land":"AT"},{"id":2,"land":"DE"},{"id":3,"land":"AT"},' \
'{"id":4,"land":"DE"},{"id":5,"land":"AT"},{"id":6,"land":"DE"},{"t":6}]' do
transform(processing: :append) { |_, n| { 't' => n } }
transform { |r| r }
transform(processing: :prepend) { |_, n| { 'h' => n } }
end
end
def test_condition_string
assert_output '[{"id":2,"land":"DE","i":1},{"id":4,"land":"DE","i":3},{"id":6,"land":"DE","i":5}]' do
transform(condition: "row['land'] == 'DE'") { |r, i| r.merge('i' => i) }
end
end
def test_two_parallel_transforms
expected = ROWS.map { |r| r.merge('a' => 1) } + ROWS.map { |r| r.merge('b' => 1) }
assert_output JSON.generate(expected) do
transform { |r| r.merge('a' => 1) }
transform { |r| [r.merge('b' => 1), nil] }
end
end
def test_nil_and_empty_array
assert_output '[{"id":1,"land":"AT"}]' do
transform { |r| r['id'] > 3 ? nil : [] }
transform { |r| r['id'] == 1 ? r : nil }
end
end
end
test/core/layout_test.rb
+74
View File
@@ -0,0 +1,74 @@
require_relative '../test_helper'
class LayoutTest < PipelineTest
LAYOUT = DataPipeline::Layout.new do
record 'HEAD', [['SATYP', 'AN', 4, true], ['DATUM', 'DAT', 8, true], ['ZEIT', 'TIM', 6]]
record 'POS', [['SATYP', 'AN', 4, true], ['NR', 'N', 3, true], ['TEXT', 'AN', 5], ['BETRAG', 'N', '13.2']]
end
def test_values_in_field_order_any_key_case
assert_equal ['POS', '1', 'abc', nil], LAYOUT.values({ SATYP: 'POS', betrag: nil, 'text' => 'abc', 'NR' => '1' })
assert_equal ['HEAD', '20251103', nil], LAYOUT.values({ 'satyp' => 'HEAD', 'datum' => '20251103' })
end
def test_unknown_field_and_record_type
assert_includes assert_raises(DataPipeline::LayoutError) { LAYOUT.values({ 'satyp' => 'POS', 'extra' => 1 }) }.message, 'extra'
assert_includes assert_raises(DataPipeline::LayoutError) { LAYOUT.values({ 'satyp' => 'XXXX' }) }.message, 'XXXX'
lenient = DataPipeline::Layout.new(strict: false) { record 'A', [['SATYP']] }
assert_equal ['A'], lenient.values({ 'satyp' => 'A', 'extra' => 1 })
end
def test_type_field
layout = DataPipeline::Layout.new(type_field: 'art') { record 'X', [['ID'], ['ART']] }
assert_equal %w[7 X], layout.values({ 'id' => '7', 'art' => 'X' })
end
def test_errors
assert_empty LAYOUT.errors({ 'satyp' => 'POS', 'nr' => '001', 'text' => 'abcde', 'betrag' => '-123.45' })
assert_equal ['NR: missing', 'TEXT: longer than 5', 'BETRAG: more than 2 decimals'],
LAYOUT.errors({ 'satyp' => 'POS', 'text' => 'abcdef', 'betrag' => '1.234' })
assert_equal ['NR: more than 3 digits', 'BETRAG: not a number'], LAYOUT.errors({ 'satyp' => 'POS', 'nr' => '1000', 'betrag' => '1,5' })
assert_equal ['DATUM: not a date YYYYMMDD', 'ZEIT: not a time HHMMSS'], LAYOUT.errors({ 'satyp' => 'HEAD', 'datum' => '20251331', 'zeit' => '250000' })
assert_equal ['record type "?" not in layout (HEAD, POS)'], LAYOUT.errors({ 'satyp' => '?' })
end
def test_csv_with_layout
with_tmpdir do |dir|
rows = [{ SATYP: 'HEAD', DATUM: '20251103' }, { 'satyp' => 'POS', 'betrag' => '1.50', 'nr' => '1' }]
DataPipeline::CsvStorage.new(file: "#{dir}/l.ok", layout: LAYOUT, csv: { col_sep: '|', quote_char: '' }).write(rows)
assert_equal "HEAD|20251103|\nPOS|1||1.50\n", File.read("#{dir}/l.ok")
end
end
def test_validate_layout_abort_warn_and_ignore
rows = [{ 'satyp' => 'POS', 'nr' => '1' }, { 'satyp' => 'POS', 'text' => 'zu lang' }]
assert_raises(DataPipeline::ValidationError) { run_pipeline(rows) { validate LAYOUT } }
target, stats = run_pipeline(rows) { validate LAYOUT, warn: true }
assert_equal 2, target.output.size
assert_equal 1, stats.warnings
assert_match(/warnings=1/, stats.to_s)
_target, stats = run_pipeline(rows) { validate LAYOUT, warn: true, ignore: %w[NR TEXT] }
assert_equal 0, stats.warnings
end
def test_warning_log_is_capped
log = StringIO.new
stats = DataPipeline.new do
logger Logger.new(log)
read MemoryStorage.new(rows: Array.new(150) { { 'satyp' => 'POS' } })
write MemoryStorage.new
validate LAYOUT, warn: true
end.run
assert_equal 150, stats.warnings
assert_equal 101, log.string.lines.count { |l| l.include?('WARN') }
assert_includes log.string, 'more than 100 warnings'
end
def test_validate_hash_without_braces
target, stats = run_pipeline([{ 'a' => 'x' }, { 'a' => '1' }]) { on_error :skip; validate 'a' => { format: /\A\d\z/ } }
assert_equal [{ 'a' => '1' }], target.output
assert_equal 1, stats.rejected
end
end
test/core/operations_test.rb
+178
View File
@@ -0,0 +1,178 @@
require_relative '../test_helper'
# on_error, validate, stats, hooks, watermark, dry run, limit
class OperationsTest < PipelineTest
ROWS = (1..5).map { |i| { 'id' => i.to_s, 'shkzg' => i == 3 ? 'X' : 'S', 'betrag' => i == 4 ? 'abc' : "#{i}.50" } }.freeze
RULES = { 'id' => { required: true, format: /\A\d+\z/ }, 'shkzg' => { in: %w[S H] }, 'betrag' => { type: :decimal } }.freeze
def failing_map = proc { map { |r| r['id'] == '2' ? raise('kaputt') : r } }
def test_abort_is_default
error = assert_raises(RuntimeError) { run_pipeline(ROWS, &failing_map) }
assert_equal 'kaputt', error.message
assert_equal 'kaputt', DataPipeline.runs.last.error.message
end
def test_skip
map = failing_map
target, stats = run_pipeline(ROWS) { on_error :skip; instance_eval(&map) }
assert_equal 4, target.output.size
assert_equal 1, stats.rejected
assert_equal 'map#1', stats.errors.first[:step]
refute stats.ok?
end
def test_collect_batch_and_stream
[false, true].each do |streamed|
rejects = MemoryStorage.new
map = failing_map
target, = run_pipeline(ROWS, streamed:) { on_error :collect, to: rejects; instance_eval(&map) }
assert_equal [ROWS[1].merge('_step' => 'map#1', '_error' => 'kaputt')], rejects.streamed_rows
assert_equal 4, target.output.size
end
end
def test_on_error_arguments
assert_raises(ArgumentError) { DataPipeline.new { on_error :collect } }
assert_raises(ArgumentError) { DataPipeline.new { on_error :ignore } }
end
def test_edge_errors_always_abort
assert_raises(RuntimeError) { run_pipeline(ROWS) { on_error :skip; header { raise 'h' }; map { |r| r } } }
end
def test_skip_in_parallel_mode
target, stats = run_pipeline(ROWS) { on_error :skip; transform { |r| r['id'] == '1' ? raise('x') : r } }
assert_equal 4, target.output.size
assert_equal 1, stats.rejected
end
def test_validate_rules
assert_raises(DataPipeline::ValidationError) { run_pipeline(ROWS) { validate RULES } }
target, stats = run_pipeline(ROWS) { on_error :skip; validate RULES }
assert_equal %w[1 2 5], target.output.map { |r| r['id'] }
assert_equal 2, stats.rejected
end
def test_validate_block
target, stats = run_pipeline(ROWS) { on_error :skip; validate { |r| 'zu gross' if r['id'].to_i > 3 } }
assert_equal 3, target.output.size
assert_equal 'zu gross', stats.errors.first[:error]
end
def test_validation_messages
errors = DataPipeline::Validation.errors({ 'a' => '12345', 'd' => '2025-13-01' },
'a' => { max: 3, format: /x/ }, 'd' => { type: :date }, 'b' => { required: true })
assert_equal ['a: longer than 3', 'a: format /x/', 'd: not date', 'b: missing'], errors
assert_empty DataPipeline::Validation.errors({ 'a' => '' }, 'a' => { max: 1, type: :integer })
end
def test_validate_turns_chaining_on
target, = run_pipeline(ROWS) { on_error :skip; transform { |r| r.merge('x' => 1) }; validate RULES }
assert_equal 3, target.output.size
assert(target.output.all? { |r| r['x'] })
end
def test_stats
_target, stats = run_pipeline(ROWS) { map { |r| [r, r] } }
assert_equal [5, 10, 10], [stats.read, stats.written, stats.output]
assert stats.ok?
assert_kind_of Float, stats.duration
map = failing_map
_target, stats = run_pipeline(ROWS) { on_error :skip; instance_eval(&map) }
assert_match(/\Aread=5 written=4 rejected=1/, stats.to_s)
end
def test_on_finish_after_success_and_abort
seen = []
run_pipeline(ROWS) { on_finish { |s| seen << s.error } }
assert_raises(RuntimeError) { run_pipeline(ROWS) { on_finish { |s| seen << s.error.class }; map { raise 'x' } } }
assert_equal [nil, RuntimeError], seen
end
def test_hook_error_is_only_logged
log = StringIO.new
stats = DataPipeline.new do
logger Logger.new(log)
read MemoryStorage.new(rows: ROWS)
write MemoryStorage.new
on_finish { raise 'hook' }
end.run
assert stats.ok?
assert_includes log.string, 'on_finish: RuntimeError: hook'
end
def test_logger_messages
log = StringIO.new
DataPipeline.new do
logger Logger.new(log)
on_error :skip
read MemoryStorage.new(rows: ROWS)
write MemoryStorage.new
map { |r| r['id'] == '1' ? raise('weg') : r }
end.run
assert_includes log.string, 'rejected by map#1: weg'
assert_includes log.string, 'finish read=5 written=4 rejected=1'
end
def test_watermark
with_tmpdir do |dir|
file = "#{dir}/state.json"
wm = nil
run_pipeline(ROWS) { wm = watermark :t, column: 'id', file:, initial: 0 }
assert_equal '5', wm.value
run_pipeline([{ 'id' => '9' }, { 'id' => '10' }]) { watermark :t, column: 'id', file: }
assert_equal '10', JSON.parse(File.read(file))['t'], 'numeric texts compared as numbers'
assert_raises(RuntimeError) { run_pipeline([{ 'id' => '99' }]) { watermark :t, column: 'id', file:; map { raise 'x' } } }
assert_equal '10', JSON.parse(File.read(file))['t'], 'not saved after abort'
run_pipeline(ROWS, streamed: true) { watermark :s, column: 'id', file: }
assert_equal({ 't' => '10', 's' => '5' }, JSON.parse(File.read(file)))
end
end
def test_dry_run
with_tmpdir do |dir|
rejects = MemoryStorage.new
target = nil
text = capture_stdout do
target, = run_pipeline(ROWS, dry_run: true, preview: 2) do
watermark :d, column: 'id', file: "#{dir}/state.json"
on_error :collect, to: rejects
map { |r| r['id'] == '1' ? raise('x') : r }
end
end
assert_nil target.written
assert_empty target.streamed_rows
assert_empty rejects.streamed_rows
refute File.exist?("#{dir}/state.json")
assert text.start_with?('DRY RUN')
assert_equal 3, text.lines.size
assert DataPipeline.runs.last.dry_run
end
end
def test_dry_run_creates_no_csv_file
with_tmpdir do |dir|
capture_stdout do
DataPipeline.new(dry_run: true) do
logger PipelineHelper::QUIET
read MemoryStorage.new(rows: ROWS)
write container(:csv, file: "#{dir}/dry.csv")
end.run
end
refute File.exist?("#{dir}/dry.csv")
end
end
def test_limit
target, stats = run_pipeline(ROWS, limit: 2) { map { |r| r } }
assert_equal [2, 2], [target.output.size, stats.read]
target, stats = run_pipeline(ROWS, streamed: true, limit: 3) { trailer { |c| { 'n' => c.read } } }
assert_equal({ 'n' => 3 }, target.output.last)
assert_equal 3, stats.read
end
end
test/core/pipeline_test.rb
+131
View File
@@ -0,0 +1,131 @@
require_relative '../test_helper'
# dsl, settings, plugin lookup, parallel mode
class PipelineCoreTest < PipelineTest
ROWS = [{ 'a' => 1 }, { 'a' => 2 }, { 'a' => 3 }].freeze
def settings_of(pipeline) = pipeline.instance_variable_get(:@settings)
def test_nil_filters_rows
assert_equal [{ 'a' => 1 }], output(ROWS) { transform { |r| r['a'] == 1 ? r : nil } }
assert_equal ROWS, output(ROWS) { transform { |r| [r, nil] } }
assert_equal ROWS, output(ROWS) { transform(processing: :prepend) { nil }; transform { |r| r }; transform(processing: :append) { nil } }
end
def test_dsl_switches
assert_equal false, settings_of(DataPipeline.new { verbose false })[:verbose]
assert_equal true, settings_of(DataPipeline.new { verbose })[:verbose]
assert_equal false, settings_of(DataPipeline.new { streamed false })[:streamed]
assert_equal '1.0', settings_of(DataPipeline.new { version '1.0' })[:version]
assert_equal true, settings_of(DataPipeline.new { configure verbose: true })[:verbose]
end
def test_typo_raises
assert_raises(NoMethodError) { DataPipeline.new { verbsoe true } }
end
def test_unknown_plugin
error = assert_raises(RuntimeError) { DataPipeline.new { container :gibtsnicht } }
assert_includes error.message, 'GibtsnichtStorage'
end
def test_plugin_errors_pass_through
assert_raises(ArgumentError) { DataPipeline.new { container :csv } }
broken = Class.new(DataPipeline::StoragePlugin) { def initialize(**) = undefined_thing }
DataPipeline.register :broken, broken
assert_includes assert_raises(NameError) { DataPipeline.new { container :broken } }.message, 'undefined_thing'
end
def test_register_lazy_path
with_tmpdir do |dir|
File.write("#{dir}/lazy_storage.rb", "class LazyStorage < MemoryStorage; end\n")
DataPipeline.register :lazy, 'LazyStorage', path: "#{dir}/lazy_storage.rb"
assert_empty $LOADED_FEATURES.grep(/lazy_storage/)
storage = nil
DataPipeline.new { storage = container :lazy }
assert_equal 'LazyStorage', storage.class.name
refute_empty $LOADED_FEATURES.grep(/lazy_storage/)
end
end
def test_csv_plugin_by_convention
assert_equal DataPipeline::CsvStorage, DataPipeline.plugin(:csv)
end
def test_settings_are_frozen_and_replaced
assert DataPipeline.settings.frozen?
pipeline = DataPipeline.new { logger PipelineHelper::QUIET }
DataPipeline.settings(verbose: true)
assert_equal false, settings_of(pipeline)[:verbose]
assert_equal true, DataPipeline.settings[:verbose]
DataPipeline.reset_settings!
assert_equal DataPipeline::DEFAULT_SETTINGS, DataPipeline.settings
end
def test_settings_per_pipeline
_target, stats = run_pipeline(ROWS, limit: 1)
assert_equal 1, stats.read
assert_nil DataPipeline.settings[:limit]
end
def test_parallel_threads
reads = Array.new(8) { |i| Thread.new { run_pipeline(ROWS, limit: (i % 3) + 1).last.read } }.map(&:value)
assert_equal Array.new(8) { |i| (i % 3) + 1 }, reads
end
def test_parallel_mode_is_side_by_side
out = output(ROWS) { transform { |r| r.merge('b' => 1) }; transform { |r| r.merge('c' => 2) } }
assert_equal 6, out.size
end
def test_parallel_mode_copies_rows_per_step
out = output(ROWS) { transform { |r| r['x'] = 1; r }; transform { |r| r.key?('x') ? nil : r } }
assert_equal 6, out.size
assert_equal [{ 'a' => 10 }, { 'a' => 20 }, { 'a' => 30 }], output(ROWS) { transform { |r| [r.transform_values! { |v| v * 10 }] } }
end
def test_stable_order_and_all_prepends_appends
out = output(ROWS) { 20.times { |k| transform { |r| r['a'] == 1 ? { 'k' => k } : nil } } }
assert_equal (0...20).to_a, out.map { |r| r['k'] }
out = output(ROWS) do
transform(processing: :prepend) { { 'h' => 1 } }
transform(processing: :prepend) { { 'h' => 2 } }
transform { |r| r }
transform(processing: :append) { { 't' => 1 } }
transform(processing: :append) { { 't' => 2 } }
end
assert_equal [{ 'h' => 1 }, { 'h' => 2 }, *ROWS, { 't' => 1 }, { 't' => 2 }], out
end
def test_append_count_parallel_vs_chained
assert_equal({ 'n' => 3 }, output(ROWS) { transform { |r| [r, r] }; transform(processing: :append) { |_, n| { 'n' => n } } }.last)
assert_equal({ 'n' => 6 }, output(ROWS) { chained; transform { |r| [r, r] }; transform(processing: :append) { |_, n| { 'n' => n } } }.last)
end
def test_conditions
assert_equal [{ 'a' => 1 }, { 'a' => 3 }], output(ROWS) { transform(condition: ->(r) { r['a'] != 2 }) { |r| r } }
assert_equal [{ 'a' => 2 }], output(ROWS) { transform(condition: "row['a'] == 2") { |r| r } }
end
def test_copy_without_steps
assert_equal ROWS, output(ROWS)
end
def test_closes_source_and_target
source = MemoryStorage.new(rows: ROWS)
target = MemoryStorage.new
DataPipeline.new { logger PipelineHelper::QUIET; read source; write target }.run
assert source.closed?
assert target.closed?
end
def test_read_and_write_required
assert_includes assert_raises(ArgumentError) { DataPipeline.new { logger PipelineHelper::QUIET; read MemoryStorage.new }.run }.message, 'write'
assert_includes assert_raises(ArgumentError) { DataPipeline.new { logger PipelineHelper::QUIET; write MemoryStorage.new }.run }.message, 'read'
end
def test_block_required
assert_raises(ArgumentError) { DataPipeline.new { map } }
end
end
test/core/streaming_test.rb
+153
View File
@@ -0,0 +1,153 @@
require_relative '../test_helper'
# streamed mode, lookups, several targets
class StreamingTest < PipelineTest
ROWS = [{ 'a' => 1 }, { 'a' => 2 }, { 'a' => 3 }].freeze
CHAIN = proc do
header { |c| { 'H' => c.written } }
filter { |r| r['a'] > 1 }
map { |r, c| [r, r.merge('k' => c.index)] }
map { |r, c| r.merge('i' => c.index) }
trailer { |c| { 'T' => c.written, 'R' => c.read } }
end
def test_batch_and_stream_same_body
batch = output(ROWS, &CHAIN)
stream = output(ROWS, streamed: true, &CHAIN)
assert_equal batch[1..-2], stream[1..-2]
assert_equal({ 'H' => 4 }, batch.first)
assert_equal({ 'H' => nil }, stream.first)
assert_equal({ 'T' => 4, 'R' => 3 }, stream.last)
end
def test_stream_prepend_append
out = output(ROWS, streamed: true) do
chained
transform(processing: :prepend) { |_, n| { 'p' => n } }
transform { |r| [r, r] }
transform(processing: :append) { |_, n| { 'n' => n } }
end
assert_equal({ 'p' => 0 }, out.first)
assert_equal({ 'n' => 6 }, out.last)
assert_equal 8, out.size
end
def test_stream_parallel_single_step_and_error_for_several
out = output(ROWS, streamed: true) { transform { |r| [r, r] }; transform(processing: :append) { |_, n| { 'n' => n } } }
assert_equal({ 'n' => 3 }, out.last)
error = assert_raises(ArgumentError) { output(ROWS, streamed: true) { transform { |r| r }; transform { |r| r } } }
assert_includes error.message, 'chained'
end
def test_stream_condition_and_copy
assert_equal [{ 'a' => 1 }, { 'a' => 3 }], output(ROWS, streamed: true) { map(condition: ->(r) { r['a'] != 2 }) { |r| r } }
assert_equal ROWS, output(ROWS, streamed: true)
end
def test_stream_csv_to_csv
with_tmpdir do |dir|
File.write("#{dir}/in.csv", "a,b\n1,x\n2,y\n3,z\n")
DataPipeline.new do
logger PipelineHelper::QUIET
streamed
read container(:csv, file: "#{dir}/in.csv")
filter { |r| r['a'] != '2' }
trailer { |c| { 'a' => 'T', 'b' => c.written } }
write container(:csv, file: "#{dir}/out.csv")
end.run
assert_equal "a,b\n1,x\n3,z\nT,2\n", File.read("#{dir}/out.csv")
end
end
def test_stream_postgres_with_query_on_same_connection
with_pg_database do |db|
pg_exec db, "create table k as select i as id, case when i % 2 = 0 then 'DE' else 'AT' end as land from generate_series(1, 5) i;
create table l (land text, name text); insert into l values ('DE','Deutschland'),('AT','Oesterreich')"
with_tmpdir do |dir|
DataPipeline.new do
logger PipelineHelper::QUIET
streamed
pg = container :postgres, **db
read pg, query: 'select id, land from k where id <= $1 order by id', params: [5], fetch_size: 2
map { |r| r.merge('name' => query(pg, query: 'select name from l where land = $1', params: [r['land']]).first['name']) }
write container(:csv, file: "#{dir}/o.csv", columns: %w[id name])
end.run
assert_equal 6, File.readlines("#{dir}/o.csv").size
assert_includes File.read("#{dir}/o.csv"), '2,Deutschland'
end
end
end
def test_lookup_grouped_unique_and_lambda
with_tmpdir do |dir|
File.write("#{dir}/lk.csv", "id,grp\n1,a\n2,b\n3,a\n")
grouped = unique = by_lambda = nil
DataPipeline.new do
csv = container(:csv, file: "#{dir}/lk.csv")
grouped = lookup csv, key: 'grp'
unique = lookup csv, key: 'id', unique: true
by_lambda = lookup csv, key: ->(r) { r['grp'].upcase }
end
assert_equal 2, grouped['a'].size
assert_equal 'b', unique['2']['grp']
assert_equal %w[A B], by_lambda.keys.sort
end
end
def test_lookup_postgres_join_without_n_plus_one
with_pg_database do |db|
pg_exec db, "create table k as select i as id, case when i % 2 = 0 then 'DE' else 'AT' end as land from generate_series(1, 3) i;
create table l (land text, name text); insert into l values ('DE','Deutschland'),('AT','Oesterreich')"
out = nil
DataPipeline.new do
logger PipelineHelper::QUIET
pg = container :postgres, **db
names = lookup pg, query: 'select * from l where land like $1', params: ['%'], key: 'land', unique: true
read pg, query: 'select id, land from k order by id'
map { |r| r.merge('name' => names.dig(r['land'], 'name')) }
write(out = MemoryStorage.new)
end.run
assert_equal %w[Oesterreich Deutschland Oesterreich], out.output.map { |r| r['name'] }
end
end
def test_several_targets_with_only
rows = [{ 'satyp' => 'DOHD', 'n' => 1 }, { 'satyp' => 'DOPS', 'n' => 2 }, { 'satyp' => 'DOPS', 'n' => 3 }]
[false, true].each do |streamed|
all = MemoryStorage.new
heads = MemoryStorage.new
positions = MemoryStorage.new
trailers = MemoryStorage.new
DataPipeline.new(streamed:) do
logger PipelineHelper::QUIET
read MemoryStorage.new(rows:)
header { { 'satyp' => 'FIRC' } }
map { |r| r }
trailer { |c| { 'satyp' => 'LARC', 'n' => c.written } }
write all
write heads, only: ->(r) { r['satyp'] == 'DOHD' }
write positions, only: ->(r) { r['satyp'] == 'DOPS' }
write trailers, only: ->(r) { r['satyp'] == 'LARC' }
end.run
assert_equal 5, all.output.size, "streamed=#{streamed}"
assert_equal [1], heads.output.map { |r| r['n'] }
assert_equal 2, positions.output.size
assert_equal [{ 'satyp' => 'LARC', 'n' => 3 }], trailers.output
end
end
def test_memory_stays_flat_when_streaming
with_tmpdir do |dir|
File.open("#{dir}/big.csv", 'w') { |f| f.puts 'id,text'; 50_000.times { |i| f.puts "#{i},#{'x' * 80}" } }
stats = DataPipeline.new(streamed: true) do
logger PipelineHelper::QUIET
read container(:csv, file: "#{dir}/big.csv")
map { |r| r.merge('len' => r['text'].size) }
write container(:csv, file: "#{dir}/out.csv", columns: %w[id len])
end.run
assert_equal 50_000, stats.written
assert_equal 50_001, File.foreach("#{dir}/out.csv").count
end
end
end
test/examples/doc_examples_test.rb
+155
View File
@@ -0,0 +1,155 @@
require_relative '../test_helper'
require_relative '../../examples/demo_api'
require 'open3'
require 'fileutils'
#
# the examples of README.md section 1: the code in the documentation is the file,
# and every example runs and produces the documented result
#
class DocExamplesTest < PipelineTest
DOC = File.read(File.join(ROOT, 'README.md'))
EXAMPLES = DOC.scan(/<!-- beispiel: (\S+) -->\n```ruby\n(.*?)\n```/m)
def run_example dir, name, *args
env = { 'BUNDLE_GEMFILE' => File.join(ROOT, 'Gemfile') }
Open3.capture3(env, RbConfig.ruby, File.join(ROOT, 'bin/etl'), 'run', File.join(ROOT, name), "--out=#{dir}", *args,
chdir: dir)
end
def test_documented_code_equals_files
assert_equal %w[examples/kunden_import.rb examples/rechnungen_satzarten.rb examples/bewegungen_inkrementell.rb
examples/eigenes_plugin.rb examples/rest_lesen.rb examples/rest_schreiben.rb
examples/rest_zu_rest.rb], EXAMPLES.map(&:first)
EXAMPLES.each do |file, code|
assert_equal File.read(File.join(ROOT, file)).rstrip, code.rstrip, "#{file} differs from README.md"
end
end
def test_kunden_import
with_tmpdir do |dir|
out, err, status = run_example(dir, 'examples/kunden_import.rb')
assert_equal 2, status.exitstatus, err
assert_includes out, 'kunden_import: read=9 written=4 rejected=4'
db = DataPipeline::SqliteStorage.new(database: "#{dir}/kunden.sqlite3", query: 'select * from kunden order by kundennr')
rows = db.read
db.close
assert_equal %w[100001 100002 100003 100009], rows.map { |r| r['kundennr'] }
assert_equal({ 'kundennr' => '100002', 'name' => 'Huber & Partner', 'land' => 'Österreich', 'eu' => 'ja', 'umsatz' => 8300.0 }, rows[1])
excel = DataPipeline::ExcelStorage.new(file: "#{dir}/eu_kunden.xlsx", sheet: 'EU-Kunden').read
assert_equal %w[100001 100002 100009], excel.map { |r| r['kundennr'] }
errors = DataPipeline::CsvStorage.new(file: "#{dir}/kunden_fehler.csv").read
assert_equal ['name: missing', 'kundennr: format /\A\d{6}\z/', 'Land XX unbekannt', 'umsatz: not decimal'],
errors.map { |r| r['_error'] }
# a second run replaces the table content (truncate)
run_example(dir, 'examples/kunden_import.rb')
db = DataPipeline::SqliteStorage.new(database: "#{dir}/kunden.sqlite3", query: 'select count(*) as n from kunden')
assert_equal 4, db.read.first['n']
db.close
end
end
def test_kunden_import_dry_run_creates_no_files
with_tmpdir do |dir|
out, _err, status = run_example(dir, 'examples/kunden_import.rb', '--dry-run')
assert_equal 2, status.exitstatus
assert_includes out, 'DRY RUN'
assert_empty Dir.children(dir)
end
end
def test_rechnungen_satzarten
with_tmpdir do |dir|
_out, err, status = run_example(dir, 'examples/rechnungen_satzarten.rb')
assert status.success?, err
assert_includes err, 'TEXT: longer than 30'
documented = DOC[/Ergebnis `tmp\/rechnungen.txt`.*?```text\n(.*?)```/m, 1]
expected = documented.sub(/\ADATEI\|\d{8}/, "DATEI|#{Date.today.strftime('%Y%m%d')}")
assert_equal expected, File.read("#{dir}/rechnungen.txt")
assert_equal 3, File.readlines("#{dir}/positionen.ndjson").size
end
end
def test_bewegungen_inkrementell
with_tmpdir do |dir|
out, err, status = run_example(dir, 'examples/bewegungen_inkrementell.rb')
assert status.success?, err
assert_includes out, 'neu: 4, Stand jetzt: 4'
file = Dir["#{dir}/bewegungen_*.ndjson"].first
assert_equal({ 'id' => 2, 'konto' => '4000', 'betrag' => 20.0, 'soll_haben' => 'H' }, JSON.parse(File.readlines(file)[1]))
FileUtils.rm(file)
out, = run_example(dir, 'examples/bewegungen_inkrementell.rb')
assert_includes out, 'neu: 0, Stand jetzt: 4'
assert_empty Dir["#{dir}/bewegungen_*.ndjson"], 'no file without new rows'
end
end
def test_eigenes_plugin
with_tmpdir do |dir|
_out, err, status = run_example(dir, 'examples/eigenes_plugin.rb')
assert status.success?, err
documented = DOC[/Ergebnis `tmp\/alle_filialen.csv`:\n\n```text\n(.*?)```/m, 1]
assert_equal documented, File.read("#{dir}/alle_filialen.csv")
end
end
def with_demo_api
api = DemoApi.start(port: 0)
yield api
ensure
api&.stop
end
def test_rest_lesen
with_demo_api do |api|
with_tmpdir do |dir|
out, err, status = run_example(dir, 'examples/rest_lesen.rb', "--url=#{api.url}")
assert_equal 2, status.exitstatus, err
assert_includes out, 'rest_lesen: read=7 written=6 rejected=1'
documented = DOC[/Ergebnis `tmp\/api_kunden.csv`:\n\n```text\n(.*?)```/m, 1]
assert_equal documented, File.read("#{dir}/api_kunden.csv")
assert_equal 6, JSON.parse(File.read("#{dir}/api_kunden.json")).size
assert_equal ['umsatz: not decimal'], DataPipeline::CsvStorage.new(file: "#{dir}/api_fehler.csv").read.map { |r| r['_error'] }
end
end
end
def test_rest_schreiben_and_dry_run
with_demo_api do |api|
with_tmpdir do |dir|
out, _err, status = run_example(dir, 'examples/rest_schreiben.rb', "--url=#{api.url}", '--dry-run')
assert_equal 2, status.exitstatus
assert_includes out, 'DRY RUN'
assert_empty api.imported, 'dry run sends nothing'
out, err, status = run_example(dir, 'examples/rest_schreiben.rb', "--url=#{api.url}")
assert_equal 2, status.exitstatus, err
assert_includes out, 'gesendet: 6 Kunden in 3 Paketen'
assert_equal %w[100001 100002 100003 100007 100008 100009], api.imported.map { |r| r['kundennr'] }
assert_equal 2, DataPipeline::CsvStorage.new(file: "#{dir}/import_fehler.csv").read.size
end
end
end
def test_rest_zu_rest
with_demo_api do |api|
with_tmpdir do |dir|
out, err, status = run_example(dir, 'examples/rest_zu_rest.rb', "--url=#{api.url}")
assert status.success?, err
assert_includes out, 'archiviert: 5, Stand jetzt: 5'
assert_equal %w[1 2 3 4 5], api.archive.keys
assert_equal Date.today.iso8601, api.archive['3']['archiviert']
api.archive.clear
out, = run_example(dir, 'examples/rest_zu_rest.rb', "--url=#{api.url}")
assert_includes out, 'archiviert: 0, Stand jetzt: 5'
assert_empty api.archive
end
end
end
end
test/plugins/csv_storage_test.rb
+89
View File
@@ -0,0 +1,89 @@
require_relative '../test_helper'
require_relative '../support/storage_contract'
class CsvStorageTest < PipelineTest
include StorageContract
def contract_storage(dir, _purpose) = DataPipeline::CsvStorage.new(file: "#{dir}/data.csv")
def test_read_with_defaults
with_tmpdir do |dir|
File.write("#{dir}/in.csv", "a,b\n1,x\n2,\"\"\n")
assert_equal [{ 'a' => '1', 'b' => 'x' }, { 'a' => '2', 'b' => '' }], DataPipeline::CsvStorage.new(file: "#{dir}/in.csv").read
end
end
def test_read_without_header_uses_columns_or_index
with_tmpdir do |dir|
File.write("#{dir}/in.csv", "1|x\n2|y\n")
csv = { headers: false, col_sep: '|' }
assert_equal [{ 'id' => '1', 'txt' => 'x' }, { 'id' => '2', 'txt' => 'y' }],
DataPipeline::CsvStorage.new(file: "#{dir}/in.csv", columns: %w[id txt], csv:).read
assert_equal({ '0' => '1', '1' => 'x' }, DataPipeline::CsvStorage.new(file: "#{dir}/in.csv", csv:).read.first)
end
end
def test_read_streaming_with_block
with_tmpdir do |dir|
File.write("#{dir}/in.csv", "a\n1\n2\n")
rows = []
DataPipeline::CsvStorage.new(file: "#{dir}/in.csv").read { |row| rows << row }
assert_equal %w[1 2], rows.map { |r| r['a'] }
end
end
def test_defaults_are_merged
options = DataPipeline::CsvStorage.new(file: 'x', csv: { col_sep: ';' }).instance_variable_get(:@options)[:csv]
assert_equal ';', options[:col_sep]
assert options[:headers]
end
def test_file_is_required
error = assert_raises(ArgumentError) { DataPipeline::CsvStorage.new }
assert_includes error.message, 'file'
end
def test_write_without_csv_options
with_tmpdir do |dir|
DataPipeline::CsvStorage.new(file: "#{dir}/o.csv").write([{ 'a' => 1, 'b' => 2 }])
assert_equal "a,b\n1,2\n", File.read("#{dir}/o.csv")
end
end
def test_columns_fix_order_missing_fields_and_key_types
with_tmpdir do |dir|
DataPipeline::CsvStorage.new(file: "#{dir}/o.csv", columns: %w[a b c]).write([{ 'b' => 2, 'a' => 1 }, { c: 3, a: 4 }])
assert_equal "a,b,c\n1,2,\n4,,3\n", File.read("#{dir}/o.csv")
end
end
def test_empty_data
with_tmpdir do |dir|
DataPipeline::CsvStorage.new(file: "#{dir}/a.csv").write([])
DataPipeline::CsvStorage.new(file: "#{dir}/b.csv", columns: %w[a b]).write([])
assert_equal '', File.read("#{dir}/a.csv")
assert_equal "a,b\n", File.read("#{dir}/b.csv")
end
end
def test_write_row_and_close
with_tmpdir do |dir|
csv = DataPipeline::CsvStorage.new(file: "#{dir}/o.csv")
csv.write_row({ 'a' => 1 })
csv.write_row({ 'a' => 2 })
csv.close
assert_equal "a\n1\n2\n", File.read("#{dir}/o.csv")
end
end
# options of the CUDO exports: no header, pipe, no quotes, mixed record types and key types
def test_cudo_format_mixed_record_types
with_tmpdir do |dir|
rows = [{ SATYP: 'FIRC', LFDNR: 0 }, { 'satyp' => 'DOHD', 'belnr' => 'AR1', 'wwert' => nil },
{ 'satyp' => 'DOPS', 'dmbtr' => '1.50' }, { 'SATYPE' => 'LARC', 'SUMME' => '0000000002' }]
DataPipeline::CsvStorage.new(file: "#{dir}/c.ok", csv: { headers: false, col_sep: '|', quote_char: '', encoding: 'utf-8' })
.write(rows)
assert_equal "FIRC|0\nDOHD|AR1|\nDOPS|1.50\nLARC|0000000002\n", File.read("#{dir}/c.ok")
end
end
end
test/plugins/excel_storage_test.rb
+57
View File
@@ -0,0 +1,57 @@
require_relative '../test_helper'
require_relative '../support/storage_contract'
class ExcelStorageTest < PipelineTest
include StorageContract
def contract_storage(dir, _purpose) = DataPipeline::ExcelStorage.new(file: "#{dir}/data.xlsx")
# an empty text is an empty cell in excel
def contract_rows(rows) = rows.map { |row| row.transform_values { |v| v == '' ? nil : v } }
def test_text_stays_text_numbers_and_dates_keep_type
with_tmpdir do |dir|
rows = [{ 'lfdnr' => '0000000001', 'betrag' => 1.5, 'menge' => 3, 'datum' => Date.new(2025, 11, 3), 'text' => '=1+1' }]
DataPipeline::ExcelStorage.new(file: "#{dir}/t.xlsx").write(rows)
row = DataPipeline::ExcelStorage.new(file: "#{dir}/t.xlsx").read.first
assert_equal '0000000001', row['lfdnr']
assert_in_delta 1.5, row['betrag']
assert_equal 3, row['menge'].to_i
assert_equal Date.new(2025, 11, 3), row['datum'].to_date
assert_equal '=1+1', row['text']
end
end
def test_columns_sheet_name_and_symbol_keys
with_tmpdir do |dir|
DataPipeline::ExcelStorage.new(file: "#{dir}/c.xlsx", sheet: 'Buchungen', columns: %w[b a])
.write([{ a: '1', b: '2' }, { 'b' => '4' }])
assert_equal [{ 'b' => '2', 'a' => '1' }, { 'b' => '4', 'a' => nil }],
DataPipeline::ExcelStorage.new(file: "#{dir}/c.xlsx", sheet: 'Buchungen').read
end
end
def test_without_header_and_skip_title_rows
with_tmpdir do |dir|
DataPipeline::ExcelStorage.new(file: "#{dir}/h.xlsx", headers: false).write([{ 'x' => 'Titel' }, { 'x' => 'Kopf' }, { 'x' => 'Wert' }])
assert_equal [{ 'name' => 'Titel' }, { 'name' => 'Kopf' }, { 'name' => 'Wert' }],
DataPipeline::ExcelStorage.new(file: "#{dir}/h.xlsx", headers: false, columns: %w[name]).read
assert_equal [{ 'Kopf' => 'Wert' }], DataPipeline::ExcelStorage.new(file: "#{dir}/h.xlsx", skip: 1).read
end
end
def test_empty_rows_are_skipped_and_short_name
with_tmpdir do |dir|
DataPipeline.new do
logger PipelineHelper::QUIET
read MemoryStorage.new(rows: [{ 'a' => '1' }, { 'a' => nil }, { 'a' => '3' }])
write container(:xlsx, file: "#{dir}/e.xlsx")
end.run
assert_equal [{ 'a' => '1' }, { 'a' => '3' }], DataPipeline::ExcelStorage.new(file: "#{dir}/e.xlsx").read
end
end
def test_file_required
assert_raises(ArgumentError) { DataPipeline::ExcelStorage.new }
end
end
test/plugins/json_storage_test.rb
+56
View File
@@ -0,0 +1,56 @@
require_relative '../test_helper'
require_relative '../support/storage_contract'
class JsonArrayStorageTest < PipelineTest
include StorageContract
def contract_storage(dir, _purpose) = DataPipeline::JsonStorage.new(file: "#{dir}/data.json")
def test_streamed_array_is_valid_json
with_tmpdir do |dir|
json = DataPipeline::JsonStorage.new(file: "#{dir}/a.json")
json.write_row({ 'a' => 1 })
json.write_row({ 'a' => 2 })
json.close
assert_equal [{ 'a' => 1 }, { 'a' => 2 }], JSON.parse(File.read("#{dir}/a.json"))
end
end
def test_root_path_and_pretty
with_tmpdir do |dir|
File.write("#{dir}/api.json", JSON.generate('data' => { 'items' => [{ 'id' => 1 }] }))
assert_equal [{ 'id' => 1 }], DataPipeline::JsonStorage.new(file: "#{dir}/api.json", root: 'data.items').read
DataPipeline::JsonStorage.new(file: "#{dir}/p.json", pretty: true).write([{ 'a' => 1 }])
assert_includes File.read("#{dir}/p.json"), "\n"
end
end
def test_single_object_is_one_row_and_scalar_raises
with_tmpdir do |dir|
File.write("#{dir}/o.json", '{"a":1}')
assert_equal [{ 'a' => 1 }], DataPipeline::JsonStorage.new(file: "#{dir}/o.json").read
File.write("#{dir}/s.json", '42')
assert_raises(ArgumentError) { DataPipeline::JsonStorage.new(file: "#{dir}/s.json").read }
end
end
def test_options
assert_raises(ArgumentError) { DataPipeline::JsonStorage.new }
assert_raises(ArgumentError) { DataPipeline::JsonStorage.new(file: 'x.json', format: :xml) }
end
end
class JsonLinesStorageTest < PipelineTest
include StorageContract
def contract_storage(dir, _purpose) = DataPipeline::JsonStorage.new(file: "#{dir}/data.ndjson")
def test_format_from_extension_and_blank_lines
with_tmpdir do |dir|
File.write("#{dir}/d.jsonl", "{\"a\":1}\n\n{\"a\":2}\n")
assert_equal [{ 'a' => 1 }, { 'a' => 2 }], DataPipeline::JsonStorage.new(file: "#{dir}/d.jsonl").read
DataPipeline::JsonStorage.new(file: "#{dir}/x.txt", format: :lines).write([{ 'a' => 1 }, { 'a' => 2 }])
assert_equal "{\"a\":1}\n{\"a\":2}\n", File.read("#{dir}/x.txt")
end
end
end
test/plugins/lazy_loading_test.rb
+45
View File
@@ -0,0 +1,45 @@
require_relative '../test_helper'
require 'open3'
# in a fresh process: plugins and their gems are loaded only when used
class LazyLoadingTest < PipelineTest
def loaded_after code
script = <<~RUBY
require 'bundler/setup'
require #{File.join(ROOT, 'lib/datapipeline').inspect}
#{code}
features = $LOADED_FEATURES.join(' ')
puts %w[pg sqlite3 roo caxlsx http].select { |gem| features.match?(%r{/gems/\#{gem}-\\d}) }.join(',')
RUBY
out, err, status = Open3.capture3({ 'BUNDLE_GEMFILE' => File.join(ROOT, 'Gemfile') }, RbConfig.ruby, '-e', script)
assert status.success?, err
out.strip.split(',')
end
def test_nothing_loaded_at_start
assert_empty loaded_after('')
end
def test_json_and_csv_need_no_gem
with_tmpdir do |dir|
code = "DataPipeline.new { logger Logger.new(IO::NULL); read container(:json, file: '#{dir}/x.json').tap { File.write('#{dir}/x.json', '[]') }; write container(:csv, file: '#{dir}/y.csv') }.run"
assert_empty loaded_after(code)
end
end
def test_each_plugin_loads_only_its_gem
with_tmpdir do |dir|
assert_equal %w[sqlite3], loaded_after("DataPipeline::SqliteStorage.new(database: ':memory:').close")
assert_equal %w[caxlsx], loaded_after("DataPipeline::ExcelStorage.new(file: '#{dir}/a.xlsx').write([{ 'a' => 1 }])")
assert_equal %w[roo caxlsx].sort, loaded_after("DataPipeline::ExcelStorage.new(file: '#{dir}/b.xlsx').write([{ 'a' => 1 }]); DataPipeline::ExcelStorage.new(file: '#{dir}/b.xlsx').read").sort
assert_equal %w[http], loaded_after("DataPipeline::RestStorage.new(url: 'http://localhost')")
end
end
def test_missing_gem_message
plugin = Class.new(DataPipeline::StoragePlugin) { def initialize(**) = self.class.require_gem('gibtsnicht_gem') }
Object.const_set(:GibtsNichtStorage, plugin) unless defined?(GibtsNichtStorage)
error = assert_raises(LoadError) { GibtsNichtStorage.new }
assert_includes error.message, "gem 'gibtsnicht_gem'"
end
end
test/plugins/postgres_storage_test.rb
+133
View File
@@ -0,0 +1,133 @@
require_relative '../test_helper'
require_relative '../support/storage_contract'
class PostgresStorageTest < PipelineTest
include StorageContract
def with_contract_env
with_pg_database do |db|
pg_exec db, 'create table rows (id text, code text, name text, note text)'
yield db
end
end
def contract_storage db, purpose
purpose == :write ? DataPipeline::PostgresStorage.new(**db, table: 'rows') : DataPipeline::PostgresStorage.new(**db, query: 'select * from rows order by id')
end
def with_table
with_pg_database do |db|
pg_exec db, 'create table t (id int, name text, note text); create schema s; create table s."Mixed Case" ("Col A" text)'
storage = DataPipeline::PostgresStorage.new(**db)
yield db, storage
ensure
storage&.close
end
end
def test_write_rows_with_table_option_quotes_and_null
with_table do |db, pg|
DataPipeline::PostgresStorage.new(**db, table: 't').tap do |w|
w.write([{ 'id' => 1, 'name' => "O'Brien", 'note' => nil }, { id: 2, name: "x'); drop table t; --" }])
w.close
end
assert_equal [{ 'id' => '1', 'name' => "O'Brien", 'note' => nil },
{ 'id' => '2', 'name' => "x'); drop table t; --", 'note' => nil }],
pg.query(query: 'select id, name, note from t order by id')
end
end
def test_write_old_signature_and_schema_quoting
with_table do |_db, pg|
pg.write('t', [{ 'id' => 3 }])
pg.write('s.Mixed Case', [{ 'Col A' => 'v' }])
assert_equal '1', pg.query(query: 'select count(*) from t').first['count']
assert_equal [{ 'Col A' => 'v' }], pg.query(query: 'select * from s."Mixed Case"')
end
end
def test_truncate_before_first_write
with_table do |db, pg|
pg.write('t', [{ 'id' => 1 }, { 'id' => 2 }])
writer = DataPipeline::PostgresStorage.new(**db, table: 't', truncate: true)
writer.write([{ 'id' => 3 }])
writer.write_row({ 'id' => 4 })
writer.close
assert_equal %w[3 4], pg.query(query: 'select id from t order by id').map { |r| r['id'] }
end
end
def test_write_is_one_transaction
with_table do |_db, pg|
assert_raises(PG::Error) { pg.write('t', [{ 'id' => 10 }, { 'id' => 'kein int' }]) }
assert_equal '0', pg.query(query: 'select count(*) from t').first['count']
end
end
def test_write_without_table
with_table do |_db, pg|
assert_includes assert_raises(RuntimeError) { pg.write([{ 'id' => 1 }]) }.message, 'Table'
end
end
def test_query_and_read_with_params
with_table do |db, pg|
pg.write('t', [{ 'id' => 1, 'name' => 'a' }, { 'id' => 2, 'name' => 'b' }, { 'id' => 3 }])
assert_equal [{ 'name' => 'a' }], pg.query(query: 'select name from t where id = $1', params: [1])
reader = DataPipeline::PostgresStorage.new(**db)
reader.set(query: 'select id from t where id > $1 order by id', params: [1])
assert_equal [{ 'id' => '2' }, { 'id' => '3' }], reader.read
reader.close
end
end
def test_typed_and_default_string_values
with_table do |db, pg|
typed = DataPipeline::PostgresStorage.new(**db, typed: true)
row = typed.query(query: 'select 1::int as i, 1.5::float8 as f, current_date as d').first
typed.close
assert_equal 1, row['i']
assert_in_delta 1.5, row['f']
assert_kind_of Date, row['d']
assert_equal '1', pg.query(query: 'select 1::int as i').first['i']
end
end
def test_cursor_read_in_batches
with_pg_database do |db|
pg_exec db, 'create table k as select i as id from generate_series(1, 2500) i'
reader = DataPipeline::PostgresStorage.new(**db, query: "select id from k where id > $1 order by id;\n", params: [500],
fetch_size: 100)
count = 0
reader.read { count += 1 }
reader.close
assert_equal 2000, count
end
end
def test_cursor_rollback_on_error_keeps_connection_usable
with_pg_database do |db|
pg_exec db, 'create table k as select 1 as id'
pg = DataPipeline::PostgresStorage.new(**db, query: 'select id from k')
assert_raises(RuntimeError) { pg.read { raise 'x' } }
assert_equal [{ 'x' => '1' }], pg.query(query: 'select 1 as x')
pg.close
end
end
def test_close_twice
with_pg_database do |db|
pg = DataPipeline::PostgresStorage.new(**db)
pg.close
pg.close
assert pg.instance_variable_get(:@conn).finished?
end
end
def test_connection_error_keeps_cause
skip 'no local postgres' unless PgHelper.available?
error = assert_raises(PG::ConnectionBad) { DataPipeline::PostgresStorage.new(host: 'localhost', port: 1, database: 'x') }
assert_includes error.message, 'Error connection to database:'
end
end
test/plugins/rest_storage_test.rb
+151
View File
@@ -0,0 +1,151 @@
require_relative '../test_helper'
require_relative '../support/storage_contract'
require_relative '../support/test_http_server'
class RestStorageTest < PipelineTest
include StorageContract
# a tiny api: POST /items stores one row or an array, GET /items returns all
def with_contract_env
store = []
server = TestHttpServer.new do |req|
if req[:method] == 'POST'
Array(JSON.parse(req[:body]).then { |b| b.is_a?(Array) ? b : [b] }).each { |row| store << row }
[201, { 'ok' => true }]
else
[200, { 'data' => { 'items' => store } }]
end
end
yield server
ensure
server&.stop
end
def contract_storage(server, _purpose) = DataPipeline::RestStorage.new(url: "#{server.url}/items", root: 'data.items')
# handler: ->(request) { [status, body] }
def server handler
s = TestHttpServer.new(&handler)
yield s
ensure
s&.stop
end
def test_page_param_paging
pages = { '1' => [{ 'id' => 1 }, { 'id' => 2 }], '2' => [{ 'id' => 3 }] }
server(->(req) { [200, pages.fetch(req[:query]['page'], [])] }) do |s|
rows = DataPipeline::RestStorage.new(url: "#{s.url}/kunden", page_param: 'page', params: { 'land' => 'DE' }).read
assert_equal [1, 2, 3], rows.map { |r| r['id'] }
assert_equal %w[1 2 3], s.requests.map { |r| r[:query]['page'] }
assert(s.requests.all? { |r| r[:query]['land'] == 'DE' })
end
end
def test_next_url_paging
base = nil
handler = lambda do |req|
if req[:path] == '/a'
[200, { 'rows' => [{ 'n' => 1 }], 'next' => "#{base}/b" }]
else
[200, { 'rows' => [{ 'n' => 2 }], 'next' => nil }]
end
end
server(handler) do |s|
base = s.url
rows = DataPipeline::RestStorage.new(url: "#{s.url}/a", root: 'rows', next: ->(body) { body['next'] }).read
assert_equal [1, 2], rows.map { |r| r['n'] }
end
end
def test_auth_and_headers
server(->(_req) { [200, []] }) do |s|
DataPipeline::RestStorage.new(url: s.url, auth: 'Bearer geheim', headers: { 'X-Mandant' => '001' }).read
assert_equal 'Bearer geheim', s.requests.first[:headers]['authorization']
assert_equal '001', s.requests.first[:headers]['x-mandant']
end
end
def test_retry_on_503_then_success
calls = 0
server(->(_req) { (calls += 1) < 3 ? [503, 'busy'] : [200, [{ 'ok' => 1 }]] }) do |s|
assert_equal [{ 'ok' => 1 }], DataPipeline::RestStorage.new(url: s.url, retries: 2, retry_wait: 0).read
assert_equal 3, calls
end
end
def test_no_retry_on_404_and_error_message
server(->(_req) { [404, { 'error' => 'nicht da' }] }) do |s|
error = assert_raises(DataPipeline::RestStorage::Error) { DataPipeline::RestStorage.new(url: "#{s.url}/x", retry_wait: 0).read }
assert_includes error.message, 'HTTP 404'
assert_includes error.message, 'nicht da'
assert_equal 1, s.requests.size
end
end
def test_retries_exhausted_and_connection_refused
server(->(_req) { [500, 'kaputt'] }) do |s|
assert_includes assert_raises(DataPipeline::RestStorage::Error) { DataPipeline::RestStorage.new(url: s.url, retries: 1, retry_wait: 0).read }.message, 'HTTP 500'
assert_equal 2, s.requests.size
end
closed = TCPServer.new('127.0.0.1', 0).then { |t| port = t.addr[1]; t.close; port }
assert_raises(DataPipeline::RestStorage::Error) { DataPipeline::RestStorage.new(url: "http://127.0.0.1:#{closed}", retries: 1, retry_wait: 0).read }
end
def test_batch_write_and_write_method
server(->(_req) { [200, ''] }) do |s|
rest = DataPipeline::RestStorage.new(url: "#{s.url}/import", batch_size: 2, write_method: :put)
rest.write([{ 'a' => 1 }, { 'a' => 2 }, { 'a' => 3 }])
assert_equal [[{ 'a' => 1 }, { 'a' => 2 }], [{ 'a' => 3 }]], s.requests.map { |r| JSON.parse(r[:body]) }
assert(s.requests.all? { |r| r[:method] == 'PUT' })
end
end
def test_write_url_per_row
server(->(_req) { [200, ''] }) do |s|
DataPipeline::RestStorage.new(url: s.url, write_method: :put, write_url: ->(row) { "#{s.url}/kunden/#{row['id']}" })
.write([{ 'id' => 7 }, { 'id' => 8 }])
assert_equal %w[/kunden/7 /kunden/8], s.requests.map { |r| r[:path] }
assert_equal({ 'id' => 8 }, JSON.parse(s.requests.last[:body]))
end
assert_raises(ArgumentError) { DataPipeline::RestStorage.new(url: 'http://x', batch_size: 2, write_url: ->(_row) { 'http://x' }) }
end
def test_call_json_status_and_retries
calls = 0
server(->(req) { calls += 1; req[:path] == '/api/fail' ? [503, ''] : [201, { 'echo' => JSON.parse(req[:body]) }] }) do |s|
rest = DataPipeline::RestStorage.new(url: "#{s.url}/api/", retries: 2, retry_wait: 0)
assert_equal({ 'echo' => { 'a' => 1 } }, rest.call(:post, 'person/create', json: { 'a' => 1 }))
assert_equal '/api/person/create', s.requests.last[:path]
calls = 0
error = assert_raises(DataPipeline::RestStorage::Error) { rest.call(:post, 'fail', json: {}, retries: 0) }
assert_equal 503, error.status
assert_equal 1, calls, 'retries: 0 sends once'
end
server(->(_req) { [404, { 'message' => 'weg' }] }) do |s|
assert_equal 404, assert_raises(DataPipeline::RestStorage::Error) { DataPipeline::RestStorage.new(url: s.url).call(:get, 'x') }.status
end
end
def test_ssl_verify_false_context
context = DataPipeline::RestStorage.new(url: 'https://192.168.178.251:50443/pds/rest/api', ssl_verify: false).send(:ssl_context)
assert_equal OpenSSL::SSL::VERIFY_NONE, context.verify_mode
end
def test_query_for_lookup
server(->(req) { [200, [{ 'land' => req[:query]['land'], 'pfad' => req[:path] }]] }) do |s|
names = nil
DataPipeline.new do
api = container :rest, url: "#{s.url}/api/"
names = lookup api, query: '/laender', params: { 'land' => 'DE' }, key: 'land', unique: true
end
assert_equal '/api/laender', names['DE']['pfad']
end
end
def test_not_an_array
server(->(_req) { [200, { 'a' => 1 }] }) do |s|
assert_raises(DataPipeline::RestStorage::Error) { DataPipeline::RestStorage.new(url: s.url).read }
end
end
end
test/plugins/slack_test.rb
+18
View File
@@ -0,0 +1,18 @@
require_relative '../test_helper'
# no network: only construction and options
class SlackTest < PipelineTest
def test_token_required
token = ENV.delete('SLACK_TOKEN')
assert_raises(ArgumentError) { DataPipeline::Slack.new }
ensure
ENV['SLACK_TOKEN'] = token if token
end
def test_options_per_instance
a = DataPipeline::Slack.new(token: 'x', channel: '#a')
b = DataPipeline::Slack.new(token: 'y')
assert_equal '#a', a.instance_variable_get(:@options)[:channel]
assert_equal '#general', b.instance_variable_get(:@options)[:channel]
end
end
test/plugins/sqlite_storage_test.rb
+95
View File
@@ -0,0 +1,95 @@
require_relative '../test_helper'
require_relative '../support/storage_contract'
class SqliteStorageTest < PipelineTest
include StorageContract
def contract_storage dir, purpose
db = "#{dir}/test.sqlite3"
unless File.exist?(db)
DataPipeline::SqliteStorage.new(database: db).tap do |s|
s.query(query: 'create table rows (id text, code text, name text, note text)')
s.close
end
end
purpose == :write ? DataPipeline::SqliteStorage.new(database: db, table: 'rows') : DataPipeline::SqliteStorage.new(database: db, query: 'select * from rows order by id')
end
def memory_db
DataPipeline::SqliteStorage.new(database: ':memory:').tap do |s|
s.query(query: 'create table t (id integer, name text, betrag real, datum text)')
end
end
def test_types_null_and_quotes
db = memory_db
db.write('t', [{ 'id' => 1, 'name' => "O'Brien", 'betrag' => 1.5, 'datum' => Date.new(2025, 11, 3) }, { id: 2, name: nil }])
assert_equal [{ 'id' => 1, 'name' => "O'Brien", 'betrag' => 1.5, 'datum' => '2025-11-03' },
{ 'id' => 2, 'name' => nil, 'betrag' => nil, 'datum' => nil }],
db.query(query: 'select * from t order by id')
db.close
end
def test_positional_and_named_params
db = memory_db
db.write('t', [{ 'id' => 1, 'name' => 'a' }, { 'id' => 2, 'name' => 'b' }])
assert_equal [{ 'name' => 'b' }], db.query(query: 'select name from t where id = ?', params: [2])
assert_equal [{ 'name' => 'a' }], db.query(query: 'select name from t where id = :id', params: { id: 1 })
db.close
end
def test_write_is_one_transaction
db = memory_db
db.query(query: 'create table u (id integer not null)')
assert_raises(SQLite3::Exception) { db.write('u', [{ 'id' => 1 }, { 'id' => nil }]) }
assert_equal 0, db.query(query: 'select count(*) as n from u').first['n']
db.close
end
def test_sql_injection_in_values_and_identifiers
db = memory_db
db.query(query: 'create table "Mixed ""Name""" ("Col A" text)')
db.write('Mixed "Name"', [{ 'Col A' => "x'); drop table t; --" }])
assert_equal [{ 'Col A' => "x'); drop table t; --" }], db.query(query: 'select * from "Mixed ""Name"""')
assert_equal 0, db.query(query: 'select count(*) as n from t').first['n']
db.close
end
def test_create_truncate_and_lazy_open
with_tmpdir do |dir|
path = "#{dir}/neu.sqlite3"
storage = DataPipeline::SqliteStorage.new(database: path, table: 'k', create: true, truncate: true)
refute File.exist?(path), 'opened only on first use'
storage.write([{ 'id' => 1, 'betrag' => 1.5, 'name' => 'a' }, { 'id' => 2, 'betrag' => 2.0, 'name' => 'b' }])
storage.close
again = DataPipeline::SqliteStorage.new(database: path, table: 'k', create: true, truncate: true)
again.write_row({ 'id' => 3, 'betrag' => 3.0, 'name' => 'c' })
assert_equal [{ 'id' => 3, 'betrag' => 3.0, 'name' => 'c' }], again.query(query: 'select * from k')
assert_equal %w[integer real text], again.query(query: 'pragma table_info(k)').map { |c| c['type'].downcase }
again.close
end
end
def test_missing_options
assert_raises(ArgumentError) { DataPipeline::SqliteStorage.new }
db = memory_db
assert_raises(RuntimeError) { db.write([{ 'id' => 1 }]) }
assert_raises(RuntimeError) { db.read }
db.close
end
def test_short_name_and_lookup
with_tmpdir do |dir|
path = "#{dir}/l.sqlite3"
names = nil
DataPipeline.new do
db = container :sqlite3, database: path
db.query(query: 'create table land (code text, name text)')
db.write('land', [{ 'code' => 'DE', 'name' => 'Deutschland' }])
names = lookup db, query: 'select * from land where code like ?', params: ['%'], key: 'code', unique: true
end
assert_equal 'Deutschland', names['DE']['name']
end
end
end
test/support/storage_contract.rb
+73
View File
@@ -0,0 +1,73 @@
#
# shared tests for every storage plugin that can write and read back
#
# a test class includes StorageContract and implements
# contract_storage(env, purpose) -> a new storage; purpose is :write or :read
# optionally
# with_contract_env { |env| ... } -> environment per test (default: a tmp dir)
# contract_rows(rows) -> rows as the plugin reads them back (default: unchanged)
#
module StorageContract
ROWS = [
{ 'id' => '1', 'code' => '007', 'name' => "O'Brien", 'note' => nil },
{ 'id' => '2', 'code' => '010', 'name' => 'Zürich | "ä", ß', 'note' => 'zwei' },
{ 'id' => '3', 'code' => '100', 'name' => 'drei', 'note' => '' }
].freeze
def with_contract_env(&block) = with_tmpdir(&block)
def contract_rows(rows) = rows
def test_contract_write_and_read_back
with_contract_env do |env|
contract_storage(env, :write).tap { |s| s.write(ROWS) }.close
reader = contract_storage(env, :read)
assert_equal contract_rows(ROWS), reader.read
reader.close
end
end
def test_contract_write_row_and_read_with_block
with_contract_env do |env|
writer = contract_storage(env, :write)
ROWS.each { |row| writer.write_row(row) }
writer.close
rows = []
reader = contract_storage(env, :read)
assert_nil reader.read { |row| rows << row }
reader.close
assert_equal contract_rows(ROWS), rows
end
end
def test_contract_as_pipeline_source_and_target
[false, true].each do |streamed|
with_contract_env do |env|
target = contract_storage(env, :write)
stats = DataPipeline.new(streamed:) do
logger PipelineHelper::QUIET
read MemoryStorage.new(rows: ROWS)
write target
end.run
assert_equal 3, stats.written
source = contract_storage(env, :read)
copy = MemoryStorage.new
DataPipeline.new(streamed:) do
logger PipelineHelper::QUIET
read source
write copy
end.run
assert_equal contract_rows(ROWS), copy.output, "streamed=#{streamed}"
end
end
end
def test_contract_close_twice
with_contract_env do |env|
storage = contract_storage(env, :write)
storage.close
storage.close
pass
end
end
end
test/support/test_http_server.rb
+52
View File
@@ -0,0 +1,52 @@
require 'socket'
require 'uri'
require 'json'
#
# minimal http server for DataPipeline::RestStorage tests, one request per connection
# server = TestHttpServer.new { |req| [200, { 'items' => [] }] }
# req: { method:, path:, query:, headers:, body: } -> [status, hash/array or string]
#
class TestHttpServer
attr_reader :requests
def initialize(&handler)
@handler = handler
@requests = []
@server = TCPServer.new('127.0.0.1', 0)
@thread = Thread.new { loop { serve(@server.accept) } }
end
def url = "http://127.0.0.1:#{@server.addr[1]}"
def stop
@thread.kill
@server.close
end
private
def serve client
method, target = client.gets.to_s.split
return unless method
headers = {}
while (line = client.gets) && line != "\r\n"
key, value = line.split(':', 2)
headers[key.downcase] = value.strip
end
body = headers['content-length'] ? client.read(headers['content-length'].to_i) : ''
path, query = target.split('?', 2)
request = { method:, path:, query: URI.decode_www_form(query.to_s).to_h, headers:, body: }
@requests << request
status, payload = @handler.call(request)
text = payload.is_a?(String) ? payload : JSON.generate(payload)
client.write "HTTP/1.1 #{status} X\r\nContent-Type: application/json\r\n" \
"Content-Length: #{text.bytesize}\r\nConnection: close\r\n\r\n#{text}"
rescue StandardError => e
warn "TestHttpServer: #{e.class}: #{e.message}"
ensure
client.close
end
end
test/test_helper.rb
+148
View File
@@ -0,0 +1,148 @@
#
# rake test or ruby -Itest test/core/chained_test.rb
#
# .env is NOT loaded: no test may reach the production database.
# postgres tests use a local server and a throw-away database per test:
# ETL_TEST_PG_HOST (default localhost), ETL_TEST_PG_PORT, ETL_TEST_PG_USER (default $USER), ETL_TEST_PG_PASSWORD
# without a reachable server they are skipped.
#
require 'bundler/setup'
require 'minitest/autorun'
require 'json'
require 'logger'
require 'stringio'
require 'tmpdir'
require 'datapipeline'
ROOT = File.expand_path('..', __dir__)
# source and target in memory
class MemoryStorage < DataPipeline::StoragePlugin
attr_reader :rows, :written, :streamed_rows
def initialize rows: [], **options
@options = options
@rows = rows
@written = nil
@streamed_rows = []
@closed = false
end
def read(&block)
return @rows.map(&:dup) unless block
@rows.map(&:dup).each(&block)
nil
end
def write(rows) = (@written = rows)
def write_row(row) = (@streamed_rows << row)
def close = (@closed = true)
def closed? = @closed
# rows of the last batch write or of the streamed rows
def output = @written || @streamed_rows
end
DataPipeline.register :memory, MemoryStorage
module PipelineHelper
QUIET = Logger.new(IO::NULL)
def setup
super
DataPipeline.reset_settings!
DataPipeline.runs.clear
DataPipeline.params.clear
end
# runs a pipeline from rows into a MemoryStorage, returns [target, stats]
def run_pipeline rows = [], **settings, &block
source = MemoryStorage.new(rows:)
target = MemoryStorage.new
stats = DataPipeline.new(**settings) do
logger QUIET
read source
write target
instance_eval(&block) if block
end.run
[target, stats]
end
def output rows = [], **settings, &block
run_pipeline(rows, **settings, &block).first.output
end
def with_tmpdir(&block)
Dir.mktmpdir('etl-test', &block)
end
def capture_stdout
old = $stdout
$stdout = StringIO.new
yield
$stdout.string
ensure
$stdout = old
end
end
module PgHelper
def self.config
{
host: ENV.fetch('ETL_TEST_PG_HOST', 'localhost'),
port: ENV.fetch('ETL_TEST_PG_PORT', '5432'),
username: ENV.fetch('ETL_TEST_PG_USER', ENV['USER']),
password: ENV.fetch('ETL_TEST_PG_PASSWORD', nil)
}
end
def self.admin
require 'pg'
c = config
PG.connect(host: c[:host], port: c[:port], user: c[:username], password: c[:password], dbname: 'postgres',
connect_timeout: 3)
end
def self.available?
return @available unless @available.nil?
@available = begin
admin.close
true
rescue StandardError, LoadError
false
end
end
# yields connection options for a fresh database, dropped afterwards
def with_pg_database
skip 'no local postgres (ETL_TEST_PG_HOST)' unless PgHelper.available?
name = "etl_test_#{Process.pid}_#{rand(1_000_000)}"
admin = PgHelper.admin
admin.exec("create database #{name}")
yield PgHelper.config.merge(database: name)
ensure
if admin
admin.exec("select pg_terminate_backend(pid) from pg_stat_activity where datname = '#{name}' and pid <> pg_backend_pid()")
admin.exec("drop database if exists #{name}")
admin.close
end
end
# plain sql, several statements allowed (setup of test tables)
def pg_exec options, sql
conn = PG.connect(host: options[:host], port: options[:port], user: options[:username],
password: options[:password], dbname: options[:database])
conn.exec(sql).to_a
ensure
conn&.close
end
end
class PipelineTest < Minitest::Test
include PipelineHelper
include PgHelper
end