90 lines
3.6 KiB
Ruby
90 lines
3.6 KiB
Ruby
require_relative '../test_helper'
|
|
require 'open3'
|
|
|
|
# bin/etl as a subprocess: arguments, params, dry run, exit codes
|
|
class EtlCliTest < PipelineTest
|
|
def etl dir, *args
|
|
env = { 'BUNDLE_GEMFILE' => File.join(ROOT, 'Gemfile') }
|
|
Open3.capture3(env, RbConfig.ruby, File.join(ROOT, 'bin/etl'), *args, chdir: dir)
|
|
end
|
|
|
|
def script dir, name, body
|
|
File.write("#{dir}/in.csv", "a,b\n1,x\n2,y\nz,q\n")
|
|
File.write("#{dir}/#{name}", "require #{File.join(ROOT, 'lib/datapipeline').inspect}\n#{body}")
|
|
"#{dir}/#{name}"
|
|
end
|
|
|
|
# like import.*.rb: own OptionParser for --from/--till, transform api
|
|
def test_legacy_script_gets_from_till_and_limit
|
|
with_tmpdir do |dir|
|
|
path = script(dir, 'legacy.rb', <<~RUBY)
|
|
require 'optparse/date'
|
|
opts = {}
|
|
OptionParser.new { |p| p.on('--from=DATE', Date) { |v| opts[:from] = v }; p.on('--till=DATE', Date) { |v| opts[:till] = v } }.parse!
|
|
File.write('args.txt', "\#{opts[:from]} \#{opts[:till]}")
|
|
DataPipeline.new do
|
|
read container(:csv, file: 'in.csv')
|
|
transform(processing: :prepend) { |_, _| { 'h' => 'HEAD' } }
|
|
transform { |r, i| r.merge('i' => i) }
|
|
write container(:csv, file: 'out.csv', csv: { headers: false, col_sep: '|' })
|
|
end.run
|
|
RUBY
|
|
_out, err, status = etl(dir, 'run', path, '--from=2025-11-01', '--till=2025-11-30', '--limit=2')
|
|
assert_equal 0, status.exitstatus, err
|
|
assert_equal '2025-11-01 2025-11-30', File.read("#{dir}/args.txt")
|
|
assert_equal "HEAD\n1|x|0\n2|y|1\n", File.read("#{dir}/out.csv")
|
|
assert_includes err, 'INFO finish read=2 written=2 rejected=0'
|
|
end
|
|
end
|
|
|
|
def test_params_and_exit_code_2_for_rejected_rows
|
|
with_tmpdir do |dir|
|
|
path = script(dir, 'neu.rb', <<~RUBY)
|
|
DataPipeline.new do
|
|
on_error :skip
|
|
read container(:csv, file: 'in.csv')
|
|
validate 'a' => { format: /\\A\\d+\\z/ }
|
|
map { |r| r.merge('land' => params[:land]) }
|
|
write container(:csv, file: 'out.csv')
|
|
end.run
|
|
RUBY
|
|
_out, err, status = etl(dir, 'run', path, '--param', 'land=DE')
|
|
assert_equal 2, status.exitstatus, err
|
|
assert_equal "a,b,land\n1,x,DE\n2,y,DE\n", File.read("#{dir}/out.csv")
|
|
|
|
File.delete("#{dir}/out.csv")
|
|
out, _err, status = etl(dir, 'run', path, '--param', 'land=AT', '--dry-run', '--quiet')
|
|
assert_equal 2, status.exitstatus
|
|
refute File.exist?("#{dir}/out.csv")
|
|
assert out.start_with?('DRY RUN')
|
|
assert_includes out, '"land":"AT"'
|
|
end
|
|
end
|
|
|
|
def test_run_is_optional
|
|
with_tmpdir do |dir|
|
|
path = script(dir, 'kopie.rb', "DataPipeline.new { read container(:csv, file: 'in.csv'); write container(:csv, file: 'out.csv') }.run\n")
|
|
_out, err, status = etl(dir, path, '--quiet')
|
|
assert_equal 0, status.exitstatus, err
|
|
assert_equal File.read("#{dir}/in.csv"), File.read("#{dir}/out.csv")
|
|
end
|
|
end
|
|
|
|
def test_exit_code_1_on_abort_and_usage
|
|
with_tmpdir do |dir|
|
|
path = script(dir, 'kaputt.rb', "DataPipeline.new { read container(:csv, file: 'in.csv'); map { raise 'boom' }; write container(:csv, file: 'k.csv') }.run\n")
|
|
_out, err, status = etl(dir, 'run', path, '--quiet')
|
|
assert_equal 1, status.exitstatus
|
|
assert_includes err, 'RuntimeError: boom'
|
|
|
|
_out, err, status = etl(dir, 'gibtsnicht.rb')
|
|
assert_equal 1, status.exitstatus
|
|
assert_includes err, 'gibtsnicht.rb not found'
|
|
|
|
_out, err, status = etl(dir)
|
|
assert_equal 1, status.exitstatus
|
|
assert_includes err, 'Usage: etl [run] PIPELINE.rb'
|
|
end
|
|
end
|
|
end
|