datapipeline/test/cli/etl_cli_test.rb

require_relative '../test_helper'
require 'open3'

# bin/etl as a subprocess: arguments, params, dry run, exit codes
class EtlCliTest < PipelineTest
  def etl dir, *args
    env = { 'BUNDLE_GEMFILE' => File.join(ROOT, 'Gemfile') }
    Open3.capture3(env, RbConfig.ruby, File.join(ROOT, 'bin/etl'), *args, chdir: dir)
  end

  def script dir, name, body
    File.write("#{dir}/in.csv", "a,b\n1,x\n2,y\nz,q\n")
    File.write("#{dir}/#{name}", "require #{File.join(ROOT, 'lib/datapipeline').inspect}\n#{body}")
    "#{dir}/#{name}"
  end

  # scripts with an own OptionParser for --from/--till
  def test_script_with_own_option_parser_gets_from_till_and_limit
    with_tmpdir do |dir|
      path = script(dir, 'legacy.rb', <<~RUBY)
        require 'optparse/date'
        opts = {}
        OptionParser.new { |p| p.on('--from=DATE', Date) { |v| opts[:from] = v }; p.on('--till=DATE', Date) { |v| opts[:till] = v } }.parse!
        File.write('args.txt', "\#{opts[:from]} \#{opts[:till]}")
        DataPipeline.new do
          read container(:csv, file: 'in.csv')
          header { { 'h' => 'HEAD' } }
          map { |r, ctx| r.merge('i' => ctx.index) }
          write container(:csv, file: 'out.csv', csv: { headers: false, col_sep: '|' })
        end.run
      RUBY
      _out, err, status = etl(dir, 'run', path, '--from=2025-11-01', '--till=2025-11-30', '--limit=2')
      assert_equal 0, status.exitstatus, err
      assert_equal '2025-11-01 2025-11-30', File.read("#{dir}/args.txt")
      assert_equal "HEAD\n1|x|0\n2|y|1\n", File.read("#{dir}/out.csv")
      assert_includes err, 'INFO finish read=2 written=2 rejected=0'
    end
  end

  def test_params_and_exit_code_2_for_rejected_rows
    with_tmpdir do |dir|
      path = script(dir, 'neu.rb', <<~RUBY)
        DataPipeline.new do
          on_error :skip
          read container(:csv, file: 'in.csv')
          validate 'a' => { format: /\\A\\d+\\z/ }
          map { |r| r.merge('land' => params[:land]) }
          write container(:csv, file: 'out.csv')
        end.run
      RUBY
      _out, err, status = etl(dir, 'run', path, '--param', 'land=DE')
      assert_equal 2, status.exitstatus, err
      assert_equal "a,b,land\n1,x,DE\n2,y,DE\n", File.read("#{dir}/out.csv")

      File.delete("#{dir}/out.csv")
      out, _err, status = etl(dir, 'run', path, '--param', 'land=AT', '--dry-run', '--quiet')
      assert_equal 2, status.exitstatus
      refute File.exist?("#{dir}/out.csv")
      assert out.start_with?('DRY RUN')
      assert_includes out, '"land":"AT"'
    end
  end

  def test_run_is_optional
    with_tmpdir do |dir|
      path = script(dir, 'kopie.rb', "DataPipeline.new { read container(:csv, file: 'in.csv'); write container(:csv, file: 'out.csv') }.run\n")
      _out, err, status = etl(dir, path, '--quiet')
      assert_equal 0, status.exitstatus, err
      assert_equal File.read("#{dir}/in.csv"), File.read("#{dir}/out.csv")
    end
  end

  def test_exit_code_1_on_abort_and_usage
    with_tmpdir do |dir|
      path = script(dir, 'kaputt.rb', "DataPipeline.new { read container(:csv, file: 'in.csv'); map { raise 'boom' }; write container(:csv, file: 'k.csv') }.run\n")
      _out, err, status = etl(dir, 'run', path, '--quiet')
      assert_equal 1, status.exitstatus
      assert_includes err, 'RuntimeError: boom'

      _out, err, status = etl(dir, 'gibtsnicht.rb')
      assert_equal 1, status.exitstatus
      assert_includes err, 'gibtsnicht.rb not found'

      _out, err, status = etl(dir)
      assert_equal 1, status.exitstatus
      assert_includes err, 'Usage: etl [run] PIPELINE.rb'
    end
  end

  def test_lock_exit_code_3
    with_tmpdir do |dir|
      path = script(dir, 'locked.rb', <<~RUBY)
        DataPipeline.new do
          lock '.pipeline.lock'
          read container(:csv, file: 'in.csv')
          write container(:csv, file: 'out.csv')
        end.run
      RUBY
      holder = DataPipeline::Lock.new("#{dir}/.script.lock").acquire
      _out, err, status = etl(dir, 'run', path, '--lock=.script.lock')
      assert_equal 3, status.exitstatus, err
      assert_includes err, 'another run is still active'
      refute File.exist?("#{dir}/out.csv")
      holder.release

      holder = DataPipeline::Lock.new("#{dir}/.pipeline.lock").acquire
      _out, err, status = etl(dir, 'run', path, '--lock=.script.lock')
      assert_equal 3, status.exitstatus, err
      holder.release

      _out, err, status = etl(dir, 'run', path, '--lock=.script.lock')
      assert_equal 0, status.exitstatus, err
    end
  end

  def test_history_with_script_name
    with_tmpdir do |dir|
      path = script(dir, 'export_kunden.rb', <<~RUBY)
        DataPipeline.new do
          read container(:csv, file: 'in.csv')
          write container(:csv, file: 'out.csv')
        end.run
      RUBY
      _out, err, status = etl(dir, 'run', path, '--history', 'runs.ndjson')
      assert_equal 0, status.exitstatus, err
      entry = JSON.parse(File.read("#{dir}/runs.ndjson"))
      assert_equal ['export_kunden', 3, true], entry.values_at('name', 'read', 'ok')
    end
  end
end