datapipeline 2.0.0: Gem aus dem Projekt etl herausgelöst

test/core/pipeline_test.rb

@@ -0,0 +1,131 @@
+require_relative '../test_helper'
+
+# dsl, settings, plugin lookup, parallel mode
+class PipelineCoreTest < PipelineTest
+  ROWS = [{ 'a' => 1 }, { 'a' => 2 }, { 'a' => 3 }].freeze
+
+  def settings_of(pipeline) = pipeline.instance_variable_get(:@settings)
+
+  def test_nil_filters_rows
+    assert_equal [{ 'a' => 1 }], output(ROWS) { transform { |r| r['a'] == 1 ? r : nil } }
+    assert_equal ROWS, output(ROWS) { transform { |r| [r, nil] } }
+    assert_equal ROWS, output(ROWS) { transform(processing: :prepend) { nil }; transform { |r| r }; transform(processing: :append) { nil } }
+  end
+
+  def test_dsl_switches
+    assert_equal false, settings_of(DataPipeline.new { verbose false })[:verbose]
+    assert_equal true, settings_of(DataPipeline.new { verbose })[:verbose]
+    assert_equal false, settings_of(DataPipeline.new { streamed false })[:streamed]
+    assert_equal '1.0', settings_of(DataPipeline.new { version '1.0' })[:version]
+    assert_equal true, settings_of(DataPipeline.new { configure verbose: true })[:verbose]
+  end
+
+  def test_typo_raises
+    assert_raises(NoMethodError) { DataPipeline.new { verbsoe true } }
+  end
+
+  def test_unknown_plugin
+    error = assert_raises(RuntimeError) { DataPipeline.new { container :gibtsnicht } }
+    assert_includes error.message, 'GibtsnichtStorage'
+  end
+
+  def test_plugin_errors_pass_through
+    assert_raises(ArgumentError) { DataPipeline.new { container :csv } }
+    broken = Class.new(DataPipeline::StoragePlugin) { def initialize(**) = undefined_thing }
+    DataPipeline.register :broken, broken
+    assert_includes assert_raises(NameError) { DataPipeline.new { container :broken } }.message, 'undefined_thing'
+  end
+
+  def test_register_lazy_path
+    with_tmpdir do |dir|
+      File.write("#{dir}/lazy_storage.rb", "class LazyStorage < MemoryStorage; end\n")
+      DataPipeline.register :lazy, 'LazyStorage', path: "#{dir}/lazy_storage.rb"
+      assert_empty $LOADED_FEATURES.grep(/lazy_storage/)
+      storage = nil
+      DataPipeline.new { storage = container :lazy }
+      assert_equal 'LazyStorage', storage.class.name
+      refute_empty $LOADED_FEATURES.grep(/lazy_storage/)
+    end
+  end
+
+  def test_csv_plugin_by_convention
+    assert_equal DataPipeline::CsvStorage, DataPipeline.plugin(:csv)
+  end
+
+  def test_settings_are_frozen_and_replaced
+    assert DataPipeline.settings.frozen?
+    pipeline = DataPipeline.new { logger PipelineHelper::QUIET }
+    DataPipeline.settings(verbose: true)
+    assert_equal false, settings_of(pipeline)[:verbose]
+    assert_equal true, DataPipeline.settings[:verbose]
+    DataPipeline.reset_settings!
+    assert_equal DataPipeline::DEFAULT_SETTINGS, DataPipeline.settings
+  end
+
+  def test_settings_per_pipeline
+    _target, stats = run_pipeline(ROWS, limit: 1)
+    assert_equal 1, stats.read
+    assert_nil DataPipeline.settings[:limit]
+  end
+
+  def test_parallel_threads
+    reads = Array.new(8) { |i| Thread.new { run_pipeline(ROWS, limit: (i % 3) + 1).last.read } }.map(&:value)
+    assert_equal Array.new(8) { |i| (i % 3) + 1 }, reads
+  end
+
+  def test_parallel_mode_is_side_by_side
+    out = output(ROWS) { transform { |r| r.merge('b' => 1) }; transform { |r| r.merge('c' => 2) } }
+    assert_equal 6, out.size
+  end
+
+  def test_parallel_mode_copies_rows_per_step
+    out = output(ROWS) { transform { |r| r['x'] = 1; r }; transform { |r| r.key?('x') ? nil : r } }
+    assert_equal 6, out.size
+    assert_equal [{ 'a' => 10 }, { 'a' => 20 }, { 'a' => 30 }], output(ROWS) { transform { |r| [r.transform_values! { |v| v * 10 }] } }
+  end
+
+  def test_stable_order_and_all_prepends_appends
+    out = output(ROWS) { 20.times { |k| transform { |r| r['a'] == 1 ? { 'k' => k } : nil } } }
+    assert_equal (0...20).to_a, out.map { |r| r['k'] }
+
+    out = output(ROWS) do
+      transform(processing: :prepend) { { 'h' => 1 } }
+      transform(processing: :prepend) { { 'h' => 2 } }
+      transform { |r| r }
+      transform(processing: :append) { { 't' => 1 } }
+      transform(processing: :append) { { 't' => 2 } }
+    end
+    assert_equal [{ 'h' => 1 }, { 'h' => 2 }, *ROWS, { 't' => 1 }, { 't' => 2 }], out
+  end
+
+  def test_append_count_parallel_vs_chained
+    assert_equal({ 'n' => 3 }, output(ROWS) { transform { |r| [r, r] }; transform(processing: :append) { |_, n| { 'n' => n } } }.last)
+    assert_equal({ 'n' => 6 }, output(ROWS) { chained; transform { |r| [r, r] }; transform(processing: :append) { |_, n| { 'n' => n } } }.last)
+  end
+
+  def test_conditions
+    assert_equal [{ 'a' => 1 }, { 'a' => 3 }], output(ROWS) { transform(condition: ->(r) { r['a'] != 2 }) { |r| r } }
+    assert_equal [{ 'a' => 2 }], output(ROWS) { transform(condition: "row['a'] == 2") { |r| r } }
+  end
+
+  def test_copy_without_steps
+    assert_equal ROWS, output(ROWS)
+  end
+
+  def test_closes_source_and_target
+    source = MemoryStorage.new(rows: ROWS)
+    target = MemoryStorage.new
+    DataPipeline.new { logger PipelineHelper::QUIET; read source; write target }.run
+    assert source.closed?
+    assert target.closed?
+  end
+
+  def test_read_and_write_required
+    assert_includes assert_raises(ArgumentError) { DataPipeline.new { logger PipelineHelper::QUIET; read MemoryStorage.new }.run }.message, 'write'
+    assert_includes assert_raises(ArgumentError) { DataPipeline.new { logger PipelineHelper::QUIET; write MemoryStorage.new }.run }.message, 'read'
+  end
+
+  def test_block_required
+    assert_raises(ArgumentError) { DataPipeline.new { map } }
+  end
+end