datapipeline 2.0.0: Gem aus dem Projekt etl herausgelöst

2026-10-08 15:30:02 +02:00
commit cce7b03ca9
56 changed files with 5205 additions and 0 deletions
@@ -0,0 +1,81 @@
+require_relative '../test_helper'
+
+#
+# parallel mode (transform without chained/map/filter) must keep the output of v1.2.1,
+# the version the CUDO importers were built on. expected values were recorded with v1.2.1.
+#
+class CompatTest < PipelineTest
+  ROWS = (1..6).map { |i| { 'id' => i, 'land' => i.even? ? 'DE' : 'AT' } }
+
+  def assert_output expected_json, &block
+    assert_equal expected_json, JSON.generate(output(ROWS, &block))
+  end
+
+  # pattern of import.customer.rb
+  def test_customer_pattern
+    assert_output '[{"SATYP":"FIRC","LFDNR":0},{"id":1,"land":"AT","lfdnr":"0000000001"},' \
+                  '{"id":2,"land":"DE","lfdnr":"0000000002"},{"id":3,"land":"AT","lfdnr":"0000000003"},' \
+                  '{"id":4,"land":"DE","lfdnr":"0000000004"},{"id":5,"land":"AT","lfdnr":"0000000005"},' \
+                  '{"id":6,"land":"DE","lfdnr":"0000000006"},{"topic":"LARC","counter":"0000000006"}]' do
+      transform(processing: :prepend) { |_row, n| { 'SATYP': 'FIRC', 'LFDNR': n } }
+      transform(processing: :append) { |_row, n| { 'topic': 'LARC', 'counter': format('%010d', n) } }
+      transform { |row, _i| row.merge('lfdnr' => format('%010d', row['id'])) }
+    end
+  end
+
+  # pattern of import.posting.rb: closure counter, 1:n
+  def test_posting_pattern
+    expected = [{ SATYP: 'FIRC', LFDNR: 0 }]
+    lfdnr = 0
+    ROWS.each do |row|
+      expected << row.merge('lfdnr' => lfdnr += 1)
+      row['id'].times { |k| expected << { 'pos' => k, 'lfdnr' => lfdnr += 1 } }
+    end
+    expected << { SATYPE: 'LARC', SUMME: '0000000027' }
+
+    assert_output JSON.generate(expected) do
+      cnt = 1
+      transform(processing: :prepend) { |_row, _n| { 'SATYP': 'FIRC', 'LFDNR': 0 } }
+      transform do |row|
+        out = [row.merge('lfdnr' => cnt)]
+        cnt += 1
+        row['id'].times do |k|
+          out << { 'pos' => k, 'lfdnr' => cnt }
+          cnt += 1
+        end
+        out
+      end
+      transform(processing: :append) { |_row| { 'SATYPE': 'LARC', 'SUMME': format('%010d', cnt - 1) } }
+    end
+  end
+
+  def test_append_registered_before_prepend
+    assert_output '[{"h":0},{"id":1,"land":"AT"},{"id":2,"land":"DE"},{"id":3,"land":"AT"},' \
+                  '{"id":4,"land":"DE"},{"id":5,"land":"AT"},{"id":6,"land":"DE"},{"t":6}]' do
+      transform(processing: :append) { |_, n| { 't' => n } }
+      transform { |r| r }
+      transform(processing: :prepend) { |_, n| { 'h' => n } }
+    end
+  end
+
+  def test_condition_string
+    assert_output '[{"id":2,"land":"DE","i":1},{"id":4,"land":"DE","i":3},{"id":6,"land":"DE","i":5}]' do
+      transform(condition: "row['land'] == 'DE'") { |r, i| r.merge('i' => i) }
+    end
+  end
+
+  def test_two_parallel_transforms
+    expected = ROWS.map { |r| r.merge('a' => 1) } + ROWS.map { |r| r.merge('b' => 1) }
+    assert_output JSON.generate(expected) do
+      transform { |r| r.merge('a' => 1) }
+      transform { |r| [r.merge('b' => 1), nil] }
+    end
+  end
+
+  def test_nil_and_empty_array
+    assert_output '[{"id":1,"land":"AT"}]' do
+      transform { |r| r['id'] > 3 ? nil : [] }
+      transform { |r| r['id'] == 1 ? r : nil }
+    end
+  end
+end