datapipeline 2.0.0: Gem aus dem Projekt etl herausgelöst

2026-10-08 15:30:02 +02:00
commit cce7b03ca9
56 changed files with 5205 additions and 0 deletions
@@ -0,0 +1,89 @@
+require_relative '../test_helper'
+require_relative '../support/storage_contract'
+
+class CsvStorageTest < PipelineTest
+  include StorageContract
+
+  def contract_storage(dir, _purpose) = DataPipeline::CsvStorage.new(file: "#{dir}/data.csv")
+
+  def test_read_with_defaults
+    with_tmpdir do |dir|
+      File.write("#{dir}/in.csv", "a,b\n1,x\n2,\"\"\n")
+      assert_equal [{ 'a' => '1', 'b' => 'x' }, { 'a' => '2', 'b' => '' }], DataPipeline::CsvStorage.new(file: "#{dir}/in.csv").read
+    end
+  end
+
+  def test_read_without_header_uses_columns_or_index
+    with_tmpdir do |dir|
+      File.write("#{dir}/in.csv", "1|x\n2|y\n")
+      csv = { headers: false, col_sep: '|' }
+      assert_equal [{ 'id' => '1', 'txt' => 'x' }, { 'id' => '2', 'txt' => 'y' }],
+                   DataPipeline::CsvStorage.new(file: "#{dir}/in.csv", columns: %w[id txt], csv:).read
+      assert_equal({ '0' => '1', '1' => 'x' }, DataPipeline::CsvStorage.new(file: "#{dir}/in.csv", csv:).read.first)
+    end
+  end
+
+  def test_read_streaming_with_block
+    with_tmpdir do |dir|
+      File.write("#{dir}/in.csv", "a\n1\n2\n")
+      rows = []
+      DataPipeline::CsvStorage.new(file: "#{dir}/in.csv").read { |row| rows << row }
+      assert_equal %w[1 2], rows.map { |r| r['a'] }
+    end
+  end
+
+  def test_defaults_are_merged
+    options = DataPipeline::CsvStorage.new(file: 'x', csv: { col_sep: ';' }).instance_variable_get(:@options)[:csv]
+    assert_equal ';', options[:col_sep]
+    assert options[:headers]
+  end
+
+  def test_file_is_required
+    error = assert_raises(ArgumentError) { DataPipeline::CsvStorage.new }
+    assert_includes error.message, 'file'
+  end
+
+  def test_write_without_csv_options
+    with_tmpdir do |dir|
+      DataPipeline::CsvStorage.new(file: "#{dir}/o.csv").write([{ 'a' => 1, 'b' => 2 }])
+      assert_equal "a,b\n1,2\n", File.read("#{dir}/o.csv")
+    end
+  end
+
+  def test_columns_fix_order_missing_fields_and_key_types
+    with_tmpdir do |dir|
+      DataPipeline::CsvStorage.new(file: "#{dir}/o.csv", columns: %w[a b c]).write([{ 'b' => 2, 'a' => 1 }, { c: 3, a: 4 }])
+      assert_equal "a,b,c\n1,2,\n4,,3\n", File.read("#{dir}/o.csv")
+    end
+  end
+
+  def test_empty_data
+    with_tmpdir do |dir|
+      DataPipeline::CsvStorage.new(file: "#{dir}/a.csv").write([])
+      DataPipeline::CsvStorage.new(file: "#{dir}/b.csv", columns: %w[a b]).write([])
+      assert_equal '', File.read("#{dir}/a.csv")
+      assert_equal "a,b\n", File.read("#{dir}/b.csv")
+    end
+  end
+
+  def test_write_row_and_close
+    with_tmpdir do |dir|
+      csv = DataPipeline::CsvStorage.new(file: "#{dir}/o.csv")
+      csv.write_row({ 'a' => 1 })
+      csv.write_row({ 'a' => 2 })
+      csv.close
+      assert_equal "a\n1\n2\n", File.read("#{dir}/o.csv")
+    end
+  end
+
+  # options of the CUDO exports: no header, pipe, no quotes, mixed record types and key types
+  def test_cudo_format_mixed_record_types
+    with_tmpdir do |dir|
+      rows = [{ SATYP: 'FIRC', LFDNR: 0 }, { 'satyp' => 'DOHD', 'belnr' => 'AR1', 'wwert' => nil },
+              { 'satyp' => 'DOPS', 'dmbtr' => '1.50' }, { 'SATYPE' => 'LARC', 'SUMME' => '0000000002' }]
+      DataPipeline::CsvStorage.new(file: "#{dir}/c.ok", csv: { headers: false, col_sep: '|', quote_char: '', encoding: 'utf-8' })
+                .write(rows)
+      assert_equal "FIRC|0\nDOHD|AR1|\nDOPS|1.50\nLARC|0000000002\n", File.read("#{dir}/c.ok")
+    end
+  end
+end