Betriebssicherheit: Sperre gegen parallele Läufe, Upsert, Batch-Schreiben für Postgres, Zeichensatz für CSV

test/plugins/csv_storage_test.rb

@@ -86,4 +86,34 @@ class CsvStorageTest < PipelineTest
       assert_equal "FIRC|0\nDOHD|AR1|\nDOPS|1.50\nLARC|0000000002\n", File.read("#{dir}/c.ok")
     end
   end
+
+  def test_encoding_read_and_write
+    with_tmpdir do |dir|
+      DataPipeline::CsvStorage.new(file: "#{dir}/l1.csv", encoding: 'ISO-8859-1').write([{ 'name' => 'Müller', 'ort' => 'Zürich' }])
+      assert_equal "name,ort\nM\xFCller,Z\xFCrich\n".b, File.binread("#{dir}/l1.csv")
+      rows = DataPipeline::CsvStorage.new(file: "#{dir}/l1.csv", encoding: 'ISO-8859-1').read
+      assert_equal [{ 'name' => 'Müller', 'ort' => 'Zürich' }], rows
+      assert_equal Encoding::UTF_8, rows.first['name'].encoding
+    end
+  end
+
+  def test_encoding_error_or_replace
+    with_tmpdir do |dir|
+      assert_raises(Encoding::UndefinedConversionError) do
+        DataPipeline::CsvStorage.new(file: "#{dir}/a.csv", encoding: 'ISO-8859-1').write([{ 'x' => '5 €' }])
+      end
+      DataPipeline::CsvStorage.new(file: "#{dir}/b.csv", encoding: 'ISO-8859-1', replace: '?').write([{ 'x' => '5 €' }])
+      assert_equal "x\n5 ?\n", File.read("#{dir}/b.csv")
+      DataPipeline::CsvStorage.new(file: "#{dir}/c.csv", encoding: 'Windows-1252').write([{ 'x' => '5 €' }])
+      assert_equal "x\n5 \x80\n".b, File.binread("#{dir}/c.csv")
+    end
+  end
+
+  def test_bom_written_and_skipped
+    with_tmpdir do |dir|
+      DataPipeline::CsvStorage.new(file: "#{dir}/bom.csv", bom: true).write([{ 'a' => '1' }])
+      assert_equal "\uFEFFa\n1\n", File.read("#{dir}/bom.csv", encoding: 'utf-8')
+      assert_equal [{ 'a' => '1' }], DataPipeline::CsvStorage.new(file: "#{dir}/bom.csv").read
+    end
+  end
 end

test/plugins/postgres_storage_test.rb

@@ -130,4 +130,84 @@ class PostgresStorageTest < PipelineTest
     error = assert_raises(PG::ConnectionBad) { DataPipeline::PostgresStorage.new(host: 'localhost', port: 1, database: 'x') }
     assert_includes error.message, 'Error connection to database:'
   end
+
+  def test_upsert_updates_existing_rows
+    with_pg_database do |db|
+      pg_exec db, 'create table u (belnr text primary key, betrag numeric, text text)'
+      writer = DataPipeline::PostgresStorage.new(**db, table: 'u', upsert: %w[belnr])
+      writer.write([{ 'belnr' => 'A', 'betrag' => 1, 'text' => 'alt' }, { 'belnr' => 'B', 'betrag' => 2, 'text' => 'b' }])
+      writer.write([{ 'belnr' => 'A', 'betrag' => 5, 'text' => 'neu' }])
+      writer.write_row({ 'belnr' => 'B' })
+      writer.write_row({ 'belnr' => 'C', 'betrag' => 3 })
+      writer.close
+      assert_equal [%w[A 5 neu], %w[B 2 b], ['C', '3', nil]],
+                   pg_exec(db, 'select belnr, betrag::int::text as betrag, text from u order by belnr').map(&:values)
+    end
+  end
+
+  def test_write_row_in_batches
+    with_pg_database do |db|
+      pg_exec db, 'create table b (id int)'
+      writer = DataPipeline::PostgresStorage.new(**db, table: 'b', batch_size: 2)
+      3.times { |i| writer.write_row({ 'id' => i }) }
+      assert_equal '2', pg_exec(db, 'select count(*) from b').first['count'] # third row still pending
+      writer.finalize
+      assert_equal '3', pg_exec(db, 'select count(*) from b').first['count']
+    end
+  end
+
+  def test_close_writes_pending_rows
+    with_pg_database do |db|
+      pg_exec db, 'create table b (id int)'
+      writer = DataPipeline::PostgresStorage.new(**db, table: 'b')
+      writer.write_row({ 'id' => 1 })
+      writer.close
+      assert_equal '1', pg_exec(db, 'select count(*) from b').first['count']
+    end
+  end
+
+  def test_batch_size_all_is_one_transaction_for_the_stream
+    with_pg_database do |db|
+      pg_exec db, 'create table b (id int); insert into b values (99)'
+      aborted = DataPipeline::PostgresStorage.new(**db, table: 'b', batch_size: :all, truncate: true)
+      2.times { |i| aborted.write_row({ 'id' => i }) }
+      aborted.close # without finalize: rolled back, truncate too
+      assert_equal ['99'], pg_exec(db, 'select id from b').map { |r| r['id'] }
+
+      done = DataPipeline::PostgresStorage.new(**db, table: 'b', batch_size: :all, truncate: true)
+      2.times { |i| done.write_row({ 'id' => i }) }
+      done.finalize
+      assert_equal %w[0 1], pg_exec(db, 'select id from b order by id').map { |r| r['id'] }
+    end
+  end
+
+  def test_streamed_pipeline_aborted_with_batch_size_all_writes_nothing
+    with_pg_database do |db|
+      pg_exec db, 'create table b (id int)'
+      assert_raises(RuntimeError) do
+        DataPipeline.new do
+          logger PipelineHelper::QUIET
+          streamed
+          read MemoryStorage.new(rows: Array.new(5) { |i| { 'id' => i } })
+          map { |row| row['id'] == 3 ? raise('kaputt') : row }
+          write DataPipeline::PostgresStorage.new(**db, table: 'b', batch_size: :all)
+        end.run
+      end
+      assert_equal '0', pg_exec(db, 'select count(*) from b').first['count']
+    end
+  end
+
+  def test_streamed_copy_on_the_same_connection_as_the_cursor
+    with_pg_database do |db|
+      pg_exec db, 'create table k as select i as id from generate_series(1, 25) i; create table c (id int)'
+      pg = DataPipeline::PostgresStorage.new(**db, query: 'select id from k', table: 'c', fetch_size: 10, batch_size: 7)
+      DataPipeline.new do
+        logger PipelineHelper::QUIET
+        streamed
+        read pg
+        write pg
+      end.run
+      assert_equal '25', pg_exec(db, 'select count(*) from c').first['count']
+    end
+  end
 end

test/plugins/sqlite_storage_test.rb

@@ -92,4 +92,15 @@ class SqliteStorageTest < PipelineTest
       assert_equal 'Deutschland', names['DE']['name']
     end
   end
+
+  def test_upsert_with_created_unique_index
+    with_tmpdir do |dir|
+      writer = DataPipeline::SqliteStorage.new(database: "#{dir}/u.sqlite3", table: 'u', create: true, upsert: %w[belnr])
+      writer.write([{ 'belnr' => 'A', 'betrag' => 1.0 }, { 'belnr' => 'B', 'betrag' => 2.0 }])
+      writer.write([{ 'belnr' => 'A', 'betrag' => 5.0 }])
+      writer.write_row({ 'belnr' => 'B' })
+      assert_equal [['A', 5.0], ['B', 2.0]], writer.query(query: 'select belnr, betrag from u order by belnr').map(&:values)
+      writer.close
+    end
+  end
 end