Betriebssicherheit: Sperre gegen parallele Läufe, Upsert, Batch-Schreiben für Postgres, Zeichensatz für CSV

test/plugins/postgres_storage_test.rb

@@ -130,4 +130,84 @@ class PostgresStorageTest < PipelineTest
     error = assert_raises(PG::ConnectionBad) { DataPipeline::PostgresStorage.new(host: 'localhost', port: 1, database: 'x') }
     assert_includes error.message, 'Error connection to database:'
   end
+
+  def test_upsert_updates_existing_rows
+    with_pg_database do |db|
+      pg_exec db, 'create table u (belnr text primary key, betrag numeric, text text)'
+      writer = DataPipeline::PostgresStorage.new(**db, table: 'u', upsert: %w[belnr])
+      writer.write([{ 'belnr' => 'A', 'betrag' => 1, 'text' => 'alt' }, { 'belnr' => 'B', 'betrag' => 2, 'text' => 'b' }])
+      writer.write([{ 'belnr' => 'A', 'betrag' => 5, 'text' => 'neu' }])
+      writer.write_row({ 'belnr' => 'B' })
+      writer.write_row({ 'belnr' => 'C', 'betrag' => 3 })
+      writer.close
+      assert_equal [%w[A 5 neu], %w[B 2 b], ['C', '3', nil]],
+                   pg_exec(db, 'select belnr, betrag::int::text as betrag, text from u order by belnr').map(&:values)
+    end
+  end
+
+  def test_write_row_in_batches
+    with_pg_database do |db|
+      pg_exec db, 'create table b (id int)'
+      writer = DataPipeline::PostgresStorage.new(**db, table: 'b', batch_size: 2)
+      3.times { |i| writer.write_row({ 'id' => i }) }
+      assert_equal '2', pg_exec(db, 'select count(*) from b').first['count'] # third row still pending
+      writer.finalize
+      assert_equal '3', pg_exec(db, 'select count(*) from b').first['count']
+    end
+  end
+
+  def test_close_writes_pending_rows
+    with_pg_database do |db|
+      pg_exec db, 'create table b (id int)'
+      writer = DataPipeline::PostgresStorage.new(**db, table: 'b')
+      writer.write_row({ 'id' => 1 })
+      writer.close
+      assert_equal '1', pg_exec(db, 'select count(*) from b').first['count']
+    end
+  end
+
+  def test_batch_size_all_is_one_transaction_for_the_stream
+    with_pg_database do |db|
+      pg_exec db, 'create table b (id int); insert into b values (99)'
+      aborted = DataPipeline::PostgresStorage.new(**db, table: 'b', batch_size: :all, truncate: true)
+      2.times { |i| aborted.write_row({ 'id' => i }) }
+      aborted.close # without finalize: rolled back, truncate too
+      assert_equal ['99'], pg_exec(db, 'select id from b').map { |r| r['id'] }
+
+      done = DataPipeline::PostgresStorage.new(**db, table: 'b', batch_size: :all, truncate: true)
+      2.times { |i| done.write_row({ 'id' => i }) }
+      done.finalize
+      assert_equal %w[0 1], pg_exec(db, 'select id from b order by id').map { |r| r['id'] }
+    end
+  end
+
+  def test_streamed_pipeline_aborted_with_batch_size_all_writes_nothing
+    with_pg_database do |db|
+      pg_exec db, 'create table b (id int)'
+      assert_raises(RuntimeError) do
+        DataPipeline.new do
+          logger PipelineHelper::QUIET
+          streamed
+          read MemoryStorage.new(rows: Array.new(5) { |i| { 'id' => i } })
+          map { |row| row['id'] == 3 ? raise('kaputt') : row }
+          write DataPipeline::PostgresStorage.new(**db, table: 'b', batch_size: :all)
+        end.run
+      end
+      assert_equal '0', pg_exec(db, 'select count(*) from b').first['count']
+    end
+  end
+
+  def test_streamed_copy_on_the_same_connection_as_the_cursor
+    with_pg_database do |db|
+      pg_exec db, 'create table k as select i as id from generate_series(1, 25) i; create table c (id int)'
+      pg = DataPipeline::PostgresStorage.new(**db, query: 'select id from k', table: 'c', fetch_size: 10, batch_size: 7)
+      DataPipeline.new do
+        logger PipelineHelper::QUIET
+        streamed
+        read pg
+        write pg
+      end.run
+      assert_equal '25', pg_exec(db, 'select count(*) from c').first['count']
+    end
+  end
 end