Betriebssicherheit: Sperre gegen parallele Läufe, Upsert, Batch-Schreiben für Postgres, Zeichensatz für CSV

test/cli/etl_cli_test.rb

@@ -86,4 +86,30 @@ class EtlCliTest < PipelineTest
       assert_includes err, 'Usage: etl [run] PIPELINE.rb'
     end
   end
+
+  def test_lock_exit_code_3
+    with_tmpdir do |dir|
+      path = script(dir, 'locked.rb', <<~RUBY)
+        DataPipeline.new do
+          lock '.pipeline.lock'
+          read container(:csv, file: 'in.csv')
+          write container(:csv, file: 'out.csv')
+        end.run
+      RUBY
+      holder = DataPipeline::Lock.new("#{dir}/.script.lock").acquire
+      _out, err, status = etl(dir, 'run', path, '--lock=.script.lock')
+      assert_equal 3, status.exitstatus, err
+      assert_includes err, 'another run is still active'
+      refute File.exist?("#{dir}/out.csv")
+      holder.release
+
+      holder = DataPipeline::Lock.new("#{dir}/.pipeline.lock").acquire
+      _out, err, status = etl(dir, 'run', path, '--lock=.script.lock')
+      assert_equal 3, status.exitstatus, err
+      holder.release
+
+      _out, err, status = etl(dir, 'run', path, '--lock=.script.lock')
+      assert_equal 0, status.exitstatus, err
+    end
+  end
 end

test/core/lock_test.rb

@@ -0,0 +1,47 @@
+require_relative '../test_helper'
+
+class LockTest < PipelineTest
+  def test_second_lock_on_the_same_file_fails
+    with_tmpdir do |dir|
+      first = DataPipeline::Lock.new("#{dir}/sub/.lock").acquire
+      assert first.locked?
+      error = assert_raises(DataPipeline::LockError) { DataPipeline::Lock.new("#{dir}/sub/.lock").acquire }
+      assert_includes error.message, "pid #{Process.pid}"
+      first.release
+      DataPipeline::Lock.new("#{dir}/sub/.lock").synchronize { assert true }
+    end
+  end
+
+  def test_pipeline_holds_the_lock_during_run_and_releases_it
+    with_tmpdir do |dir|
+      file = "#{dir}/.lock"
+      inside = nil
+      run_pipeline([{ 'a' => 1 }]) do
+        lock file
+        map { |row| inside = (DataPipeline::Lock.new(file).acquire rescue $!); row }
+      end
+      assert_kind_of DataPipeline::LockError, inside
+      DataPipeline::Lock.new(file).synchronize { assert true } # released after run
+    end
+  end
+
+  def test_locked_pipeline_aborts_with_lock_error_and_writes_nothing
+    with_tmpdir do |dir|
+      holder = DataPipeline::Lock.new("#{dir}/.lock").acquire
+      target = MemoryStorage.new
+      finished = nil
+      assert_raises(DataPipeline::LockError) do
+        DataPipeline.new do
+          logger PipelineHelper::QUIET
+          lock "#{dir}/.lock"
+          read MemoryStorage.new(rows: [{ 'a' => 1 }])
+          write target
+          on_finish { |stats| finished = stats }
+        end.run
+      end
+      assert_nil target.written
+      assert_kind_of DataPipeline::LockError, finished.error
+      holder.release
+    end
+  end
+end

test/plugins/csv_storage_test.rb

@@ -86,4 +86,34 @@ class CsvStorageTest < PipelineTest
       assert_equal "FIRC|0\nDOHD|AR1|\nDOPS|1.50\nLARC|0000000002\n", File.read("#{dir}/c.ok")
     end
   end
+
+  def test_encoding_read_and_write
+    with_tmpdir do |dir|
+      DataPipeline::CsvStorage.new(file: "#{dir}/l1.csv", encoding: 'ISO-8859-1').write([{ 'name' => 'Müller', 'ort' => 'Zürich' }])
+      assert_equal "name,ort\nM\xFCller,Z\xFCrich\n".b, File.binread("#{dir}/l1.csv")
+      rows = DataPipeline::CsvStorage.new(file: "#{dir}/l1.csv", encoding: 'ISO-8859-1').read
+      assert_equal [{ 'name' => 'Müller', 'ort' => 'Zürich' }], rows
+      assert_equal Encoding::UTF_8, rows.first['name'].encoding
+    end
+  end
+
+  def test_encoding_error_or_replace
+    with_tmpdir do |dir|
+      assert_raises(Encoding::UndefinedConversionError) do
+        DataPipeline::CsvStorage.new(file: "#{dir}/a.csv", encoding: 'ISO-8859-1').write([{ 'x' => '5 €' }])
+      end
+      DataPipeline::CsvStorage.new(file: "#{dir}/b.csv", encoding: 'ISO-8859-1', replace: '?').write([{ 'x' => '5 €' }])
+      assert_equal "x\n5 ?\n", File.read("#{dir}/b.csv")
+      DataPipeline::CsvStorage.new(file: "#{dir}/c.csv", encoding: 'Windows-1252').write([{ 'x' => '5 €' }])
+      assert_equal "x\n5 \x80\n".b, File.binread("#{dir}/c.csv")
+    end
+  end
+
+  def test_bom_written_and_skipped
+    with_tmpdir do |dir|
+      DataPipeline::CsvStorage.new(file: "#{dir}/bom.csv", bom: true).write([{ 'a' => '1' }])
+      assert_equal "\uFEFFa\n1\n", File.read("#{dir}/bom.csv", encoding: 'utf-8')
+      assert_equal [{ 'a' => '1' }], DataPipeline::CsvStorage.new(file: "#{dir}/bom.csv").read
+    end
+  end
 end

test/plugins/postgres_storage_test.rb

@@ -130,4 +130,84 @@ class PostgresStorageTest < PipelineTest
     error = assert_raises(PG::ConnectionBad) { DataPipeline::PostgresStorage.new(host: 'localhost', port: 1, database: 'x') }
     assert_includes error.message, 'Error connection to database:'
   end
+
+  def test_upsert_updates_existing_rows
+    with_pg_database do |db|
+      pg_exec db, 'create table u (belnr text primary key, betrag numeric, text text)'
+      writer = DataPipeline::PostgresStorage.new(**db, table: 'u', upsert: %w[belnr])
+      writer.write([{ 'belnr' => 'A', 'betrag' => 1, 'text' => 'alt' }, { 'belnr' => 'B', 'betrag' => 2, 'text' => 'b' }])
+      writer.write([{ 'belnr' => 'A', 'betrag' => 5, 'text' => 'neu' }])
+      writer.write_row({ 'belnr' => 'B' })
+      writer.write_row({ 'belnr' => 'C', 'betrag' => 3 })
+      writer.close
+      assert_equal [%w[A 5 neu], %w[B 2 b], ['C', '3', nil]],
+                   pg_exec(db, 'select belnr, betrag::int::text as betrag, text from u order by belnr').map(&:values)
+    end
+  end
+
+  def test_write_row_in_batches
+    with_pg_database do |db|
+      pg_exec db, 'create table b (id int)'
+      writer = DataPipeline::PostgresStorage.new(**db, table: 'b', batch_size: 2)
+      3.times { |i| writer.write_row({ 'id' => i }) }
+      assert_equal '2', pg_exec(db, 'select count(*) from b').first['count'] # third row still pending
+      writer.finalize
+      assert_equal '3', pg_exec(db, 'select count(*) from b').first['count']
+    end
+  end
+
+  def test_close_writes_pending_rows
+    with_pg_database do |db|
+      pg_exec db, 'create table b (id int)'
+      writer = DataPipeline::PostgresStorage.new(**db, table: 'b')
+      writer.write_row({ 'id' => 1 })
+      writer.close
+      assert_equal '1', pg_exec(db, 'select count(*) from b').first['count']
+    end
+  end
+
+  def test_batch_size_all_is_one_transaction_for_the_stream
+    with_pg_database do |db|
+      pg_exec db, 'create table b (id int); insert into b values (99)'
+      aborted = DataPipeline::PostgresStorage.new(**db, table: 'b', batch_size: :all, truncate: true)
+      2.times { |i| aborted.write_row({ 'id' => i }) }
+      aborted.close # without finalize: rolled back, truncate too
+      assert_equal ['99'], pg_exec(db, 'select id from b').map { |r| r['id'] }
+
+      done = DataPipeline::PostgresStorage.new(**db, table: 'b', batch_size: :all, truncate: true)
+      2.times { |i| done.write_row({ 'id' => i }) }
+      done.finalize
+      assert_equal %w[0 1], pg_exec(db, 'select id from b order by id').map { |r| r['id'] }
+    end
+  end
+
+  def test_streamed_pipeline_aborted_with_batch_size_all_writes_nothing
+    with_pg_database do |db|
+      pg_exec db, 'create table b (id int)'
+      assert_raises(RuntimeError) do
+        DataPipeline.new do
+          logger PipelineHelper::QUIET
+          streamed
+          read MemoryStorage.new(rows: Array.new(5) { |i| { 'id' => i } })
+          map { |row| row['id'] == 3 ? raise('kaputt') : row }
+          write DataPipeline::PostgresStorage.new(**db, table: 'b', batch_size: :all)
+        end.run
+      end
+      assert_equal '0', pg_exec(db, 'select count(*) from b').first['count']
+    end
+  end
+
+  def test_streamed_copy_on_the_same_connection_as_the_cursor
+    with_pg_database do |db|
+      pg_exec db, 'create table k as select i as id from generate_series(1, 25) i; create table c (id int)'
+      pg = DataPipeline::PostgresStorage.new(**db, query: 'select id from k', table: 'c', fetch_size: 10, batch_size: 7)
+      DataPipeline.new do
+        logger PipelineHelper::QUIET
+        streamed
+        read pg
+        write pg
+      end.run
+      assert_equal '25', pg_exec(db, 'select count(*) from c').first['count']
+    end
+  end
 end

test/plugins/sqlite_storage_test.rb

@@ -92,4 +92,15 @@ class SqliteStorageTest < PipelineTest
       assert_equal 'Deutschland', names['DE']['name']
     end
   end
+
+  def test_upsert_with_created_unique_index
+    with_tmpdir do |dir|
+      writer = DataPipeline::SqliteStorage.new(database: "#{dir}/u.sqlite3", table: 'u', create: true, upsert: %w[belnr])
+      writer.write([{ 'belnr' => 'A', 'betrag' => 1.0 }, { 'belnr' => 'B', 'betrag' => 2.0 }])
+      writer.write([{ 'belnr' => 'A', 'betrag' => 5.0 }])
+      writer.write_row({ 'belnr' => 'B' })
+      assert_equal [['A', 5.0], ['B', 2.0]], writer.query(query: 'select belnr, betrag from u order by belnr').map(&:values)
+      writer.close
+    end
+  end
 end