datapipeline 3.1.0: SQLite schreibt im Streaming in Paketen, Lizenz BSD-3-Clause

test/core/connections_totals_history_test.rb

@@ -1,4 +1,5 @@
 require_relative '../test_helper'
+require 'time'
 
 # connections file, totals with reconcile, run history
 class ConnectionsTotalsHistoryTest < PipelineTest

test/plugins/sqlite_storage_test.rb

@@ -103,4 +103,60 @@ class SqliteStorageTest < PipelineTest
       writer.close
     end
   end
+
+  def count(file) = DataPipeline::SqliteStorage.new(database: file).then { |db| db.query(query: 'select count(*) as n from b').first['n'].tap { db.close } }
+
+  def test_write_row_in_batches_and_query_sees_pending_rows
+    with_tmpdir do |dir|
+      file = "#{dir}/b.sqlite3"
+      writer = DataPipeline::SqliteStorage.new(database: file, table: 'b', create: true, batch_size: 2)
+      3.times { |i| writer.write_row({ 'id' => i }) }
+      assert_equal 2, count(file) # third row still pending
+      assert_equal 3, writer.query(query: 'select count(*) as n from b').first['n']
+      writer.write_row({ 'id' => 3 })
+      writer.finalize
+      assert_equal 4, count(file)
+    end
+  end
+
+  def test_close_writes_pending_rows
+    with_tmpdir do |dir|
+      writer = DataPipeline::SqliteStorage.new(database: "#{dir}/b.sqlite3", table: 'b', create: true)
+      writer.write_row({ 'id' => 1 })
+      writer.close
+      assert_equal 1, count("#{dir}/b.sqlite3")
+    end
+  end
+
+  def test_batch_size_all_is_one_transaction_for_the_stream
+    with_tmpdir do |dir|
+      file = "#{dir}/b.sqlite3"
+      DataPipeline::SqliteStorage.new(database: file, table: 'b', create: true).tap { |w| w.write([{ 'id' => 99 }]); w.close }
+      aborted = DataPipeline::SqliteStorage.new(database: file, table: 'b', truncate: true, batch_size: :all)
+      2.times { |i| aborted.write_row({ 'id' => i }) }
+      aborted.close # without finalize: rolled back, truncate too
+      assert_equal 1, count(file)
+
+      done = DataPipeline::SqliteStorage.new(database: file, table: 'b', truncate: true, batch_size: :all)
+      2.times { |i| done.write_row({ 'id' => i }) }
+      done.finalize
+      assert_equal 2, count(file)
+    end
+  end
+
+  def test_streamed_pipeline_aborted_with_batch_size_all_writes_nothing
+    with_tmpdir do |dir|
+      file = "#{dir}/b.sqlite3"
+      assert_raises(RuntimeError) do
+        DataPipeline.new do
+          logger PipelineHelper::QUIET
+          streamed
+          read MemoryStorage.new(rows: Array.new(5) { |i| { 'id' => i } })
+          map { |row| row['id'] == 3 ? raise('kaputt') : row }
+          write DataPipeline::SqliteStorage.new(database: file, table: 'b', create: true, batch_size: :all)
+        end.run
+      end
+      refute DataPipeline::SqliteStorage.new(database: file).query(query: "select name from sqlite_master where name = 'b'").any?
+    end
+  end
 end