Betrieb: Verbindungsdatei, Summen mit Abgleich vor dem Schreiben, Laufhistorie

test/cli/etl_cli_test.rb

@@ -112,4 +112,19 @@ class EtlCliTest < PipelineTest
       assert_equal 0, status.exitstatus, err
     end
   end
+
+  def test_history_with_script_name
+    with_tmpdir do |dir|
+      path = script(dir, 'export_kunden.rb', <<~RUBY)
+        DataPipeline.new do
+          read container(:csv, file: 'in.csv')
+          write container(:csv, file: 'out.csv')
+        end.run
+      RUBY
+      _out, err, status = etl(dir, 'run', path, '--history', 'runs.ndjson')
+      assert_equal 0, status.exitstatus, err
+      entry = JSON.parse(File.read("#{dir}/runs.ndjson"))
+      assert_equal ['export_kunden', 3, true], entry.values_at('name', 'read', 'ok')
+    end
+  end
 end

test/core/connections_totals_history_test.rb

@@ -0,0 +1,97 @@
+require_relative '../test_helper'
+
+# connections file, totals with reconcile, run history
+class ConnectionsTotalsHistoryTest < PipelineTest
+  def teardown
+    DataPipeline.connections_file = nil
+    ENV.delete('ETL_TEST_SECRET')
+    super
+  end
+
+  def write_connections dir
+    File.write("#{dir}/connections.yml", <<~YAML)
+      mem:
+        type: memory
+        rows: [{ a: 1 }]
+      secret:
+        type: csv
+        file: <%= ENV.fetch('ETL_TEST_SECRET', 'default.csv') %>
+    YAML
+    DataPipeline.connections_file = "#{dir}/connections.yml"
+  end
+
+  def test_connection_and_container_with_connection
+    with_tmpdir do |dir|
+      write_connections(dir)
+      ENV['ETL_TEST_SECRET'] = "#{dir}/from_env.csv"
+      pipeline = DataPipeline.new { logger PipelineHelper::QUIET }
+      assert_kind_of MemoryStorage, pipeline.connection(:mem)
+      csv = pipeline.container(:csv, connection: 'secret')
+      assert_equal "#{dir}/from_env.csv", csv.instance_variable_get(:@file_name)
+      csv = pipeline.connection(:secret, file: "#{dir}/override.csv")
+      assert_equal "#{dir}/override.csv", csv.instance_variable_get(:@file_name)
+
+      assert_includes assert_raises(ArgumentError) { pipeline.container(:json, connection: :secret) }.message, 'is csv, not json'
+      assert_includes assert_raises(ArgumentError) { pipeline.connection(:nope) }.message, 'mem, secret'
+      DataPipeline.connections_file = "#{dir}/missing.yml"
+      assert_includes assert_raises(ArgumentError) { pipeline.connection(:mem) }.message, 'missing.yml missing'
+    end
+  end
+
+  ROWS = [{ 'beleg' => '1', 'dmbtr' => '10.10' }, { 'beleg' => '2', 'dmbtr' => 5.05 },
+          { 'beleg' => '3', 'dmbtr' => '' }, { 'beleg' => '4', 'dmbtr' => 'x' }].freeze
+
+  def test_totals_batch_and_streaming
+    [false, true].each do |streamed|
+      _target, stats = run_pipeline(ROWS, streamed:) do
+        totals 'dmbtr'
+        filter { |row| row['beleg'] != '2' }
+      end
+      assert_equal Rational(1515, 100), stats.total('dmbtr', :read)
+      assert_equal Rational(1010, 100), stats.total('dmbtr')
+      assert_includes stats.to_s, 'dmbtr=15.15/10.1'
+    end
+  end
+
+  def test_reconcile_aborts_before_writing
+    target = nil
+    error = assert_raises(DataPipeline::ReconcileError) do
+      target, = run_pipeline(ROWS) do
+        totals 'dmbtr'
+        filter { |row| row['beleg'] != '2' }
+        reconcile { |stats| "Summe: #{stats.total('dmbtr', :read).to_f} statt #{stats.total('dmbtr').to_f}" if stats.total('dmbtr', :read) != stats.total('dmbtr') }
+        reconcile { |stats| ['zu wenig Zeilen'] if stats.written < 4 }
+      end
+    end
+    assert_equal 'Summe: 15.15 statt 10.1; zu wenig Zeilen', error.message
+    assert_nil target
+
+    _target, stats = run_pipeline(ROWS) { totals 'dmbtr'; reconcile { |s| nil if s.total('dmbtr', :read) == s.total('dmbtr') } }
+    assert stats.ok?
+  end
+
+  def test_history_lines_also_for_aborted_runs
+    with_tmpdir do |dir|
+      file = "#{dir}/log/runs.ndjson"
+      run_pipeline(ROWS) { name 'buchungen'; version '1.0'; history file; totals 'dmbtr' }
+      assert_raises(RuntimeError) { run_pipeline(ROWS) { name 'buchungen'; history file; map { raise 'kaputt' } } }
+      run_pipeline(ROWS) { name 'andere'; history file }
+
+      entries = DataPipeline.history(file, name: 'buchungen')
+      assert_equal 2, entries.size
+      first, second = entries
+      assert_equal({ 'name' => 'buchungen', 'version' => '1.0', 'read' => 4, 'written' => 4, 'rejected' => 0, 'ok' => true,
+                     'error' => nil, 'totals' => { 'dmbtr' => { 'read' => '15.15', 'written' => '15.15' } } },
+                   first.slice('name', 'version', 'read', 'written', 'rejected', 'ok', 'error', 'totals'))
+      assert_equal [false, 'RuntimeError: kaputt'], second.values_at('ok', 'error')
+      assert Time.iso8601(first['started_at'])
+      assert_equal 3, DataPipeline.history(file).size
+      assert_equal [], DataPipeline.history("#{dir}/none.ndjson")
+    end
+  end
+
+  def test_decimal
+    assert_equal ['1234.5', '0.333333', '7', '12'], [DataPipeline.decimal(Rational(12_345, 10)), DataPipeline.decimal(Rational(1, 3)),
+                                                     DataPipeline.decimal(7r), DataPipeline.decimal(12)]
+  end
+end