datapipeline 2.0.0: Gem aus dem Projekt etl herausgelöst

test/plugins/csv_storage_test.rb

@@ -0,0 +1,89 @@
+require_relative '../test_helper'
+require_relative '../support/storage_contract'
+
+class CsvStorageTest < PipelineTest
+  include StorageContract
+
+  def contract_storage(dir, _purpose) = DataPipeline::CsvStorage.new(file: "#{dir}/data.csv")
+
+  def test_read_with_defaults
+    with_tmpdir do |dir|
+      File.write("#{dir}/in.csv", "a,b\n1,x\n2,\"\"\n")
+      assert_equal [{ 'a' => '1', 'b' => 'x' }, { 'a' => '2', 'b' => '' }], DataPipeline::CsvStorage.new(file: "#{dir}/in.csv").read
+    end
+  end
+
+  def test_read_without_header_uses_columns_or_index
+    with_tmpdir do |dir|
+      File.write("#{dir}/in.csv", "1|x\n2|y\n")
+      csv = { headers: false, col_sep: '|' }
+      assert_equal [{ 'id' => '1', 'txt' => 'x' }, { 'id' => '2', 'txt' => 'y' }],
+                   DataPipeline::CsvStorage.new(file: "#{dir}/in.csv", columns: %w[id txt], csv:).read
+      assert_equal({ '0' => '1', '1' => 'x' }, DataPipeline::CsvStorage.new(file: "#{dir}/in.csv", csv:).read.first)
+    end
+  end
+
+  def test_read_streaming_with_block
+    with_tmpdir do |dir|
+      File.write("#{dir}/in.csv", "a\n1\n2\n")
+      rows = []
+      DataPipeline::CsvStorage.new(file: "#{dir}/in.csv").read { |row| rows << row }
+      assert_equal %w[1 2], rows.map { |r| r['a'] }
+    end
+  end
+
+  def test_defaults_are_merged
+    options = DataPipeline::CsvStorage.new(file: 'x', csv: { col_sep: ';' }).instance_variable_get(:@options)[:csv]
+    assert_equal ';', options[:col_sep]
+    assert options[:headers]
+  end
+
+  def test_file_is_required
+    error = assert_raises(ArgumentError) { DataPipeline::CsvStorage.new }
+    assert_includes error.message, 'file'
+  end
+
+  def test_write_without_csv_options
+    with_tmpdir do |dir|
+      DataPipeline::CsvStorage.new(file: "#{dir}/o.csv").write([{ 'a' => 1, 'b' => 2 }])
+      assert_equal "a,b\n1,2\n", File.read("#{dir}/o.csv")
+    end
+  end
+
+  def test_columns_fix_order_missing_fields_and_key_types
+    with_tmpdir do |dir|
+      DataPipeline::CsvStorage.new(file: "#{dir}/o.csv", columns: %w[a b c]).write([{ 'b' => 2, 'a' => 1 }, { c: 3, a: 4 }])
+      assert_equal "a,b,c\n1,2,\n4,,3\n", File.read("#{dir}/o.csv")
+    end
+  end
+
+  def test_empty_data
+    with_tmpdir do |dir|
+      DataPipeline::CsvStorage.new(file: "#{dir}/a.csv").write([])
+      DataPipeline::CsvStorage.new(file: "#{dir}/b.csv", columns: %w[a b]).write([])
+      assert_equal '', File.read("#{dir}/a.csv")
+      assert_equal "a,b\n", File.read("#{dir}/b.csv")
+    end
+  end
+
+  def test_write_row_and_close
+    with_tmpdir do |dir|
+      csv = DataPipeline::CsvStorage.new(file: "#{dir}/o.csv")
+      csv.write_row({ 'a' => 1 })
+      csv.write_row({ 'a' => 2 })
+      csv.close
+      assert_equal "a\n1\n2\n", File.read("#{dir}/o.csv")
+    end
+  end
+
+  # options of the CUDO exports: no header, pipe, no quotes, mixed record types and key types
+  def test_cudo_format_mixed_record_types
+    with_tmpdir do |dir|
+      rows = [{ SATYP: 'FIRC', LFDNR: 0 }, { 'satyp' => 'DOHD', 'belnr' => 'AR1', 'wwert' => nil },
+              { 'satyp' => 'DOPS', 'dmbtr' => '1.50' }, { 'SATYPE' => 'LARC', 'SUMME' => '0000000002' }]
+      DataPipeline::CsvStorage.new(file: "#{dir}/c.ok", csv: { headers: false, col_sep: '|', quote_char: '', encoding: 'utf-8' })
+                .write(rows)
+      assert_equal "FIRC|0\nDOHD|AR1|\nDOPS|1.50\nLARC|0000000002\n", File.read("#{dir}/c.ok")
+    end
+  end
+end

test/plugins/excel_storage_test.rb

@@ -0,0 +1,57 @@
+require_relative '../test_helper'
+require_relative '../support/storage_contract'
+
+class ExcelStorageTest < PipelineTest
+  include StorageContract
+
+  def contract_storage(dir, _purpose) = DataPipeline::ExcelStorage.new(file: "#{dir}/data.xlsx")
+
+  # an empty text is an empty cell in excel
+  def contract_rows(rows) = rows.map { |row| row.transform_values { |v| v == '' ? nil : v } }
+
+  def test_text_stays_text_numbers_and_dates_keep_type
+    with_tmpdir do |dir|
+      rows = [{ 'lfdnr' => '0000000001', 'betrag' => 1.5, 'menge' => 3, 'datum' => Date.new(2025, 11, 3), 'text' => '=1+1' }]
+      DataPipeline::ExcelStorage.new(file: "#{dir}/t.xlsx").write(rows)
+      row = DataPipeline::ExcelStorage.new(file: "#{dir}/t.xlsx").read.first
+      assert_equal '0000000001', row['lfdnr']
+      assert_in_delta 1.5, row['betrag']
+      assert_equal 3, row['menge'].to_i
+      assert_equal Date.new(2025, 11, 3), row['datum'].to_date
+      assert_equal '=1+1', row['text']
+    end
+  end
+
+  def test_columns_sheet_name_and_symbol_keys
+    with_tmpdir do |dir|
+      DataPipeline::ExcelStorage.new(file: "#{dir}/c.xlsx", sheet: 'Buchungen', columns: %w[b a])
+                  .write([{ a: '1', b: '2' }, { 'b' => '4' }])
+      assert_equal [{ 'b' => '2', 'a' => '1' }, { 'b' => '4', 'a' => nil }],
+                   DataPipeline::ExcelStorage.new(file: "#{dir}/c.xlsx", sheet: 'Buchungen').read
+    end
+  end
+
+  def test_without_header_and_skip_title_rows
+    with_tmpdir do |dir|
+      DataPipeline::ExcelStorage.new(file: "#{dir}/h.xlsx", headers: false).write([{ 'x' => 'Titel' }, { 'x' => 'Kopf' }, { 'x' => 'Wert' }])
+      assert_equal [{ 'name' => 'Titel' }, { 'name' => 'Kopf' }, { 'name' => 'Wert' }],
+                   DataPipeline::ExcelStorage.new(file: "#{dir}/h.xlsx", headers: false, columns: %w[name]).read
+      assert_equal [{ 'Kopf' => 'Wert' }], DataPipeline::ExcelStorage.new(file: "#{dir}/h.xlsx", skip: 1).read
+    end
+  end
+
+  def test_empty_rows_are_skipped_and_short_name
+    with_tmpdir do |dir|
+      DataPipeline.new do
+        logger PipelineHelper::QUIET
+        read MemoryStorage.new(rows: [{ 'a' => '1' }, { 'a' => nil }, { 'a' => '3' }])
+        write container(:xlsx, file: "#{dir}/e.xlsx")
+      end.run
+      assert_equal [{ 'a' => '1' }, { 'a' => '3' }], DataPipeline::ExcelStorage.new(file: "#{dir}/e.xlsx").read
+    end
+  end
+
+  def test_file_required
+    assert_raises(ArgumentError) { DataPipeline::ExcelStorage.new }
+  end
+end

test/plugins/json_storage_test.rb

@@ -0,0 +1,56 @@
+require_relative '../test_helper'
+require_relative '../support/storage_contract'
+
+class JsonArrayStorageTest < PipelineTest
+  include StorageContract
+
+  def contract_storage(dir, _purpose) = DataPipeline::JsonStorage.new(file: "#{dir}/data.json")
+
+  def test_streamed_array_is_valid_json
+    with_tmpdir do |dir|
+      json = DataPipeline::JsonStorage.new(file: "#{dir}/a.json")
+      json.write_row({ 'a' => 1 })
+      json.write_row({ 'a' => 2 })
+      json.close
+      assert_equal [{ 'a' => 1 }, { 'a' => 2 }], JSON.parse(File.read("#{dir}/a.json"))
+    end
+  end
+
+  def test_root_path_and_pretty
+    with_tmpdir do |dir|
+      File.write("#{dir}/api.json", JSON.generate('data' => { 'items' => [{ 'id' => 1 }] }))
+      assert_equal [{ 'id' => 1 }], DataPipeline::JsonStorage.new(file: "#{dir}/api.json", root: 'data.items').read
+      DataPipeline::JsonStorage.new(file: "#{dir}/p.json", pretty: true).write([{ 'a' => 1 }])
+      assert_includes File.read("#{dir}/p.json"), "\n"
+    end
+  end
+
+  def test_single_object_is_one_row_and_scalar_raises
+    with_tmpdir do |dir|
+      File.write("#{dir}/o.json", '{"a":1}')
+      assert_equal [{ 'a' => 1 }], DataPipeline::JsonStorage.new(file: "#{dir}/o.json").read
+      File.write("#{dir}/s.json", '42')
+      assert_raises(ArgumentError) { DataPipeline::JsonStorage.new(file: "#{dir}/s.json").read }
+    end
+  end
+
+  def test_options
+    assert_raises(ArgumentError) { DataPipeline::JsonStorage.new }
+    assert_raises(ArgumentError) { DataPipeline::JsonStorage.new(file: 'x.json', format: :xml) }
+  end
+end
+
+class JsonLinesStorageTest < PipelineTest
+  include StorageContract
+
+  def contract_storage(dir, _purpose) = DataPipeline::JsonStorage.new(file: "#{dir}/data.ndjson")
+
+  def test_format_from_extension_and_blank_lines
+    with_tmpdir do |dir|
+      File.write("#{dir}/d.jsonl", "{\"a\":1}\n\n{\"a\":2}\n")
+      assert_equal [{ 'a' => 1 }, { 'a' => 2 }], DataPipeline::JsonStorage.new(file: "#{dir}/d.jsonl").read
+      DataPipeline::JsonStorage.new(file: "#{dir}/x.txt", format: :lines).write([{ 'a' => 1 }, { 'a' => 2 }])
+      assert_equal "{\"a\":1}\n{\"a\":2}\n", File.read("#{dir}/x.txt")
+    end
+  end
+end

test/plugins/lazy_loading_test.rb

@@ -0,0 +1,45 @@
+require_relative '../test_helper'
+require 'open3'
+
+# in a fresh process: plugins and their gems are loaded only when used
+class LazyLoadingTest < PipelineTest
+  def loaded_after code
+    script = <<~RUBY
+      require 'bundler/setup'
+      require #{File.join(ROOT, 'lib/datapipeline').inspect}
+      #{code}
+      features = $LOADED_FEATURES.join(' ')
+      puts %w[pg sqlite3 roo caxlsx http].select { |gem| features.match?(%r{/gems/\#{gem}-\\d}) }.join(',')
+    RUBY
+    out, err, status = Open3.capture3({ 'BUNDLE_GEMFILE' => File.join(ROOT, 'Gemfile') }, RbConfig.ruby, '-e', script)
+    assert status.success?, err
+    out.strip.split(',')
+  end
+
+  def test_nothing_loaded_at_start
+    assert_empty loaded_after('')
+  end
+
+  def test_json_and_csv_need_no_gem
+    with_tmpdir do |dir|
+      code = "DataPipeline.new { logger Logger.new(IO::NULL); read container(:json, file: '#{dir}/x.json').tap { File.write('#{dir}/x.json', '[]') }; write container(:csv, file: '#{dir}/y.csv') }.run"
+      assert_empty loaded_after(code)
+    end
+  end
+
+  def test_each_plugin_loads_only_its_gem
+    with_tmpdir do |dir|
+      assert_equal %w[sqlite3], loaded_after("DataPipeline::SqliteStorage.new(database: ':memory:').close")
+      assert_equal %w[caxlsx], loaded_after("DataPipeline::ExcelStorage.new(file: '#{dir}/a.xlsx').write([{ 'a' => 1 }])")
+      assert_equal %w[roo caxlsx].sort, loaded_after("DataPipeline::ExcelStorage.new(file: '#{dir}/b.xlsx').write([{ 'a' => 1 }]); DataPipeline::ExcelStorage.new(file: '#{dir}/b.xlsx').read").sort
+      assert_equal %w[http], loaded_after("DataPipeline::RestStorage.new(url: 'http://localhost')")
+    end
+  end
+
+  def test_missing_gem_message
+    plugin = Class.new(DataPipeline::StoragePlugin) { def initialize(**) = self.class.require_gem('gibtsnicht_gem') }
+    Object.const_set(:GibtsNichtStorage, plugin) unless defined?(GibtsNichtStorage)
+    error = assert_raises(LoadError) { GibtsNichtStorage.new }
+    assert_includes error.message, "gem 'gibtsnicht_gem'"
+  end
+end

test/plugins/postgres_storage_test.rb

@@ -0,0 +1,133 @@
+require_relative '../test_helper'
+require_relative '../support/storage_contract'
+
+class PostgresStorageTest < PipelineTest
+  include StorageContract
+
+  def with_contract_env
+    with_pg_database do |db|
+      pg_exec db, 'create table rows (id text, code text, name text, note text)'
+      yield db
+    end
+  end
+
+  def contract_storage db, purpose
+    purpose == :write ? DataPipeline::PostgresStorage.new(**db, table: 'rows') : DataPipeline::PostgresStorage.new(**db, query: 'select * from rows order by id')
+  end
+
+  def with_table
+    with_pg_database do |db|
+      pg_exec db, 'create table t (id int, name text, note text); create schema s; create table s."Mixed Case" ("Col A" text)'
+      storage = DataPipeline::PostgresStorage.new(**db)
+      yield db, storage
+    ensure
+      storage&.close
+    end
+  end
+
+  def test_write_rows_with_table_option_quotes_and_null
+    with_table do |db, pg|
+      DataPipeline::PostgresStorage.new(**db, table: 't').tap do |w|
+        w.write([{ 'id' => 1, 'name' => "O'Brien", 'note' => nil }, { id: 2, name: "x'); drop table t; --" }])
+        w.close
+      end
+      assert_equal [{ 'id' => '1', 'name' => "O'Brien", 'note' => nil },
+                    { 'id' => '2', 'name' => "x'); drop table t; --", 'note' => nil }],
+                   pg.query(query: 'select id, name, note from t order by id')
+    end
+  end
+
+  def test_write_old_signature_and_schema_quoting
+    with_table do |_db, pg|
+      pg.write('t', [{ 'id' => 3 }])
+      pg.write('s.Mixed Case', [{ 'Col A' => 'v' }])
+      assert_equal '1', pg.query(query: 'select count(*) from t').first['count']
+      assert_equal [{ 'Col A' => 'v' }], pg.query(query: 'select * from s."Mixed Case"')
+    end
+  end
+
+  def test_truncate_before_first_write
+    with_table do |db, pg|
+      pg.write('t', [{ 'id' => 1 }, { 'id' => 2 }])
+      writer = DataPipeline::PostgresStorage.new(**db, table: 't', truncate: true)
+      writer.write([{ 'id' => 3 }])
+      writer.write_row({ 'id' => 4 })
+      writer.close
+      assert_equal %w[3 4], pg.query(query: 'select id from t order by id').map { |r| r['id'] }
+    end
+  end
+
+  def test_write_is_one_transaction
+    with_table do |_db, pg|
+      assert_raises(PG::Error) { pg.write('t', [{ 'id' => 10 }, { 'id' => 'kein int' }]) }
+      assert_equal '0', pg.query(query: 'select count(*) from t').first['count']
+    end
+  end
+
+  def test_write_without_table
+    with_table do |_db, pg|
+      assert_includes assert_raises(RuntimeError) { pg.write([{ 'id' => 1 }]) }.message, 'Table'
+    end
+  end
+
+  def test_query_and_read_with_params
+    with_table do |db, pg|
+      pg.write('t', [{ 'id' => 1, 'name' => 'a' }, { 'id' => 2, 'name' => 'b' }, { 'id' => 3 }])
+      assert_equal [{ 'name' => 'a' }], pg.query(query: 'select name from t where id = $1', params: [1])
+
+      reader = DataPipeline::PostgresStorage.new(**db)
+      reader.set(query: 'select id from t where id > $1 order by id', params: [1])
+      assert_equal [{ 'id' => '2' }, { 'id' => '3' }], reader.read
+      reader.close
+    end
+  end
+
+  def test_typed_and_default_string_values
+    with_table do |db, pg|
+      typed = DataPipeline::PostgresStorage.new(**db, typed: true)
+      row = typed.query(query: 'select 1::int as i, 1.5::float8 as f, current_date as d').first
+      typed.close
+      assert_equal 1, row['i']
+      assert_in_delta 1.5, row['f']
+      assert_kind_of Date, row['d']
+      assert_equal '1', pg.query(query: 'select 1::int as i').first['i']
+    end
+  end
+
+  def test_cursor_read_in_batches
+    with_pg_database do |db|
+      pg_exec db, 'create table k as select i as id from generate_series(1, 2500) i'
+      reader = DataPipeline::PostgresStorage.new(**db, query: "select id from k where id > $1 order by id;\n", params: [500],
+                                         fetch_size: 100)
+      count = 0
+      reader.read { count += 1 }
+      reader.close
+      assert_equal 2000, count
+    end
+  end
+
+  def test_cursor_rollback_on_error_keeps_connection_usable
+    with_pg_database do |db|
+      pg_exec db, 'create table k as select 1 as id'
+      pg = DataPipeline::PostgresStorage.new(**db, query: 'select id from k')
+      assert_raises(RuntimeError) { pg.read { raise 'x' } }
+      assert_equal [{ 'x' => '1' }], pg.query(query: 'select 1 as x')
+      pg.close
+    end
+  end
+
+  def test_close_twice
+    with_pg_database do |db|
+      pg = DataPipeline::PostgresStorage.new(**db)
+      pg.close
+      pg.close
+      assert pg.instance_variable_get(:@conn).finished?
+    end
+  end
+
+  def test_connection_error_keeps_cause
+    skip 'no local postgres' unless PgHelper.available?
+    error = assert_raises(PG::ConnectionBad) { DataPipeline::PostgresStorage.new(host: 'localhost', port: 1, database: 'x') }
+    assert_includes error.message, 'Error connection to database:'
+  end
+end

test/plugins/rest_storage_test.rb

@@ -0,0 +1,151 @@
+require_relative '../test_helper'
+require_relative '../support/storage_contract'
+require_relative '../support/test_http_server'
+
+class RestStorageTest < PipelineTest
+  include StorageContract
+
+  # a tiny api: POST /items stores one row or an array, GET /items returns all
+  def with_contract_env
+    store = []
+    server = TestHttpServer.new do |req|
+      if req[:method] == 'POST'
+        Array(JSON.parse(req[:body]).then { |b| b.is_a?(Array) ? b : [b] }).each { |row| store << row }
+        [201, { 'ok' => true }]
+      else
+        [200, { 'data' => { 'items' => store } }]
+      end
+    end
+    yield server
+  ensure
+    server&.stop
+  end
+
+  def contract_storage(server, _purpose) = DataPipeline::RestStorage.new(url: "#{server.url}/items", root: 'data.items')
+
+  # handler: ->(request) { [status, body] }
+  def server handler
+    s = TestHttpServer.new(&handler)
+    yield s
+  ensure
+    s&.stop
+  end
+
+  def test_page_param_paging
+    pages = { '1' => [{ 'id' => 1 }, { 'id' => 2 }], '2' => [{ 'id' => 3 }] }
+    server(->(req) { [200, pages.fetch(req[:query]['page'], [])] }) do |s|
+      rows = DataPipeline::RestStorage.new(url: "#{s.url}/kunden", page_param: 'page', params: { 'land' => 'DE' }).read
+      assert_equal [1, 2, 3], rows.map { |r| r['id'] }
+      assert_equal %w[1 2 3], s.requests.map { |r| r[:query]['page'] }
+      assert(s.requests.all? { |r| r[:query]['land'] == 'DE' })
+    end
+  end
+
+  def test_next_url_paging
+    base = nil
+    handler = lambda do |req|
+      if req[:path] == '/a'
+        [200, { 'rows' => [{ 'n' => 1 }], 'next' => "#{base}/b" }]
+      else
+        [200, { 'rows' => [{ 'n' => 2 }], 'next' => nil }]
+      end
+    end
+    server(handler) do |s|
+      base = s.url
+      rows = DataPipeline::RestStorage.new(url: "#{s.url}/a", root: 'rows', next: ->(body) { body['next'] }).read
+      assert_equal [1, 2], rows.map { |r| r['n'] }
+    end
+  end
+
+  def test_auth_and_headers
+    server(->(_req) { [200, []] }) do |s|
+      DataPipeline::RestStorage.new(url: s.url, auth: 'Bearer geheim', headers: { 'X-Mandant' => '001' }).read
+      assert_equal 'Bearer geheim', s.requests.first[:headers]['authorization']
+      assert_equal '001', s.requests.first[:headers]['x-mandant']
+    end
+  end
+
+  def test_retry_on_503_then_success
+    calls = 0
+    server(->(_req) { (calls += 1) < 3 ? [503, 'busy'] : [200, [{ 'ok' => 1 }]] }) do |s|
+      assert_equal [{ 'ok' => 1 }], DataPipeline::RestStorage.new(url: s.url, retries: 2, retry_wait: 0).read
+      assert_equal 3, calls
+    end
+  end
+
+  def test_no_retry_on_404_and_error_message
+    server(->(_req) { [404, { 'error' => 'nicht da' }] }) do |s|
+      error = assert_raises(DataPipeline::RestStorage::Error) { DataPipeline::RestStorage.new(url: "#{s.url}/x", retry_wait: 0).read }
+      assert_includes error.message, 'HTTP 404'
+      assert_includes error.message, 'nicht da'
+      assert_equal 1, s.requests.size
+    end
+  end
+
+  def test_retries_exhausted_and_connection_refused
+    server(->(_req) { [500, 'kaputt'] }) do |s|
+      assert_includes assert_raises(DataPipeline::RestStorage::Error) { DataPipeline::RestStorage.new(url: s.url, retries: 1, retry_wait: 0).read }.message, 'HTTP 500'
+      assert_equal 2, s.requests.size
+    end
+    closed = TCPServer.new('127.0.0.1', 0).then { |t| port = t.addr[1]; t.close; port }
+    assert_raises(DataPipeline::RestStorage::Error) { DataPipeline::RestStorage.new(url: "http://127.0.0.1:#{closed}", retries: 1, retry_wait: 0).read }
+  end
+
+  def test_batch_write_and_write_method
+    server(->(_req) { [200, ''] }) do |s|
+      rest = DataPipeline::RestStorage.new(url: "#{s.url}/import", batch_size: 2, write_method: :put)
+      rest.write([{ 'a' => 1 }, { 'a' => 2 }, { 'a' => 3 }])
+      assert_equal [[{ 'a' => 1 }, { 'a' => 2 }], [{ 'a' => 3 }]], s.requests.map { |r| JSON.parse(r[:body]) }
+      assert(s.requests.all? { |r| r[:method] == 'PUT' })
+    end
+  end
+
+  def test_write_url_per_row
+    server(->(_req) { [200, ''] }) do |s|
+      DataPipeline::RestStorage.new(url: s.url, write_method: :put, write_url: ->(row) { "#{s.url}/kunden/#{row['id']}" })
+                 .write([{ 'id' => 7 }, { 'id' => 8 }])
+      assert_equal %w[/kunden/7 /kunden/8], s.requests.map { |r| r[:path] }
+      assert_equal({ 'id' => 8 }, JSON.parse(s.requests.last[:body]))
+    end
+    assert_raises(ArgumentError) { DataPipeline::RestStorage.new(url: 'http://x', batch_size: 2, write_url: ->(_row) { 'http://x' }) }
+  end
+
+  def test_call_json_status_and_retries
+    calls = 0
+    server(->(req) { calls += 1; req[:path] == '/api/fail' ? [503, ''] : [201, { 'echo' => JSON.parse(req[:body]) }] }) do |s|
+      rest = DataPipeline::RestStorage.new(url: "#{s.url}/api/", retries: 2, retry_wait: 0)
+      assert_equal({ 'echo' => { 'a' => 1 } }, rest.call(:post, 'person/create', json: { 'a' => 1 }))
+      assert_equal '/api/person/create', s.requests.last[:path]
+
+      calls = 0
+      error = assert_raises(DataPipeline::RestStorage::Error) { rest.call(:post, 'fail', json: {}, retries: 0) }
+      assert_equal 503, error.status
+      assert_equal 1, calls, 'retries: 0 sends once'
+    end
+    server(->(_req) { [404, { 'message' => 'weg' }] }) do |s|
+      assert_equal 404, assert_raises(DataPipeline::RestStorage::Error) { DataPipeline::RestStorage.new(url: s.url).call(:get, 'x') }.status
+    end
+  end
+
+  def test_ssl_verify_false_context
+    context = DataPipeline::RestStorage.new(url: 'https://192.168.178.251:50443/pds/rest/api', ssl_verify: false).send(:ssl_context)
+    assert_equal OpenSSL::SSL::VERIFY_NONE, context.verify_mode
+  end
+
+  def test_query_for_lookup
+    server(->(req) { [200, [{ 'land' => req[:query]['land'], 'pfad' => req[:path] }]] }) do |s|
+      names = nil
+      DataPipeline.new do
+        api = container :rest, url: "#{s.url}/api/"
+        names = lookup api, query: '/laender', params: { 'land' => 'DE' }, key: 'land', unique: true
+      end
+      assert_equal '/api/laender', names['DE']['pfad']
+    end
+  end
+
+  def test_not_an_array
+    server(->(_req) { [200, { 'a' => 1 }] }) do |s|
+      assert_raises(DataPipeline::RestStorage::Error) { DataPipeline::RestStorage.new(url: s.url).read }
+    end
+  end
+end

test/plugins/slack_test.rb

@@ -0,0 +1,18 @@
+require_relative '../test_helper'
+
+# no network: only construction and options
+class SlackTest < PipelineTest
+  def test_token_required
+    token = ENV.delete('SLACK_TOKEN')
+    assert_raises(ArgumentError) { DataPipeline::Slack.new }
+  ensure
+    ENV['SLACK_TOKEN'] = token if token
+  end
+
+  def test_options_per_instance
+    a = DataPipeline::Slack.new(token: 'x', channel: '#a')
+    b = DataPipeline::Slack.new(token: 'y')
+    assert_equal '#a', a.instance_variable_get(:@options)[:channel]
+    assert_equal '#general', b.instance_variable_get(:@options)[:channel]
+  end
+end

test/plugins/sqlite_storage_test.rb

@@ -0,0 +1,95 @@
+require_relative '../test_helper'
+require_relative '../support/storage_contract'
+
+class SqliteStorageTest < PipelineTest
+  include StorageContract
+
+  def contract_storage dir, purpose
+    db = "#{dir}/test.sqlite3"
+    unless File.exist?(db)
+      DataPipeline::SqliteStorage.new(database: db).tap do |s|
+        s.query(query: 'create table rows (id text, code text, name text, note text)')
+        s.close
+      end
+    end
+    purpose == :write ? DataPipeline::SqliteStorage.new(database: db, table: 'rows') : DataPipeline::SqliteStorage.new(database: db, query: 'select * from rows order by id')
+  end
+
+  def memory_db
+    DataPipeline::SqliteStorage.new(database: ':memory:').tap do |s|
+      s.query(query: 'create table t (id integer, name text, betrag real, datum text)')
+    end
+  end
+
+  def test_types_null_and_quotes
+    db = memory_db
+    db.write('t', [{ 'id' => 1, 'name' => "O'Brien", 'betrag' => 1.5, 'datum' => Date.new(2025, 11, 3) }, { id: 2, name: nil }])
+    assert_equal [{ 'id' => 1, 'name' => "O'Brien", 'betrag' => 1.5, 'datum' => '2025-11-03' },
+                  { 'id' => 2, 'name' => nil, 'betrag' => nil, 'datum' => nil }],
+                 db.query(query: 'select * from t order by id')
+    db.close
+  end
+
+  def test_positional_and_named_params
+    db = memory_db
+    db.write('t', [{ 'id' => 1, 'name' => 'a' }, { 'id' => 2, 'name' => 'b' }])
+    assert_equal [{ 'name' => 'b' }], db.query(query: 'select name from t where id = ?', params: [2])
+    assert_equal [{ 'name' => 'a' }], db.query(query: 'select name from t where id = :id', params: { id: 1 })
+    db.close
+  end
+
+  def test_write_is_one_transaction
+    db = memory_db
+    db.query(query: 'create table u (id integer not null)')
+    assert_raises(SQLite3::Exception) { db.write('u', [{ 'id' => 1 }, { 'id' => nil }]) }
+    assert_equal 0, db.query(query: 'select count(*) as n from u').first['n']
+    db.close
+  end
+
+  def test_sql_injection_in_values_and_identifiers
+    db = memory_db
+    db.query(query: 'create table "Mixed ""Name""" ("Col A" text)')
+    db.write('Mixed "Name"', [{ 'Col A' => "x'); drop table t; --" }])
+    assert_equal [{ 'Col A' => "x'); drop table t; --" }], db.query(query: 'select * from "Mixed ""Name"""')
+    assert_equal 0, db.query(query: 'select count(*) as n from t').first['n']
+    db.close
+  end
+
+  def test_create_truncate_and_lazy_open
+    with_tmpdir do |dir|
+      path = "#{dir}/neu.sqlite3"
+      storage = DataPipeline::SqliteStorage.new(database: path, table: 'k', create: true, truncate: true)
+      refute File.exist?(path), 'opened only on first use'
+      storage.write([{ 'id' => 1, 'betrag' => 1.5, 'name' => 'a' }, { 'id' => 2, 'betrag' => 2.0, 'name' => 'b' }])
+      storage.close
+
+      again = DataPipeline::SqliteStorage.new(database: path, table: 'k', create: true, truncate: true)
+      again.write_row({ 'id' => 3, 'betrag' => 3.0, 'name' => 'c' })
+      assert_equal [{ 'id' => 3, 'betrag' => 3.0, 'name' => 'c' }], again.query(query: 'select * from k')
+      assert_equal %w[integer real text], again.query(query: 'pragma table_info(k)').map { |c| c['type'].downcase }
+      again.close
+    end
+  end
+
+  def test_missing_options
+    assert_raises(ArgumentError) { DataPipeline::SqliteStorage.new }
+    db = memory_db
+    assert_raises(RuntimeError) { db.write([{ 'id' => 1 }]) }
+    assert_raises(RuntimeError) { db.read }
+    db.close
+  end
+
+  def test_short_name_and_lookup
+    with_tmpdir do |dir|
+      path = "#{dir}/l.sqlite3"
+      names = nil
+      DataPipeline.new do
+        db = container :sqlite3, database: path
+        db.query(query: 'create table land (code text, name text)')
+        db.write('land', [{ 'code' => 'DE', 'name' => 'Deutschland' }])
+        names = lookup db, query: 'select * from land where code like ?', params: ['%'], key: 'code', unique: true
+      end
+      assert_equal 'Deutschland', names['DE']['name']
+    end
+  end
+end