datapipeline 2.0.0: Gem aus dem Projekt etl herausgelöst

test/support/storage_contract.rb

@@ -0,0 +1,73 @@
+#
+# shared tests for every storage plugin that can write and read back
+#
+# a test class includes StorageContract and implements
+#   contract_storage(env, purpose) -> a new storage; purpose is :write or :read
+# optionally
+#   with_contract_env { |env| ... } -> environment per test (default: a tmp dir)
+#   contract_rows(rows)             -> rows as the plugin reads them back (default: unchanged)
+#
+module StorageContract
+  ROWS = [
+    { 'id' => '1', 'code' => '007', 'name' => "O'Brien", 'note' => nil },
+    { 'id' => '2', 'code' => '010', 'name' => 'Zürich | "ä", ß', 'note' => 'zwei' },
+    { 'id' => '3', 'code' => '100', 'name' => 'drei', 'note' => '' }
+  ].freeze
+
+  def with_contract_env(&block) = with_tmpdir(&block)
+  def contract_rows(rows) = rows
+
+  def test_contract_write_and_read_back
+    with_contract_env do |env|
+      contract_storage(env, :write).tap { |s| s.write(ROWS) }.close
+      reader = contract_storage(env, :read)
+      assert_equal contract_rows(ROWS), reader.read
+      reader.close
+    end
+  end
+
+  def test_contract_write_row_and_read_with_block
+    with_contract_env do |env|
+      writer = contract_storage(env, :write)
+      ROWS.each { |row| writer.write_row(row) }
+      writer.close
+      rows = []
+      reader = contract_storage(env, :read)
+      assert_nil reader.read { |row| rows << row }
+      reader.close
+      assert_equal contract_rows(ROWS), rows
+    end
+  end
+
+  def test_contract_as_pipeline_source_and_target
+    [false, true].each do |streamed|
+      with_contract_env do |env|
+        target = contract_storage(env, :write)
+        stats = DataPipeline.new(streamed:) do
+          logger PipelineHelper::QUIET
+          read MemoryStorage.new(rows: ROWS)
+          write target
+        end.run
+        assert_equal 3, stats.written
+
+        source = contract_storage(env, :read)
+        copy = MemoryStorage.new
+        DataPipeline.new(streamed:) do
+          logger PipelineHelper::QUIET
+          read source
+          write copy
+        end.run
+        assert_equal contract_rows(ROWS), copy.output, "streamed=#{streamed}"
+      end
+    end
+  end
+
+  def test_contract_close_twice
+    with_contract_env do |env|
+      storage = contract_storage(env, :write)
+      storage.close
+      storage.close
+      pass
+    end
+  end
+end

test/support/test_http_server.rb

@@ -0,0 +1,52 @@
+require 'socket'
+require 'uri'
+require 'json'
+
+#
+# minimal http server for DataPipeline::RestStorage tests, one request per connection
+#   server = TestHttpServer.new { |req| [200, { 'items' => [] }] }
+#   req: { method:, path:, query:, headers:, body: }  -> [status, hash/array or string]
+#
+class TestHttpServer
+  attr_reader :requests
+
+  def initialize(&handler)
+    @handler = handler
+    @requests = []
+    @server = TCPServer.new('127.0.0.1', 0)
+    @thread = Thread.new { loop { serve(@server.accept) } }
+  end
+
+  def url = "http://127.0.0.1:#{@server.addr[1]}"
+
+  def stop
+    @thread.kill
+    @server.close
+  end
+
+  private
+
+  def serve client
+    method, target = client.gets.to_s.split
+    return unless method
+
+    headers = {}
+    while (line = client.gets) && line != "\r\n"
+      key, value = line.split(':', 2)
+      headers[key.downcase] = value.strip
+    end
+    body = headers['content-length'] ? client.read(headers['content-length'].to_i) : ''
+    path, query = target.split('?', 2)
+    request = { method:, path:, query: URI.decode_www_form(query.to_s).to_h, headers:, body: }
+    @requests << request
+
+    status, payload = @handler.call(request)
+    text = payload.is_a?(String) ? payload : JSON.generate(payload)
+    client.write "HTTP/1.1 #{status} X\r\nContent-Type: application/json\r\n" \
+                 "Content-Length: #{text.bytesize}\r\nConnection: close\r\n\r\n#{text}"
+  rescue StandardError => e
+    warn "TestHttpServer: #{e.class}: #{e.message}"
+  ensure
+    client.close
+  end
+end