datapipeline/test/plugins/rest_storage_test.rb

require_relative '../test_helper'
require_relative '../support/storage_contract'
require_relative '../support/test_http_server'

class RestStorageTest < PipelineTest
  include StorageContract

  # a tiny api: POST /items stores one row or an array, GET /items returns all
  def with_contract_env
    store = []
    server = TestHttpServer.new do |req|
      if req[:method] == 'POST'
        Array(JSON.parse(req[:body]).then { |b| b.is_a?(Array) ? b : [b] }).each { |row| store << row }
        [201, { 'ok' => true }]
      else
        [200, { 'data' => { 'items' => store } }]
      end
    end
    yield server
  ensure
    server&.stop
  end

  def contract_storage(server, _purpose) = DataPipeline::RestStorage.new(url: "#{server.url}/items", root: 'data.items')

  # handler: ->(request) { [status, body] }
  def server handler
    s = TestHttpServer.new(&handler)
    yield s
  ensure
    s&.stop
  end

  def test_page_param_paging
    pages = { '1' => [{ 'id' => 1 }, { 'id' => 2 }], '2' => [{ 'id' => 3 }] }
    server(->(req) { [200, pages.fetch(req[:query]['page'], [])] }) do |s|
      rows = DataPipeline::RestStorage.new(url: "#{s.url}/kunden", page_param: 'page', params: { 'land' => 'DE' }).read
      assert_equal [1, 2, 3], rows.map { |r| r['id'] }
      assert_equal %w[1 2 3], s.requests.map { |r| r[:query]['page'] }
      assert(s.requests.all? { |r| r[:query]['land'] == 'DE' })
    end
  end

  def test_next_url_paging
    base = nil
    handler = lambda do |req|
      if req[:path] == '/a'
        [200, { 'rows' => [{ 'n' => 1 }], 'next' => "#{base}/b" }]
      else
        [200, { 'rows' => [{ 'n' => 2 }], 'next' => nil }]
      end
    end
    server(handler) do |s|
      base = s.url
      rows = DataPipeline::RestStorage.new(url: "#{s.url}/a", root: 'rows', next: ->(body) { body['next'] }).read
      assert_equal [1, 2], rows.map { |r| r['n'] }
    end
  end

  def test_auth_and_headers
    server(->(_req) { [200, []] }) do |s|
      DataPipeline::RestStorage.new(url: s.url, auth: 'Bearer geheim', headers: { 'X-Mandant' => '001' }).read
      assert_equal 'Bearer geheim', s.requests.first[:headers]['authorization']
      assert_equal '001', s.requests.first[:headers]['x-mandant']
    end
  end

  def test_retry_on_503_then_success
    calls = 0
    server(->(_req) { (calls += 1) < 3 ? [503, 'busy'] : [200, [{ 'ok' => 1 }]] }) do |s|
      assert_equal [{ 'ok' => 1 }], DataPipeline::RestStorage.new(url: s.url, retries: 2, retry_wait: 0).read
      assert_equal 3, calls
    end
  end

  def test_no_retry_on_404_and_error_message
    server(->(_req) { [404, { 'error' => 'nicht da' }] }) do |s|
      error = assert_raises(DataPipeline::RestStorage::Error) { DataPipeline::RestStorage.new(url: "#{s.url}/x", retry_wait: 0).read }
      assert_includes error.message, 'HTTP 404'
      assert_includes error.message, 'nicht da'
      assert_equal 1, s.requests.size
    end
  end

  def test_retries_exhausted_and_connection_refused
    server(->(_req) { [500, 'kaputt'] }) do |s|
      assert_includes assert_raises(DataPipeline::RestStorage::Error) { DataPipeline::RestStorage.new(url: s.url, retries: 1, retry_wait: 0).read }.message, 'HTTP 500'
      assert_equal 2, s.requests.size
    end
    closed = TCPServer.new('127.0.0.1', 0).then { |t| port = t.addr[1]; t.close; port }
    assert_raises(DataPipeline::RestStorage::Error) { DataPipeline::RestStorage.new(url: "http://127.0.0.1:#{closed}", retries: 1, retry_wait: 0).read }
  end

  def test_batch_write_and_write_method
    server(->(_req) { [200, ''] }) do |s|
      rest = DataPipeline::RestStorage.new(url: "#{s.url}/import", batch_size: 2, write_method: :put)
      rest.write([{ 'a' => 1 }, { 'a' => 2 }, { 'a' => 3 }])
      assert_equal [[{ 'a' => 1 }, { 'a' => 2 }], [{ 'a' => 3 }]], s.requests.map { |r| JSON.parse(r[:body]) }
      assert(s.requests.all? { |r| r[:method] == 'PUT' })
    end
  end

  def test_write_url_per_row
    server(->(_req) { [200, ''] }) do |s|
      DataPipeline::RestStorage.new(url: s.url, write_method: :put, write_url: ->(row) { "#{s.url}/kunden/#{row['id']}" })
                 .write([{ 'id' => 7 }, { 'id' => 8 }])
      assert_equal %w[/kunden/7 /kunden/8], s.requests.map { |r| r[:path] }
      assert_equal({ 'id' => 8 }, JSON.parse(s.requests.last[:body]))
    end
    assert_raises(ArgumentError) { DataPipeline::RestStorage.new(url: 'http://x', batch_size: 2, write_url: ->(_row) { 'http://x' }) }
  end

  def test_call_json_status_and_retries
    calls = 0
    server(->(req) { calls += 1; req[:path] == '/api/fail' ? [503, ''] : [201, { 'echo' => JSON.parse(req[:body]) }] }) do |s|
      rest = DataPipeline::RestStorage.new(url: "#{s.url}/api/", retries: 2, retry_wait: 0)
      assert_equal({ 'echo' => { 'a' => 1 } }, rest.call(:post, 'person/create', json: { 'a' => 1 }))
      assert_equal '/api/person/create', s.requests.last[:path]

      calls = 0
      error = assert_raises(DataPipeline::RestStorage::Error) { rest.call(:post, 'fail', json: {}, retries: 0) }
      assert_equal 503, error.status
      assert_equal 1, calls, 'retries: 0 sends once'
    end
    server(->(_req) { [404, { 'message' => 'weg' }] }) do |s|
      assert_equal 404, assert_raises(DataPipeline::RestStorage::Error) { DataPipeline::RestStorage.new(url: s.url).call(:get, 'x') }.status
    end
  end

  def test_ssl_verify_false_context
    context = DataPipeline::RestStorage.new(url: 'https://192.168.178.251:50443/pds/rest/api', ssl_verify: false).send(:ssl_context)
    assert_equal OpenSSL::SSL::VERIFY_NONE, context.verify_mode
  end

  def test_query_for_lookup
    server(->(req) { [200, [{ 'land' => req[:query]['land'], 'pfad' => req[:path] }]] }) do |s|
      names = nil
      DataPipeline.new do
        api = container :rest, url: "#{s.url}/api/"
        names = lookup api, query: '/laender', params: { 'land' => 'DE' }, key: 'land', unique: true
      end
      assert_equal '/api/laender', names['DE']['pfad']
    end
  end

  def test_not_an_array
    server(->(_req) { [200, { 'a' => 1 }] }) do |s|
      assert_raises(DataPipeline::RestStorage::Error) { DataPipeline::RestStorage.new(url: s.url).read }
    end
  end
end