require_relative '../test_helper' require_relative '../support/storage_contract' require_relative '../support/test_http_server' class RestStorageTest < PipelineTest include StorageContract # a tiny api: POST /items stores one row or an array, GET /items returns all def with_contract_env store = [] server = TestHttpServer.new do |req| if req[:method] == 'POST' Array(JSON.parse(req[:body]).then { |b| b.is_a?(Array) ? b : [b] }).each { |row| store << row } [201, { 'ok' => true }] else [200, { 'data' => { 'items' => store } }] end end yield server ensure server&.stop end def contract_storage(server, _purpose) = DataPipeline::RestStorage.new(url: "#{server.url}/items", root: 'data.items') # handler: ->(request) { [status, body] } def server handler s = TestHttpServer.new(&handler) yield s ensure s&.stop end def test_page_param_paging pages = { '1' => [{ 'id' => 1 }, { 'id' => 2 }], '2' => [{ 'id' => 3 }] } server(->(req) { [200, pages.fetch(req[:query]['page'], [])] }) do |s| rows = DataPipeline::RestStorage.new(url: "#{s.url}/kunden", page_param: 'page', params: { 'land' => 'DE' }).read assert_equal [1, 2, 3], rows.map { |r| r['id'] } assert_equal %w[1 2 3], s.requests.map { |r| r[:query]['page'] } assert(s.requests.all? { |r| r[:query]['land'] == 'DE' }) end end def test_next_url_paging base = nil handler = lambda do |req| if req[:path] == '/a' [200, { 'rows' => [{ 'n' => 1 }], 'next' => "#{base}/b" }] else [200, { 'rows' => [{ 'n' => 2 }], 'next' => nil }] end end server(handler) do |s| base = s.url rows = DataPipeline::RestStorage.new(url: "#{s.url}/a", root: 'rows', next: ->(body) { body['next'] }).read assert_equal [1, 2], rows.map { |r| r['n'] } end end def test_auth_and_headers server(->(_req) { [200, []] }) do |s| DataPipeline::RestStorage.new(url: s.url, auth: 'Bearer geheim', headers: { 'X-Mandant' => '001' }).read assert_equal 'Bearer geheim', s.requests.first[:headers]['authorization'] assert_equal '001', s.requests.first[:headers]['x-mandant'] end end def test_retry_on_503_then_success calls = 0 server(->(_req) { (calls += 1) < 3 ? [503, 'busy'] : [200, [{ 'ok' => 1 }]] }) do |s| assert_equal [{ 'ok' => 1 }], DataPipeline::RestStorage.new(url: s.url, retries: 2, retry_wait: 0).read assert_equal 3, calls end end def test_no_retry_on_404_and_error_message server(->(_req) { [404, { 'error' => 'nicht da' }] }) do |s| error = assert_raises(DataPipeline::RestStorage::Error) { DataPipeline::RestStorage.new(url: "#{s.url}/x", retry_wait: 0).read } assert_includes error.message, 'HTTP 404' assert_includes error.message, 'nicht da' assert_equal 1, s.requests.size end end def test_retries_exhausted_and_connection_refused server(->(_req) { [500, 'kaputt'] }) do |s| assert_includes assert_raises(DataPipeline::RestStorage::Error) { DataPipeline::RestStorage.new(url: s.url, retries: 1, retry_wait: 0).read }.message, 'HTTP 500' assert_equal 2, s.requests.size end closed = TCPServer.new('127.0.0.1', 0).then { |t| port = t.addr[1]; t.close; port } assert_raises(DataPipeline::RestStorage::Error) { DataPipeline::RestStorage.new(url: "http://127.0.0.1:#{closed}", retries: 1, retry_wait: 0).read } end def test_batch_write_and_write_method server(->(_req) { [200, ''] }) do |s| rest = DataPipeline::RestStorage.new(url: "#{s.url}/import", batch_size: 2, write_method: :put) rest.write([{ 'a' => 1 }, { 'a' => 2 }, { 'a' => 3 }]) assert_equal [[{ 'a' => 1 }, { 'a' => 2 }], [{ 'a' => 3 }]], s.requests.map { |r| JSON.parse(r[:body]) } assert(s.requests.all? { |r| r[:method] == 'PUT' }) end end def test_write_url_per_row server(->(_req) { [200, ''] }) do |s| DataPipeline::RestStorage.new(url: s.url, write_method: :put, write_url: ->(row) { "#{s.url}/kunden/#{row['id']}" }) .write([{ 'id' => 7 }, { 'id' => 8 }]) assert_equal %w[/kunden/7 /kunden/8], s.requests.map { |r| r[:path] } assert_equal({ 'id' => 8 }, JSON.parse(s.requests.last[:body])) end assert_raises(ArgumentError) { DataPipeline::RestStorage.new(url: 'http://x', batch_size: 2, write_url: ->(_row) { 'http://x' }) } end def test_call_json_status_and_retries calls = 0 server(->(req) { calls += 1; req[:path] == '/api/fail' ? [503, ''] : [201, { 'echo' => JSON.parse(req[:body]) }] }) do |s| rest = DataPipeline::RestStorage.new(url: "#{s.url}/api/", retries: 2, retry_wait: 0) assert_equal({ 'echo' => { 'a' => 1 } }, rest.call(:post, 'person/create', json: { 'a' => 1 })) assert_equal '/api/person/create', s.requests.last[:path] calls = 0 error = assert_raises(DataPipeline::RestStorage::Error) { rest.call(:post, 'fail', json: {}, retries: 0) } assert_equal 503, error.status assert_equal 1, calls, 'retries: 0 sends once' end server(->(_req) { [404, { 'message' => 'weg' }] }) do |s| assert_equal 404, assert_raises(DataPipeline::RestStorage::Error) { DataPipeline::RestStorage.new(url: s.url).call(:get, 'x') }.status end end def test_ssl_verify_false_context context = DataPipeline::RestStorage.new(url: 'https://192.168.178.251:50443/pds/rest/api', ssl_verify: false).send(:ssl_context) assert_equal OpenSSL::SSL::VERIFY_NONE, context.verify_mode end def test_query_for_lookup server(->(req) { [200, [{ 'land' => req[:query]['land'], 'pfad' => req[:path] }]] }) do |s| names = nil DataPipeline.new do api = container :rest, url: "#{s.url}/api/" names = lookup api, query: '/laender', params: { 'land' => 'DE' }, key: 'land', unique: true end assert_equal '/api/laender', names['DE']['pfad'] end end def test_not_an_array server(->(_req) { [200, { 'a' => 1 }] }) do |s| assert_raises(DataPipeline::RestStorage::Error) { DataPipeline::RestStorage.new(url: s.url).read } end end end