require_relative '../test_helper' require_relative '../support/storage_contract' class JsonArrayStorageTest < PipelineTest include StorageContract def contract_storage(dir, _purpose) = DataPipeline::JsonStorage.new(file: "#{dir}/data.json") def test_streamed_array_is_valid_json with_tmpdir do |dir| json = DataPipeline::JsonStorage.new(file: "#{dir}/a.json") json.write_row({ 'a' => 1 }) json.write_row({ 'a' => 2 }) json.close assert_equal [{ 'a' => 1 }, { 'a' => 2 }], JSON.parse(File.read("#{dir}/a.json")) end end def test_root_path_and_pretty with_tmpdir do |dir| File.write("#{dir}/api.json", JSON.generate('data' => { 'items' => [{ 'id' => 1 }] })) assert_equal [{ 'id' => 1 }], DataPipeline::JsonStorage.new(file: "#{dir}/api.json", root: 'data.items').read DataPipeline::JsonStorage.new(file: "#{dir}/p.json", pretty: true).write([{ 'a' => 1 }]) assert_includes File.read("#{dir}/p.json"), "\n" end end def test_single_object_is_one_row_and_scalar_raises with_tmpdir do |dir| File.write("#{dir}/o.json", '{"a":1}') assert_equal [{ 'a' => 1 }], DataPipeline::JsonStorage.new(file: "#{dir}/o.json").read File.write("#{dir}/s.json", '42') assert_raises(ArgumentError) { DataPipeline::JsonStorage.new(file: "#{dir}/s.json").read } end end def test_options assert_raises(ArgumentError) { DataPipeline::JsonStorage.new } assert_raises(ArgumentError) { DataPipeline::JsonStorage.new(file: 'x.json', format: :xml) } end end class JsonLinesStorageTest < PipelineTest include StorageContract def contract_storage(dir, _purpose) = DataPipeline::JsonStorage.new(file: "#{dir}/data.ndjson") def test_format_from_extension_and_blank_lines with_tmpdir do |dir| File.write("#{dir}/d.jsonl", "{\"a\":1}\n\n{\"a\":2}\n") assert_equal [{ 'a' => 1 }, { 'a' => 2 }], DataPipeline::JsonStorage.new(file: "#{dir}/d.jsonl").read DataPipeline::JsonStorage.new(file: "#{dir}/x.txt", format: :lines).write([{ 'a' => 1 }, { 'a' => 2 }]) assert_equal "{\"a\":1}\n{\"a\":2}\n", File.read("#{dir}/x.txt") end end end