datapipeline 2.0.0: Gem aus dem Projekt etl herausgelöst

2026-10-08 15:30:02 +02:00
commit cce7b03ca9
56 changed files with 5205 additions and 0 deletions
@@ -0,0 +1,56 @@
+require_relative '../test_helper'
+require_relative '../support/storage_contract'
+
+class JsonArrayStorageTest < PipelineTest
+  include StorageContract
+
+  def contract_storage(dir, _purpose) = DataPipeline::JsonStorage.new(file: "#{dir}/data.json")
+
+  def test_streamed_array_is_valid_json
+    with_tmpdir do |dir|
+      json = DataPipeline::JsonStorage.new(file: "#{dir}/a.json")
+      json.write_row({ 'a' => 1 })
+      json.write_row({ 'a' => 2 })
+      json.close
+      assert_equal [{ 'a' => 1 }, { 'a' => 2 }], JSON.parse(File.read("#{dir}/a.json"))
+    end
+  end
+
+  def test_root_path_and_pretty
+    with_tmpdir do |dir|
+      File.write("#{dir}/api.json", JSON.generate('data' => { 'items' => [{ 'id' => 1 }] }))
+      assert_equal [{ 'id' => 1 }], DataPipeline::JsonStorage.new(file: "#{dir}/api.json", root: 'data.items').read
+      DataPipeline::JsonStorage.new(file: "#{dir}/p.json", pretty: true).write([{ 'a' => 1 }])
+      assert_includes File.read("#{dir}/p.json"), "\n"
+    end
+  end
+
+  def test_single_object_is_one_row_and_scalar_raises
+    with_tmpdir do |dir|
+      File.write("#{dir}/o.json", '{"a":1}')
+      assert_equal [{ 'a' => 1 }], DataPipeline::JsonStorage.new(file: "#{dir}/o.json").read
+      File.write("#{dir}/s.json", '42')
+      assert_raises(ArgumentError) { DataPipeline::JsonStorage.new(file: "#{dir}/s.json").read }
+    end
+  end
+
+  def test_options
+    assert_raises(ArgumentError) { DataPipeline::JsonStorage.new }
+    assert_raises(ArgumentError) { DataPipeline::JsonStorage.new(file: 'x.json', format: :xml) }
+  end
+end
+
+class JsonLinesStorageTest < PipelineTest
+  include StorageContract
+
+  def contract_storage(dir, _purpose) = DataPipeline::JsonStorage.new(file: "#{dir}/data.ndjson")
+
+  def test_format_from_extension_and_blank_lines
+    with_tmpdir do |dir|
+      File.write("#{dir}/d.jsonl", "{\"a\":1}\n\n{\"a\":2}\n")
+      assert_equal [{ 'a' => 1 }, { 'a' => 2 }], DataPipeline::JsonStorage.new(file: "#{dir}/d.jsonl").read
+      DataPipeline::JsonStorage.new(file: "#{dir}/x.txt", format: :lines).write([{ 'a' => 1 }, { 'a' => 2 }])
+      assert_equal "{\"a\":1}\n{\"a\":2}\n", File.read("#{dir}/x.txt")
+    end
+  end
+end