Betriebssicherheit: Sperre gegen parallele Läufe, Upsert, Batch-Schreiben für Postgres, Zeichensatz für CSV

test/plugins/csv_storage_test.rb

@@ -86,4 +86,34 @@ class CsvStorageTest < PipelineTest
       assert_equal "FIRC|0\nDOHD|AR1|\nDOPS|1.50\nLARC|0000000002\n", File.read("#{dir}/c.ok")
     end
   end
+
+  def test_encoding_read_and_write
+    with_tmpdir do |dir|
+      DataPipeline::CsvStorage.new(file: "#{dir}/l1.csv", encoding: 'ISO-8859-1').write([{ 'name' => 'Müller', 'ort' => 'Zürich' }])
+      assert_equal "name,ort\nM\xFCller,Z\xFCrich\n".b, File.binread("#{dir}/l1.csv")
+      rows = DataPipeline::CsvStorage.new(file: "#{dir}/l1.csv", encoding: 'ISO-8859-1').read
+      assert_equal [{ 'name' => 'Müller', 'ort' => 'Zürich' }], rows
+      assert_equal Encoding::UTF_8, rows.first['name'].encoding
+    end
+  end
+
+  def test_encoding_error_or_replace
+    with_tmpdir do |dir|
+      assert_raises(Encoding::UndefinedConversionError) do
+        DataPipeline::CsvStorage.new(file: "#{dir}/a.csv", encoding: 'ISO-8859-1').write([{ 'x' => '5 €' }])
+      end
+      DataPipeline::CsvStorage.new(file: "#{dir}/b.csv", encoding: 'ISO-8859-1', replace: '?').write([{ 'x' => '5 €' }])
+      assert_equal "x\n5 ?\n", File.read("#{dir}/b.csv")
+      DataPipeline::CsvStorage.new(file: "#{dir}/c.csv", encoding: 'Windows-1252').write([{ 'x' => '5 €' }])
+      assert_equal "x\n5 \x80\n".b, File.binread("#{dir}/c.csv")
+    end
+  end
+
+  def test_bom_written_and_skipped
+    with_tmpdir do |dir|
+      DataPipeline::CsvStorage.new(file: "#{dir}/bom.csv", bom: true).write([{ 'a' => '1' }])
+      assert_equal "\uFEFFa\n1\n", File.read("#{dir}/bom.csv", encoding: 'utf-8')
+      assert_equal [{ 'a' => '1' }], DataPipeline::CsvStorage.new(file: "#{dir}/bom.csv").read
+    end
+  end
 end