require_relative '../test_helper' require_relative '../support/storage_contract' class CsvStorageTest < PipelineTest include StorageContract def contract_storage(dir, _purpose) = DataPipeline::CsvStorage.new(file: "#{dir}/data.csv") def test_read_with_defaults with_tmpdir do |dir| File.write("#{dir}/in.csv", "a,b\n1,x\n2,\"\"\n") assert_equal [{ 'a' => '1', 'b' => 'x' }, { 'a' => '2', 'b' => '' }], DataPipeline::CsvStorage.new(file: "#{dir}/in.csv").read end end def test_read_without_header_uses_columns_or_index with_tmpdir do |dir| File.write("#{dir}/in.csv", "1|x\n2|y\n") csv = { headers: false, col_sep: '|' } assert_equal [{ 'id' => '1', 'txt' => 'x' }, { 'id' => '2', 'txt' => 'y' }], DataPipeline::CsvStorage.new(file: "#{dir}/in.csv", columns: %w[id txt], csv:).read assert_equal({ '0' => '1', '1' => 'x' }, DataPipeline::CsvStorage.new(file: "#{dir}/in.csv", csv:).read.first) end end def test_read_streaming_with_block with_tmpdir do |dir| File.write("#{dir}/in.csv", "a\n1\n2\n") rows = [] DataPipeline::CsvStorage.new(file: "#{dir}/in.csv").read { |row| rows << row } assert_equal %w[1 2], rows.map { |r| r['a'] } end end def test_defaults_are_merged options = DataPipeline::CsvStorage.new(file: 'x', csv: { col_sep: ';' }).instance_variable_get(:@options)[:csv] assert_equal ';', options[:col_sep] assert options[:headers] end def test_file_is_required error = assert_raises(ArgumentError) { DataPipeline::CsvStorage.new } assert_includes error.message, 'file' end def test_write_without_csv_options with_tmpdir do |dir| DataPipeline::CsvStorage.new(file: "#{dir}/o.csv").write([{ 'a' => 1, 'b' => 2 }]) assert_equal "a,b\n1,2\n", File.read("#{dir}/o.csv") end end def test_columns_fix_order_missing_fields_and_key_types with_tmpdir do |dir| DataPipeline::CsvStorage.new(file: "#{dir}/o.csv", columns: %w[a b c]).write([{ 'b' => 2, 'a' => 1 }, { c: 3, a: 4 }]) assert_equal "a,b,c\n1,2,\n4,,3\n", File.read("#{dir}/o.csv") end end def test_empty_data with_tmpdir do |dir| DataPipeline::CsvStorage.new(file: "#{dir}/a.csv").write([]) DataPipeline::CsvStorage.new(file: "#{dir}/b.csv", columns: %w[a b]).write([]) assert_equal '', File.read("#{dir}/a.csv") assert_equal "a,b\n", File.read("#{dir}/b.csv") end end def test_write_row_and_close with_tmpdir do |dir| csv = DataPipeline::CsvStorage.new(file: "#{dir}/o.csv") csv.write_row({ 'a' => 1 }) csv.write_row({ 'a' => 2 }) csv.close assert_equal "a\n1\n2\n", File.read("#{dir}/o.csv") end end # options of the CUDO exports: no header, pipe, no quotes, mixed record types and key types def test_cudo_format_mixed_record_types with_tmpdir do |dir| rows = [{ SATYP: 'FIRC', LFDNR: 0 }, { 'satyp' => 'DOHD', 'belnr' => 'AR1', 'wwert' => nil }, { 'satyp' => 'DOPS', 'dmbtr' => '1.50' }, { 'SATYPE' => 'LARC', 'SUMME' => '0000000002' }] DataPipeline::CsvStorage.new(file: "#{dir}/c.ok", csv: { headers: false, col_sep: '|', quote_char: '', encoding: 'utf-8' }) .write(rows) assert_equal "FIRC|0\nDOHD|AR1|\nDOPS|1.50\nLARC|0000000002\n", File.read("#{dir}/c.ok") end end end