datapipeline 2.0.0: Gem aus dem Projekt etl herausgelöst
This commit is contained in:
@@ -0,0 +1,155 @@
|
||||
require_relative '../test_helper'
|
||||
require_relative '../../examples/demo_api'
|
||||
require 'open3'
|
||||
require 'fileutils'
|
||||
|
||||
#
|
||||
# the examples of README.md section 1: the code in the documentation is the file,
|
||||
# and every example runs and produces the documented result
|
||||
#
|
||||
class DocExamplesTest < PipelineTest
|
||||
DOC = File.read(File.join(ROOT, 'README.md'))
|
||||
EXAMPLES = DOC.scan(/<!-- beispiel: (\S+) -->\n```ruby\n(.*?)\n```/m)
|
||||
|
||||
def run_example dir, name, *args
|
||||
env = { 'BUNDLE_GEMFILE' => File.join(ROOT, 'Gemfile') }
|
||||
Open3.capture3(env, RbConfig.ruby, File.join(ROOT, 'bin/etl'), 'run', File.join(ROOT, name), "--out=#{dir}", *args,
|
||||
chdir: dir)
|
||||
end
|
||||
|
||||
def test_documented_code_equals_files
|
||||
assert_equal %w[examples/kunden_import.rb examples/rechnungen_satzarten.rb examples/bewegungen_inkrementell.rb
|
||||
examples/eigenes_plugin.rb examples/rest_lesen.rb examples/rest_schreiben.rb
|
||||
examples/rest_zu_rest.rb], EXAMPLES.map(&:first)
|
||||
EXAMPLES.each do |file, code|
|
||||
assert_equal File.read(File.join(ROOT, file)).rstrip, code.rstrip, "#{file} differs from README.md"
|
||||
end
|
||||
end
|
||||
|
||||
def test_kunden_import
|
||||
with_tmpdir do |dir|
|
||||
out, err, status = run_example(dir, 'examples/kunden_import.rb')
|
||||
assert_equal 2, status.exitstatus, err
|
||||
assert_includes out, 'kunden_import: read=9 written=4 rejected=4'
|
||||
|
||||
db = DataPipeline::SqliteStorage.new(database: "#{dir}/kunden.sqlite3", query: 'select * from kunden order by kundennr')
|
||||
rows = db.read
|
||||
db.close
|
||||
assert_equal %w[100001 100002 100003 100009], rows.map { |r| r['kundennr'] }
|
||||
assert_equal({ 'kundennr' => '100002', 'name' => 'Huber & Partner', 'land' => 'Österreich', 'eu' => 'ja', 'umsatz' => 8300.0 }, rows[1])
|
||||
|
||||
excel = DataPipeline::ExcelStorage.new(file: "#{dir}/eu_kunden.xlsx", sheet: 'EU-Kunden').read
|
||||
assert_equal %w[100001 100002 100009], excel.map { |r| r['kundennr'] }
|
||||
|
||||
errors = DataPipeline::CsvStorage.new(file: "#{dir}/kunden_fehler.csv").read
|
||||
assert_equal ['name: missing', 'kundennr: format /\A\d{6}\z/', 'Land XX unbekannt', 'umsatz: not decimal'],
|
||||
errors.map { |r| r['_error'] }
|
||||
|
||||
# a second run replaces the table content (truncate)
|
||||
run_example(dir, 'examples/kunden_import.rb')
|
||||
db = DataPipeline::SqliteStorage.new(database: "#{dir}/kunden.sqlite3", query: 'select count(*) as n from kunden')
|
||||
assert_equal 4, db.read.first['n']
|
||||
db.close
|
||||
end
|
||||
end
|
||||
|
||||
def test_kunden_import_dry_run_creates_no_files
|
||||
with_tmpdir do |dir|
|
||||
out, _err, status = run_example(dir, 'examples/kunden_import.rb', '--dry-run')
|
||||
assert_equal 2, status.exitstatus
|
||||
assert_includes out, 'DRY RUN'
|
||||
assert_empty Dir.children(dir)
|
||||
end
|
||||
end
|
||||
|
||||
def test_rechnungen_satzarten
|
||||
with_tmpdir do |dir|
|
||||
_out, err, status = run_example(dir, 'examples/rechnungen_satzarten.rb')
|
||||
assert status.success?, err
|
||||
assert_includes err, 'TEXT: longer than 30'
|
||||
documented = DOC[/Ergebnis `tmp\/rechnungen.txt`.*?```text\n(.*?)```/m, 1]
|
||||
expected = documented.sub(/\ADATEI\|\d{8}/, "DATEI|#{Date.today.strftime('%Y%m%d')}")
|
||||
assert_equal expected, File.read("#{dir}/rechnungen.txt")
|
||||
assert_equal 3, File.readlines("#{dir}/positionen.ndjson").size
|
||||
end
|
||||
end
|
||||
|
||||
def test_bewegungen_inkrementell
|
||||
with_tmpdir do |dir|
|
||||
out, err, status = run_example(dir, 'examples/bewegungen_inkrementell.rb')
|
||||
assert status.success?, err
|
||||
assert_includes out, 'neu: 4, Stand jetzt: 4'
|
||||
file = Dir["#{dir}/bewegungen_*.ndjson"].first
|
||||
assert_equal({ 'id' => 2, 'konto' => '4000', 'betrag' => 20.0, 'soll_haben' => 'H' }, JSON.parse(File.readlines(file)[1]))
|
||||
|
||||
FileUtils.rm(file)
|
||||
out, = run_example(dir, 'examples/bewegungen_inkrementell.rb')
|
||||
assert_includes out, 'neu: 0, Stand jetzt: 4'
|
||||
assert_empty Dir["#{dir}/bewegungen_*.ndjson"], 'no file without new rows'
|
||||
end
|
||||
end
|
||||
|
||||
def test_eigenes_plugin
|
||||
with_tmpdir do |dir|
|
||||
_out, err, status = run_example(dir, 'examples/eigenes_plugin.rb')
|
||||
assert status.success?, err
|
||||
documented = DOC[/Ergebnis `tmp\/alle_filialen.csv`:\n\n```text\n(.*?)```/m, 1]
|
||||
assert_equal documented, File.read("#{dir}/alle_filialen.csv")
|
||||
end
|
||||
end
|
||||
|
||||
def with_demo_api
|
||||
api = DemoApi.start(port: 0)
|
||||
yield api
|
||||
ensure
|
||||
api&.stop
|
||||
end
|
||||
|
||||
def test_rest_lesen
|
||||
with_demo_api do |api|
|
||||
with_tmpdir do |dir|
|
||||
out, err, status = run_example(dir, 'examples/rest_lesen.rb', "--url=#{api.url}")
|
||||
assert_equal 2, status.exitstatus, err
|
||||
assert_includes out, 'rest_lesen: read=7 written=6 rejected=1'
|
||||
documented = DOC[/Ergebnis `tmp\/api_kunden.csv`:\n\n```text\n(.*?)```/m, 1]
|
||||
assert_equal documented, File.read("#{dir}/api_kunden.csv")
|
||||
assert_equal 6, JSON.parse(File.read("#{dir}/api_kunden.json")).size
|
||||
assert_equal ['umsatz: not decimal'], DataPipeline::CsvStorage.new(file: "#{dir}/api_fehler.csv").read.map { |r| r['_error'] }
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def test_rest_schreiben_and_dry_run
|
||||
with_demo_api do |api|
|
||||
with_tmpdir do |dir|
|
||||
out, _err, status = run_example(dir, 'examples/rest_schreiben.rb', "--url=#{api.url}", '--dry-run')
|
||||
assert_equal 2, status.exitstatus
|
||||
assert_includes out, 'DRY RUN'
|
||||
assert_empty api.imported, 'dry run sends nothing'
|
||||
|
||||
out, err, status = run_example(dir, 'examples/rest_schreiben.rb', "--url=#{api.url}")
|
||||
assert_equal 2, status.exitstatus, err
|
||||
assert_includes out, 'gesendet: 6 Kunden in 3 Paketen'
|
||||
assert_equal %w[100001 100002 100003 100007 100008 100009], api.imported.map { |r| r['kundennr'] }
|
||||
assert_equal 2, DataPipeline::CsvStorage.new(file: "#{dir}/import_fehler.csv").read.size
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def test_rest_zu_rest
|
||||
with_demo_api do |api|
|
||||
with_tmpdir do |dir|
|
||||
out, err, status = run_example(dir, 'examples/rest_zu_rest.rb', "--url=#{api.url}")
|
||||
assert status.success?, err
|
||||
assert_includes out, 'archiviert: 5, Stand jetzt: 5'
|
||||
assert_equal %w[1 2 3 4 5], api.archive.keys
|
||||
assert_equal Date.today.iso8601, api.archive['3']['archiviert']
|
||||
|
||||
api.archive.clear
|
||||
out, = run_example(dir, 'examples/rest_zu_rest.rb', "--url=#{api.url}")
|
||||
assert_includes out, 'archiviert: 0, Stand jetzt: 5'
|
||||
assert_empty api.archive
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user