datapipeline 2.0.0: Gem aus dem Projekt etl herausgelöst

2026-10-08 15:30:02 +02:00
commit cce7b03ca9
56 changed files with 5205 additions and 0 deletions
@@ -0,0 +1,70 @@
+#!/usr/bin/env ruby
+#
+# etl [run] <pipeline.rb> [options]      ('run' is optional)
+#
+# executable of the gem datapipeline; in this repository bin/etl loads the bundle first
+#
+#   --from=DATE --till=DATE   in DataPipeline.params, also passed on to the pipeline script
+#   --param key=value         in DataPipeline.params[:key], repeatable
+#   --dry-run                 run everything, write nothing, print the first rows
+#   --limit=N                 only the first N source rows
+#   --verbose                 log level debug, --quiet only warnings (default info)
+#   --name=value              any other option: DataPipeline.params[:name] and passed on to the script
+#
+# exit code: 0 ok, 1 aborted, 2 finished with rejected rows
+#
+require 'optparse'
+require 'logger'
+require_relative '../lib/datapipeline'
+
+options = { settings: { log_level: Logger::INFO }, passthrough: [] }
+DataPipeline.params = {}
+parser = OptionParser.new do |opts|
+  opts.banner = "Usage: etl [run] PIPELINE.rb [options]\n\n  etl examples/kunden_import.rb --dry-run\n"
+  opts.on('--from=DATE', String) { |v| DataPipeline.params[:from] = v; options[:passthrough] << "--from=#{v}" }
+  opts.on('--till=DATE', String) { |v| DataPipeline.params[:till] = v; options[:passthrough] << "--till=#{v}" }
+  opts.on('--param KEY=VALUE', String) do |v|
+    key, value = v.split('=', 2)
+    DataPipeline.params[key.to_sym] = value
+  end
+  opts.on('--dry-run') { options[:settings][:dry_run] = true }
+  opts.on('--limit=N', Integer) { |v| options[:settings][:limit] = v }
+  opts.on('--verbose') { options[:settings][:log_level] = Logger::DEBUG }
+  opts.on('--quiet') { options[:settings][:log_level] = Logger::WARN }
+end
+
+# own options for the parser, everything else (also --name=value of the script) goes to the script
+own = []
+args = []
+argv = ARGV.dup
+while (arg = argv.shift)
+  if arg.match?(/\A--(from|till|param|dry-run|limit|verbose|quiet)(=|\z)/)
+    own << arg
+    own << argv.shift if %w[--param --limit --from --till].include?(arg)
+  else
+    args << arg
+  end
+end
+parser.parse(own)
+args.shift if args.first == 'run'
+script = args.shift
+DataPipeline.params.merge!(DataPipeline.parse_params(args)) { |_key, own_value, _| own_value }
+abort parser.help if script.nil?
+abort "etl: #{script} not found\n\n#{parser.help}" unless File.exist?(script)
+
+DataPipeline.settings(options[:settings])
+
+# the pipeline script sees only its own arguments
+ARGV.replace(options[:passthrough] + args)
+
+begin
+  load File.expand_path(script)
+rescue StandardError => e
+  warn "#{File.basename(script)}: #{e.class}: #{e.message}"
+  exit 1
+end
+
+runs = DataPipeline.runs
+exit 1 if runs.any?(&:error)
+exit 2 if runs.any? { |stats| stats.rejected.positive? }
+exit 0