Betriebssicherheit: Sperre gegen parallele Läufe, Upsert, Batch-Schreiben für Postgres, Zeichensatz für CSV

exe/etl

@@ -9,9 +9,10 @@
 #   --dry-run                 run everything, write nothing, print the first rows
 #   --limit=N                 only the first N source rows
 #   --verbose                 log level debug, --quiet only warnings (default info)
+#   --lock=FILE               exclusive lock for the whole script, exit 3 when another run holds it
 #   --name=value              any other option: DataPipeline.params[:name] and passed on to the script
 #
-# exit code: 0 ok, 1 aborted, 2 finished with rejected rows
+# exit code: 0 ok, 1 aborted, 2 finished with rejected rows, 3 locked by another run
 #
 require 'optparse'
 require 'logger'
@@ -31,6 +32,7 @@ parser = OptionParser.new do |opts|
   opts.on('--limit=N', Integer) { |v| options[:settings][:limit] = v }
   opts.on('--verbose') { options[:settings][:log_level] = Logger::DEBUG }
   opts.on('--quiet') { options[:settings][:log_level] = Logger::WARN }
+  opts.on('--lock=FILE', String) { |v| options[:lock] = v }
 end
 
 # own options for the parser, everything else (also --name=value of the script) goes to the script
@@ -38,7 +40,7 @@ own = []
 args = []
 argv = ARGV.dup
 while (arg = argv.shift)
-  if arg.match?(/\A--(from|till|param|dry-run|limit|verbose|quiet)(=|\z)/)
+  if arg.match?(/\A--(from|till|param|dry-run|limit|verbose|quiet|lock)(=|\z)/)
     own << arg
     own << argv.shift if %w[--param --limit --from --till].include?(arg)
   else
@@ -54,6 +56,13 @@ abort "etl: #{script} not found\n\n#{parser.help}" unless File.exist?(script)
 
 DataPipeline.settings(options[:settings])
 
+begin
+  script_lock = DataPipeline::Lock.new(options[:lock]).acquire if options[:lock]   # held until the process ends
+rescue DataPipeline::LockError => e
+  warn "#{File.basename(script)}: #{e.message}"
+  exit 3
+end
+
 # the pipeline script sees only its own arguments
 ARGV.replace(options[:passthrough] + args)
 
@@ -61,10 +70,11 @@ begin
   load File.expand_path(script)
 rescue StandardError => e
   warn "#{File.basename(script)}: #{e.class}: #{e.message}"
-  exit 1
+  exit(e.is_a?(DataPipeline::LockError) ? 3 : 1)
 end
 
 runs = DataPipeline.runs
+exit 3 if runs.any? { |stats| stats.error.is_a?(DataPipeline::LockError) }
 exit 1 if runs.any?(&:error)
 exit 2 if runs.any? { |stats| stats.rejected.positive? }
 exit 0