Betriebssicherheit: Sperre gegen parallele Läufe, Upsert, Batch-Schreiben für Postgres, Zeichensatz für CSV

lib/datapipeline.rb

@@ -6,6 +6,7 @@ require_relative 'datapipeline/storage/storage_plugin'
 require_relative 'datapipeline/validation'
 require_relative 'datapipeline/layout'
 require_relative 'datapipeline/watermark'
+require_relative 'datapipeline/lock'
 
 class DataPipeline
   # plugins are loaded on first use, their gems too: pg (postgres), sqlite3 (sqlite), roo/caxlsx (excel),
@@ -58,6 +59,7 @@ class DataPipeline
     limit: nil,
     preview: 10,
     on_error: :abort,
+    lock: nil,
     log_level: Logger::WARN
   }.freeze
 
@@ -204,6 +206,11 @@ class DataPipeline
     @on_finish << block
   end
 
+  # lock 'data/.cudo.lock' -> held during run; a second run at the same time aborts with LockError
+  def lock file
+    @settings[:lock] = file
+  end
+
   # see DataPipeline::Watermark
   def watermark name, column:, file:, initial: nil
     @watermark = Watermark.new(name, column:, file:, initial:)
@@ -303,6 +310,7 @@ class DataPipeline
     @logger.info "start#{" #{@settings[:version]}" if @settings[:version]}#{@settings[:dry_run] ? ' (dry run)' : ''}"
     @preview = []
     raise ArgumentError, 'read storage missing' unless @source
+    (@lock = Lock.new(@settings[:lock])).acquire if @settings[:lock]
     raise ArgumentError, 'write storage missing' if @targets.empty?
 
     before = @transformations.select { |t| EDGE_BEFORE.include?(t[:kind]) }
@@ -342,6 +350,7 @@ class DataPipeline
   def finish
     @stats.finished_at = Time.now
     [@source, *@targets.map { |t| t[:storage] }, @reject_target].compact.uniq.each(&:close)
+    @lock&.release
     self.class.runs << @stats
     print_preview if @stats.dry_run && @stats.error.nil?
     @logger.info "finish #{@stats}"