DSL: Kurzschritte rename, select, cast, default; transform, Parallelmodus und String-Bedingungen veraltet

lib/datapipeline.rb

@@ -51,6 +51,18 @@ class DataPipeline
   ON_ERROR    = %i[abort skip collect].freeze
   MAX_ERRORS  = 100
 
+  # removed in 3.0, logged once per process as a warning
+  DEPRECATIONS = {
+    transform: 'transform is deprecated and will be removed in 3.0: use map/filter, ' \
+               'header/trailer instead of processing: :prepend/:append',
+    parallel: 'parallel mode (several transform steps without chained) is deprecated and will be removed in 3.0: ' \
+              'set chained or use map',
+    condition: 'condition: as string (eval) is deprecated and will be removed in 3.0: use a lambda ->(row) { ... }'
+  }.freeze
+
+  # value conversions for cast, empty values become nil
+  CASTS = Validation::TYPES.merge(string: ->(v) { v.to_s }).freeze
+
   DEFAULT_SETTINGS = {
     version: nil,
     streamed: false,
@@ -67,6 +79,7 @@ class DataPipeline
   SETTINGS_LOCK = Mutex.new
   @defaults = DEFAULT_SETTINGS
   @plugins = {}
+  @deprecations_logged = []
 
   #
   # defaults for every following DataPipeline.new, e.g. from bin/etl
@@ -132,6 +145,18 @@ class DataPipeline
     @runs ||= []
   end
 
+  # true the first time a deprecation is reported in this process
+  def self.deprecation_once key
+    SETTINGS_LOCK.synchronize do
+      next false if @deprecations_logged.include?(key)
+
+      @deprecations_logged << key
+      true
+    end
+  end
+
+  def self.reset_deprecations! = SETTINGS_LOCK.synchronize { @deprecations_logged = [] }
+
   # DataPipeline.new(verbose: true) { ... } overrides the defaults for this pipeline only
   def initialize(**settings, &block)
     @settings = DataPipeline.settings.merge(settings)
@@ -247,6 +272,46 @@ class DataPipeline
     add_step :filter, options, &block
   end
 
+  #
+  # short steps, each one a map step (turns chaining on)
+  #
+  #   rename 'kunnr' => 'kundennr'               -> new key at the same position
+  #   select %w[belnr betrag]                    -> only these keys in this order, missing ones nil
+  #   cast 'betrag' => :decimal                  -> :integer, :decimal (Float), :date, :string; empty -> nil
+  #   default 'land' => 'CH'                     -> for nil or blank values; a lambda gets the row
+  #
+  def rename mapping
+    mapping = mapping.to_h { |from, to| [from.to_s, to.to_s] }
+    add_step(:map, { name: 'rename' }) { |row| row.to_h { |key, value| [mapping.fetch(key.to_s, key), value] } }
+  end
+
+  def select *keys
+    keys = keys.flatten.map(&:to_s)
+    add_step(:map, { name: 'select' }) { |row| keys.to_h { |key| [key, value_of(row, key)] } }
+  end
+
+  def cast types
+    types = types.to_h { |key, type| [key.to_s, type.to_sym] }
+    unknown = types.values - CASTS.keys
+    raise ArgumentError, "cast: #{unknown.join(', ')} unknown, use #{CASTS.keys.join(', ')}" if unknown.any?
+
+    add_step(:map, { name: 'cast' }) do |row|
+      types.each_with_object(row.dup) do |(key, type), out|
+        value = value_of(row, key)
+        out[key] = blank?(value) ? nil : cast_value(key, value, type)
+      end
+    end
+  end
+
+  def default values
+    values = values.to_h { |key, value| [key.to_s, value] }
+    add_step(:map, { name: 'default' }) do |row|
+      values.each_with_object(row.dup) do |(key, value), out|
+        out[key] = value.respond_to?(:call) ? value.call(row) : value if blank?(value_of(row, key))
+      end
+    end
+  end
+
   # header { |ctx| ... } -> rows in front of the body, called after the body with full counters
   def header(&block)
     add_step :header, {}, &block
@@ -317,6 +382,7 @@ class DataPipeline
     before = @transformations.select { |t| EDGE_BEFORE.include?(t[:kind]) }
     after  = @transformations.select { |t| EDGE_AFTER.include?(t[:kind]) }
     body   = @transformations - before - after
+    log_deprecations body
 
     @settings[:streamed] ? run_streamed(before, body, after) : run_batch(before, body, after)
 
@@ -338,6 +404,29 @@ class DataPipeline
     @transformations << { kind:, block:, options: }
   end
 
+  def value_of row, key
+    row.key?(key) ? row[key] : row[key.to_sym]
+  end
+
+  def blank? value
+    value.nil? || (value.is_a?(String) && value.strip.empty?)
+  end
+
+  def cast_value key, value, type
+    CASTS.fetch(type).call(value)
+  rescue ArgumentError, TypeError, Date::Error
+    raise ArgumentError, "#{key}: #{value.inspect} is not #{type}"
+  end
+
+  # each deprecated feature once per process, with the logger of the pipeline
+  def log_deprecations body
+    used = []
+    used << :transform if @transformations.any? { |t| %i[prepend normal append].include?(t[:kind]) }
+    used << :parallel if !chained? && body.size > 1
+    used << :condition if @transformations.any? { |t| t[:options].is_a?(Hash) && t[:options][:condition].is_a?(String) }
+    used.each { |key| @logger.warn "deprecated: #{DEPRECATIONS[key]}" if DataPipeline.deprecation_once(key) }
+  end
+
   # without any transformation the source is copied unchanged
   def chained?
     @settings[:chained] || @transformations.empty? ||
@@ -489,7 +578,8 @@ class DataPipeline
   end
 
   def step_name step
-    "#{step[:kind]}##{@transformations.index(step) + 1}"
+    name = step[:options].is_a?(Hash) && step[:options][:name] || step[:kind]
+    "#{name}##{@transformations.index(step) + 1}"
   end
 
   # rows produced by one step for one input row