datapipeline 3.2.0: Benachrichtigungen mit Webhook, Slack und E-Mail

lib/datapipeline.rb

@@ -8,10 +8,11 @@ require_relative 'datapipeline/layout'
 require_relative 'datapipeline/watermark'
 require_relative 'datapipeline/lock'
 require_relative 'datapipeline/connections'
+require_relative 'datapipeline/notification/notifier'
 
 class DataPipeline
   # plugins are loaded on first use, their gems too: pg (postgres), sqlite3 (sqlite), roo/caxlsx (excel),
-  # http (rest, slack)
+  # http (rest, slack, webhook), net-smtp (email)
   autoload :CsvStorage,      File.expand_path('datapipeline/storage/csv', __dir__)
   autoload :PostgresStorage, File.expand_path('datapipeline/storage/postgres', __dir__)
   autoload :JsonStorage,     File.expand_path('datapipeline/storage/json', __dir__)
@@ -20,6 +21,8 @@ class DataPipeline
   autoload :RestStorage,     File.expand_path('datapipeline/storage/rest', __dir__)
   autoload :FixedStorage,    File.expand_path('datapipeline/storage/fixed', __dir__)
   autoload :Slack,           File.expand_path('datapipeline/notification/slack', __dir__)
+  autoload :Webhook,         File.expand_path('datapipeline/notification/webhook', __dir__)
+  autoload :Email,           File.expand_path('datapipeline/notification/email', __dir__)
 
   # passed to header, trailer, map and filter
   #   read:    number of source rows (nil while streaming, until the trailer)
@@ -240,6 +243,26 @@ class DataPipeline
     @reject_target = to
   end
 
+  #
+  # notify notifier, on: :problems           -> a report after the run, see DataPipeline::Notifier
+  #   on:           :failure (abort), :problems (abort, rejected rows or warnings, default), :always
+  #   title:        default the name of the pipeline
+  #   dry_run:      true -> also after a dry run (default: not)
+  #   include_rows: true -> rejected rows in the report (default: not, they may hold customer data)
+  #
+  # an error while sending is only logged
+  #
+  def notify notifier, on: :problems, title: nil, dry_run: false, include_rows: false
+    raise ArgumentError, 'notify: notifier needs deliver(report)' unless notifier.respond_to?(:deliver)
+
+    (@notifiers ||= []) << { notifier:, when: Notifier.condition(on), title:, dry_run:, include_rows: }
+  end
+
+  # notifier(:alerts) -> Webhook, Slack or Email from the connections file (type webhook, slack, email)
+  def notifier name
+    Notifier.from_connection(name)
+  end
+
   # on_finish { |stats| ... } -> after every run, also after an abort (stats.error)
   def on_finish(&block)
     @on_finish << block
@@ -484,6 +507,7 @@ class DataPipeline
     print_preview if @stats.dry_run && @stats.error.nil?
     @logger.info "finish #{@stats}"
     append_history
+    send_notifications
     @on_finish.each do |hook|
       hook.call(@stats)
     rescue StandardError => e
@@ -504,6 +528,20 @@ class DataPipeline
     @logger.error "history: #{e.class}: #{e.message}"
   end
 
+  def send_notifications
+    Array(@notifiers).each do |entry|
+      next if @stats.dry_run && !entry[:dry_run]
+
+      report = Report.new([@stats], title: entry[:title], params:)
+      next unless entry[:when].call(report)
+
+      entry[:notifier].deliver(report, include_rows: entry[:include_rows])
+      @logger.info "notified #{entry[:notifier].class.name.split('::').last}: #{report.event}"
+    rescue StandardError => e
+      @logger.error "notify #{entry[:notifier].class}: #{e.class}: #{e.message}"
+    end
+  end
+
   def add_total side, row
     @stats.totals&.each do |col, sum|
       value = row.key?(col) ? row[col] : row[col.to_sym]