datapipeline 3.2.0: Benachrichtigungen mit Webhook, Slack und E-Mail

lib/datapipeline.rb

@@ -8,10 +8,11 @@ require_relative 'datapipeline/layout'
 require_relative 'datapipeline/watermark'
 require_relative 'datapipeline/lock'
 require_relative 'datapipeline/connections'
+require_relative 'datapipeline/notification/notifier'
 
 class DataPipeline
   # plugins are loaded on first use, their gems too: pg (postgres), sqlite3 (sqlite), roo/caxlsx (excel),
-  # http (rest, slack)
+  # http (rest, slack, webhook), net-smtp (email)
   autoload :CsvStorage,      File.expand_path('datapipeline/storage/csv', __dir__)
   autoload :PostgresStorage, File.expand_path('datapipeline/storage/postgres', __dir__)
   autoload :JsonStorage,     File.expand_path('datapipeline/storage/json', __dir__)
@@ -20,6 +21,8 @@ class DataPipeline
   autoload :RestStorage,     File.expand_path('datapipeline/storage/rest', __dir__)
   autoload :FixedStorage,    File.expand_path('datapipeline/storage/fixed', __dir__)
   autoload :Slack,           File.expand_path('datapipeline/notification/slack', __dir__)
+  autoload :Webhook,         File.expand_path('datapipeline/notification/webhook', __dir__)
+  autoload :Email,           File.expand_path('datapipeline/notification/email', __dir__)
 
   # passed to header, trailer, map and filter
   #   read:    number of source rows (nil while streaming, until the trailer)
@@ -240,6 +243,26 @@ class DataPipeline
     @reject_target = to
   end
 
+  #
+  # notify notifier, on: :problems           -> a report after the run, see DataPipeline::Notifier
+  #   on:           :failure (abort), :problems (abort, rejected rows or warnings, default), :always
+  #   title:        default the name of the pipeline
+  #   dry_run:      true -> also after a dry run (default: not)
+  #   include_rows: true -> rejected rows in the report (default: not, they may hold customer data)
+  #
+  # an error while sending is only logged
+  #
+  def notify notifier, on: :problems, title: nil, dry_run: false, include_rows: false
+    raise ArgumentError, 'notify: notifier needs deliver(report)' unless notifier.respond_to?(:deliver)
+
+    (@notifiers ||= []) << { notifier:, when: Notifier.condition(on), title:, dry_run:, include_rows: }
+  end
+
+  # notifier(:alerts) -> Webhook, Slack or Email from the connections file (type webhook, slack, email)
+  def notifier name
+    Notifier.from_connection(name)
+  end
+
   # on_finish { |stats| ... } -> after every run, also after an abort (stats.error)
   def on_finish(&block)
     @on_finish << block
@@ -484,6 +507,7 @@ class DataPipeline
     print_preview if @stats.dry_run && @stats.error.nil?
     @logger.info "finish #{@stats}"
     append_history
+    send_notifications
     @on_finish.each do |hook|
       hook.call(@stats)
     rescue StandardError => e
@@ -504,6 +528,20 @@ class DataPipeline
     @logger.error "history: #{e.class}: #{e.message}"
   end
 
+  def send_notifications
+    Array(@notifiers).each do |entry|
+      next if @stats.dry_run && !entry[:dry_run]
+
+      report = Report.new([@stats], title: entry[:title], params:)
+      next unless entry[:when].call(report)
+
+      entry[:notifier].deliver(report, include_rows: entry[:include_rows])
+      @logger.info "notified #{entry[:notifier].class.name.split('::').last}: #{report.event}"
+    rescue StandardError => e
+      @logger.error "notify #{entry[:notifier].class}: #{e.class}: #{e.message}"
+    end
+  end
+
   def add_total side, row
     @stats.totals&.each do |col, sum|
       value = row.key?(col) ? row[col] : row[col.to_sym]

lib/datapipeline/notification/email.rb

@@ -0,0 +1,93 @@
+require 'securerandom'
+require_relative 'notifier'
+require_relative '../storage/storage_plugin'
+
+class DataPipeline
+  #
+  # e-mail over SMTP, text/plain UTF-8
+  #
+  #   notify DataPipeline::Email.new(to: 'etl@example.com'), on: :failure
+  #
+  # options (defaults from ENV)
+  #   to:        one address or an array (required)
+  #   from:      sender, default ENV['SMTP_FROM'] or etl@<hostname>
+  #   host:      ENV['SMTP_HOST'], default localhost
+  #   port:      ENV['SMTP_PORT'], default 587 (465 with tls: true, 25 without starttls)
+  #   username:  ENV['SMTP_USERNAME'], password: ENV['SMTP_PASSWORD']; without username no login
+  #   tls:       true -> TLS from the start (port 465)
+  #   starttls:  :auto (default, if the server offers it), :always or false
+  #   subject_prefix: default '[etl] '
+  #   timeout:   seconds for open and read (default 15)
+  #
+  class Email < Notifier
+    class Error < StandardError; end
+
+    def initialize to:, from: nil, host: nil, port: nil, username: nil, password: nil, tls: false,
+                   starttls: :auto, subject_prefix: '[etl] ', timeout: 15
+      StoragePlugin.require_gem 'net/smtp', 'net-smtp'
+      @to = Array(to).map(&:to_s).reject(&:empty?)
+      raise ArgumentError, 'Email: to missing' if @to.empty?
+
+      @from = from || ENV['SMTP_FROM'] || "etl@#{Socket.gethostname}"
+      @host = host || ENV.fetch('SMTP_HOST', 'localhost')
+      @tls = tls
+      @starttls = tls || !starttls ? false : starttls.to_sym
+      @port = Integer(port || ENV['SMTP_PORT'] || default_port)
+      @username = username || ENV['SMTP_USERNAME']
+      @password = password || ENV['SMTP_PASSWORD']
+      @subject_prefix = subject_prefix
+      @timeout = timeout
+    end
+
+    def deliver report, include_rows: false
+      send_mail("#{@subject_prefix}#{report.subject}", body_of(report, include_rows))
+    end
+
+    # any text, e.g. from on_finish
+    def send_mail subject, body
+      message = build(subject, body)
+      smtp = Net::SMTP.new(@host, @port, tls: @tls, starttls: @starttls || false)
+      smtp.open_timeout = @timeout
+      smtp.read_timeout = @timeout
+      login = @username ? { user: @username, secret: @password, authtype: :login } : {}
+      smtp.start(helo: Socket.gethostname, **login) { |session| session.send_message(message, @from, *@to) }
+      message
+    rescue Net::SMTPError, SystemCallError, IOError, Timeout::Error, OpenSSL::SSL::SSLError => e
+      raise Error, "#{@host}:#{@port}: #{e.class}: #{e.message}"
+    end
+
+    private
+
+    def default_port
+      return 465 if @tls
+
+      @starttls ? 587 : 25
+    end
+
+    # rows of rejected lines only on request, they may hold customer data
+    def body_of report, include_rows
+      text = report.to_s
+      return text unless include_rows
+
+      rows = report.runs.flat_map { |stats| stats.errors.first(Report::MAX_ERRORS).map { |e| "#{e[:step]}: #{JSON.generate(e[:row])}" } }
+      rows.empty? ? text : "#{text}\n\nZeilen:\n#{rows.join("\n")}"
+    end
+
+    # RFC 5322 with a base64 body, the subject RFC 2047 encoded when it is not ascii
+    def build subject, body
+      encoded_subject = subject.ascii_only? ? subject : "=?UTF-8?B?#{[subject].pack('m0')}?="
+      [
+        "From: #{@from}",
+        "To: #{@to.join(', ')}",
+        "Subject: #{encoded_subject}",
+        "Date: #{Time.now.strftime('%a, %d %b %Y %H:%M:%S %z')}",
+        "Message-ID: <#{SecureRandom.uuid}@#{@from.split('@').last}>",
+        'MIME-Version: 1.0',
+        'Content-Type: text/plain; charset=UTF-8',
+        'Content-Transfer-Encoding: base64',
+        '',
+        [body].pack('m')
+      ].join("\r\n")
+    end
+  end
+end

lib/datapipeline/notification/notifier.rb

@@ -0,0 +1,141 @@
+require 'json'
+require 'securerandom'
+require 'socket'
+
+class DataPipeline
+  #
+  # what a notifier gets: the stats of one or more runs (one script), with event, title and text lines
+  #
+  #   event: run.failed (an abort), run.rejected (rejected rows), run.warnings, run.succeeded
+  #   to_h:  stable payload for webhooks (schema 1); rows of stats.errors only with include_rows: true,
+  #          because they may hold customer data
+  #
+  class Report
+    EVENTS = %w[run.succeeded run.warnings run.rejected run.failed].freeze
+    MAX_ERRORS = 20
+
+    attr_reader :id, :title, :runs, :params, :host, :error
+
+    # error: an exception outside of a run (e.g. the script failed before or between pipelines)
+    def initialize runs, title: nil, params: {}, error: nil
+      @id = SecureRandom.uuid
+      @runs = Array(runs)
+      @title = title || @runs.filter_map(&:name).uniq.join(', ').then { |t| t.empty? ? 'datapipeline' : t }
+      @params = params.to_h
+      @host = Socket.gethostname
+      @error = error
+    end
+
+    # the worst event of all runs
+    def event
+      return 'run.failed' if @error
+
+      @runs.map { |stats| event_of(stats) }.max_by { |event| EVENTS.index(event) } || 'run.succeeded'
+    end
+
+    def failed? = event == 'run.failed'
+    def ok? = event == 'run.succeeded'
+
+    def status
+      { 'run.failed' => 'abgebrochen', 'run.rejected' => 'mit verworfenen Zeilen',
+        'run.warnings' => 'mit Warnungen', 'run.succeeded' => 'ok' }.fetch(event)
+    end
+
+    def subject = "#{@title}: #{status}"
+
+    # text for mail, slack and logs: one block per run, the first errors without row data
+    def lines errors: 5
+      lines = ["#{subject} (#{@host})"]
+      lines << "Parameter: #{@params.map { |key, value| "#{key}=#{value}" }.join(' ')}" if @params.any?
+      lines << "Fehler: #{@error.class}: #{@error.message}" if @error
+      @runs.each do |stats|
+        lines << ''
+        lines << "#{stats.name || 'Pipeline'}#{" #{stats.version}" if stats.version}: #{stats}"
+        lines << "Start #{stats.started_at&.strftime('%d.%m.%Y %H:%M:%S')}, Dauer #{stats.duration}s"
+        stats.errors.first(errors).each { |e| lines << "- #{e[:step]}: #{e[:error]}" }
+        lines << "- … und #{stats.rejected - errors} weitere" if stats.rejected > errors
+      end
+      lines
+    end
+
+    def to_s = lines.join("\n")
+
+    def to_h include_rows: false
+      {
+        schema: 1, id: @id, event:, title: @title, host: @host, params: @params,
+        error: @error && "#{@error.class}: #{@error.message}",
+        runs: @runs.map do |stats|
+          stats.to_h.merge(errors: stats.errors.first(MAX_ERRORS).map { |e| include_rows ? e : e.except(:row) })
+        end
+      }
+    end
+
+    private
+
+    def event_of stats
+      return 'run.failed' if stats.error
+      return 'run.rejected' if stats.rejected.positive?
+      return 'run.warnings' if stats.warnings.to_i.positive?
+
+      'run.succeeded'
+    end
+  end
+
+  #
+  # a notifier only needs deliver(report, include_rows: false); see Webhook, Slack, Email
+  #
+  #   notify DataPipeline::Webhook.new(url: ...), on: :problems
+  #
+  class Notifier
+    # when a notification is sent
+    WHEN = {
+      always: ->(_report) { true },
+      failure: ->(report) { report.failed? },
+      problems: ->(report) { !report.ok? }
+    }.freeze
+
+    TYPES = { webhook: 'DataPipeline::Webhook', slack: 'DataPipeline::Slack', email: 'DataPipeline::Email' }.freeze
+
+    # notifier from the connections file: type webhook, slack or email, the other keys are its options
+    def self.from_connection name
+      options = Connections.fetch(name)
+      type = options[:type].to_s.to_sym
+      klass = TYPES.fetch(type) { raise ArgumentError, "connection #{name.inspect}: type #{type} is no notifier (#{TYPES.keys.join(', ')})" }
+      Object.const_get(klass).new(**options.except(:type))
+    end
+
+    def self.condition on
+      WHEN.fetch(on.to_sym) { raise ArgumentError, "notify on: #{on} unknown, use #{WHEN.keys.join(', ')}" }
+    end
+
+    def deliver(_report, include_rows: false)
+      raise NotImplementedError, "#{self.class}#deliver is missing"
+    end
+
+    private
+
+    # POST with retries on 429/5xx (Retry-After honoured) and network errors; other 4xx raise at once
+    def post_with_retries url, body:, headers:, timeout:, retries:, retry_wait:, error:
+      attempt = 0
+      loop do
+        attempt += 1
+        begin
+          response = HTTP.timeout(timeout).headers(headers).post(url, body:)
+        rescue HTTP::Error, SystemCallError, IOError => e
+          raise error, "#{url}: #{e.class}: #{e.message}" if attempt > retries
+
+          sleep(retry_wait * attempt)
+          next
+        end
+        return response if response.status.success?
+
+        code = response.code
+        unless attempt <= retries && (code == 429 || code >= 500)
+          raise error, "#{url}: HTTP #{code} #{response.body.to_s[0, 200]}"
+        end
+
+        sleep(response.headers['Retry-After']&.to_f || retry_wait * attempt)
+      end
+    end
+  end
+end

lib/datapipeline/notification/slack.rb

@@ -1,34 +1,88 @@
-require 'http'
 require 'json'
+require_relative 'notifier'
+require_relative '../storage/storage_plugin'
 
 class DataPipeline
   #
   # Slack.new(channel: '#etl').send_message('posting export done')
+  # notify DataPipeline::Slack.new(channel: '#etl'), on: :problems
+  # notify DataPipeline::Slack.new(webhook: ENV['SLACK_WEBHOOK'])      # incoming webhook, no token needed
   #
   # options
-  #   token:   bot token, default ENV['SLACK_TOKEN']
-  #   channel: default '#general'
+  #   token:      bot token for chat.postMessage, default ENV['SLACK_TOKEN'] (not needed with webhook)
+  #   channel:    default '#general' (with webhook the channel of the webhook)
+  #   webhook:    url of an incoming webhook instead of token and channel
+  #   mention:    e.g. '<!here>' or '<!subteam^ID>', only in reports of failed runs
+  #   timeout:    seconds per request (default 10)
+  #   retries:    repetitions on 429, 5xx and network errors (default 3)
+  #   retry_wait: seconds, multiplied by the attempt (default 2)
+  #   api_url:    default https://slack.com/api/chat.postMessage
   #
-  class Slack
+  class Slack < Notifier
     API_URL = 'https://slack.com/api/chat.postMessage'.freeze
+    MAX_TEXT = 2900 # a section block holds 3000 characters
+
+    class Error < StandardError; end
 
     DEFAULTS = {
       token: nil,
-      channel: '#general'
+      channel: '#general',
+      webhook: nil,
+      mention: nil,
+      timeout: 10,
+      retries: 3,
+      retry_wait: 2,
+      api_url: API_URL
     }.freeze
 
+    ICONS = { 'run.failed' => ':red_circle:', 'run.rejected' => ':large_orange_circle:',
+              'run.warnings' => ':large_yellow_circle:', 'run.succeeded' => ':large_green_circle:' }.freeze
+
+    # & < > have a meaning in slack texts
+    def self.escape(text) = text.to_s.gsub('&', '&amp;').gsub('<', '&lt;').gsub('>', '&gt;')
+
     def initialize options = {}
+      StoragePlugin.require_gem 'http'
       @options = DEFAULTS.merge(token: ENV['SLACK_TOKEN']).merge(options)
-      raise ArgumentError, 'Slack: token missing (option token: or ENV SLACK_TOKEN)' unless @options[:token]
+      return if @options[:webhook] || @options[:token]
+
+      raise ArgumentError, 'Slack: token missing (option token: or ENV SLACK_TOKEN, or webhook:)'
     end
 
-    # returns the parsed slack response, raises when slack reports an error
+    # plain text (slack markup allowed); returns the parsed slack response, raises when slack reports an error
     def send_message(content, verbose: false)
-      response = HTTP.auth("Bearer #{@options[:token]}")
-                     .post(API_URL, json: { channel: @options[:channel], text: content })
-      body = JSON.parse(response.body.to_s)
-      puts JSON.pretty_generate(body) if verbose
-      raise "Slack#send_message failed: #{body['error']}" unless body['ok']
+      post({ text: content }, verbose:)
+    end
+
+    # the report as blocks: status line, then the runs as preformatted text
+    def deliver report, include_rows: false
+      icon = ICONS.fetch(report.event)
+      mention = report.failed? && @options[:mention] ? " #{@options[:mention]}" : ''
+      head = "#{icon} *#{self.class.escape(report.title)}*: #{report.status}#{mention}"
+      details = self.class.escape(report.lines.drop(1).join("\n").strip)
+      details = "#{details[0, MAX_TEXT]}\n…" if details.length > MAX_TEXT
+      blocks = [{ type: 'section', text: { type: 'mrkdwn', text: head } }]
+      blocks << { type: 'section', text: { type: 'mrkdwn', text: "```#{details}```" } } unless details.empty?
+      post({ text: "#{report.subject}#{mention}", blocks: })
+    end
+
+    private
+
+    def post payload, verbose: false
+      url = @options[:webhook] || @options[:api_url]
+      headers = { 'Content-Type' => 'application/json; charset=utf-8' }
+      unless @options[:webhook]
+        headers['Authorization'] = "Bearer #{@options[:token]}"
+        payload = payload.merge(channel: @options[:channel])
+      end
+      response = post_with_retries(url, body: JSON.generate(payload), headers:, timeout: @options[:timeout],
+                                        retries: @options[:retries], retry_wait: @options[:retry_wait], error: Error)
+      text = response.body.to_s
+      puts text if verbose
+      return text if @options[:webhook] # incoming webhooks answer "ok"
+
+      body = JSON.parse(text)
+      raise Error, "Slack#send_message failed: #{body['error']}" unless body['ok']
 
       body
     end

lib/datapipeline/notification/webhook.rb

@@ -0,0 +1,63 @@
+require 'openssl'
+require_relative 'notifier'
+require_relative '../storage/storage_plugin'
+
+class DataPipeline
+  #
+  # POST of the report as json (Report#to_h, schema 1)
+  #
+  #   notify DataPipeline::Webhook.new(url: ENV['ALERT_URL'], secret: ENV['ALERT_SECRET'])
+  #
+  # options
+  #   url:        target (required)
+  #   secret:     signs the body: X-DataPipeline-Signature: sha256=HMAC_SHA256(secret, "<timestamp>.<body>")
+  #               with X-DataPipeline-Timestamp (unix seconds)
+  #   headers:    more headers, e.g. { 'Authorization' => 'Bearer ...' }
+  #   timeout:    seconds per request (default 10)
+  #   retries:    repetitions on 429, 5xx and network errors (default 3), Retry-After is honoured
+  #   retry_wait: seconds, multiplied by the attempt (default 2)
+  #
+  # every request carries X-DataPipeline-Event and X-DataPipeline-Delivery (the report id, the same on retries)
+  #
+  class Webhook < Notifier
+    class Error < StandardError; end
+
+    def initialize url:, secret: nil, headers: {}, timeout: 10, retries: 3, retry_wait: 2
+      StoragePlugin.require_gem 'http'
+      @url = url.to_s
+      raise ArgumentError, 'Webhook: url missing' if @url.empty?
+
+      @secret = secret
+      @headers = headers.to_h { |key, value| [key.to_s, value.to_s] }
+      @timeout = timeout
+      @retries = retries
+      @retry_wait = retry_wait
+    end
+
+    def deliver report, include_rows: false
+      body = JSON.generate(report.to_h(include_rows:))
+      post_with_retries(@url, body:, headers: headers_for(report, body), timeout: @timeout, retries: @retries,
+                              retry_wait: @retry_wait, error: Error)
+    end
+
+    # for receivers: true when signature and timestamp (max_age seconds) of a request are valid
+    def self.valid_signature? secret:, body:, timestamp:, signature:, max_age: 300
+      return false if timestamp.to_s.empty? || (Time.now.to_i - timestamp.to_i).abs > max_age
+
+      expected = "sha256=#{OpenSSL::HMAC.hexdigest('SHA256', secret, "#{timestamp}.#{body}")}"
+      expected.bytesize == signature.to_s.bytesize && OpenSSL.fixed_length_secure_compare(expected, signature.to_s)
+    end
+
+    private
+
+    def headers_for report, body
+      headers = { 'Content-Type' => 'application/json', 'User-Agent' => "datapipeline/#{VERSION}",
+                  'X-DataPipeline-Event' => report.event, 'X-DataPipeline-Delivery' => report.id }.merge(@headers)
+      return headers unless @secret
+
+      timestamp = Time.now.to_i.to_s
+      signature = OpenSSL::HMAC.hexdigest('SHA256', @secret, "#{timestamp}.#{body}")
+      headers.merge('X-DataPipeline-Timestamp' => timestamp, 'X-DataPipeline-Signature' => "sha256=#{signature}")
+    end
+  end
+end

lib/datapipeline/version.rb

@@ -1,5 +1,5 @@
 class DataPipeline
-  VERSION = '3.1.0'.freeze
+  VERSION = '3.2.0'.freeze
 
   # kept for scripts that print the version (rake info:info); the history is in CHANGELOG.md
   class Version