datapipeline 3.2.0: Benachrichtigungen mit Webhook, Slack und E-Mail
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
require_relative '../test_helper'
|
||||
require 'open3'
|
||||
require_relative '../support/test_http_server'
|
||||
|
||||
# bin/etl as a subprocess: arguments, params, dry run, exit codes
|
||||
class EtlCliTest < PipelineTest
|
||||
@@ -127,4 +128,44 @@ class EtlCliTest < PipelineTest
|
||||
assert_equal ['export_kunden', 3, true], entry.values_at('name', 'read', 'ok')
|
||||
end
|
||||
end
|
||||
|
||||
def test_notify_one_report_for_all_pipelines_of_the_script
|
||||
server = TestHttpServer.new { [200, 'ok'] }
|
||||
with_tmpdir do |dir|
|
||||
Dir.mkdir("#{dir}/config")
|
||||
File.write("#{dir}/config/connections.yml", "alerts:\n type: webhook\n url: #{server.url}/hook\n")
|
||||
path = script(dir, 'zwei.rb', <<~RUBY)
|
||||
2.times do |i|
|
||||
DataPipeline.new do
|
||||
name "teil\#{i}"
|
||||
on_error :skip
|
||||
read container(:csv, file: 'in.csv')
|
||||
validate 'a' => { format: /\\A\\d+\\z/ } if i == 1
|
||||
write container(:csv, file: "out\#{i}.csv")
|
||||
end.run
|
||||
end
|
||||
RUBY
|
||||
_out, err, status = etl(dir, 'run', path, '--notify=alerts')
|
||||
assert_equal 2, status.exitstatus, err
|
||||
assert_equal 1, server.requests.size
|
||||
payload = JSON.parse(server.requests.last[:body])
|
||||
assert_equal ['run.rejected', 'zwei', %w[teil0 teil1]], [payload['event'], payload['title'], payload['runs'].map { |r| r['name'] }]
|
||||
|
||||
_out, err, status = etl(dir, 'run', path, '--notify=alerts', '--notify-on=failure')
|
||||
assert_equal 2, status.exitstatus, err
|
||||
assert_equal 1, server.requests.size # nothing failed
|
||||
|
||||
broken = script(dir, 'kaputt.rb', "raise 'Skriptfehler'\n")
|
||||
_out, _err, status = etl(dir, 'run', broken, '--notify=alerts')
|
||||
assert_equal 1, status.exitstatus
|
||||
payload = JSON.parse(server.requests.last[:body])
|
||||
assert_equal ['run.failed', 'RuntimeError: Skriptfehler', []], payload.values_at('event', 'error', 'runs')
|
||||
|
||||
_out, err, status = etl(dir, 'run', path, '--notify=gibtsnicht')
|
||||
assert_equal 1, status.exitstatus
|
||||
assert_includes err, 'etl: --notify=gibtsnicht: connection "gibtsnicht" not in config/connections.yml (alerts)'
|
||||
end
|
||||
ensure
|
||||
server&.stop
|
||||
end
|
||||
end
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
require_relative '../test_helper'
|
||||
require_relative '../support/test_http_server'
|
||||
require_relative '../support/test_smtp_server'
|
||||
|
||||
# Report, Webhook, Slack, Email and notify against local test servers
|
||||
class NotificationTest < PipelineTest
|
||||
ROWS = [{ 'id' => '1', 'konto' => '1000' }, { 'id' => '2', 'konto' => '' }].freeze
|
||||
|
||||
# collects reports, for notify without network
|
||||
class MemoryNotifier < DataPipeline::Notifier
|
||||
attr_reader :reports
|
||||
|
||||
def initialize(fail: false) = (@reports = []; @fail = fail)
|
||||
|
||||
def deliver report, include_rows: false
|
||||
raise 'kaputt' if @fail
|
||||
|
||||
@reports << [report, include_rows]
|
||||
end
|
||||
end
|
||||
|
||||
def teardown
|
||||
@server&.stop
|
||||
DataPipeline.connections_file = nil
|
||||
super
|
||||
end
|
||||
|
||||
def stats_with_rejected
|
||||
run_pipeline(ROWS.map(&:dup)) { name 'export'; on_error :skip; validate 'konto' => { required: true } }.last
|
||||
end
|
||||
|
||||
def report(stats = stats_with_rejected) = DataPipeline::Report.new([stats], params: { from: '2026-10-01' })
|
||||
|
||||
def test_report_event_lines_and_payload_without_rows
|
||||
report = report()
|
||||
assert_equal ['run.rejected', 'export: mit verworfenen Zeilen'], [report.event, report.subject]
|
||||
assert_includes report.to_s, '- validate#1: konto: missing'
|
||||
assert_includes report.to_s, 'Parameter: from=2026-10-01'
|
||||
payload = report.to_h
|
||||
assert_equal [1, 'run.rejected', 'export'], payload.values_at(:schema, :event, :title)
|
||||
assert_equal [{ step: 'validate#1', error: 'konto: missing' }], payload[:runs].first[:errors]
|
||||
assert_equal({ 'id' => '2', 'konto' => '' }, report.to_h(include_rows: true)[:runs].first[:errors].first[:row])
|
||||
end
|
||||
|
||||
def test_report_worst_event_of_several_runs_and_script_error
|
||||
ok = run_pipeline(ROWS.map(&:dup)).last
|
||||
assert_equal 'run.succeeded', DataPipeline::Report.new([ok]).event
|
||||
assert_equal 'run.rejected', DataPipeline::Report.new([ok, stats_with_rejected]).event
|
||||
failed = DataPipeline::Report.new([ok], title: 'skript', error: RuntimeError.new('kaputt'))
|
||||
assert_equal %w[run.failed skript], [failed.event, failed.title]
|
||||
assert_includes failed.to_s, 'Fehler: RuntimeError: kaputt'
|
||||
end
|
||||
|
||||
def test_webhook_signed_post
|
||||
@server = TestHttpServer.new { [200, 'ok'] }
|
||||
report = report()
|
||||
DataPipeline::Webhook.new(url: "#{@server.url}/hook", secret: 's3cret', headers: { 'X-Team' => 'etl' }).deliver(report)
|
||||
request = @server.requests.last
|
||||
headers = request[:headers]
|
||||
assert_equal ['/hook', 'run.rejected', report.id, 'etl'],
|
||||
[request[:path], headers['x-datapipeline-event'], headers['x-datapipeline-delivery'], headers['x-team']]
|
||||
assert_equal 1, JSON.parse(request[:body])['schema']
|
||||
refute_includes request[:body], '"row"'
|
||||
assert DataPipeline::Webhook.valid_signature?(secret: 's3cret', body: request[:body],
|
||||
timestamp: headers['x-datapipeline-timestamp'],
|
||||
signature: headers['x-datapipeline-signature'])
|
||||
refute DataPipeline::Webhook.valid_signature?(secret: 'falsch', body: request[:body],
|
||||
timestamp: headers['x-datapipeline-timestamp'],
|
||||
signature: headers['x-datapipeline-signature'])
|
||||
refute DataPipeline::Webhook.valid_signature?(secret: 's3cret', body: request[:body], timestamp: (Time.now.to_i - 600).to_s,
|
||||
signature: headers['x-datapipeline-signature'])
|
||||
end
|
||||
|
||||
def test_webhook_retries_429_and_5xx_but_not_4xx
|
||||
answers = [[503, 'busy'], [429, 'slow', { 'Retry-After' => '0' }], [200, 'ok']]
|
||||
@server = TestHttpServer.new { answers.shift }
|
||||
DataPipeline::Webhook.new(url: @server.url, retry_wait: 0).deliver(report)
|
||||
assert_equal 3, @server.requests.size
|
||||
assert_equal 1, @server.requests.map { |r| r[:headers]['x-datapipeline-delivery'] }.uniq.size
|
||||
|
||||
@server.stop
|
||||
@server = TestHttpServer.new { [400, 'bad'] }
|
||||
error = assert_raises(DataPipeline::Webhook::Error) { DataPipeline::Webhook.new(url: @server.url, retry_wait: 0).deliver(report) }
|
||||
assert_includes error.message, 'HTTP 400'
|
||||
assert_equal 1, @server.requests.size
|
||||
end
|
||||
|
||||
def test_webhook_network_error_after_retries
|
||||
port = TCPServer.new('127.0.0.1', 0).then { |s| s.addr[1].tap { s.close } }
|
||||
error = assert_raises(DataPipeline::Webhook::Error) do
|
||||
DataPipeline::Webhook.new(url: "http://127.0.0.1:#{port}", retries: 1, retry_wait: 0, timeout: 2).deliver(report)
|
||||
end
|
||||
assert_includes error.message, '127.0.0.1'
|
||||
end
|
||||
|
||||
def test_slack_api_blocks_escape_and_mention
|
||||
@server = TestHttpServer.new { [200, { 'ok' => true }] }
|
||||
slack = DataPipeline::Slack.new(token: 'xoxb-1', channel: '#etl', api_url: @server.url, mention: '<!here>')
|
||||
failed = DataPipeline::Report.new([stats_with_rejected], title: 'A & <B>', error: RuntimeError.new('x < y'))
|
||||
slack.deliver(failed)
|
||||
request = @server.requests.last
|
||||
body = JSON.parse(request[:body])
|
||||
assert_equal ['Bearer xoxb-1', '#etl'], [request[:headers]['authorization'], body['channel']]
|
||||
assert_equal ':red_circle: *A & <B>*: abgebrochen <!here>', body['blocks'][0]['text']['text']
|
||||
assert_includes body['blocks'][1]['text']['text'], 'RuntimeError: x < y'
|
||||
|
||||
slack.deliver(report)
|
||||
refute_includes JSON.parse(@server.requests.last[:body])['blocks'][0]['text']['text'], '<!here>' # mention only on failure
|
||||
end
|
||||
|
||||
def test_slack_error_and_incoming_webhook
|
||||
@server = TestHttpServer.new { |req| req[:path] == '/api' ? [200, { 'ok' => false, 'error' => 'not_in_channel' }] : [200, 'ok'] }
|
||||
error = assert_raises(DataPipeline::Slack::Error) { DataPipeline::Slack.new(token: 't', api_url: "#{@server.url}/api").send_message('x') }
|
||||
assert_includes error.message, 'not_in_channel'
|
||||
|
||||
token = ENV.delete('SLACK_TOKEN')
|
||||
hook = DataPipeline::Slack.new(webhook: "#{@server.url}/hook")
|
||||
assert_equal 'ok', hook.deliver(report)
|
||||
request = @server.requests.last
|
||||
assert_nil request[:headers]['authorization']
|
||||
refute JSON.parse(request[:body]).key?('channel')
|
||||
ensure
|
||||
ENV['SLACK_TOKEN'] = token if token
|
||||
end
|
||||
|
||||
def test_email_over_smtp_with_login
|
||||
@server = TestSmtpServer.new
|
||||
mail = DataPipeline::Email.new(to: %w[a@example.com b@example.com], from: 'etl@example.com', host: '127.0.0.1',
|
||||
port: @server.port, starttls: false, username: 'user', password: 'geheim')
|
||||
mail.deliver(DataPipeline::Report.new([stats_with_rejected], title: 'Grösse Ä'))
|
||||
sent = @server.mails.last
|
||||
assert_equal ['etl@example.com', %w[a@example.com b@example.com], %w[user geheim]], sent.values_at(:from, :to, :auth)
|
||||
head, body = sent[:data].split("\r\n\r\n", 2)
|
||||
assert_includes head, "Subject: =?UTF-8?B?#{['[etl] Grösse Ä: mit verworfenen Zeilen'].pack('m0')}?="
|
||||
assert_includes head, 'Content-Type: text/plain; charset=UTF-8'
|
||||
text = body.unpack1('m').force_encoding('UTF-8')
|
||||
assert_includes text, '- validate#1: konto: missing'
|
||||
refute_includes text, 'Zeilen:'
|
||||
end
|
||||
|
||||
def test_email_rows_on_request_and_errors
|
||||
@server = TestSmtpServer.new
|
||||
mail = DataPipeline::Email.new(to: 'a@example.com', host: '127.0.0.1', port: @server.port, starttls: false)
|
||||
mail.deliver(report, include_rows: true)
|
||||
assert_includes @server.mails.last[:data].split("\r\n\r\n", 2).last.unpack1('m').force_encoding('UTF-8'), '{"id":"2","konto":""}'
|
||||
assert_nil @server.mails.last[:auth]
|
||||
|
||||
assert_raises(ArgumentError) { DataPipeline::Email.new(to: []) }
|
||||
port = TCPServer.new('127.0.0.1', 0).then { |s| s.addr[1].tap { s.close } }
|
||||
assert_raises(DataPipeline::Email::Error) { DataPipeline::Email.new(to: 'a@b', host: '127.0.0.1', port:, starttls: false).send_mail('x', 'y') }
|
||||
end
|
||||
|
||||
def test_notify_conditions_dry_run_and_rows
|
||||
always = MemoryNotifier.new
|
||||
failure = MemoryNotifier.new
|
||||
problems = MemoryNotifier.new
|
||||
rows = MemoryNotifier.new
|
||||
run_pipeline(ROWS.map(&:dup)) do
|
||||
name 'export'
|
||||
on_error :skip
|
||||
validate 'konto' => { required: true }
|
||||
notify always, on: :always
|
||||
notify failure, on: :failure
|
||||
notify problems
|
||||
notify rows, include_rows: true, title: 'Export Konten'
|
||||
end
|
||||
assert_equal [1, 0, 1], [always, failure, problems].map { |n| n.reports.size }
|
||||
assert_equal ['Export Konten', true], [rows.reports.first[0].title, rows.reports.first[1]]
|
||||
|
||||
dry = MemoryNotifier.new
|
||||
dry_too = MemoryNotifier.new
|
||||
capture_stdout { run_pipeline(ROWS.map(&:dup), dry_run: true) { notify dry, on: :always; notify dry_too, on: :always, dry_run: true } }
|
||||
assert_equal [0, 1], [dry.reports.size, dry_too.reports.size]
|
||||
|
||||
assert_raises(ArgumentError) { run_pipeline { notify Object.new } }
|
||||
assert_raises(ArgumentError) { run_pipeline { notify MemoryNotifier.new, on: :sometimes } }
|
||||
end
|
||||
|
||||
def test_notify_error_is_only_logged
|
||||
log = StringIO.new
|
||||
stats = DataPipeline.new do
|
||||
logger Logger.new(log)
|
||||
read MemoryStorage.new(rows: ROWS.map(&:dup))
|
||||
write MemoryStorage.new
|
||||
notify MemoryNotifier.new(fail: true), on: :always
|
||||
end.run
|
||||
assert stats.ok?
|
||||
assert_includes log.string, 'notify NotificationTest::MemoryNotifier: RuntimeError: kaputt'
|
||||
end
|
||||
|
||||
def test_notifier_from_connections_file
|
||||
with_tmpdir do |dir|
|
||||
File.write("#{dir}/c.yml", <<~YAML)
|
||||
alerts: { type: webhook, url: "http://127.0.0.1:9/hook", secret: "<%= 'abc' %>" }
|
||||
mail: { type: email, to: [etl@example.com], host: smtp.example.com }
|
||||
db: { type: sqlite, database: x.sqlite3 }
|
||||
YAML
|
||||
DataPipeline.connections_file = "#{dir}/c.yml"
|
||||
pipeline = DataPipeline.new { logger PipelineHelper::QUIET }
|
||||
assert_kind_of DataPipeline::Webhook, pipeline.notifier(:alerts)
|
||||
assert_equal 'abc', pipeline.notifier(:alerts).instance_variable_get(:@secret)
|
||||
assert_kind_of DataPipeline::Email, pipeline.notifier(:mail)
|
||||
assert_includes assert_raises(ArgumentError) { pipeline.notifier(:db) }.message, 'is no notifier'
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -5,7 +5,7 @@ require 'json'
|
||||
#
|
||||
# minimal http server for DataPipeline::RestStorage tests, one request per connection
|
||||
# server = TestHttpServer.new { |req| [200, { 'items' => [] }] }
|
||||
# req: { method:, path:, query:, headers:, body: } -> [status, hash/array or string]
|
||||
# req: { method:, path:, query:, headers:, body: } -> [status, hash/array or string, optional headers]
|
||||
#
|
||||
class TestHttpServer
|
||||
attr_reader :requests
|
||||
@@ -40,9 +40,10 @@ class TestHttpServer
|
||||
request = { method:, path:, query: URI.decode_www_form(query.to_s).to_h, headers:, body: }
|
||||
@requests << request
|
||||
|
||||
status, payload = @handler.call(request)
|
||||
status, payload, extra = @handler.call(request)
|
||||
text = payload.is_a?(String) ? payload : JSON.generate(payload)
|
||||
client.write "HTTP/1.1 #{status} X\r\nContent-Type: application/json\r\n" \
|
||||
extra = Hash(extra).map { |key, value| "#{key}: #{value}\r\n" }.join
|
||||
client.write "HTTP/1.1 #{status} X\r\nContent-Type: application/json\r\n#{extra}" \
|
||||
"Content-Length: #{text.bytesize}\r\nConnection: close\r\n\r\n#{text}"
|
||||
rescue StandardError => e
|
||||
warn "TestHttpServer: #{e.class}: #{e.message}"
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
require 'socket'
|
||||
|
||||
#
|
||||
# minimal smtp server for DataPipeline::Email tests: EHLO, AUTH LOGIN, MAIL, RCPT, DATA, QUIT
|
||||
# server = TestSmtpServer.new
|
||||
# server.mails -> [{ from:, to: [...], data:, auth: [user, password] }]
|
||||
#
|
||||
class TestSmtpServer
|
||||
attr_reader :mails
|
||||
|
||||
def initialize
|
||||
@mails = []
|
||||
@server = TCPServer.new('127.0.0.1', 0)
|
||||
@thread = Thread.new { loop { serve(@server.accept) } }
|
||||
end
|
||||
|
||||
def port = @server.addr[1]
|
||||
|
||||
def stop
|
||||
@thread.kill
|
||||
@server.close
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def serve client
|
||||
mail = { to: [] }
|
||||
client.write "220 test ESMTP\r\n"
|
||||
while (line = client.gets)
|
||||
case line
|
||||
when /\AEHLO/i then client.write "250-test\r\n250 AUTH LOGIN PLAIN\r\n"
|
||||
when /\AAUTH LOGIN/i
|
||||
client.write "334 VXNlcm5hbWU6\r\n"
|
||||
user = client.gets.unpack1('m')
|
||||
client.write "334 UGFzc3dvcmQ6\r\n"
|
||||
mail[:auth] = [user, client.gets.unpack1('m')]
|
||||
client.write "235 ok\r\n"
|
||||
when /\AMAIL FROM:<(.*)>/i then mail[:from] = $1; client.write "250 ok\r\n"
|
||||
when /\ARCPT TO:<(.*)>/i then mail[:to] << $1; client.write "250 ok\r\n"
|
||||
when /\ADATA/i
|
||||
client.write "354 go\r\n"
|
||||
data = +''
|
||||
while (part = client.gets) && part != ".\r\n"
|
||||
data << part
|
||||
end
|
||||
mail[:data] = data
|
||||
@mails << mail
|
||||
mail = { to: [] }
|
||||
client.write "250 queued\r\n"
|
||||
when /\AQUIT/i then client.write "221 bye\r\n"; break
|
||||
else client.write "250 ok\r\n"
|
||||
end
|
||||
end
|
||||
rescue StandardError => e
|
||||
warn "TestSmtpServer: #{e.class}: #{e.message}"
|
||||
ensure
|
||||
client.close
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user