207 lines
9.9 KiB
Ruby
207 lines
9.9 KiB
Ruby
require_relative '../test_helper'
|
|
require_relative '../support/test_http_server'
|
|
require_relative '../support/test_smtp_server'
|
|
|
|
# Report, Webhook, Slack, Email and notify against local test servers
|
|
class NotificationTest < PipelineTest
|
|
ROWS = [{ 'id' => '1', 'konto' => '1000' }, { 'id' => '2', 'konto' => '' }].freeze
|
|
|
|
# collects reports, for notify without network
|
|
class MemoryNotifier < DataPipeline::Notifier
|
|
attr_reader :reports
|
|
|
|
def initialize(fail: false) = (@reports = []; @fail = fail)
|
|
|
|
def deliver report, include_rows: false
|
|
raise 'kaputt' if @fail
|
|
|
|
@reports << [report, include_rows]
|
|
end
|
|
end
|
|
|
|
def teardown
|
|
@server&.stop
|
|
DataPipeline.connections_file = nil
|
|
super
|
|
end
|
|
|
|
def stats_with_rejected
|
|
run_pipeline(ROWS.map(&:dup)) { name 'export'; on_error :skip; validate 'konto' => { required: true } }.last
|
|
end
|
|
|
|
def report(stats = stats_with_rejected) = DataPipeline::Report.new([stats], params: { from: '2026-10-01' })
|
|
|
|
def test_report_event_lines_and_payload_without_rows
|
|
report = report()
|
|
assert_equal ['run.rejected', 'export: mit verworfenen Zeilen'], [report.event, report.subject]
|
|
assert_includes report.to_s, '- validate#1: konto: missing'
|
|
assert_includes report.to_s, 'Parameter: from=2026-10-01'
|
|
payload = report.to_h
|
|
assert_equal [1, 'run.rejected', 'export'], payload.values_at(:schema, :event, :title)
|
|
assert_equal [{ step: 'validate#1', error: 'konto: missing' }], payload[:runs].first[:errors]
|
|
assert_equal({ 'id' => '2', 'konto' => '' }, report.to_h(include_rows: true)[:runs].first[:errors].first[:row])
|
|
end
|
|
|
|
def test_report_worst_event_of_several_runs_and_script_error
|
|
ok = run_pipeline(ROWS.map(&:dup)).last
|
|
assert_equal 'run.succeeded', DataPipeline::Report.new([ok]).event
|
|
assert_equal 'run.rejected', DataPipeline::Report.new([ok, stats_with_rejected]).event
|
|
failed = DataPipeline::Report.new([ok], title: 'skript', error: RuntimeError.new('kaputt'))
|
|
assert_equal %w[run.failed skript], [failed.event, failed.title]
|
|
assert_includes failed.to_s, 'Fehler: RuntimeError: kaputt'
|
|
end
|
|
|
|
def test_webhook_signed_post
|
|
@server = TestHttpServer.new { [200, 'ok'] }
|
|
report = report()
|
|
DataPipeline::Webhook.new(url: "#{@server.url}/hook", secret: 's3cret', headers: { 'X-Team' => 'etl' }).deliver(report)
|
|
request = @server.requests.last
|
|
headers = request[:headers]
|
|
assert_equal ['/hook', 'run.rejected', report.id, 'etl'],
|
|
[request[:path], headers['x-datapipeline-event'], headers['x-datapipeline-delivery'], headers['x-team']]
|
|
assert_equal 1, JSON.parse(request[:body])['schema']
|
|
refute_includes request[:body], '"row"'
|
|
assert DataPipeline::Webhook.valid_signature?(secret: 's3cret', body: request[:body],
|
|
timestamp: headers['x-datapipeline-timestamp'],
|
|
signature: headers['x-datapipeline-signature'])
|
|
refute DataPipeline::Webhook.valid_signature?(secret: 'falsch', body: request[:body],
|
|
timestamp: headers['x-datapipeline-timestamp'],
|
|
signature: headers['x-datapipeline-signature'])
|
|
refute DataPipeline::Webhook.valid_signature?(secret: 's3cret', body: request[:body], timestamp: (Time.now.to_i - 600).to_s,
|
|
signature: headers['x-datapipeline-signature'])
|
|
end
|
|
|
|
def test_webhook_retries_429_and_5xx_but_not_4xx
|
|
answers = [[503, 'busy'], [429, 'slow', { 'Retry-After' => '0' }], [200, 'ok']]
|
|
@server = TestHttpServer.new { answers.shift }
|
|
DataPipeline::Webhook.new(url: @server.url, retry_wait: 0).deliver(report)
|
|
assert_equal 3, @server.requests.size
|
|
assert_equal 1, @server.requests.map { |r| r[:headers]['x-datapipeline-delivery'] }.uniq.size
|
|
|
|
@server.stop
|
|
@server = TestHttpServer.new { [400, 'bad'] }
|
|
error = assert_raises(DataPipeline::Webhook::Error) { DataPipeline::Webhook.new(url: @server.url, retry_wait: 0).deliver(report) }
|
|
assert_includes error.message, 'HTTP 400'
|
|
assert_equal 1, @server.requests.size
|
|
end
|
|
|
|
def test_webhook_network_error_after_retries
|
|
port = TCPServer.new('127.0.0.1', 0).then { |s| s.addr[1].tap { s.close } }
|
|
error = assert_raises(DataPipeline::Webhook::Error) do
|
|
DataPipeline::Webhook.new(url: "http://127.0.0.1:#{port}", retries: 1, retry_wait: 0, timeout: 2).deliver(report)
|
|
end
|
|
assert_includes error.message, '127.0.0.1'
|
|
end
|
|
|
|
def test_slack_api_blocks_escape_and_mention
|
|
@server = TestHttpServer.new { [200, { 'ok' => true }] }
|
|
slack = DataPipeline::Slack.new(token: 'xoxb-1', channel: '#etl', api_url: @server.url, mention: '<!here>')
|
|
failed = DataPipeline::Report.new([stats_with_rejected], title: 'A & <B>', error: RuntimeError.new('x < y'))
|
|
slack.deliver(failed)
|
|
request = @server.requests.last
|
|
body = JSON.parse(request[:body])
|
|
assert_equal ['Bearer xoxb-1', '#etl'], [request[:headers]['authorization'], body['channel']]
|
|
assert_equal ':red_circle: *A & <B>*: abgebrochen <!here>', body['blocks'][0]['text']['text']
|
|
assert_includes body['blocks'][1]['text']['text'], 'RuntimeError: x < y'
|
|
|
|
slack.deliver(report)
|
|
refute_includes JSON.parse(@server.requests.last[:body])['blocks'][0]['text']['text'], '<!here>' # mention only on failure
|
|
end
|
|
|
|
def test_slack_error_and_incoming_webhook
|
|
@server = TestHttpServer.new { |req| req[:path] == '/api' ? [200, { 'ok' => false, 'error' => 'not_in_channel' }] : [200, 'ok'] }
|
|
error = assert_raises(DataPipeline::Slack::Error) { DataPipeline::Slack.new(token: 't', api_url: "#{@server.url}/api").send_message('x') }
|
|
assert_includes error.message, 'not_in_channel'
|
|
|
|
token = ENV.delete('SLACK_TOKEN')
|
|
hook = DataPipeline::Slack.new(webhook: "#{@server.url}/hook")
|
|
assert_equal 'ok', hook.deliver(report)
|
|
request = @server.requests.last
|
|
assert_nil request[:headers]['authorization']
|
|
refute JSON.parse(request[:body]).key?('channel')
|
|
ensure
|
|
ENV['SLACK_TOKEN'] = token if token
|
|
end
|
|
|
|
def test_email_over_smtp_with_login
|
|
@server = TestSmtpServer.new
|
|
mail = DataPipeline::Email.new(to: %w[a@example.com b@example.com], from: 'etl@example.com', host: '127.0.0.1',
|
|
port: @server.port, starttls: false, username: 'user', password: 'geheim')
|
|
mail.deliver(DataPipeline::Report.new([stats_with_rejected], title: 'Grösse Ä'))
|
|
sent = @server.mails.last
|
|
assert_equal ['etl@example.com', %w[a@example.com b@example.com], %w[user geheim]], sent.values_at(:from, :to, :auth)
|
|
head, body = sent[:data].split("\r\n\r\n", 2)
|
|
assert_includes head, "Subject: =?UTF-8?B?#{['[etl] Grösse Ä: mit verworfenen Zeilen'].pack('m0')}?="
|
|
assert_includes head, 'Content-Type: text/plain; charset=UTF-8'
|
|
text = body.unpack1('m').force_encoding('UTF-8')
|
|
assert_includes text, '- validate#1: konto: missing'
|
|
refute_includes text, 'Zeilen:'
|
|
end
|
|
|
|
def test_email_rows_on_request_and_errors
|
|
@server = TestSmtpServer.new
|
|
mail = DataPipeline::Email.new(to: 'a@example.com', host: '127.0.0.1', port: @server.port, starttls: false)
|
|
mail.deliver(report, include_rows: true)
|
|
assert_includes @server.mails.last[:data].split("\r\n\r\n", 2).last.unpack1('m').force_encoding('UTF-8'), '{"id":"2","konto":""}'
|
|
assert_nil @server.mails.last[:auth]
|
|
|
|
assert_raises(ArgumentError) { DataPipeline::Email.new(to: []) }
|
|
port = TCPServer.new('127.0.0.1', 0).then { |s| s.addr[1].tap { s.close } }
|
|
assert_raises(DataPipeline::Email::Error) { DataPipeline::Email.new(to: 'a@b', host: '127.0.0.1', port:, starttls: false).send_mail('x', 'y') }
|
|
end
|
|
|
|
def test_notify_conditions_dry_run_and_rows
|
|
always = MemoryNotifier.new
|
|
failure = MemoryNotifier.new
|
|
problems = MemoryNotifier.new
|
|
rows = MemoryNotifier.new
|
|
run_pipeline(ROWS.map(&:dup)) do
|
|
name 'export'
|
|
on_error :skip
|
|
validate 'konto' => { required: true }
|
|
notify always, on: :always
|
|
notify failure, on: :failure
|
|
notify problems
|
|
notify rows, include_rows: true, title: 'Export Konten'
|
|
end
|
|
assert_equal [1, 0, 1], [always, failure, problems].map { |n| n.reports.size }
|
|
assert_equal ['Export Konten', true], [rows.reports.first[0].title, rows.reports.first[1]]
|
|
|
|
dry = MemoryNotifier.new
|
|
dry_too = MemoryNotifier.new
|
|
capture_stdout { run_pipeline(ROWS.map(&:dup), dry_run: true) { notify dry, on: :always; notify dry_too, on: :always, dry_run: true } }
|
|
assert_equal [0, 1], [dry.reports.size, dry_too.reports.size]
|
|
|
|
assert_raises(ArgumentError) { run_pipeline { notify Object.new } }
|
|
assert_raises(ArgumentError) { run_pipeline { notify MemoryNotifier.new, on: :sometimes } }
|
|
end
|
|
|
|
def test_notify_error_is_only_logged
|
|
log = StringIO.new
|
|
stats = DataPipeline.new do
|
|
logger Logger.new(log)
|
|
read MemoryStorage.new(rows: ROWS.map(&:dup))
|
|
write MemoryStorage.new
|
|
notify MemoryNotifier.new(fail: true), on: :always
|
|
end.run
|
|
assert stats.ok?
|
|
assert_includes log.string, 'notify NotificationTest::MemoryNotifier: RuntimeError: kaputt'
|
|
end
|
|
|
|
def test_notifier_from_connections_file
|
|
with_tmpdir do |dir|
|
|
File.write("#{dir}/c.yml", <<~YAML)
|
|
alerts: { type: webhook, url: "http://127.0.0.1:9/hook", secret: "<%= 'abc' %>" }
|
|
mail: { type: email, to: [etl@example.com], host: smtp.example.com }
|
|
db: { type: sqlite, database: x.sqlite3 }
|
|
YAML
|
|
DataPipeline.connections_file = "#{dir}/c.yml"
|
|
pipeline = DataPipeline.new { logger PipelineHelper::QUIET }
|
|
assert_kind_of DataPipeline::Webhook, pipeline.notifier(:alerts)
|
|
assert_equal 'abc', pipeline.notifier(:alerts).instance_variable_get(:@secret)
|
|
assert_kind_of DataPipeline::Email, pipeline.notifier(:mail)
|
|
assert_includes assert_raises(ArgumentError) { pipeline.notifier(:db) }.message, 'is no notifier'
|
|
end
|
|
end
|
|
end
|