require_relative '../test_helper' require_relative '../support/test_http_server' require_relative '../support/test_smtp_server' # Report, Webhook, Slack, Email and notify against local test servers class NotificationTest < PipelineTest ROWS = [{ 'id' => '1', 'konto' => '1000' }, { 'id' => '2', 'konto' => '' }].freeze # collects reports, for notify without network class MemoryNotifier < DataPipeline::Notifier attr_reader :reports def initialize(fail: false) = (@reports = []; @fail = fail) def deliver report, include_rows: false raise 'kaputt' if @fail @reports << [report, include_rows] end end def teardown @server&.stop DataPipeline.connections_file = nil super end def stats_with_rejected run_pipeline(ROWS.map(&:dup)) { name 'export'; on_error :skip; validate 'konto' => { required: true } }.last end def report(stats = stats_with_rejected) = DataPipeline::Report.new([stats], params: { from: '2026-10-01' }) def test_report_event_lines_and_payload_without_rows report = report() assert_equal ['run.rejected', 'export: mit verworfenen Zeilen'], [report.event, report.subject] assert_includes report.to_s, '- validate#1: konto: missing' assert_includes report.to_s, 'Parameter: from=2026-10-01' payload = report.to_h assert_equal [1, 'run.rejected', 'export'], payload.values_at(:schema, :event, :title) assert_equal [{ step: 'validate#1', error: 'konto: missing' }], payload[:runs].first[:errors] assert_equal({ 'id' => '2', 'konto' => '' }, report.to_h(include_rows: true)[:runs].first[:errors].first[:row]) end def test_report_worst_event_of_several_runs_and_script_error ok = run_pipeline(ROWS.map(&:dup)).last assert_equal 'run.succeeded', DataPipeline::Report.new([ok]).event assert_equal 'run.rejected', DataPipeline::Report.new([ok, stats_with_rejected]).event failed = DataPipeline::Report.new([ok], title: 'skript', error: RuntimeError.new('kaputt')) assert_equal %w[run.failed skript], [failed.event, failed.title] assert_includes failed.to_s, 'Fehler: RuntimeError: kaputt' end def test_webhook_signed_post @server = TestHttpServer.new { [200, 'ok'] } report = report() DataPipeline::Webhook.new(url: "#{@server.url}/hook", secret: 's3cret', headers: { 'X-Team' => 'etl' }).deliver(report) request = @server.requests.last headers = request[:headers] assert_equal ['/hook', 'run.rejected', report.id, 'etl'], [request[:path], headers['x-datapipeline-event'], headers['x-datapipeline-delivery'], headers['x-team']] assert_equal 1, JSON.parse(request[:body])['schema'] refute_includes request[:body], '"row"' assert DataPipeline::Webhook.valid_signature?(secret: 's3cret', body: request[:body], timestamp: headers['x-datapipeline-timestamp'], signature: headers['x-datapipeline-signature']) refute DataPipeline::Webhook.valid_signature?(secret: 'falsch', body: request[:body], timestamp: headers['x-datapipeline-timestamp'], signature: headers['x-datapipeline-signature']) refute DataPipeline::Webhook.valid_signature?(secret: 's3cret', body: request[:body], timestamp: (Time.now.to_i - 600).to_s, signature: headers['x-datapipeline-signature']) end def test_webhook_retries_429_and_5xx_but_not_4xx answers = [[503, 'busy'], [429, 'slow', { 'Retry-After' => '0' }], [200, 'ok']] @server = TestHttpServer.new { answers.shift } DataPipeline::Webhook.new(url: @server.url, retry_wait: 0).deliver(report) assert_equal 3, @server.requests.size assert_equal 1, @server.requests.map { |r| r[:headers]['x-datapipeline-delivery'] }.uniq.size @server.stop @server = TestHttpServer.new { [400, 'bad'] } error = assert_raises(DataPipeline::Webhook::Error) { DataPipeline::Webhook.new(url: @server.url, retry_wait: 0).deliver(report) } assert_includes error.message, 'HTTP 400' assert_equal 1, @server.requests.size end def test_webhook_network_error_after_retries port = TCPServer.new('127.0.0.1', 0).then { |s| s.addr[1].tap { s.close } } error = assert_raises(DataPipeline::Webhook::Error) do DataPipeline::Webhook.new(url: "http://127.0.0.1:#{port}", retries: 1, retry_wait: 0, timeout: 2).deliver(report) end assert_includes error.message, '127.0.0.1' end def test_slack_api_blocks_escape_and_mention @server = TestHttpServer.new { [200, { 'ok' => true }] } slack = DataPipeline::Slack.new(token: 'xoxb-1', channel: '#etl', api_url: @server.url, mention: '') failed = DataPipeline::Report.new([stats_with_rejected], title: 'A & ', error: RuntimeError.new('x < y')) slack.deliver(failed) request = @server.requests.last body = JSON.parse(request[:body]) assert_equal ['Bearer xoxb-1', '#etl'], [request[:headers]['authorization'], body['channel']] assert_equal ':red_circle: *A & <B>*: abgebrochen ', body['blocks'][0]['text']['text'] assert_includes body['blocks'][1]['text']['text'], 'RuntimeError: x < y' slack.deliver(report) refute_includes JSON.parse(@server.requests.last[:body])['blocks'][0]['text']['text'], '' # mention only on failure end def test_slack_error_and_incoming_webhook @server = TestHttpServer.new { |req| req[:path] == '/api' ? [200, { 'ok' => false, 'error' => 'not_in_channel' }] : [200, 'ok'] } error = assert_raises(DataPipeline::Slack::Error) { DataPipeline::Slack.new(token: 't', api_url: "#{@server.url}/api").send_message('x') } assert_includes error.message, 'not_in_channel' token = ENV.delete('SLACK_TOKEN') hook = DataPipeline::Slack.new(webhook: "#{@server.url}/hook") assert_equal 'ok', hook.deliver(report) request = @server.requests.last assert_nil request[:headers]['authorization'] refute JSON.parse(request[:body]).key?('channel') ensure ENV['SLACK_TOKEN'] = token if token end def test_email_over_smtp_with_login @server = TestSmtpServer.new mail = DataPipeline::Email.new(to: %w[a@example.com b@example.com], from: 'etl@example.com', host: '127.0.0.1', port: @server.port, starttls: false, username: 'user', password: 'geheim') mail.deliver(DataPipeline::Report.new([stats_with_rejected], title: 'Grösse Ä')) sent = @server.mails.last assert_equal ['etl@example.com', %w[a@example.com b@example.com], %w[user geheim]], sent.values_at(:from, :to, :auth) head, body = sent[:data].split("\r\n\r\n", 2) assert_includes head, "Subject: =?UTF-8?B?#{['[etl] Grösse Ä: mit verworfenen Zeilen'].pack('m0')}?=" assert_includes head, 'Content-Type: text/plain; charset=UTF-8' text = body.unpack1('m').force_encoding('UTF-8') assert_includes text, '- validate#1: konto: missing' refute_includes text, 'Zeilen:' end def test_email_rows_on_request_and_errors @server = TestSmtpServer.new mail = DataPipeline::Email.new(to: 'a@example.com', host: '127.0.0.1', port: @server.port, starttls: false) mail.deliver(report, include_rows: true) assert_includes @server.mails.last[:data].split("\r\n\r\n", 2).last.unpack1('m').force_encoding('UTF-8'), '{"id":"2","konto":""}' assert_nil @server.mails.last[:auth] assert_raises(ArgumentError) { DataPipeline::Email.new(to: []) } port = TCPServer.new('127.0.0.1', 0).then { |s| s.addr[1].tap { s.close } } assert_raises(DataPipeline::Email::Error) { DataPipeline::Email.new(to: 'a@b', host: '127.0.0.1', port:, starttls: false).send_mail('x', 'y') } end def test_notify_conditions_dry_run_and_rows always = MemoryNotifier.new failure = MemoryNotifier.new problems = MemoryNotifier.new rows = MemoryNotifier.new run_pipeline(ROWS.map(&:dup)) do name 'export' on_error :skip validate 'konto' => { required: true } notify always, on: :always notify failure, on: :failure notify problems notify rows, include_rows: true, title: 'Export Konten' end assert_equal [1, 0, 1], [always, failure, problems].map { |n| n.reports.size } assert_equal ['Export Konten', true], [rows.reports.first[0].title, rows.reports.first[1]] dry = MemoryNotifier.new dry_too = MemoryNotifier.new capture_stdout { run_pipeline(ROWS.map(&:dup), dry_run: true) { notify dry, on: :always; notify dry_too, on: :always, dry_run: true } } assert_equal [0, 1], [dry.reports.size, dry_too.reports.size] assert_raises(ArgumentError) { run_pipeline { notify Object.new } } assert_raises(ArgumentError) { run_pipeline { notify MemoryNotifier.new, on: :sometimes } } end def test_notify_error_is_only_logged log = StringIO.new stats = DataPipeline.new do logger Logger.new(log) read MemoryStorage.new(rows: ROWS.map(&:dup)) write MemoryStorage.new notify MemoryNotifier.new(fail: true), on: :always end.run assert stats.ok? assert_includes log.string, 'notify NotificationTest::MemoryNotifier: RuntimeError: kaputt' end def test_notifier_from_connections_file with_tmpdir do |dir| File.write("#{dir}/c.yml", <<~YAML) alerts: { type: webhook, url: "http://127.0.0.1:9/hook", secret: "<%= 'abc' %>" } mail: { type: email, to: [etl@example.com], host: smtp.example.com } db: { type: sqlite, database: x.sqlite3 } YAML DataPipeline.connections_file = "#{dir}/c.yml" pipeline = DataPipeline.new { logger PipelineHelper::QUIET } assert_kind_of DataPipeline::Webhook, pipeline.notifier(:alerts) assert_equal 'abc', pipeline.notifier(:alerts).instance_variable_get(:@secret) assert_kind_of DataPipeline::Email, pipeline.notifier(:mail) assert_includes assert_raises(ArgumentError) { pipeline.notifier(:db) }.message, 'is no notifier' end end end