datapipeline 2.0.0: Gem aus dem Projekt etl herausgelöst
This commit is contained in:
@@ -0,0 +1,553 @@
|
||||
require 'json'
|
||||
require 'logger'
|
||||
|
||||
require_relative 'datapipeline/version'
|
||||
require_relative 'datapipeline/storage/storage_plugin'
|
||||
require_relative 'datapipeline/validation'
|
||||
require_relative 'datapipeline/layout'
|
||||
require_relative 'datapipeline/watermark'
|
||||
|
||||
class DataPipeline
|
||||
# plugins are loaded on first use, their gems too: pg (postgres), sqlite3 (sqlite), roo/caxlsx (excel),
|
||||
# http (rest, slack)
|
||||
autoload :CsvStorage, File.expand_path('datapipeline/storage/csv', __dir__)
|
||||
autoload :PostgresStorage, File.expand_path('datapipeline/storage/postgres', __dir__)
|
||||
autoload :JsonStorage, File.expand_path('datapipeline/storage/json', __dir__)
|
||||
autoload :SqliteStorage, File.expand_path('datapipeline/storage/sqlite', __dir__)
|
||||
autoload :ExcelStorage, File.expand_path('datapipeline/storage/excel', __dir__)
|
||||
autoload :RestStorage, File.expand_path('datapipeline/storage/rest', __dir__)
|
||||
autoload :Slack, File.expand_path('datapipeline/notification/slack', __dir__)
|
||||
|
||||
# passed to header, trailer, map and filter
|
||||
# read: number of source rows (nil while streaming, until the trailer)
|
||||
# written: number of body rows after all steps (nil while the body is processed)
|
||||
# index: index of the current row within the current step
|
||||
Context = Struct.new(:read, :written, :index, keyword_init: true)
|
||||
|
||||
# result of a run, returned by run and passed to on_finish
|
||||
# read: source rows, written: body rows, output: rows to the target incl. header/trailer,
|
||||
# rejected: rows dropped by on_error :skip/:collect, errors: first 100 of them,
|
||||
# warnings: rows that failed a validate step with warn: true (kept), error: exception of an aborted run
|
||||
Stats = Struct.new(:version, :read, :written, :output, :rejected, :errors, :warnings, :started_at, :finished_at,
|
||||
:dry_run, :error, keyword_init: true) do
|
||||
def duration = finished_at && started_at ? (finished_at - started_at).round(3) : nil
|
||||
def ok? = error.nil? && rejected.zero?
|
||||
|
||||
def to_s
|
||||
"read=#{read} written=#{written} rejected=#{rejected}#{warnings.to_i.positive? ? " warnings=#{warnings}" : ''} " \
|
||||
"#{duration}s#{dry_run ? ' (dry run)' : ''}" \
|
||||
"#{error ? " ABORTED: #{error.class}: #{error.message}" : ''}"
|
||||
end
|
||||
end
|
||||
|
||||
# stops reading at the limit, not an error
|
||||
class LimitReached < StandardError; end
|
||||
|
||||
EDGE_BEFORE = %i[prepend header].freeze
|
||||
EDGE_AFTER = %i[append trailer].freeze
|
||||
CHAINING = %i[map filter validate].freeze
|
||||
ON_ERROR = %i[abort skip collect].freeze
|
||||
MAX_ERRORS = 100
|
||||
|
||||
DEFAULT_SETTINGS = {
|
||||
version: nil,
|
||||
streamed: false,
|
||||
chained: false,
|
||||
verbose: false,
|
||||
dry_run: false,
|
||||
limit: nil,
|
||||
preview: 10,
|
||||
on_error: :abort,
|
||||
log_level: Logger::WARN
|
||||
}.freeze
|
||||
|
||||
SETTINGS_LOCK = Mutex.new
|
||||
@defaults = DEFAULT_SETTINGS
|
||||
@plugins = {}
|
||||
|
||||
#
|
||||
# defaults for every following DataPipeline.new, e.g. from bin/etl
|
||||
# the hash is replaced, never changed: a running pipeline keeps its own copy
|
||||
#
|
||||
def self.settings(options = {})
|
||||
SETTINGS_LOCK.synchronize { @defaults = @defaults.merge(options).freeze } if options.any?
|
||||
@defaults
|
||||
end
|
||||
|
||||
def self.reset_settings!
|
||||
SETTINGS_LOCK.synchronize { @defaults = DEFAULT_SETTINGS }
|
||||
end
|
||||
|
||||
#
|
||||
# container :name finds a registered plugin first, then <Name>Storage by convention
|
||||
# (DataPipeline::<Name>Storage of the gem, else a top level <Name>Storage of the application)
|
||||
# DataPipeline.register :mem, MemStorage
|
||||
# DataPipeline.register :sftp, 'SftpStorage', path: '/path/to/sftp_storage' # loaded on first use
|
||||
#
|
||||
def self.register type, klass, path: nil
|
||||
Object.autoload(klass.to_sym, File.expand_path(path)) if path && klass.is_a?(String)
|
||||
SETTINGS_LOCK.synchronize { @plugins = @plugins.merge(type.to_sym => klass).freeze }
|
||||
end
|
||||
|
||||
def self.plugin type
|
||||
klass = @plugins[type.to_sym]
|
||||
klass = Object.const_get(klass) if klass.is_a?(String)
|
||||
return klass if klass
|
||||
|
||||
class_name = type.to_s.capitalize + 'Storage'
|
||||
return const_get(class_name, false) if const_defined?(class_name, false)
|
||||
|
||||
Object.const_get(class_name) if Object.const_defined?(class_name)
|
||||
end
|
||||
|
||||
#
|
||||
# parameters from the command line: bin/etl sets them; a script started with ruby directly
|
||||
# reads them from ARGV on first use: --name=value -> params[:name], --param key=value -> params[:key]
|
||||
#
|
||||
def self.params
|
||||
@params ||= parse_params(ARGV)
|
||||
end
|
||||
|
||||
def self.params= hash
|
||||
@params = hash
|
||||
end
|
||||
|
||||
def self.parse_params args
|
||||
params = {}
|
||||
args.each_with_index do |arg, i|
|
||||
case arg
|
||||
when /\A--param=(\w+)=(.*)\z/ then params[$1.to_sym] = $2
|
||||
when '--param' then key, value = args[i + 1].to_s.split('=', 2); params[key.to_sym] = value if value
|
||||
when /\A--([a-z][\w-]*)=(.*)\z/ then params[$1.tr('-', '_').to_sym] = $2
|
||||
end
|
||||
end
|
||||
params
|
||||
end
|
||||
|
||||
# stats of every run in this process, used by bin/etl for the exit code
|
||||
def self.runs
|
||||
@runs ||= []
|
||||
end
|
||||
|
||||
# DataPipeline.new(verbose: true) { ... } overrides the defaults for this pipeline only
|
||||
def initialize(**settings, &block)
|
||||
@settings = DataPipeline.settings.merge(settings)
|
||||
@transformations = []
|
||||
@targets = []
|
||||
@on_finish = []
|
||||
@logger = Logger.new($stderr, level: @settings[:log_level], progname: 'etl')
|
||||
@logger.formatter = proc { |severity, time, _prog, msg| "#{time.strftime('%F %T')} #{severity} #{msg}\n" }
|
||||
instance_eval(&block)
|
||||
end
|
||||
|
||||
def params
|
||||
self.class.params
|
||||
end
|
||||
|
||||
def configure(options = {})
|
||||
@settings.merge!(options)
|
||||
end
|
||||
|
||||
def version value
|
||||
@settings[:version] = value
|
||||
end
|
||||
|
||||
def verbose flag = true
|
||||
@settings[:verbose] = flag
|
||||
end
|
||||
|
||||
def streamed flag = true
|
||||
@settings[:streamed] = flag
|
||||
end
|
||||
|
||||
# each body step gets the output of the previous step instead of the source rows
|
||||
def chained flag = true
|
||||
@settings[:chained] = flag
|
||||
end
|
||||
|
||||
# run everything, write nothing (target, rejects, watermark); prints the first rows
|
||||
def dry_run flag = true
|
||||
@settings[:dry_run] = flag
|
||||
end
|
||||
|
||||
# for steps with side effects (api calls): true in a dry run
|
||||
def dry_run?
|
||||
@settings[:dry_run] ? true : false
|
||||
end
|
||||
|
||||
# process only the first n source rows
|
||||
def limit count
|
||||
@settings[:limit] = count
|
||||
end
|
||||
|
||||
def logger logger = nil
|
||||
@logger = logger if logger
|
||||
@logger
|
||||
end
|
||||
|
||||
#
|
||||
# what happens when a body step raises (or a validate step fails) for a row
|
||||
# on_error :abort -> stop the run (default)
|
||||
# on_error :skip -> drop the row, log a warning, count it
|
||||
# on_error :collect, to: storage -> like skip, plus the row with _step and _error into storage
|
||||
#
|
||||
def on_error mode, to: nil
|
||||
raise ArgumentError, "on_error: #{mode} unknown, use #{ON_ERROR.join(', ')}" unless ON_ERROR.include?(mode)
|
||||
raise ArgumentError, 'on_error :collect needs to: storage' if mode == :collect && to.nil?
|
||||
|
||||
@settings[:on_error] = mode
|
||||
@reject_target = to
|
||||
end
|
||||
|
||||
# on_finish { |stats| ... } -> after every run, also after an abort (stats.error)
|
||||
def on_finish(&block)
|
||||
@on_finish << block
|
||||
end
|
||||
|
||||
# see DataPipeline::Watermark
|
||||
def watermark name, column:, file:, initial: nil
|
||||
@watermark = Watermark.new(name, column:, file:, initial:)
|
||||
end
|
||||
|
||||
# only a missing plugin is reported as unsupported, errors while loading it
|
||||
# (missing gem) or from the plugin itself (connection, options) are passed through
|
||||
def container type, **options, &block
|
||||
klass = DataPipeline.plugin(type)
|
||||
raise "Unsupported storage type or class missing: #{type.to_s.capitalize}Storage" unless klass
|
||||
|
||||
klass.new(**options, &block)
|
||||
end
|
||||
|
||||
#
|
||||
# transform(options) { |row, index| ... }
|
||||
# processing: :prepend | :normal (default) | :append
|
||||
# condition: ruby expression as string (row is visible) or lambda { |row| ... }
|
||||
#
|
||||
def transform(options = {}, &block)
|
||||
kind = options.fetch(:processing, :normal)
|
||||
kind = :normal unless %i[prepend append].include?(kind)
|
||||
add_step kind, options, &block
|
||||
end
|
||||
|
||||
# map { |row, ctx| ... } -> hash, array of hashes or nil; turns chaining on
|
||||
def map(options = {}, &block)
|
||||
add_step :map, options, &block
|
||||
end
|
||||
|
||||
# filter { |row, ctx| ... } -> keeps the row when the block is truthy; turns chaining on
|
||||
def filter(options = {}, &block)
|
||||
add_step :filter, options, &block
|
||||
end
|
||||
|
||||
# header { |ctx| ... } -> rows in front of the body, called after the body with full counters
|
||||
def header(&block)
|
||||
add_step :header, {}, &block
|
||||
end
|
||||
|
||||
# trailer { |ctx| ... } -> rows after the body
|
||||
def trailer(&block)
|
||||
add_step :trailer, {}, &block
|
||||
end
|
||||
|
||||
#
|
||||
# validate 'kostl' => { required: true, max: 10 }, 'shkzg' => { in: %w[S H] }
|
||||
# validate layout -> fields of a DataPipeline::Layout
|
||||
# validate { |row| 'soll ungleich haben' if ... } -> nil, message or array of messages
|
||||
#
|
||||
# a failing row raises ValidationError, handled by on_error; turns chaining on
|
||||
# warn: true -> only a warning in the log and stats.warnings, the row is kept
|
||||
# ignore: %w[A B] -> messages for these fields are left out (known gaps)
|
||||
#
|
||||
def validate(rules = nil, **options, &block)
|
||||
warn = options.delete(:warn) || false
|
||||
ignore = Array(options.delete(:ignore)).map(&:to_s)
|
||||
rules ||= options # validate 'konto' => { ... } arrives as keywords
|
||||
@transformations << { kind: :validate, block:, options: { rules:, warn:, ignore: } }
|
||||
end
|
||||
|
||||
def read storage, options = {}
|
||||
@source = storage
|
||||
@source.set options if options.any?
|
||||
end
|
||||
|
||||
#
|
||||
# write storage -> all rows
|
||||
# write storage, only: ->(row) { ... } -> only matching rows (also header/trailer rows)
|
||||
#
|
||||
# can be called several times, every target gets its rows
|
||||
#
|
||||
def write storage, only: nil
|
||||
@targets << { storage:, only: }
|
||||
end
|
||||
|
||||
def query storage, options = {}
|
||||
storage.query options
|
||||
end
|
||||
|
||||
#
|
||||
# loads a table once, for lookups in map/transform instead of one query per row
|
||||
#
|
||||
# lookup pg, query: 'select ...', params: [...], key: 'rechnung_dbid' -> { key => [rows] }
|
||||
# lookup pg, query: 'select ...', key: 'id', unique: true -> { key => row }
|
||||
# lookup csv, key: ->(row) { row['land'] } -> reads the whole storage
|
||||
#
|
||||
def lookup storage, key:, unique: false, **options
|
||||
rows = options[:query] ? storage.query(options) : storage.read
|
||||
key_of = key.respond_to?(:call) ? key : ->(row) { row[key] }
|
||||
unique ? rows.to_h { |row| [key_of.call(row), row] } : rows.group_by(&key_of)
|
||||
end
|
||||
|
||||
def run
|
||||
@stats = Stats.new(version: @settings[:version], read: 0, written: 0, output: 0, rejected: 0, errors: [], warnings: 0,
|
||||
started_at: Time.now, dry_run: @settings[:dry_run])
|
||||
@logger.info "start#{" #{@settings[:version]}" if @settings[:version]}#{@settings[:dry_run] ? ' (dry run)' : ''}"
|
||||
@preview = []
|
||||
raise ArgumentError, 'read storage missing' unless @source
|
||||
raise ArgumentError, 'write storage missing' if @targets.empty?
|
||||
|
||||
before = @transformations.select { |t| EDGE_BEFORE.include?(t[:kind]) }
|
||||
after = @transformations.select { |t| EDGE_AFTER.include?(t[:kind]) }
|
||||
body = @transformations - before - after
|
||||
|
||||
@settings[:streamed] ? run_streamed(before, body, after) : run_batch(before, body, after)
|
||||
|
||||
@watermark&.save unless @settings[:dry_run]
|
||||
@stats
|
||||
rescue StandardError => e
|
||||
@stats.error = e
|
||||
@logger.error "abort: #{e.class}: #{e.message}"
|
||||
raise
|
||||
ensure
|
||||
finish
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def add_step kind, options, &block
|
||||
raise ArgumentError, "#{kind}: block missing" unless block
|
||||
|
||||
@transformations << { kind:, block:, options: }
|
||||
end
|
||||
|
||||
# without any transformation the source is copied unchanged
|
||||
def chained?
|
||||
@settings[:chained] || @transformations.empty? ||
|
||||
@transformations.any? { |t| CHAINING.include?(t[:kind]) }
|
||||
end
|
||||
|
||||
def write_row row
|
||||
@targets.each { |t| t[:storage].write_row(row) if t[:only].nil? || t[:only].call(row) }
|
||||
end
|
||||
|
||||
def finish
|
||||
@stats.finished_at = Time.now
|
||||
[@source, *@targets.map { |t| t[:storage] }, @reject_target].compact.uniq.each(&:close)
|
||||
self.class.runs << @stats
|
||||
print_preview if @stats.dry_run && @stats.error.nil?
|
||||
@logger.info "finish #{@stats}"
|
||||
@on_finish.each do |hook|
|
||||
hook.call(@stats)
|
||||
rescue StandardError => e
|
||||
@logger.error "on_finish: #{e.class}: #{e.message}"
|
||||
end
|
||||
end
|
||||
|
||||
def source_rows
|
||||
data = @source.read
|
||||
data = data.first(@settings[:limit]) if @settings[:limit]
|
||||
data
|
||||
end
|
||||
|
||||
def output rows
|
||||
@stats.output += rows.size
|
||||
@preview.concat(rows.first(@settings[:preview] - @preview.size)) if @preview.size < @settings[:preview]
|
||||
return if @settings[:dry_run]
|
||||
|
||||
yield rows
|
||||
end
|
||||
|
||||
def print_preview
|
||||
puts "DRY RUN – nothing written: #{@stats}"
|
||||
@preview.each { |row| puts JSON.generate(row) }
|
||||
end
|
||||
|
||||
# whole source in memory, one target.write at the end
|
||||
def run_batch before, body, after
|
||||
data = source_rows
|
||||
@stats.read = data.length
|
||||
data.each { |row| @watermark.track(row) } if @watermark
|
||||
dump 'Source#data', data
|
||||
|
||||
# prepend keeps its old timing: called before the body
|
||||
results = {}.compare_by_identity
|
||||
before.select { |t| t[:kind] == :prepend }.each { |t| results[t] = edge(t, nil, 0) }
|
||||
|
||||
rows = []
|
||||
if chained?
|
||||
counters = Array.new(body.size, 0)
|
||||
data.each { |row| push(row, body, counters, data.length) { |out| rows << out } }
|
||||
else
|
||||
body.each { |step| rows.concat apply(step, data) }
|
||||
end
|
||||
dump 'Transformation#data', rows
|
||||
|
||||
ctx = Context.new(read: data.length, written: rows.length)
|
||||
before.select { |t| t[:kind] == :header }.each { |t| results[t] = edge(t, ctx, 0) }
|
||||
|
||||
# parallel mode keeps the old count of source rows for append
|
||||
count = chained? ? rows.length : data.length
|
||||
new_data = []
|
||||
before.each { |t| add_rows new_data, results[t] }
|
||||
new_data.concat rows
|
||||
after.each { |t| add_rows new_data, edge(t, ctx, count) }
|
||||
|
||||
@stats.written = rows.length
|
||||
dump 'Target#data', new_data
|
||||
output(new_data) do |all|
|
||||
@targets.each { |t| t[:storage].write(t[:only] ? all.select(&t[:only]) : all) }
|
||||
end
|
||||
end
|
||||
|
||||
# row by row from source.read { } to target.write_row, header only knows nothing yet
|
||||
def run_streamed before, body, after
|
||||
if !chained? && body.size > 1
|
||||
raise ArgumentError, 'streamed: several transformations need chained (or map/filter)'
|
||||
end
|
||||
|
||||
emit = ->(result) { output(add_rows([], result)) { |rows| rows.each { |row| write_row(row) } } }
|
||||
|
||||
before.each { |t| emit.call edge(t, Context.new, 0) }
|
||||
|
||||
read = written = 0
|
||||
counters = Array.new(body.size, 0)
|
||||
begin
|
||||
@source.read do |row|
|
||||
raise LimitReached if @settings[:limit] && read >= @settings[:limit]
|
||||
|
||||
read += 1
|
||||
@watermark&.track(row)
|
||||
push(row, body, counters, nil) do |out|
|
||||
written += 1
|
||||
emit.call out
|
||||
end
|
||||
end
|
||||
rescue LimitReached
|
||||
@logger.info "limit #{@settings[:limit]} reached"
|
||||
end
|
||||
@stats.read = read
|
||||
@stats.written = written
|
||||
|
||||
ctx = Context.new(read:, written:)
|
||||
after.each { |t| emit.call edge(t, ctx, chained? ? written : read) }
|
||||
@targets.each { |t| t[:storage].finalize }
|
||||
end
|
||||
|
||||
# one row through the chain, depth first; counters hold the index per step
|
||||
def push row, body, counters, read, level = 0, &emit
|
||||
return emit.call(row) if level == body.size
|
||||
|
||||
index = counters[level]
|
||||
counters[level] += 1
|
||||
safe_step_rows(body[level], row, index, read).each do |out|
|
||||
push(out, body, counters, read, level + 1, &emit)
|
||||
end
|
||||
end
|
||||
|
||||
# parallel mode: every step reads its own copy of the source rows
|
||||
def apply step, rows
|
||||
rows.each_with_index.flat_map { |row, index| safe_step_rows(step, row.dup, index, rows.length) }
|
||||
end
|
||||
|
||||
# errors of a step for one row go to on_error
|
||||
def safe_step_rows step, row, index, read
|
||||
step_rows(step, row, index, read)
|
||||
rescue StandardError => e
|
||||
reject(step, row, e)
|
||||
[]
|
||||
end
|
||||
|
||||
def reject step, row, error
|
||||
raise error if @settings[:on_error] == :abort
|
||||
|
||||
name = step_name(step)
|
||||
@stats.rejected += 1
|
||||
@stats.errors << { step: name, error: error.message, row: } if @stats.errors.size < MAX_ERRORS
|
||||
@logger.warn "rejected by #{name}: #{error.message}"
|
||||
return if @settings[:dry_run] || @reject_target.nil?
|
||||
|
||||
@reject_target.write_row(row.merge('_step' => name, '_error' => error.message))
|
||||
end
|
||||
|
||||
def step_name step
|
||||
"#{step[:kind]}##{@transformations.index(step) + 1}"
|
||||
end
|
||||
|
||||
# rows produced by one step for one input row
|
||||
def step_rows step, row, index, read
|
||||
return [] unless condition?(step, row)
|
||||
|
||||
case step[:kind]
|
||||
when :validate
|
||||
rules = step[:options][:rules]
|
||||
messages = rules.is_a?(Layout) ? rules.errors(row) : Validation.errors(row, rules)
|
||||
messages += Array(step[:block].call(row)).compact if step[:block]
|
||||
ignore = step[:options][:ignore]
|
||||
messages = messages.reject { |m| ignore.any? { |field| m.start_with?("#{field}:") } } if ignore.any?
|
||||
return [row] if messages.empty?
|
||||
raise ValidationError, messages.join('; ') unless step[:options][:warn]
|
||||
|
||||
@stats.warnings += 1
|
||||
if @stats.warnings <= MAX_ERRORS
|
||||
@logger.warn "#{step_name(step)}: #{messages.join('; ')}"
|
||||
elsif @stats.warnings == MAX_ERRORS + 1
|
||||
@logger.warn "more than #{MAX_ERRORS} warnings, further ones only counted"
|
||||
end
|
||||
[row]
|
||||
when :filter
|
||||
step[:block].call(row, Context.new(read:, index:)) ? [row] : []
|
||||
when :map
|
||||
add_rows [], step[:block].call(row, Context.new(read:, index:))
|
||||
else
|
||||
add_rows [], step[:block].call(row, index)
|
||||
end
|
||||
end
|
||||
|
||||
def edge step, ctx, count
|
||||
case step[:kind]
|
||||
when :prepend then step[:block].call(nil, 0)
|
||||
when :append then step[:block].call(nil, count)
|
||||
else step[:block].call(ctx)
|
||||
end
|
||||
end
|
||||
|
||||
# string conditions are evaluated with the local variable row
|
||||
def condition? step, row
|
||||
condition = step[:options].fetch(:condition, true)
|
||||
return true if condition == true
|
||||
return condition.call(row) if condition.respond_to?(:call)
|
||||
|
||||
eval(condition)
|
||||
end
|
||||
|
||||
# a hash is one row, an array are several rows, nil is no row
|
||||
def add_rows new_data, result
|
||||
if result.is_a?(Array)
|
||||
new_data.concat(result.compact)
|
||||
elsif result
|
||||
new_data << result
|
||||
end
|
||||
new_data
|
||||
end
|
||||
|
||||
def dump title, data
|
||||
return unless @settings[:verbose]
|
||||
|
||||
puts "#{title}".ljust(28, '-')
|
||||
puts JSON.pretty_generate data
|
||||
puts "----------------------------"
|
||||
end
|
||||
end
|
||||
|
||||
# short names besides the <Name>Storage convention
|
||||
DataPipeline.register :sqlite3, 'DataPipeline::SqliteStorage'
|
||||
DataPipeline.register :xlsx, 'DataPipeline::ExcelStorage'
|
||||
@@ -0,0 +1,113 @@
|
||||
require 'date'
|
||||
|
||||
class DataPipeline
|
||||
class LayoutError < StandardError; end
|
||||
|
||||
#
|
||||
# fixed field list per record type, for files with several record types (e.g. SAP CUDO)
|
||||
#
|
||||
# layout = DataPipeline::Layout.new do
|
||||
# record 'DOHD', [['SATYP', 'AN', 4, true], ['LFDNR', 'N', 10, true], ['BELNR', 'AN', 10], ...]
|
||||
# record 'LARC', [['SATYP', 'AN', 4, true], ['SUMME', 'N', 10, true]]
|
||||
# end
|
||||
#
|
||||
# field: [name, type, length, required]
|
||||
# type AN (text), N (number, length 10 or 13.2 = digits.decimals), DAT (YYYYMMDD), TIM (HHMMSS)
|
||||
#
|
||||
# the record type of a row is the value of type_field (default: the first value of the row).
|
||||
# keys of a row are matched without regard to case, string or symbol.
|
||||
#
|
||||
# write container(:csv, file: ..., layout: layout) -> values in field order, unknown keys raise
|
||||
# validate layout -> required, length, format per field
|
||||
#
|
||||
class Layout
|
||||
Field = Struct.new(:name, :type, :length, :required) do
|
||||
def key = name.downcase
|
||||
|
||||
# 13.2 -> [11, 2] digits before and after the point, 10 -> [10, 0]
|
||||
def digits
|
||||
before, after = length.to_s.split('.').map(&:to_i)
|
||||
after ||= 0
|
||||
[before - after, after]
|
||||
end
|
||||
end
|
||||
|
||||
attr_reader :records
|
||||
|
||||
def initialize type_field: nil, strict: true, &block
|
||||
@type_field = type_field&.to_s&.downcase
|
||||
@strict = strict
|
||||
@records = {}
|
||||
instance_eval(&block) if block
|
||||
end
|
||||
|
||||
def record type, fields
|
||||
@records[type.to_s] = fields.map do |name, kind, length, required|
|
||||
Field.new(name.to_s, (kind || 'AN').to_s.upcase, length, required ? true : false)
|
||||
end
|
||||
end
|
||||
|
||||
def fields_for row
|
||||
type = record_type(row)
|
||||
@records.fetch(type.to_s) { raise LayoutError, "record type #{type.inspect} not in layout (#{@records.keys.join(', ')})" }
|
||||
end
|
||||
|
||||
# values in the order of the layout; unknown keys raise (strict), missing keys are nil
|
||||
def values row
|
||||
fields = fields_for(row)
|
||||
normalized = normalize(row)
|
||||
if @strict
|
||||
unknown = normalized.keys - fields.map(&:key)
|
||||
raise LayoutError, "#{record_type(row)}: fields not in layout: #{unknown.join(', ')}" if unknown.any?
|
||||
end
|
||||
fields.map { |field| normalized[field.key] }
|
||||
end
|
||||
|
||||
def errors row
|
||||
fields = fields_for(row)
|
||||
normalized = normalize(row)
|
||||
fields.flat_map { |field| check(field, normalized[field.key]) }
|
||||
rescue LayoutError => e
|
||||
[e.message]
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def record_type row
|
||||
@type_field ? normalize(row)[@type_field] : row.values.first
|
||||
end
|
||||
|
||||
def normalize row
|
||||
row.to_h { |key, value| [key.to_s.downcase, value] }
|
||||
end
|
||||
|
||||
def check field, value
|
||||
text = value.to_s
|
||||
return(field.required ? ["#{field.name}: missing"] : []) if text.strip.empty?
|
||||
|
||||
case field.type
|
||||
when 'N' then check_number(field, text)
|
||||
when 'DAT' then valid_date?(text) ? [] : ["#{field.name}: not a date YYYYMMDD"]
|
||||
when 'TIM' then text.match?(/\A([01]\d|2[0-3])[0-5]\d[0-5]\d\z/) ? [] : ["#{field.name}: not a time HHMMSS"]
|
||||
else
|
||||
field.length && text.length > field.length.to_i ? ["#{field.name}: longer than #{field.length}"] : []
|
||||
end
|
||||
end
|
||||
|
||||
def check_number field, text
|
||||
return ["#{field.name}: not a number"] unless text.match?(/\A[+-]?\d+(\.\d+)?\z/)
|
||||
return [] unless field.length
|
||||
|
||||
before, after = field.digits
|
||||
int, dec = text.delete('+-').split('.')
|
||||
return ["#{field.name}: more than #{before} digits"] if int.length > before
|
||||
return ["#{field.name}: more than #{after} decimals"] if dec && dec.length > after
|
||||
|
||||
[]
|
||||
end
|
||||
|
||||
def valid_date? text
|
||||
text.match?(/\A\d{8}\z/) && Date.valid_date?(text[0, 4].to_i, text[4, 2].to_i, text[6, 2].to_i)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,36 @@
|
||||
require 'http'
|
||||
require 'json'
|
||||
|
||||
class DataPipeline
|
||||
#
|
||||
# Slack.new(channel: '#etl').send_message('posting export done')
|
||||
#
|
||||
# options
|
||||
# token: bot token, default ENV['SLACK_TOKEN']
|
||||
# channel: default '#general'
|
||||
#
|
||||
class Slack
|
||||
API_URL = 'https://slack.com/api/chat.postMessage'.freeze
|
||||
|
||||
DEFAULTS = {
|
||||
token: nil,
|
||||
channel: '#general'
|
||||
}.freeze
|
||||
|
||||
def initialize options = {}
|
||||
@options = DEFAULTS.merge(token: ENV['SLACK_TOKEN']).merge(options)
|
||||
raise ArgumentError, 'Slack: token missing (option token: or ENV SLACK_TOKEN)' unless @options[:token]
|
||||
end
|
||||
|
||||
# returns the parsed slack response, raises when slack reports an error
|
||||
def send_message(content, verbose: false)
|
||||
response = HTTP.auth("Bearer #{@options[:token]}")
|
||||
.post(API_URL, json: { channel: @options[:channel], text: content })
|
||||
body = JSON.parse(response.body.to_s)
|
||||
puts JSON.pretty_generate(body) if verbose
|
||||
raise "Slack#send_message failed: #{body['error']}" unless body['ok']
|
||||
|
||||
body
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,121 @@
|
||||
require 'csv'
|
||||
require 'json'
|
||||
require_relative 'storage_plugin'
|
||||
|
||||
class DataPipeline
|
||||
#
|
||||
# options
|
||||
# file: path of the csv file
|
||||
# csv: options passed to ruby CSV, merged with DEFAULTS
|
||||
# columns: fixed column order for writing (and keys for reading without headers)
|
||||
# without columns the values are written in hash order
|
||||
# layout: DataPipeline::Layout, field order per record type (several record types in one file),
|
||||
# unknown fields raise; no header line
|
||||
# counter: print a dot per written row
|
||||
# verbose: print settings
|
||||
#
|
||||
class CsvStorage < StoragePlugin
|
||||
DEFAULTS = {
|
||||
verbose: false,
|
||||
counter: false,
|
||||
columns: nil,
|
||||
csv: {
|
||||
headers: true,
|
||||
col_sep: ',',
|
||||
empty_value: ''
|
||||
}
|
||||
}.freeze
|
||||
|
||||
def initialize **options, &block
|
||||
@options = DEFAULTS.merge(options) do |_key, default, value|
|
||||
default.is_a?(Hash) && value.is_a?(Hash) ? default.merge(value) : value
|
||||
end
|
||||
@file_name = @options.fetch(:file) { raise ArgumentError, 'CsvStorage: option file missing' }
|
||||
@verbose = @options[:verbose]
|
||||
|
||||
if @verbose
|
||||
puts "CsvStorage#options--------"
|
||||
puts JSON.pretty_generate @options
|
||||
puts "---------------------------"
|
||||
end
|
||||
|
||||
yield if block_given?
|
||||
end
|
||||
|
||||
def read(&block)
|
||||
if block_given?
|
||||
CSV.foreach(@file_name, **csv_options) { |row| yield to_hash(row) }
|
||||
return
|
||||
end
|
||||
|
||||
CSV.read(@file_name, **csv_options).map { |row| to_hash(row) }
|
||||
end
|
||||
|
||||
def write data
|
||||
puts "> CsvStorage#write" if @verbose
|
||||
open_file
|
||||
data.each { |row| write_row row }
|
||||
ensure
|
||||
close
|
||||
end
|
||||
|
||||
def write_row row
|
||||
open_file
|
||||
if @options[:layout]
|
||||
@file << @options[:layout].values(row)
|
||||
print '.' if @options[:counter]
|
||||
return
|
||||
end
|
||||
|
||||
if !@header_written && csv_options[:headers]
|
||||
header = columns || row.keys
|
||||
@file << header.map(&:to_s) if header
|
||||
end
|
||||
@header_written = true
|
||||
|
||||
@file << (columns ? columns.map { |col| fetch(row, col) } : row.values)
|
||||
print "." if @options[:counter]
|
||||
end
|
||||
|
||||
def close
|
||||
@file&.close
|
||||
@file = nil
|
||||
@header_written = false
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def csv_options
|
||||
@options[:csv]
|
||||
end
|
||||
|
||||
def columns
|
||||
@options[:columns]
|
||||
end
|
||||
|
||||
def open_file
|
||||
return if @file
|
||||
|
||||
@file = CSV.open(@file_name, 'w', **csv_options)
|
||||
@header_written = false
|
||||
return unless columns && csv_options[:headers] && !@options[:layout]
|
||||
|
||||
@file << columns.map(&:to_s)
|
||||
@header_written = true
|
||||
end
|
||||
|
||||
# rows with headers are CSV::Row, without headers plain arrays
|
||||
def to_hash row
|
||||
return row.to_h if row.is_a?(CSV::Row)
|
||||
|
||||
keys = columns&.map(&:to_s) || (0...row.length).map(&:to_s)
|
||||
keys.zip(row).to_h
|
||||
end
|
||||
|
||||
def fetch row, col
|
||||
return row[col] if row.key?(col)
|
||||
|
||||
row.key?(col.to_s) ? row[col.to_s] : row[col.to_s.to_sym]
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,108 @@
|
||||
require_relative 'storage_plugin'
|
||||
|
||||
class DataPipeline
|
||||
#
|
||||
# options
|
||||
# file: path of the .xlsx file
|
||||
# sheet: read: name or index (0 = first, default), write: name (default 'Daten')
|
||||
# headers: first row holds the column names (default true)
|
||||
# columns: column names when there is no header row, fixed column order for writing
|
||||
# skip: read: rows before the header row (title lines), default 0
|
||||
#
|
||||
# read needs the gem roo, write the gem caxlsx; both are loaded on first use.
|
||||
# text stays text when writing (no number conversion, leading zeros are kept).
|
||||
#
|
||||
class ExcelStorage < StoragePlugin
|
||||
def initialize **options, &block
|
||||
@options = { headers: true, skip: 0 }.merge(options)
|
||||
@file_name = @options.fetch(:file) { raise ArgumentError, 'ExcelStorage: option file missing' }
|
||||
|
||||
yield if block_given?
|
||||
end
|
||||
|
||||
# without block all rows, with block row by row (xlsx is streamed)
|
||||
def read(&block)
|
||||
return enum_for(:each_row).to_a unless block
|
||||
|
||||
each_row(&block)
|
||||
nil
|
||||
end
|
||||
|
||||
def write rows
|
||||
open_sheet
|
||||
rows.each { |row| write_row row }
|
||||
ensure
|
||||
close
|
||||
end
|
||||
|
||||
def write_row row
|
||||
open_sheet
|
||||
if @columns.nil?
|
||||
@columns = (@options[:columns] || row.keys).map(&:to_s)
|
||||
add_row @columns, header: true if @options[:headers]
|
||||
end
|
||||
add_row(@columns.map { |col| fetch(row, col) })
|
||||
end
|
||||
|
||||
def close
|
||||
return unless @package
|
||||
|
||||
@package.serialize(@file_name)
|
||||
@package = @sheet = @columns = nil
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def each_row
|
||||
self.class.require_gem 'roo'
|
||||
book = Roo::Spreadsheet.open(@file_name, extension: File.extname(@file_name).delete('.').to_sym)
|
||||
sheet = book.sheet(@options.fetch(:sheet, 0))
|
||||
keys = @options[:headers] ? nil : @options[:columns]&.map(&:to_s)
|
||||
skip = @options[:skip]
|
||||
|
||||
rows_of(sheet).each do |values|
|
||||
next if (skip -= 1) >= 0
|
||||
next if values.all? { |v| v.nil? || v.to_s.strip.empty? }
|
||||
|
||||
if keys.nil?
|
||||
keys = values.each_with_index.map { |v, i| v.nil? ? i.to_s : v.to_s }
|
||||
next
|
||||
end
|
||||
yield keys.zip(values).to_h
|
||||
end
|
||||
ensure
|
||||
book&.close
|
||||
end
|
||||
|
||||
# xlsx row by row without loading the sheet, other formats over the row api
|
||||
def rows_of sheet
|
||||
if sheet.respond_to?(:each_row_streaming)
|
||||
Enumerator.new do |y|
|
||||
sheet.each_row_streaming(pad_cells: true) { |cells| y << cells.map { |c| c&.value } }
|
||||
end
|
||||
else
|
||||
(sheet.first_row.to_i..sheet.last_row.to_i).lazy.map { |i| sheet.row(i) }
|
||||
end
|
||||
end
|
||||
|
||||
def open_sheet
|
||||
return if @package
|
||||
|
||||
self.class.require_gem 'axlsx', 'caxlsx'
|
||||
@package = Axlsx::Package.new
|
||||
@sheet = @package.workbook.add_worksheet(name: @options.fetch(:sheet, 'Daten').to_s[0, 31])
|
||||
@bold = @package.workbook.styles.add_style(b: true)
|
||||
end
|
||||
|
||||
def add_row values, header: false
|
||||
types = values.map { |v| v.is_a?(String) ? :string : nil }
|
||||
@sheet.add_row(values, types:, style: header ? @bold : nil)
|
||||
end
|
||||
|
||||
def fetch row, col
|
||||
return row[col] if row.key?(col)
|
||||
|
||||
row.key?(col.to_sym) ? row[col.to_sym] : nil
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,79 @@
|
||||
require 'json'
|
||||
require_relative 'storage_plugin'
|
||||
|
||||
class DataPipeline
|
||||
#
|
||||
# options
|
||||
# file: path of the file
|
||||
# format: :array (one json array) or :lines (one object per line, ndjson)
|
||||
# default from the extension: .ndjson / .jsonl -> :lines, else :array
|
||||
# root: read only: path to the array inside the document, e.g. 'data.items'
|
||||
# a single object (instead of an array) is read as one row
|
||||
# pretty: write :array pretty printed (default false)
|
||||
#
|
||||
class JsonStorage < StoragePlugin
|
||||
LINE_EXTENSIONS = %w[.ndjson .jsonl].freeze
|
||||
|
||||
def initialize **options, &block
|
||||
@options = options
|
||||
@file_name = options.fetch(:file) { raise ArgumentError, 'JsonStorage: option file missing' }
|
||||
@format = options.fetch(:format) { LINE_EXTENSIONS.include?(File.extname(@file_name)) ? :lines : :array }
|
||||
raise ArgumentError, "JsonStorage: format #{@format} unknown, use :array or :lines" unless %i[array lines].include?(@format)
|
||||
|
||||
yield if block_given?
|
||||
end
|
||||
|
||||
# without block all rows, with block row by row (ndjson is read line by line)
|
||||
def read(&block)
|
||||
rows = @format == :lines ? read_lines : read_array
|
||||
return rows.to_a unless block
|
||||
|
||||
rows.each(&block)
|
||||
nil
|
||||
end
|
||||
|
||||
def write rows
|
||||
if @format == :lines
|
||||
File.open(@file_name, 'w') { |f| rows.each { |row| f.puts JSON.generate(row) } }
|
||||
else
|
||||
File.write(@file_name, @options[:pretty] ? JSON.pretty_generate(rows) : JSON.generate(rows))
|
||||
end
|
||||
end
|
||||
|
||||
# streaming: ndjson line by line, array as "[row,row,...]" closed in close
|
||||
def write_row row
|
||||
@file ||= File.open(@file_name, 'w').tap { |f| f.write('[') if @format == :array }
|
||||
if @format == :array
|
||||
@file.write(',') if @rows_written
|
||||
@file.write(JSON.generate(row))
|
||||
else
|
||||
@file.puts JSON.generate(row)
|
||||
end
|
||||
@rows_written = true
|
||||
end
|
||||
|
||||
def close
|
||||
return unless @file
|
||||
|
||||
@file.write(']') if @format == :array
|
||||
@file.close
|
||||
@file = nil
|
||||
@rows_written = false
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def read_lines
|
||||
File.foreach(@file_name).lazy.map(&:strip).reject(&:empty?).map { |line| JSON.parse(line) }
|
||||
end
|
||||
|
||||
def read_array
|
||||
data = JSON.parse(File.read(@file_name))
|
||||
@options[:root].to_s.split('.').each { |key| data = data.fetch(key) } if @options[:root]
|
||||
data = [data] if data.is_a?(Hash) # a single object is one row
|
||||
raise ArgumentError, "JsonStorage: #{@file_name} does not contain an array" unless data.is_a?(Array)
|
||||
|
||||
data
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,125 @@
|
||||
require_relative 'storage_plugin'
|
||||
|
||||
class DataPipeline
|
||||
#
|
||||
# options
|
||||
# host, port, database, username, password
|
||||
# query: sql for read, placeholders $1, $2, ...
|
||||
# params: values for the placeholders of query
|
||||
# table: target table for write
|
||||
# truncate: true -> delete all rows of the table before the first write (same transaction)
|
||||
# typed: true -> values come back as ruby types (Integer, Float, Date, ...)
|
||||
# false (default) -> all values as String, NULL as nil
|
||||
# fetch_size: rows per fetch when reading with a block (default 1000)
|
||||
#
|
||||
class PostgresStorage < StoragePlugin
|
||||
def initialize **options, &block
|
||||
self.class.require_gem 'pg'
|
||||
@options = options
|
||||
|
||||
begin
|
||||
@conn = PG.connect(
|
||||
host: options[:host],
|
||||
port: options[:port] || '5432',
|
||||
dbname: options[:database],
|
||||
user: options[:username],
|
||||
password: options[:password]
|
||||
)
|
||||
rescue PG::ConnectionBad => e
|
||||
raise PG::ConnectionBad, "Error connection to database: #{e.message}"
|
||||
end
|
||||
|
||||
@conn.type_map_for_queries = PG::BasicTypeMapForQueries.new(@conn)
|
||||
@conn.type_map_for_results = PG::BasicTypeMapForResults.new(@conn) if options[:typed]
|
||||
|
||||
yield if block_given?
|
||||
end
|
||||
|
||||
# without block all rows at once, with block row by row through a server side cursor
|
||||
def read(&block)
|
||||
raise "Query for PostgresStorage missing" unless @options[:query]
|
||||
|
||||
params = @options.fetch(:params, [])
|
||||
return query(query: @options[:query], params:) unless block_given?
|
||||
|
||||
each_row(@options[:query], params, &block)
|
||||
end
|
||||
|
||||
def query options = {}
|
||||
raise "Query for PostgresStorage:query missing" unless options[:query]
|
||||
|
||||
result = @conn.exec_params(options[:query], options.fetch(:params, []))
|
||||
return result.to_a unless block_given?
|
||||
|
||||
result.each { |row| yield row }
|
||||
nil
|
||||
end
|
||||
|
||||
#
|
||||
# write rows -> table from option table:
|
||||
# write table_name, rows -> explicit table (old signature)
|
||||
#
|
||||
# all rows in one transaction, values as bind parameters, nil -> NULL
|
||||
#
|
||||
def write table_or_data, data = nil
|
||||
table_name, data = data.nil? ? [@options[:table], table_or_data] : [table_or_data, data]
|
||||
raise "Table for PostgresStorage#write missing" unless table_name
|
||||
|
||||
@conn.transaction do
|
||||
truncate_once table_name
|
||||
data.each { |row| insert table_name, row }
|
||||
end
|
||||
end
|
||||
|
||||
def write_row row
|
||||
raise "Table for PostgresStorage#write_row missing" unless @options[:table]
|
||||
|
||||
truncate_once @options[:table]
|
||||
insert @options[:table], row
|
||||
end
|
||||
|
||||
def close
|
||||
@conn.close if @conn && !@conn.finished?
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
# rows are fetched in batches inside a transaction; other queries on the
|
||||
# same connection (lookups, subqueries) are possible while reading
|
||||
def each_row sql, params
|
||||
size = Integer(@options.fetch(:fetch_size, 1000))
|
||||
@conn.transaction do
|
||||
@conn.exec_params("declare etl_read no scroll cursor for #{sql.strip.chomp(';')}", params)
|
||||
loop do
|
||||
result = @conn.exec("fetch #{size} from etl_read")
|
||||
break if result.ntuples.zero?
|
||||
|
||||
result.each { |row| yield row }
|
||||
end
|
||||
@conn.exec('close etl_read')
|
||||
end
|
||||
nil
|
||||
end
|
||||
|
||||
def truncate_once table_name
|
||||
return unless @options[:truncate] && !(@truncated ||= {})[table_name]
|
||||
|
||||
@conn.exec("delete from #{quote_table(table_name)}")
|
||||
@truncated[table_name] = true
|
||||
end
|
||||
|
||||
def insert table_name, row
|
||||
columns = row.keys.map { |col| @conn.quote_ident(col.to_s) }.join(',')
|
||||
placeholders = (1..row.size).map { |i| "$#{i}" }.join(',')
|
||||
@conn.exec_params(
|
||||
"insert into #{quote_table(table_name)} (#{columns}) values (#{placeholders})",
|
||||
row.values
|
||||
)
|
||||
end
|
||||
|
||||
# schema.table -> "schema"."table"
|
||||
def quote_table name
|
||||
name.to_s.split('.').map { |part| @conn.quote_ident(part) }.join('.')
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,180 @@
|
||||
require 'json'
|
||||
require_relative 'storage_plugin'
|
||||
|
||||
class DataPipeline
|
||||
#
|
||||
# options
|
||||
# url: base url (read and write)
|
||||
# headers: hash of request headers
|
||||
# auth: value of the Authorization header, e.g. "Bearer #{ENV['API_TOKEN']}"
|
||||
# params: query parameters for read
|
||||
# root: path to the array in the response, e.g. 'data.items' (default: the response itself)
|
||||
# timeout: seconds (default 30)
|
||||
# ssl_verify: false -> accept any server certificate (own CA, IP address); default true
|
||||
# retries: repeats on 5xx, 429 and network errors (default 2), wait retry_wait * attempt seconds
|
||||
#
|
||||
# paging for read (one of)
|
||||
# page_param: 'page' -> page_start (default 1), page_start + 1, ... until a page returns no rows
|
||||
# next: ->(body) { body['next'] } -> url of the next page or nil
|
||||
# max_pages: safety stop (default 1000)
|
||||
#
|
||||
# write
|
||||
# write_url: default url; or ->(row) { "#{base}/kunden/#{row['id']}" } for one url per row
|
||||
# write_method: :post (default), :put, :patch
|
||||
# batch_size: nil -> one request per row, n -> json array of n rows per request
|
||||
#
|
||||
# query(query: 'path', params: {...}) -> GET url + path, rows like read (for lookups)
|
||||
#
|
||||
# call(:post, 'person/listkunden', json: {...}) -> one request, parsed json back (for rpc style apis)
|
||||
# retries: 0 for requests that create something, so a 5xx is not sent twice
|
||||
# a failed request raises RestStorage::Error with #status (nil for network errors)
|
||||
#
|
||||
class RestStorage < StoragePlugin
|
||||
class Error < StandardError
|
||||
attr_reader :status
|
||||
|
||||
def initialize message = nil, status: nil
|
||||
super(message)
|
||||
@status = status
|
||||
end
|
||||
end
|
||||
|
||||
class RetryableError < Error; end
|
||||
|
||||
RETRY_STATUS = [429, 500, 502, 503, 504].freeze
|
||||
|
||||
def initialize **options, &block
|
||||
self.class.require_gem 'http'
|
||||
@options = { timeout: 30, retries: 2, retry_wait: 1, max_pages: 1000, write_method: :post }.merge(options)
|
||||
@url = @options.fetch(:url) { raise ArgumentError, 'RestStorage: option url missing' }
|
||||
if @options[:batch_size] && @options[:write_url].respond_to?(:call)
|
||||
raise ArgumentError, 'RestStorage: write_url per row does not work with batch_size'
|
||||
end
|
||||
@buffer = []
|
||||
|
||||
yield if block_given?
|
||||
end
|
||||
|
||||
def read(&block)
|
||||
rows = enum_for(:each_page_row, @url, @options.fetch(:params, {}))
|
||||
return rows.to_a unless block
|
||||
|
||||
rows.each(&block)
|
||||
nil
|
||||
end
|
||||
|
||||
def query options = {}
|
||||
path = options[:query].to_s
|
||||
url = path.start_with?('http') ? path : "#{@url.chomp('/')}/#{path.delete_prefix('/')}"
|
||||
rows = enum_for(:each_page_row, url, options.fetch(:params, {}))
|
||||
return rows.to_a unless block_given?
|
||||
|
||||
rows.each { |row| yield row }
|
||||
nil
|
||||
end
|
||||
|
||||
# one request against url + path; json body and/or query params
|
||||
def call method, path, json: nil, params: nil, retries: nil
|
||||
url = path.to_s.start_with?('http') ? path.to_s : "#{@url.chomp('/')}/#{path.to_s.delete_prefix('/')}"
|
||||
args = { json:, params: }.compact
|
||||
request(method, url, retries:, **args)
|
||||
end
|
||||
|
||||
def write rows
|
||||
rows.each { |row| write_row row }
|
||||
ensure
|
||||
close
|
||||
end
|
||||
|
||||
def write_row row
|
||||
if @options[:batch_size]
|
||||
@buffer << row
|
||||
flush if @buffer.size >= @options[:batch_size]
|
||||
else
|
||||
send_rows row
|
||||
end
|
||||
end
|
||||
|
||||
def close
|
||||
flush
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def flush
|
||||
return if @buffer.empty?
|
||||
|
||||
send_rows @buffer
|
||||
@buffer = []
|
||||
end
|
||||
|
||||
def send_rows payload
|
||||
url = @options.fetch(:write_url, @url)
|
||||
url = url.call(payload) if url.respond_to?(:call)
|
||||
request(@options[:write_method], url, json: payload)
|
||||
end
|
||||
|
||||
def each_page_row url, params
|
||||
page = @options.fetch(:page_start, 1)
|
||||
@options[:max_pages].times do
|
||||
query = @options[:page_param] ? params.merge(@options[:page_param] => page) : params
|
||||
body = request(:get, url, params: query)
|
||||
rows = rows_of(body)
|
||||
rows.each { |row| yield row }
|
||||
|
||||
if @options[:next]
|
||||
url = @options[:next].call(body)
|
||||
params = {}
|
||||
break if url.nil? || url.to_s.empty?
|
||||
elsif @options[:page_param]
|
||||
break if rows.empty?
|
||||
|
||||
page += 1
|
||||
else
|
||||
break
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def rows_of body
|
||||
data = body
|
||||
@options[:root].to_s.split('.').each { |key| data = data.fetch(key) } if @options[:root]
|
||||
data = [] if data.nil?
|
||||
raise Error, "RestStorage: response is not an array (root: #{@options[:root].inspect})" unless data.is_a?(Array)
|
||||
|
||||
data
|
||||
end
|
||||
|
||||
def request method, url, retries: nil, **args
|
||||
retries ||= @options[:retries]
|
||||
args[:ssl_context] = ssl_context if @options[:ssl_verify] == false && url.start_with?('https')
|
||||
attempt = 0
|
||||
begin
|
||||
attempt += 1
|
||||
response = client.request(method, url, **args)
|
||||
status = response.status.code
|
||||
text = response.body.to_s
|
||||
raise RetryableError.new("HTTP #{status}", status:) if RETRY_STATUS.include?(status)
|
||||
raise Error.new("#{method.upcase} #{url}: HTTP #{status} #{text[0, 200]}", status:) unless response.status.success?
|
||||
|
||||
text.strip.empty? ? nil : JSON.parse(text)
|
||||
rescue RetryableError, HTTP::Error, SystemCallError => e
|
||||
raise Error.new("#{method.upcase} #{url}: #{e.message}", status: e.respond_to?(:status) ? e.status : nil) if attempt > retries
|
||||
|
||||
sleep(@options[:retry_wait] * attempt)
|
||||
retry
|
||||
end
|
||||
end
|
||||
|
||||
def ssl_context
|
||||
require 'openssl'
|
||||
OpenSSL::SSL::SSLContext.new.tap { |ctx| ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE }
|
||||
end
|
||||
|
||||
def client
|
||||
headers = { 'Accept' => 'application/json' }.merge(@options.fetch(:headers, {}))
|
||||
headers['Authorization'] = @options[:auth] if @options[:auth]
|
||||
HTTP.timeout(@options[:timeout]).headers(headers)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,122 @@
|
||||
require_relative 'storage_plugin'
|
||||
|
||||
class DataPipeline
|
||||
#
|
||||
# options
|
||||
# database: path of the database file or ':memory:'
|
||||
# query: sql for read, placeholders ? or $1 / :name
|
||||
# params: values for the placeholders (array, or hash for :name)
|
||||
# table: target table for write
|
||||
# create: true -> create the table on the first write, columns from the first row
|
||||
# (Integer -> integer, Float -> real, else text)
|
||||
# truncate: true -> delete all rows of the table before the first write
|
||||
#
|
||||
# rows come back as Hash<String, value> with sqlite types (Integer, Float, String, nil).
|
||||
# the database file is opened on first use: a dry run creates no file.
|
||||
#
|
||||
class SqliteStorage < StoragePlugin
|
||||
def initialize **options, &block
|
||||
self.class.require_gem 'sqlite3'
|
||||
@options = options
|
||||
@database = options.fetch(:database) { raise ArgumentError, 'SqliteStorage: option database missing' }.to_s
|
||||
@prepared = {}
|
||||
|
||||
yield if block_given?
|
||||
end
|
||||
|
||||
# without block all rows at once, with block row by row
|
||||
def read(&block)
|
||||
raise 'Query for SqliteStorage missing' unless @options[:query]
|
||||
|
||||
query(query: @options[:query], params: @options.fetch(:params, []), &block)
|
||||
end
|
||||
|
||||
def query options = {}
|
||||
raise 'Query for SqliteStorage:query missing' unless options[:query]
|
||||
|
||||
statement = db.prepare(options[:query])
|
||||
result = statement.execute(*bind_args(options.fetch(:params, [])))
|
||||
return result.map(&:to_h) unless block_given?
|
||||
|
||||
result.each { |row| yield row.to_h }
|
||||
nil
|
||||
ensure
|
||||
statement&.close
|
||||
end
|
||||
|
||||
#
|
||||
# write rows -> table from option table:
|
||||
# write table_name, rows -> explicit table
|
||||
#
|
||||
# all rows in one transaction, values as bind parameters, nil -> NULL
|
||||
#
|
||||
def write table_or_data, data = nil
|
||||
table_name, data = data.nil? ? [@options[:table], table_or_data] : [table_or_data, data]
|
||||
raise 'Table for SqliteStorage#write missing' unless table_name
|
||||
|
||||
return if data.empty?
|
||||
|
||||
db.transaction do
|
||||
prepare_table table_name, data.first
|
||||
data.each { |row| insert table_name, row }
|
||||
end
|
||||
end
|
||||
|
||||
def write_row row
|
||||
raise 'Table for SqliteStorage#write_row missing' unless @options[:table]
|
||||
|
||||
prepare_table @options[:table], row
|
||||
insert @options[:table], row
|
||||
end
|
||||
|
||||
def close
|
||||
@db.close if @db && !@db.closed?
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def db
|
||||
@db ||= SQLite3::Database.new(@database).tap { |d| d.results_as_hash = true }
|
||||
end
|
||||
|
||||
# create / truncate once per table, before its first row
|
||||
def prepare_table table_name, row
|
||||
return if @prepared[table_name]
|
||||
|
||||
if @options[:create]
|
||||
columns = row.map { |key, value| "#{quote(key)} #{column_type(value)}" }.join(', ')
|
||||
db.execute("create table if not exists #{quote_table(table_name)} (#{columns})")
|
||||
end
|
||||
db.execute("delete from #{quote_table(table_name)}") if @options[:truncate]
|
||||
@prepared[table_name] = true
|
||||
end
|
||||
|
||||
def column_type value
|
||||
case value
|
||||
when Integer then 'integer'
|
||||
when Float then 'real'
|
||||
else 'text'
|
||||
end
|
||||
end
|
||||
|
||||
# a hash binds named parameters, an array positional ones
|
||||
def bind_args params
|
||||
params.is_a?(Hash) ? [params] : Array(params)
|
||||
end
|
||||
|
||||
def insert table_name, row
|
||||
columns = row.keys.map { |col| quote(col) }.join(',')
|
||||
placeholders = (['?'] * row.size).join(',')
|
||||
values = row.values.map { |v| v.is_a?(Date) || v.is_a?(Time) ? v.iso8601 : v }
|
||||
db.execute("insert into #{quote_table(table_name)} (#{columns}) values (#{placeholders})", values)
|
||||
end
|
||||
|
||||
def quote name
|
||||
%("#{name.to_s.gsub('"', '""')}")
|
||||
end
|
||||
|
||||
def quote_table name
|
||||
name.to_s.split('.').map { |part| quote(part) }.join('.')
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,58 @@
|
||||
|
||||
|
||||
class DataPipeline
|
||||
#
|
||||
# common interface for all storage plugins
|
||||
#
|
||||
# read -> Array<Hash<String, value>>, with block: yields each row
|
||||
# write rows -> writes Array<Hash>
|
||||
# write_row row -> writes a single row (streaming)
|
||||
# query opts -> Array<Hash>, ad hoc queries from transformations
|
||||
# set opts -> merge options after creation (e.g. read pg, query: ...)
|
||||
# close -> release connections and files, safe to call more than once
|
||||
#
|
||||
class StoragePlugin
|
||||
# loads the gem of a plugin on first use, with a hint when it is missing
|
||||
def self.require_gem feature, gem_name = feature
|
||||
require feature
|
||||
rescue LoadError
|
||||
raise LoadError, "#{name} needs the gem '#{gem_name}': add `gem '#{gem_name}'` to the Gemfile and run bundle install"
|
||||
end
|
||||
|
||||
def initialize **options
|
||||
@options = options
|
||||
end
|
||||
|
||||
def connect
|
||||
raise NotImplementedError, "#{self.class}#connect is missing"
|
||||
end
|
||||
|
||||
def read
|
||||
raise NotImplementedError, "#{self.class}#read is missing"
|
||||
end
|
||||
|
||||
def write data = nil
|
||||
raise NotImplementedError, "#{self.class}#write is missing"
|
||||
end
|
||||
|
||||
def write_row row
|
||||
raise NotImplementedError, "#{self.class}#write_row is missing"
|
||||
end
|
||||
|
||||
def query options = {}
|
||||
raise NotImplementedError, "#{self.class}#query is missing"
|
||||
end
|
||||
|
||||
def set options
|
||||
(@options ||= {}).merge! options
|
||||
self
|
||||
end
|
||||
|
||||
def close
|
||||
end
|
||||
|
||||
def finalize
|
||||
close
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,53 @@
|
||||
require 'date'
|
||||
|
||||
class DataPipeline
|
||||
# raised by a validate step, handled like any other row error (on_error)
|
||||
class ValidationError < StandardError; end
|
||||
|
||||
#
|
||||
# rules per column
|
||||
# required: true value present and not empty
|
||||
# max: 35, min: 1 length of the value as text
|
||||
# format: /\A\d+\z/ regex on the value as text
|
||||
# in: %w[S H] allowed values
|
||||
# type: :integer | :decimal | :date
|
||||
#
|
||||
# empty values are only checked by required
|
||||
#
|
||||
module Validation
|
||||
TYPES = {
|
||||
integer: ->(v) { Integer(v.to_s, 10) },
|
||||
decimal: ->(v) { Float(v.to_s) },
|
||||
date: ->(v) { v.is_a?(Date) ? v : Date.iso8601(v.to_s) }
|
||||
}.freeze
|
||||
|
||||
module_function
|
||||
|
||||
def errors row, rules
|
||||
rules.flat_map do |column, rule|
|
||||
value = row.key?(column) ? row[column] : row[column.to_s]
|
||||
check(column, value, rule)
|
||||
end
|
||||
end
|
||||
|
||||
def check column, value, rule
|
||||
empty = value.nil? || value.to_s.strip.empty?
|
||||
return(rule[:required] ? ["#{column}: missing"] : []) if empty
|
||||
|
||||
text = value.to_s
|
||||
messages = []
|
||||
messages << "#{column}: longer than #{rule[:max]}" if rule[:max] && text.length > rule[:max]
|
||||
messages << "#{column}: shorter than #{rule[:min]}" if rule[:min] && text.length < rule[:min]
|
||||
messages << "#{column}: format #{rule[:format].inspect}" if rule[:format] && !text.match?(rule[:format])
|
||||
messages << "#{column}: not in #{rule[:in].inspect}" if rule[:in] && !rule[:in].include?(value)
|
||||
if rule[:type]
|
||||
begin
|
||||
TYPES.fetch(rule[:type]).call(value)
|
||||
rescue ArgumentError, TypeError, Date::Error
|
||||
messages << "#{column}: not #{rule[:type]}"
|
||||
end
|
||||
end
|
||||
messages
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,8 @@
|
||||
class DataPipeline
|
||||
VERSION = '2.0.0'.freeze
|
||||
|
||||
# kept for scripts that print the version (rake info:info); the history is in CHANGELOG.md
|
||||
class Version
|
||||
def self.version = "datapipeline #{VERSION}"
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,60 @@
|
||||
require 'json'
|
||||
require 'fileutils'
|
||||
|
||||
class DataPipeline
|
||||
#
|
||||
# remembers the highest value of a column between runs (json file, one entry per name)
|
||||
#
|
||||
# wm = watermark :posting, column: 'rechnung_dbid', file: 'data/.state.json', initial: 0
|
||||
# read pg, query: 'select ... where rechnung_dbid > $1', params: [wm.value]
|
||||
#
|
||||
# the pipeline tracks every source row and saves the new value after a successful run
|
||||
# (not after an abort, not in dry run). numeric texts are compared as numbers.
|
||||
#
|
||||
class Watermark
|
||||
attr_reader :name, :column, :file, :value, :current
|
||||
|
||||
def initialize name, column:, file:, initial: nil
|
||||
@name = name.to_s
|
||||
@column = column
|
||||
@file = file
|
||||
@value = load.fetch(@name, initial)
|
||||
@current = @value
|
||||
end
|
||||
|
||||
def track row
|
||||
candidate = row.key?(@column) ? row[@column] : row[@column.to_s]
|
||||
return if candidate.nil?
|
||||
|
||||
@current = candidate if @current.nil? || greater?(candidate, @current)
|
||||
end
|
||||
|
||||
def changed?
|
||||
@current != @value
|
||||
end
|
||||
|
||||
def save
|
||||
return unless changed?
|
||||
|
||||
state = load.merge(@name => @current)
|
||||
FileUtils.mkdir_p(File.dirname(@file))
|
||||
File.write("#{@file}.tmp", JSON.pretty_generate(state))
|
||||
File.rename("#{@file}.tmp", @file)
|
||||
@value = @current
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def load
|
||||
File.exist?(@file) ? JSON.parse(File.read(@file)) : {}
|
||||
end
|
||||
|
||||
def greater? a, b
|
||||
numeric?(a) && numeric?(b) ? a.to_s.to_r > b.to_s.to_r : a.to_s > b.to_s
|
||||
end
|
||||
|
||||
def numeric? value
|
||||
value.is_a?(Numeric) || value.to_s.match?(/\A-?\d+(\.\d+)?\z/)
|
||||
end
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user