61 lines
1.6 KiB
Ruby
61 lines
1.6 KiB
Ruby
require 'json'
|
|
require 'fileutils'
|
|
|
|
class DataPipeline
|
|
#
|
|
# remembers the highest value of a column between runs (json file, one entry per name)
|
|
#
|
|
# wm = watermark :posting, column: 'rechnung_dbid', file: 'data/.state.json', initial: 0
|
|
# read pg, query: 'select ... where rechnung_dbid > $1', params: [wm.value]
|
|
#
|
|
# the pipeline tracks every source row and saves the new value after a successful run
|
|
# (not after an abort, not in dry run). numeric texts are compared as numbers.
|
|
#
|
|
class Watermark
|
|
attr_reader :name, :column, :file, :value, :current
|
|
|
|
def initialize name, column:, file:, initial: nil
|
|
@name = name.to_s
|
|
@column = column
|
|
@file = file
|
|
@value = load.fetch(@name, initial)
|
|
@current = @value
|
|
end
|
|
|
|
def track row
|
|
candidate = row.key?(@column) ? row[@column] : row[@column.to_s]
|
|
return if candidate.nil?
|
|
|
|
@current = candidate if @current.nil? || greater?(candidate, @current)
|
|
end
|
|
|
|
def changed?
|
|
@current != @value
|
|
end
|
|
|
|
def save
|
|
return unless changed?
|
|
|
|
state = load.merge(@name => @current)
|
|
FileUtils.mkdir_p(File.dirname(@file))
|
|
File.write("#{@file}.tmp", JSON.pretty_generate(state))
|
|
File.rename("#{@file}.tmp", @file)
|
|
@value = @current
|
|
end
|
|
|
|
private
|
|
|
|
def load
|
|
File.exist?(@file) ? JSON.parse(File.read(@file)) : {}
|
|
end
|
|
|
|
def greater? a, b
|
|
numeric?(a) && numeric?(b) ? a.to_s.to_r > b.to_s.to_r : a.to_s > b.to_s
|
|
end
|
|
|
|
def numeric? value
|
|
value.is_a?(Numeric) || value.to_s.match?(/\A-?\d+(\.\d+)?\z/)
|
|
end
|
|
end
|
|
end
|