2019-12-12 00:44:05 +01:00
|
|
|
|
require "file_utils"
|
|
|
|
|
require "json"
|
|
|
|
|
|
|
|
|
|
require "./dodb/*"
|
|
|
|
|
|
2019-12-28 03:24:05 +01:00
|
|
|
|
class DODB::DataBase(V)
|
2019-12-12 00:44:05 +01:00
|
|
|
|
@indexers = [] of Indexer(V)
|
|
|
|
|
|
|
|
|
|
def initialize(@directory_name : String)
|
|
|
|
|
Dir.mkdir_p data_path
|
2019-12-18 03:43:09 +01:00
|
|
|
|
|
2020-01-07 16:06:18 +01:00
|
|
|
|
begin
|
|
|
|
|
self.last_index
|
|
|
|
|
rescue
|
|
|
|
|
self.last_index = -1
|
|
|
|
|
end
|
2019-12-18 03:43:09 +01:00
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
private def index_file
|
|
|
|
|
"#{@directory_name}/last-index"
|
|
|
|
|
end
|
|
|
|
|
def last_index : Int32
|
|
|
|
|
File.read(index_file).to_i
|
|
|
|
|
end
|
|
|
|
|
def last_index=(x : Int32)
|
|
|
|
|
file = File.open(index_file, "w")
|
|
|
|
|
|
2020-01-07 16:06:18 +01:00
|
|
|
|
file << x.to_s
|
2019-12-18 03:43:09 +01:00
|
|
|
|
|
|
|
|
|
file.close
|
|
|
|
|
|
|
|
|
|
x
|
|
|
|
|
rescue
|
|
|
|
|
raise Exception.new "could not update index file"
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
def stringify_key(key : Int32)
|
|
|
|
|
# Negative numbers give strange results with Crystal’s printf.
|
|
|
|
|
if key >= 0
|
|
|
|
|
"%010i" % key
|
|
|
|
|
else
|
|
|
|
|
key.to_s
|
|
|
|
|
end
|
2019-12-12 00:44:05 +01:00
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
##
|
|
|
|
|
# name is the name that will be used on the file system.
|
|
|
|
|
def new_partition(name : String, &block : Proc(V, String))
|
|
|
|
|
Partition(V).new(@directory_name, name, block).tap do |table|
|
|
|
|
|
@indexers << table
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
##
|
|
|
|
|
# name is the name that will be used on the file system.
|
|
|
|
|
def new_index(name : String, &block : Proc(V, String))
|
2020-01-03 09:36:41 +01:00
|
|
|
|
Index(V).new(self, @directory_name, name, block).tap do |indexer|
|
2019-12-12 00:44:05 +01:00
|
|
|
|
@indexers << indexer
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
def new_tags(name : String, &block : Proc(V, Array(String)))
|
|
|
|
|
Tags(V).new(@directory_name, name, block).tap do |tags|
|
|
|
|
|
@indexers << tags
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
def get_index(name : String, key)
|
|
|
|
|
index = @indexers.find &.name.==(name)
|
|
|
|
|
|
|
|
|
|
index.not_nil!.as(DODB::Index).get key
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
def get_partition(table_name : String, partition_name : String)
|
|
|
|
|
partition = @indexers.find &.name.==(table_name)
|
|
|
|
|
|
|
|
|
|
partition.not_nil!.as(DODB::Partition).get partition_name
|
|
|
|
|
end
|
|
|
|
|
|
2019-12-28 03:24:05 +01:00
|
|
|
|
def get_tags(name, key : String)
|
2019-12-12 00:44:05 +01:00
|
|
|
|
partition = @indexers.find &.name.==(name)
|
|
|
|
|
|
|
|
|
|
partition.not_nil!.as(DODB::Tags).get name, key
|
|
|
|
|
end
|
|
|
|
|
|
2019-12-18 03:43:09 +01:00
|
|
|
|
def <<(item : V)
|
|
|
|
|
index = last_index + 1
|
|
|
|
|
|
|
|
|
|
self[index] = item
|
|
|
|
|
|
|
|
|
|
self.last_index = index
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
def []?(key : Int32) : V?
|
2019-12-12 00:44:05 +01:00
|
|
|
|
self[key]
|
|
|
|
|
rescue MissingEntry
|
|
|
|
|
# FIXME: Only rescue JSON and “no such file” errors.
|
|
|
|
|
return nil
|
|
|
|
|
end
|
|
|
|
|
|
2019-12-18 03:43:09 +01:00
|
|
|
|
def [](key : Int32) : V
|
2019-12-12 00:44:05 +01:00
|
|
|
|
raise MissingEntry.new(key) unless ::File.exists? file_path key
|
|
|
|
|
|
|
|
|
|
read file_path key
|
|
|
|
|
end
|
|
|
|
|
|
2019-12-18 03:43:09 +01:00
|
|
|
|
def []=(index : Int32, value : V)
|
|
|
|
|
old_value = self.[index]?
|
2019-12-12 00:44:05 +01:00
|
|
|
|
|
2019-12-18 03:43:09 +01:00
|
|
|
|
check_collisions! index, value, old_value
|
2019-12-12 00:44:05 +01:00
|
|
|
|
|
|
|
|
|
# Removes any old indices or partitions pointing to a value about
|
|
|
|
|
# to be replaced.
|
|
|
|
|
if old_value
|
2019-12-18 03:43:09 +01:00
|
|
|
|
remove_partitions index, old_value
|
2019-12-12 00:44:05 +01:00
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
# Avoids corruption in case the application crashes while writing.
|
2019-12-18 03:43:09 +01:00
|
|
|
|
file_path(index).tap do |path|
|
2019-12-12 00:44:05 +01:00
|
|
|
|
::File.write "#{path}.new", value.to_json
|
|
|
|
|
::FileUtils.mv "#{path}.new", path
|
|
|
|
|
end
|
|
|
|
|
|
2019-12-18 03:43:09 +01:00
|
|
|
|
write_partitions index, value
|
|
|
|
|
|
|
|
|
|
if index > last_index
|
|
|
|
|
self.last_index = index
|
|
|
|
|
end
|
2019-12-12 00:44:05 +01:00
|
|
|
|
end
|
|
|
|
|
|
2019-12-18 03:43:09 +01:00
|
|
|
|
def check_collisions!(key : Int32, value : V, old_value : V?)
|
|
|
|
|
@indexers.each &.check!(stringify_key(key), value, old_value)
|
2019-12-12 00:44:05 +01:00
|
|
|
|
end
|
|
|
|
|
|
2019-12-18 03:43:09 +01:00
|
|
|
|
def write_partitions(key : Int32, value : V)
|
|
|
|
|
@indexers.each &.index(stringify_key(key), value)
|
2019-12-12 00:44:05 +01:00
|
|
|
|
end
|
|
|
|
|
|
2019-12-18 03:43:09 +01:00
|
|
|
|
def pop
|
|
|
|
|
index = last_index
|
|
|
|
|
|
|
|
|
|
# Some entries may have been removed. We’ll skip over those.
|
|
|
|
|
# Not the most efficient if a large number of indices are empty.
|
|
|
|
|
while index >= 0 && self[index]?.nil?
|
|
|
|
|
index = index - 1
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
if index < 0
|
|
|
|
|
return nil
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
poped = self[index]
|
|
|
|
|
|
|
|
|
|
self.delete index
|
|
|
|
|
|
|
|
|
|
last_index = index - 1
|
|
|
|
|
|
|
|
|
|
poped
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
def delete(key : Int32)
|
2019-12-12 00:44:05 +01:00
|
|
|
|
value = self[key]?
|
|
|
|
|
|
|
|
|
|
return if value.nil?
|
|
|
|
|
|
|
|
|
|
begin
|
|
|
|
|
::File.delete file_path key
|
|
|
|
|
rescue
|
|
|
|
|
# FIXME: Only intercept “no such file" errors
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
remove_partitions key, value
|
|
|
|
|
|
|
|
|
|
value
|
|
|
|
|
end
|
|
|
|
|
|
2019-12-18 03:43:09 +01:00
|
|
|
|
def remove_partitions(key : Int32, value : V)
|
|
|
|
|
@indexers.each &.deindex(stringify_key(key), value)
|
2019-12-12 00:44:05 +01:00
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
##
|
|
|
|
|
# CAUTION: Very slow. Try not to use.
|
|
|
|
|
# Can be useful for making dumps or to restore a database, however.
|
2019-12-18 03:43:09 +01:00
|
|
|
|
def each_with_index
|
2019-12-12 00:44:05 +01:00
|
|
|
|
dirname = data_path
|
|
|
|
|
Dir.each_child dirname do |child|
|
|
|
|
|
next if child.match /^\./
|
|
|
|
|
|
|
|
|
|
full_path = "#{dirname}/#{child}"
|
|
|
|
|
|
|
|
|
|
begin
|
|
|
|
|
# FIXME: Only intercept JSON parsing errors.
|
|
|
|
|
field = read full_path
|
|
|
|
|
rescue
|
|
|
|
|
next
|
|
|
|
|
end
|
|
|
|
|
|
2019-12-18 03:43:09 +01:00
|
|
|
|
key = child.gsub(/\.json$/, "").to_i
|
2019-12-12 00:44:05 +01:00
|
|
|
|
|
2019-12-18 03:43:09 +01:00
|
|
|
|
yield field, key
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
def each
|
|
|
|
|
each_with_index do |item, index|
|
|
|
|
|
yield item
|
2019-12-12 00:44:05 +01:00
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
2020-01-10 17:20:10 +01:00
|
|
|
|
def reverse_each_with_index
|
|
|
|
|
(last_index..0).each do |key|
|
|
|
|
|
full_path = file_path key
|
|
|
|
|
|
|
|
|
|
next unless File.exists? full_path
|
|
|
|
|
|
|
|
|
|
begin
|
|
|
|
|
# FIXME: Only intercept JSON parsing errors.
|
|
|
|
|
item = read full_path
|
|
|
|
|
rescue
|
|
|
|
|
next
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
yield item, key
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
def reverse_each
|
|
|
|
|
reverse_each_with_index do |item, index|
|
|
|
|
|
yield item
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
2019-12-18 03:43:09 +01:00
|
|
|
|
##
|
|
|
|
|
# CAUTION: Very slow. Try not to use.
|
|
|
|
|
def to_a
|
|
|
|
|
array = ::Array(V).new
|
|
|
|
|
|
|
|
|
|
each do |value|
|
|
|
|
|
array << value
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
array
|
|
|
|
|
end
|
|
|
|
|
|
2019-12-12 00:44:05 +01:00
|
|
|
|
##
|
|
|
|
|
# CAUTION: Very slow. Try not to use.
|
|
|
|
|
def to_h
|
2019-12-18 03:43:09 +01:00
|
|
|
|
hash = ::Hash(Int32, V).new
|
2019-12-12 00:44:05 +01:00
|
|
|
|
|
2019-12-18 03:43:09 +01:00
|
|
|
|
each_with_index do |element, index|
|
|
|
|
|
hash[index] = element
|
2019-12-12 00:44:05 +01:00
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
hash
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
private def data_path
|
|
|
|
|
"#{@directory_name}/data"
|
|
|
|
|
end
|
|
|
|
|
|
2019-12-18 03:43:09 +01:00
|
|
|
|
private def file_path(key : Int32)
|
|
|
|
|
"#{data_path}/%010i.json" % key
|
2019-12-12 00:44:05 +01:00
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
private def read(file_path : String)
|
|
|
|
|
V.from_json ::File.read file_path
|
|
|
|
|
end
|
|
|
|
|
|
2019-12-18 03:43:09 +01:00
|
|
|
|
private def remove_data!
|
|
|
|
|
FileUtils.rm_rf data_path
|
|
|
|
|
Dir.mkdir_p data_path
|
|
|
|
|
end
|
|
|
|
|
|
2019-12-19 04:22:14 +01:00
|
|
|
|
private def remove_indexing!
|
|
|
|
|
@indexers.each do |indexer|
|
|
|
|
|
FileUtils.rm_rf indexer.indexing_directory
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
2019-12-12 00:44:05 +01:00
|
|
|
|
# A very slow operation that removes all indices and then rewrites
|
|
|
|
|
# them all.
|
2019-12-18 03:43:09 +01:00
|
|
|
|
# FIXME: Is this really useful in its current form? We should remove the
|
|
|
|
|
# index directories, not the indices based on our current (and
|
|
|
|
|
# possiblly different from what’s stored) data.
|
2019-12-12 00:44:05 +01:00
|
|
|
|
def reindex_everything!
|
|
|
|
|
old_data = to_h
|
|
|
|
|
|
2019-12-19 04:22:14 +01:00
|
|
|
|
remove_indexing!
|
|
|
|
|
remove_data!
|
|
|
|
|
|
2019-12-18 03:43:09 +01:00
|
|
|
|
old_data.each do |index, item|
|
|
|
|
|
self[index] = item
|
2019-12-12 00:44:05 +01:00
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|