Factoring.
parent
51ebb40953
commit
fd73bc437f
206
src/cached.cr
206
src/cached.cr
|
|
@ -37,14 +37,6 @@ class DODB::CachedDataBase(V) < DODB::Storage(V)
|
|||
end
|
||||
end
|
||||
|
||||
# Data should be written on disk AND in RAM.
|
||||
def <<(item : V)
|
||||
index = last_index + 1
|
||||
self[index] = item
|
||||
self.last_index = index
|
||||
self.last_index
|
||||
end
|
||||
|
||||
# Getting data from the hash in RAM.
|
||||
def []?(key : Int32) : V?
|
||||
@data[key]
|
||||
|
|
@ -85,8 +77,7 @@ class DODB::CachedDataBase(V) < DODB::Storage(V)
|
|||
end
|
||||
|
||||
##
|
||||
# CAUTION: Very slow. Try not to use.
|
||||
# Can be useful for making dumps or to restore a database, however.
|
||||
# Can be useful for making dumps or to restore a database.
|
||||
def each_with_index(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil)
|
||||
i = -1 # do not trust key to be the right index
|
||||
(reversed ? @data.reverse : @data).each do |index, v|
|
||||
|
|
@ -97,149 +88,6 @@ class DODB::CachedDataBase(V) < DODB::Storage(V)
|
|||
yield v, index
|
||||
end
|
||||
end
|
||||
def each(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil)
|
||||
each_with_index(
|
||||
reversed: reversed,
|
||||
start_offset: start_offset,
|
||||
end_offset: end_offset
|
||||
) do |item, index|
|
||||
yield item
|
||||
end
|
||||
end
|
||||
|
||||
def to_a(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil)
|
||||
res = Array(V).new
|
||||
each(reversed, start_offset, end_offset) do |v|
|
||||
res << v
|
||||
end
|
||||
res
|
||||
end
|
||||
|
||||
def to_h(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil)
|
||||
res = Hash(Int32, V).new
|
||||
|
||||
(reversed ? @data.reverse : @data).each do |index, v|
|
||||
next if start_offset > index
|
||||
break unless end_offset.nil? || index < end_offset
|
||||
|
||||
res[index] = v
|
||||
end
|
||||
|
||||
res
|
||||
end
|
||||
|
||||
private def index_file
|
||||
"#{@directory_name}/last-index"
|
||||
end
|
||||
def last_index : Int32
|
||||
File.read(index_file).to_i
|
||||
end
|
||||
def last_index=(x : Int32)
|
||||
file = File.open(index_file, "w")
|
||||
|
||||
file << x.to_s
|
||||
|
||||
file.close
|
||||
|
||||
x
|
||||
rescue
|
||||
raise Exception.new "could not update index file"
|
||||
end
|
||||
|
||||
def stringify_key(key : Int32)
|
||||
# Negative numbers give strange results with Crystal’s printf.
|
||||
if key >= 0
|
||||
"%010i" % key
|
||||
else
|
||||
key.to_s
|
||||
end
|
||||
end
|
||||
|
||||
def request_lock(name)
|
||||
r = -1
|
||||
file_path = get_lock_file_path name
|
||||
file_perms = 0o644
|
||||
|
||||
flags = LibC::O_EXCL | LibC::O_CREAT
|
||||
while (r = LibC.open file_path, flags, file_perms) == -1
|
||||
sleep 1.milliseconds
|
||||
end
|
||||
|
||||
LibC.close r
|
||||
end
|
||||
def release_lock(name)
|
||||
File.delete get_lock_file_path name
|
||||
end
|
||||
|
||||
##
|
||||
# name is the name that will be used on the file system.
|
||||
def new_partition(name : String, &block : Proc(V, String))
|
||||
Partition(V).new(self, @directory_name, name, block).tap do |table|
|
||||
@indexers << table
|
||||
end
|
||||
end
|
||||
|
||||
##
|
||||
# name is the name that will be used on the file system.
|
||||
def new_index(name : String, &block : Proc(V, String))
|
||||
Index(V).new(self, @directory_name, name, block).tap do |indexer|
|
||||
@indexers << indexer
|
||||
end
|
||||
end
|
||||
|
||||
def new_tags(name : String, &block : Proc(V, Array(String)))
|
||||
Tags(V).new(@directory_name, name, block).tap do |tags|
|
||||
@indexers << tags
|
||||
end
|
||||
end
|
||||
|
||||
def get_index(name : String, key)
|
||||
index = @indexers.find &.name.==(name)
|
||||
|
||||
index.not_nil!.as(DODB::Index).get key
|
||||
end
|
||||
|
||||
def get_partition(table_name : String, partition_name : String)
|
||||
partition = @indexers.find &.name.==(table_name)
|
||||
|
||||
partition.not_nil!.as(DODB::Partition).get partition_name
|
||||
end
|
||||
|
||||
def get_tags(name, key : String)
|
||||
partition = @indexers.find &.name.==(name)
|
||||
|
||||
partition.not_nil!.as(DODB::Tags).get name, key
|
||||
end
|
||||
|
||||
def check_collisions!(key : Int32, value : V, old_value : V?)
|
||||
@indexers.each &.check!(stringify_key(key), value, old_value)
|
||||
end
|
||||
|
||||
def write_partitions(key : Int32, value : V)
|
||||
@indexers.each &.index(stringify_key(key), value)
|
||||
end
|
||||
|
||||
def pop
|
||||
index = last_index
|
||||
|
||||
# Some entries may have been removed. We’ll skip over those.
|
||||
# Not the most efficient if a large number of indices are empty.
|
||||
while index >= 0 && self[index]?.nil?
|
||||
index = index - 1
|
||||
end
|
||||
|
||||
if index < 0
|
||||
return nil
|
||||
end
|
||||
|
||||
poped = self[index]
|
||||
|
||||
self.delete index
|
||||
|
||||
last_index = index - 1
|
||||
|
||||
poped
|
||||
end
|
||||
|
||||
def delete(key : Int32)
|
||||
value = self[key]?
|
||||
|
|
@ -257,56 +105,4 @@ class DODB::CachedDataBase(V) < DODB::Storage(V)
|
|||
@data.delete key
|
||||
value
|
||||
end
|
||||
|
||||
def remove_partitions(key : Int32, value : V)
|
||||
@indexers.each &.deindex(stringify_key(key), value)
|
||||
end
|
||||
|
||||
private def data_path
|
||||
"#{@directory_name}/data"
|
||||
end
|
||||
|
||||
private def file_path(key : Int32)
|
||||
"#{data_path}/%010i.json" % key
|
||||
end
|
||||
|
||||
private def locks_directory : String
|
||||
"#{@directory_name}/locks"
|
||||
end
|
||||
|
||||
private def get_lock_file_path(name : String)
|
||||
"#{locks_directory}/#{name}.lock"
|
||||
end
|
||||
|
||||
private def read(file_path : String)
|
||||
V.from_json ::File.read file_path
|
||||
end
|
||||
|
||||
private def remove_data!
|
||||
FileUtils.rm_rf data_path
|
||||
Dir.mkdir_p data_path
|
||||
@data = Hash(Int32, V).new
|
||||
end
|
||||
|
||||
private def remove_indexing!
|
||||
@indexers.each do |indexer|
|
||||
FileUtils.rm_rf indexer.indexing_directory
|
||||
end
|
||||
end
|
||||
|
||||
# A very slow operation that removes all indices and then rewrites
|
||||
# them all.
|
||||
# FIXME: Is this really useful in its current form? We should remove the
|
||||
# index directories, not the indices based on our current (and
|
||||
# possiblly different from what’s stored) data.
|
||||
def reindex_everything!
|
||||
old_data = to_h
|
||||
|
||||
remove_indexing!
|
||||
remove_data!
|
||||
|
||||
old_data.each do |index, item|
|
||||
self[index] = item
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
303
src/dodb.cr
303
src/dodb.cr
|
|
@ -4,6 +4,11 @@ require "json"
|
|||
require "./dodb/*"
|
||||
|
||||
abstract class DODB::Storage(V)
|
||||
property directory_name : String
|
||||
|
||||
def initialize(@directory_name : String)
|
||||
end
|
||||
|
||||
def request_lock(name)
|
||||
r = -1
|
||||
file_path = get_lock_file_path name
|
||||
|
|
@ -20,25 +25,6 @@ abstract class DODB::Storage(V)
|
|||
File.delete get_lock_file_path name
|
||||
end
|
||||
|
||||
abstract def [](key : Int32)
|
||||
abstract def <<(item : V)
|
||||
abstract def delete(key : Int32)
|
||||
end
|
||||
|
||||
class DODB::DataBase(V) < DODB::Storage(V)
|
||||
@indexers = [] of Indexer(V)
|
||||
|
||||
def initialize(@directory_name : String)
|
||||
Dir.mkdir_p data_path
|
||||
Dir.mkdir_p locks_directory
|
||||
|
||||
begin
|
||||
self.last_index
|
||||
rescue
|
||||
self.last_index = -1
|
||||
end
|
||||
end
|
||||
|
||||
private def index_file
|
||||
"#{@directory_name}/last-index"
|
||||
end
|
||||
|
|
@ -66,14 +52,54 @@ class DODB::DataBase(V) < DODB::Storage(V)
|
|||
end
|
||||
end
|
||||
|
||||
##
|
||||
# name is the name that will be used on the file system.
|
||||
def new_partition(name : String, &block : Proc(V, String))
|
||||
Partition(V).new(self, @directory_name, name, block).tap do |table|
|
||||
@indexers << table
|
||||
def <<(item : V)
|
||||
index = last_index + 1
|
||||
self[index] = item
|
||||
self.last_index = index
|
||||
end
|
||||
|
||||
def each(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil)
|
||||
each_with_index(
|
||||
reversed: reversed,
|
||||
start_offset: start_offset,
|
||||
end_offset: end_offset
|
||||
) do |item, index|
|
||||
yield item
|
||||
end
|
||||
end
|
||||
|
||||
##
|
||||
# CAUTION: Very slow. Try not to use.
|
||||
def to_a(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil)
|
||||
array = ::Array(V).new
|
||||
|
||||
each(
|
||||
reversed: reversed,
|
||||
start_offset: start_offset,
|
||||
end_offset: end_offset
|
||||
) do |value|
|
||||
array << value
|
||||
end
|
||||
|
||||
array
|
||||
end
|
||||
|
||||
##
|
||||
# CAUTION: Very slow. Try not to use.
|
||||
def to_h(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil)
|
||||
hash = ::Hash(Int32, V).new
|
||||
|
||||
each_with_index(
|
||||
reversed: reversed,
|
||||
start_offset: start_offset,
|
||||
end_offset: end_offset
|
||||
) do |element, index|
|
||||
hash[index] = element
|
||||
end
|
||||
|
||||
hash
|
||||
end
|
||||
|
||||
##
|
||||
# name is the name that will be used on the file system.
|
||||
def new_index(name : String, &block : Proc(V, String))
|
||||
|
|
@ -82,6 +108,14 @@ class DODB::DataBase(V) < DODB::Storage(V)
|
|||
end
|
||||
end
|
||||
|
||||
##
|
||||
# name is the name that will be used on the file system.
|
||||
def new_partition(name : String, &block : Proc(V, String))
|
||||
Partition(V).new(self, @directory_name, name, block).tap do |table|
|
||||
@indexers << table
|
||||
end
|
||||
end
|
||||
|
||||
def new_tags(name : String, &block : Proc(V, Array(String)))
|
||||
Tags(V).new(@directory_name, name, block).tap do |tags|
|
||||
@indexers << tags
|
||||
|
|
@ -106,12 +140,103 @@ class DODB::DataBase(V) < DODB::Storage(V)
|
|||
partition.not_nil!.as(DODB::Tags).get name, key
|
||||
end
|
||||
|
||||
def <<(item : V)
|
||||
index = last_index + 1
|
||||
def check_collisions!(key : Int32, value : V, old_value : V?)
|
||||
@indexers.each &.check!(stringify_key(key), value, old_value)
|
||||
end
|
||||
|
||||
self[index] = item
|
||||
def write_partitions(key : Int32, value : V)
|
||||
@indexers.each &.index(stringify_key(key), value)
|
||||
end
|
||||
|
||||
self.last_index = index
|
||||
def pop
|
||||
index = last_index
|
||||
|
||||
# Some entries may have been removed. We’ll skip over those.
|
||||
# Not the most efficient if a large number of indices are empty.
|
||||
while index >= 0 && self[index]?.nil?
|
||||
index = index - 1
|
||||
end
|
||||
|
||||
if index < 0
|
||||
return nil
|
||||
end
|
||||
|
||||
poped = self[index]
|
||||
|
||||
self.delete index
|
||||
|
||||
last_index = index - 1
|
||||
|
||||
poped
|
||||
end
|
||||
|
||||
private def data_path
|
||||
"#{@directory_name}/data"
|
||||
end
|
||||
|
||||
private def file_path(key : Int32)
|
||||
"#{data_path}/%010i.json" % key
|
||||
end
|
||||
|
||||
private def locks_directory : String
|
||||
"#{@directory_name}/locks"
|
||||
end
|
||||
|
||||
private def get_lock_file_path(name : String)
|
||||
"#{locks_directory}/#{name}.lock"
|
||||
end
|
||||
|
||||
private def read(file_path : String)
|
||||
V.from_json ::File.read file_path
|
||||
end
|
||||
|
||||
private def remove_data!
|
||||
FileUtils.rm_rf data_path
|
||||
Dir.mkdir_p data_path
|
||||
end
|
||||
|
||||
private def remove_indexing!
|
||||
@indexers.each do |indexer|
|
||||
FileUtils.rm_rf indexer.indexing_directory
|
||||
end
|
||||
end
|
||||
|
||||
# A very slow operation that removes all indices and then rewrites
|
||||
# them all.
|
||||
# FIXME: Is this really useful in its current form? We should remove the
|
||||
# index directories, not the indices based on our current (and
|
||||
# possiblly different from what’s stored) data.
|
||||
def reindex_everything!
|
||||
old_data = to_h
|
||||
|
||||
remove_indexing!
|
||||
remove_data!
|
||||
|
||||
old_data.each do |index, item|
|
||||
self[index] = item
|
||||
end
|
||||
end
|
||||
|
||||
def remove_partitions(key : Int32, value : V)
|
||||
@indexers.each &.deindex(stringify_key(key), value)
|
||||
end
|
||||
|
||||
abstract def [](key : Int32)
|
||||
abstract def delete(key : Int32)
|
||||
end
|
||||
|
||||
class DODB::DataBase(V) < DODB::Storage(V)
|
||||
@indexers = [] of Indexer(V)
|
||||
|
||||
def initialize(@directory_name : String)
|
||||
Dir.mkdir_p data_path
|
||||
Dir.mkdir_p locks_directory
|
||||
|
||||
begin
|
||||
self.last_index
|
||||
rescue
|
||||
self.last_index = -1
|
||||
end
|
||||
end
|
||||
|
||||
def []?(key : Int32) : V?
|
||||
|
|
@ -151,36 +276,6 @@ class DODB::DataBase(V) < DODB::Storage(V)
|
|||
end
|
||||
end
|
||||
|
||||
def check_collisions!(key : Int32, value : V, old_value : V?)
|
||||
@indexers.each &.check!(stringify_key(key), value, old_value)
|
||||
end
|
||||
|
||||
def write_partitions(key : Int32, value : V)
|
||||
@indexers.each &.index(stringify_key(key), value)
|
||||
end
|
||||
|
||||
def pop
|
||||
index = last_index
|
||||
|
||||
# Some entries may have been removed. We’ll skip over those.
|
||||
# Not the most efficient if a large number of indices are empty.
|
||||
while index >= 0 && self[index]?.nil?
|
||||
index = index - 1
|
||||
end
|
||||
|
||||
if index < 0
|
||||
return nil
|
||||
end
|
||||
|
||||
poped = self[index]
|
||||
|
||||
self.delete index
|
||||
|
||||
last_index = index - 1
|
||||
|
||||
poped
|
||||
end
|
||||
|
||||
def delete(key : Int32)
|
||||
value = self[key]?
|
||||
|
||||
|
|
@ -197,10 +292,6 @@ class DODB::DataBase(V) < DODB::Storage(V)
|
|||
value
|
||||
end
|
||||
|
||||
def remove_partitions(key : Int32, value : V)
|
||||
@indexers.each &.deindex(stringify_key(key), value)
|
||||
end
|
||||
|
||||
private def each_key(reversed = false)
|
||||
start = 0
|
||||
_end = last_index
|
||||
|
|
@ -252,94 +343,6 @@ class DODB::DataBase(V) < DODB::Storage(V)
|
|||
yield field, key
|
||||
end
|
||||
end
|
||||
def each(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil)
|
||||
each_with_index(
|
||||
reversed: reversed,
|
||||
start_offset: start_offset,
|
||||
end_offset: end_offset
|
||||
) do |item, index|
|
||||
yield item
|
||||
end
|
||||
end
|
||||
|
||||
##
|
||||
# CAUTION: Very slow. Try not to use.
|
||||
def to_a(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil)
|
||||
array = ::Array(V).new
|
||||
|
||||
each(
|
||||
reversed: reversed,
|
||||
start_offset: start_offset,
|
||||
end_offset: end_offset
|
||||
) do |value|
|
||||
array << value
|
||||
end
|
||||
|
||||
array
|
||||
end
|
||||
|
||||
##
|
||||
# CAUTION: Very slow. Try not to use.
|
||||
def to_h(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil)
|
||||
hash = ::Hash(Int32, V).new
|
||||
|
||||
each_with_index(
|
||||
reversed: reversed,
|
||||
start_offset: start_offset,
|
||||
end_offset: end_offset
|
||||
) do |element, index|
|
||||
hash[index] = element
|
||||
end
|
||||
|
||||
hash
|
||||
end
|
||||
|
||||
private def data_path
|
||||
"#{@directory_name}/data"
|
||||
end
|
||||
|
||||
private def file_path(key : Int32)
|
||||
"#{data_path}/%010i.json" % key
|
||||
end
|
||||
|
||||
private def locks_directory : String
|
||||
"#{@directory_name}/locks"
|
||||
end
|
||||
|
||||
private def get_lock_file_path(name : String)
|
||||
"#{locks_directory}/#{name}.lock"
|
||||
end
|
||||
|
||||
private def read(file_path : String)
|
||||
V.from_json ::File.read file_path
|
||||
end
|
||||
|
||||
private def remove_data!
|
||||
FileUtils.rm_rf data_path
|
||||
Dir.mkdir_p data_path
|
||||
end
|
||||
|
||||
private def remove_indexing!
|
||||
@indexers.each do |indexer|
|
||||
FileUtils.rm_rf indexer.indexing_directory
|
||||
end
|
||||
end
|
||||
|
||||
# A very slow operation that removes all indices and then rewrites
|
||||
# them all.
|
||||
# FIXME: Is this really useful in its current form? We should remove the
|
||||
# index directories, not the indices based on our current (and
|
||||
# possiblly different from what’s stored) data.
|
||||
def reindex_everything!
|
||||
old_data = to_h
|
||||
|
||||
remove_indexing!
|
||||
remove_data!
|
||||
|
||||
old_data.each do |index, item|
|
||||
self[index] = item
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
require "./cached.cr"
|
||||
|
|
|
|||
Loading…
Reference in New Issue