Baguette
/
dodb.cr
3
0
Fork 1
Code Issues Pull Requests Releases Wiki Activity

Factoring.

remotes/1719344442898712274/ramdb
Karchnu 2020-07-18 00:37:29 +02:00
parent 51ebb40953
commit fd73bc437f
2 changed files with 154 additions and 355 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

206
src/cached.cr
View File

@ -37,14 +37,6 @@ class DODB::CachedDataBase(V) < DODB::Storage(V)
end end
end end
# Data should be written on disk AND in RAM.
def <<(item : V)
index = last_index + 1
self[index] = item
self.last_index = index
self.last_index
end
# Getting data from the hash in RAM. # Getting data from the hash in RAM.
def []?(key : Int32) : V? def []?(key : Int32) : V?
@data[key] @data[key]
@ -85,8 +77,7 @@ class DODB::CachedDataBase(V) < DODB::Storage(V)
end end
## ##
# CAUTION: Very slow. Try not to use. # Can be useful for making dumps or to restore a database.
# Can be useful for making dumps or to restore a database, however.
def each_with_index(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil) def each_with_index(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil)
i = -1 # do not trust key to be the right index i = -1 # do not trust key to be the right index
(reversed ? @data.reverse : @data).each do |index, v| (reversed ? @data.reverse : @data).each do |index, v|
@ -97,149 +88,6 @@ class DODB::CachedDataBase(V) < DODB::Storage(V)
yield v, index yield v, index
end end
end end
def each(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil)
each_with_index(
reversed: reversed,
start_offset: start_offset,
end_offset: end_offset
) do |item, index|
yield item
end
end
def to_a(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil)
res = Array(V).new
each(reversed, start_offset, end_offset) do |v|
res << v
end
res
end
def to_h(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil)
res = Hash(Int32, V).new
(reversed ? @data.reverse : @data).each do |index, v|
next if start_offset > index
break unless end_offset.nil? || index < end_offset
res[index] = v
end
res
end
private def index_file
"#{@directory_name}/last-index"
end
def last_index : Int32
File.read(index_file).to_i
end
def last_index=(x : Int32)
file = File.open(index_file, "w")
file << x.to_s
file.close
x
rescue
raise Exception.new "could not update index file"
end
def stringify_key(key : Int32)
# Negative numbers give strange results with Crystal’s printf.
if key >= 0
"%010i" % key
else
key.to_s
end
end
def request_lock(name)
r = -1
file_path = get_lock_file_path name
file_perms = 0o644
flags = LibC::O_EXCL | LibC::O_CREAT
while (r = LibC.open file_path, flags, file_perms) == -1
sleep 1.milliseconds
end
LibC.close r
end
def release_lock(name)
File.delete get_lock_file_path name
end
##
# name is the name that will be used on the file system.
def new_partition(name : String, &block : Proc(V, String))
Partition(V).new(self, @directory_name, name, block).tap do |table|
@indexers << table
end
end
##
# name is the name that will be used on the file system.
def new_index(name : String, &block : Proc(V, String))
Index(V).new(self, @directory_name, name, block).tap do |indexer|
@indexers << indexer
end
end
def new_tags(name : String, &block : Proc(V, Array(String)))
Tags(V).new(@directory_name, name, block).tap do |tags|
@indexers << tags
end
end
def get_index(name : String, key)
index = @indexers.find &.name.==(name)
index.not_nil!.as(DODB::Index).get key
end
def get_partition(table_name : String, partition_name : String)
partition = @indexers.find &.name.==(table_name)
partition.not_nil!.as(DODB::Partition).get partition_name
end
def get_tags(name, key : String)
partition = @indexers.find &.name.==(name)
partition.not_nil!.as(DODB::Tags).get name, key
end
def check_collisions!(key : Int32, value : V, old_value : V?)
@indexers.each &.check!(stringify_key(key), value, old_value)
end
def write_partitions(key : Int32, value : V)
@indexers.each &.index(stringify_key(key), value)
end
def pop
index = last_index
# Some entries may have been removed. We’ll skip over those.
# Not the most efficient if a large number of indices are empty.
while index >= 0 && self[index]?.nil?
index = index - 1
end
if index < 0
return nil
end
poped = self[index]
self.delete index
last_index = index - 1
poped
end
def delete(key : Int32) def delete(key : Int32)
value = self[key]? value = self[key]?
@ -257,56 +105,4 @@ class DODB::CachedDataBase(V) < DODB::Storage(V)
@data.delete key @data.delete key
value value
end end
def remove_partitions(key : Int32, value : V)
@indexers.each &.deindex(stringify_key(key), value)
end
private def data_path
"#{@directory_name}/data"
end
private def file_path(key : Int32)
"#{data_path}/%010i.json" % key
end
private def locks_directory : String
"#{@directory_name}/locks"
end
private def get_lock_file_path(name : String)
"#{locks_directory}/#{name}.lock"
end
private def read(file_path : String)
V.from_json ::File.read file_path
end
private def remove_data!
FileUtils.rm_rf data_path
Dir.mkdir_p data_path
@data = Hash(Int32, V).new
end
private def remove_indexing!
@indexers.each do |indexer|
FileUtils.rm_rf indexer.indexing_directory
end
end
# A very slow operation that removes all indices and then rewrites
# them all.
# FIXME: Is this really useful in its current form? We should remove the
# index directories, not the indices based on our current (and
# possiblly different from what’s stored) data.
def reindex_everything!
old_data = to_h
remove_indexing!
remove_data!
old_data.each do |index, item|
self[index] = item
end
end
end end

303
src/dodb.cr
View File

@ -4,6 +4,11 @@ require "json"
require "./dodb/*" require "./dodb/*"
abstract class DODB::Storage(V) abstract class DODB::Storage(V)
property directory_name : String
def initialize(@directory_name : String)
end
def request_lock(name) def request_lock(name)
r = -1 r = -1
file_path = get_lock_file_path name file_path = get_lock_file_path name
@ -20,25 +25,6 @@ abstract class DODB::Storage(V)
File.delete get_lock_file_path name File.delete get_lock_file_path name
end end
abstract def [](key : Int32)
abstract def <<(item : V)
abstract def delete(key : Int32)
end
class DODB::DataBase(V) < DODB::Storage(V)
@indexers = [] of Indexer(V)
def initialize(@directory_name : String)
Dir.mkdir_p data_path
Dir.mkdir_p locks_directory
begin
self.last_index
rescue
self.last_index = -1
end
end
private def index_file private def index_file
"#{@directory_name}/last-index" "#{@directory_name}/last-index"
end end
@ -66,14 +52,54 @@ class DODB::DataBase(V) < DODB::Storage(V)
end end
end end
## def <<(item : V)
# name is the name that will be used on the file system. index = last_index + 1
def new_partition(name : String, &block : Proc(V, String)) self[index] = item
Partition(V).new(self, @directory_name, name, block).tap do |table| self.last_index = index
@indexers << table end
def each(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil)
each_with_index(
reversed: reversed,
start_offset: start_offset,
end_offset: end_offset
) do |item, index|
yield item
end end
end end
##
# CAUTION: Very slow. Try not to use.
def to_a(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil)
array = ::Array(V).new
each(
reversed: reversed,
start_offset: start_offset,
end_offset: end_offset
) do |value|
array << value
end
array
end
##
# CAUTION: Very slow. Try not to use.
def to_h(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil)
hash = ::Hash(Int32, V).new
each_with_index(
reversed: reversed,
start_offset: start_offset,
end_offset: end_offset
) do |element, index|
hash[index] = element
end
hash
end
## ##
# name is the name that will be used on the file system. # name is the name that will be used on the file system.
def new_index(name : String, &block : Proc(V, String)) def new_index(name : String, &block : Proc(V, String))
@ -82,6 +108,14 @@ class DODB::DataBase(V) < DODB::Storage(V)
end end
end end
##
# name is the name that will be used on the file system.
def new_partition(name : String, &block : Proc(V, String))
Partition(V).new(self, @directory_name, name, block).tap do |table|
@indexers << table
end
end
def new_tags(name : String, &block : Proc(V, Array(String))) def new_tags(name : String, &block : Proc(V, Array(String)))
Tags(V).new(@directory_name, name, block).tap do |tags| Tags(V).new(@directory_name, name, block).tap do |tags|
@indexers << tags @indexers << tags
@ -106,12 +140,103 @@ class DODB::DataBase(V) < DODB::Storage(V)
partition.not_nil!.as(DODB::Tags).get name, key partition.not_nil!.as(DODB::Tags).get name, key
end end
def <<(item : V) def check_collisions!(key : Int32, value : V, old_value : V?)
index = last_index + 1 @indexers.each &.check!(stringify_key(key), value, old_value)
end
self[index] = item def write_partitions(key : Int32, value : V)
@indexers.each &.index(stringify_key(key), value)
end
self.last_index = index def pop
index = last_index
# Some entries may have been removed. We’ll skip over those.
# Not the most efficient if a large number of indices are empty.
while index >= 0 && self[index]?.nil?
index = index - 1
end
if index < 0
return nil
end
poped = self[index]
self.delete index
last_index = index - 1
poped
end
private def data_path
"#{@directory_name}/data"
end
private def file_path(key : Int32)
"#{data_path}/%010i.json" % key
end
private def locks_directory : String
"#{@directory_name}/locks"
end
private def get_lock_file_path(name : String)
"#{locks_directory}/#{name}.lock"
end
private def read(file_path : String)
V.from_json ::File.read file_path
end
private def remove_data!
FileUtils.rm_rf data_path
Dir.mkdir_p data_path
end
private def remove_indexing!
@indexers.each do |indexer|
FileUtils.rm_rf indexer.indexing_directory
end
end
# A very slow operation that removes all indices and then rewrites
# them all.
# FIXME: Is this really useful in its current form? We should remove the
# index directories, not the indices based on our current (and
# possiblly different from what’s stored) data.
def reindex_everything!
old_data = to_h
remove_indexing!
remove_data!
old_data.each do |index, item|
self[index] = item
end
end
def remove_partitions(key : Int32, value : V)
@indexers.each &.deindex(stringify_key(key), value)
end
abstract def [](key : Int32)
abstract def delete(key : Int32)
end
class DODB::DataBase(V) < DODB::Storage(V)
@indexers = [] of Indexer(V)
def initialize(@directory_name : String)
Dir.mkdir_p data_path
Dir.mkdir_p locks_directory
begin
self.last_index
rescue
self.last_index = -1
end
end end
def []?(key : Int32) : V? def []?(key : Int32) : V?
@ -151,36 +276,6 @@ class DODB::DataBase(V) < DODB::Storage(V)
end end
end end
def check_collisions!(key : Int32, value : V, old_value : V?)
@indexers.each &.check!(stringify_key(key), value, old_value)
end
def write_partitions(key : Int32, value : V)
@indexers.each &.index(stringify_key(key), value)
end
def pop
index = last_index
# Some entries may have been removed. We’ll skip over those.
# Not the most efficient if a large number of indices are empty.
while index >= 0 && self[index]?.nil?
index = index - 1
end
if index < 0
return nil
end
poped = self[index]
self.delete index
last_index = index - 1
poped
end
def delete(key : Int32) def delete(key : Int32)
value = self[key]? value = self[key]?
@ -197,10 +292,6 @@ class DODB::DataBase(V) < DODB::Storage(V)
value value
end end
def remove_partitions(key : Int32, value : V)
@indexers.each &.deindex(stringify_key(key), value)
end
private def each_key(reversed = false) private def each_key(reversed = false)
start = 0 start = 0
_end = last_index _end = last_index
@ -252,94 +343,6 @@ class DODB::DataBase(V) < DODB::Storage(V)
yield field, key yield field, key
end end
end end
def each(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil)
each_with_index(
reversed: reversed,
start_offset: start_offset,
end_offset: end_offset
) do |item, index|
yield item
end
end
##
# CAUTION: Very slow. Try not to use.
def to_a(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil)
array = ::Array(V).new
each(
reversed: reversed,
start_offset: start_offset,
end_offset: end_offset
) do |value|
array << value
end
array
end
##
# CAUTION: Very slow. Try not to use.
def to_h(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil)
hash = ::Hash(Int32, V).new
each_with_index(
reversed: reversed,
start_offset: start_offset,
end_offset: end_offset
) do |element, index|
hash[index] = element
end
hash
end
private def data_path
"#{@directory_name}/data"
end
private def file_path(key : Int32)
"#{data_path}/%010i.json" % key
end
private def locks_directory : String
"#{@directory_name}/locks"
end
private def get_lock_file_path(name : String)
"#{locks_directory}/#{name}.lock"
end
private def read(file_path : String)
V.from_json ::File.read file_path
end
private def remove_data!
FileUtils.rm_rf data_path
Dir.mkdir_p data_path
end
private def remove_indexing!
@indexers.each do |indexer|
FileUtils.rm_rf indexer.indexing_directory
end
end
# A very slow operation that removes all indices and then rewrites
# them all.
# FIXME: Is this really useful in its current form? We should remove the
# index directories, not the indices based on our current (and
# possiblly different from what’s stored) data.
def reindex_everything!
old_data = to_h
remove_indexing!
remove_data!
old_data.each do |index, item|
self[index] = item
end
end
end end
require "./cached.cr" require "./cached.cr"