Baguette/dodb.cr
3
0
Fork 1
Code Issues Pull Requests Releases Wiki Activity

Remove nilable indexes.

Browse Source
This commit is contained in:
Philippe PITTOLI 2024-05-17 19:47:11 +02:00
parent 700f86a7f5
commit ab139f4f52
5 changed files with 49 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
Makefile
View File

@ -15,3 +15,13 @@ wipe-db:
release:
make build OPTS="--release --progress"
doc:
crystal docs
HTTPD_ACCESS_LOGS ?= /tmp/access-dodb-docs.log
HTTPD_ADDR ?= 127.0.0.1
HTTPD_PORT ?= 9000
DIR ?= docs
serve-doc:
darkhttpd $(DIR) --addr $(HTTPD_ADDR) --port $(HTTPD_PORT) --log $(HTTPD_ACCESS_LOGS)

25
graphs/graphs.ms
View File

@ -850,25 +850,27 @@ T}:400 to 500x slower
as expected, retrieving a single value is fast and the size of the database doesn't matter much.
Each deserialization and, more importantly, each disk access is a pain point.
Caching the value enables a massive performance gain, data can be retrieved several hundred times quicker.
.bp
.SS Partitions (1 to n relations)
.LP
.so graph_query_partition.grap
.bp
.SS Tags (n to n relations)
.LP
.so graph_query_tag.grap
.
.
.
.SECTION Future work
This section presents all the features I want to see in a future version of the DODB library.
.
.SS Cached database and indexes with selective memory
Right now, both cached database and cached indexes will store any cached value indefinitively.
Giving the cache the ability to select the values to keep in memory would enable a massive speed-up even in memory-constrained environments.
The policy could be as simple as keeping in memory only the most recently requested values.
These new versions of cached database and indexes will become the standard, default DODB behavior.
.
.SS Pagination via the indexes: offset and limit
Right now, browsing the entire database by requesting a limited list at a time is possible, thanks to some functions accepting an
.I offset
@ -876,5 +878,24 @@ and a
.I size .
However, this is not possible with the indexes, thus when querying for example a partition the API provides the entire list of matching values.
This is not acceptable for databases with large partitions and tags: memory will be over-used and requests will be slow.
.
.SS DODB and security
Right now, security isn't managed in DODB, at all.
Sure, DODB isn't vulnerable to SQL injections, but an internet-facing application may encounter a few other problems including, but not limited to, code injection, buffer overflows, etc.
Of course, DODB isn't a mechanism to protect applications from any possible attack, so most of the vulnerabilities cannot be countered by the library.
However, a few security mechanisms exist to prevent data leak or data modification from an outsider, and the DODB library should implement some of them in the future.
.B "Preventing data leak" .
Since DODB is a library, any attack on the application using it can lead to a data leak.
For the moment, any part of the application can access data stored in memory.
Operating systems provide system calls to protect parts of the allocated memory;
.FUNCTION_CALL mlock ,
.FUNCTION_CALL mprotect
prevents a region of memory from being put in the swap.
.B "Discussion on security, not related to DODB" .
No authorization mechanism prevents the application to access un-authorized data, including, but not limited to, any file on the file-system.
.
.
.SECTION Conclusion
.TBD

1
src/cached.cr
View File

@ -1,5 +1,6 @@
require "file_utils"
require "json"
require "./dodb.cr"
class Hash(K,V)
def reverse

72
src/dodb.cr
View File

@ -111,37 +111,19 @@ abstract class DODB::Storage(V)
##
# name is the name that will be used on the file system.
def new_index(name : String, &block : Proc(V, String))
def new_index(name : String, &block : Proc(V, String) | Proc(V, String | DODB::NoIndex))
CachedIndex(V).new(self, @directory_name, name, block).tap do |indexer|
@indexers << indexer
end
end
def new_nilable_index(name : String, &block : Proc(V, String | DODB::NoIndex))
CachedIndex(V).new(self, @directory_name, name, block).tap do |indexer|
@indexers << indexer
end
end
def new_uncached_index(name : String, &block : Proc(V, String))
def new_uncached_index(name : String, &block : Proc(V, String) | Proc(V, String | DODB::NoIndex))
Index(V).new(self, @directory_name, name, block).tap do |indexer|
@indexers << indexer
end
end
def new_nilable_uncached_index(name : String, &block : Proc(V, String | DODB::NoIndex))
Index(V).new(self, @directory_name, name, block).tap do |indexer|
@indexers << indexer
end
end
def new_RAM_index(name : String, &block : Proc(V, String))
RAMOnlyIndex(V).new(self, @directory_name, name, block).tap do |indexer|
@indexers << indexer
end
end
def new_nilable_RAM_index(name : String, &block : Proc(V, String | DODB::NoIndex))
def new_RAM_index(name : String, &block : Proc(V, String) | Proc(V, String | DODB::NoIndex))
RAMOnlyIndex(V).new(self, @directory_name, name, block).tap do |indexer|
@indexers << indexer
end
@ -155,37 +137,19 @@ abstract class DODB::Storage(V)
##
# name is the name that will be used on the file system.
def new_partition(name : String, &block : Proc(V, String))
def new_partition(name : String, &block : Proc(V, String) | Proc(V, String | DODB::NoIndex))
CachedPartition(V).new(self, @directory_name, name, block).tap do |table|
@indexers << table
end
end
def new_uncached_partition(name : String, &block : Proc(V, String))
def new_uncached_partition(name : String, &block : Proc(V, String) | Proc(V, String | DODB::NoIndex))
Partition(V).new(self, @directory_name, name, block).tap do |table|
@indexers << table
end
end
def new_RAM_partition(name : String, &block : Proc(V, String))
RAMOnlyPartition(V).new(self, @directory_name, name, block).tap do |table|
@indexers << table
end
end
def new_nilable_partition(name : String, &block : Proc(V, String | DODB::NoIndex))
CachedPartition(V).new(self, @directory_name, name, block).tap do |table|
@indexers << table
end
end
def new_nilable_uncached_partition(name : String, &block : Proc(V, String | DODB::NoIndex))
Partition(V).new(self, @directory_name, name, block).tap do |table|
@indexers << table
end
end
def new_nilable_RAM_partition(name : String, &block : Proc(V, String | DODB::NoIndex))
def new_RAM_partition(name : String, &block : Proc(V, String) | Proc(V, String | DODB::NoIndex))
RAMOnlyPartition(V).new(self, @directory_name, name, block).tap do |table|
@indexers << table
end
@ -201,37 +165,19 @@ abstract class DODB::Storage(V)
@indexers.each &.index(stringify_key(key), value)
end
def new_tags(name : String, &block : Proc(V, Array(String)))
def new_tags(name : String, &block : Proc(V, Array(String)) | Proc(V, Array(String) | DODB::NoIndex))
CachedTags(V).new(self, @directory_name, name, block).tap do |tags|
@indexers << tags
end
end
def new_uncached_tags(name : String, &block : Proc(V, Array(String)))
def new_uncached_tags(name : String, &block : Proc(V, Array(String)) | Proc(V, Array(String) | DODB::NoIndex))
Tags(V).new(self, @directory_name, name, block).tap do |tags|
@indexers << tags
end
end
def new_RAM_tags(name : String, &block : Proc(V, Array(String)))
RAMOnlyTags(V).new(self, @directory_name, name, block).tap do |tags|
@indexers << tags
end
end
def new_nilable_tags(name : String, &block : Proc(V, Array(String) | DODB::NoIndex))
CachedTags(V).new(self, @directory_name, name, block).tap do |tags|
@indexers << tags
end
end
def new_nilable_uncached_tags(name : String, &block : Proc(V, Array(String) | DODB::NoIndex))
Tags(V).new(self, @directory_name, name, block).tap do |tags|
@indexers << tags
end
end
def new_nilable_RAM_tags(name : String, &block : Proc(V, Array(String) | DODB::NoIndex))
def new_RAM_tags(name : String, &block : Proc(V, Array(String)) | Proc(V, Array(String) | DODB::NoIndex))
RAMOnlyTags(V).new(self, @directory_name, name, block).tap do |tags|
@indexers << tags
end

6
src/dodb/index.cr
View File

@ -119,6 +119,12 @@ class DODB::Index(V) < DODB::Indexer(V)
@storage[key] = new_value
end
def update_or_create(new_value : V)
update new_value
rescue MissingEntry
@storage << new_value
end
def update_or_create(index : String, new_value : V)
update index, new_value
rescue MissingEntry