From ab139f4f52b2f471d6e75c18257d3cf4de687160 Mon Sep 17 00:00:00 2001 From: Philippe PITTOLI Date: Fri, 17 May 2024 19:47:11 +0200 Subject: [PATCH] Remove nilable indexes. --- Makefile | 10 +++++++ graphs/graphs.ms | 25 ++++++++++++++-- src/cached.cr | 1 + src/dodb.cr | 72 ++++++----------------------------------------- src/dodb/index.cr | 6 ++++ 5 files changed, 49 insertions(+), 65 deletions(-) diff --git a/Makefile b/Makefile index 9ebf96a..919f689 100644 --- a/Makefile +++ b/Makefile @@ -15,3 +15,13 @@ wipe-db: release: make build OPTS="--release --progress" + +doc: + crystal docs + +HTTPD_ACCESS_LOGS ?= /tmp/access-dodb-docs.log +HTTPD_ADDR ?= 127.0.0.1 +HTTPD_PORT ?= 9000 +DIR ?= docs +serve-doc: + darkhttpd $(DIR) --addr $(HTTPD_ADDR) --port $(HTTPD_PORT) --log $(HTTPD_ACCESS_LOGS) diff --git a/graphs/graphs.ms b/graphs/graphs.ms index 77899f7..0f0ae2f 100644 --- a/graphs/graphs.ms +++ b/graphs/graphs.ms @@ -850,25 +850,27 @@ T}:400 to 500x slower as expected, retrieving a single value is fast and the size of the database doesn't matter much. Each deserialization and, more importantly, each disk access is a pain point. Caching the value enables a massive performance gain, data can be retrieved several hundred times quicker. -.bp .SS Partitions (1 to n relations) .LP .so graph_query_partition.grap -.bp .SS Tags (n to n relations) .LP .so graph_query_tag.grap . +. +. .SECTION Future work This section presents all the features I want to see in a future version of the DODB library. +. .SS Cached database and indexes with selective memory Right now, both cached database and cached indexes will store any cached value indefinitively. Giving the cache the ability to select the values to keep in memory would enable a massive speed-up even in memory-constrained environments. The policy could be as simple as keeping in memory only the most recently requested values. These new versions of cached database and indexes will become the standard, default DODB behavior. +. .SS Pagination via the indexes: offset and limit Right now, browsing the entire database by requesting a limited list at a time is possible, thanks to some functions accepting an .I offset @@ -876,5 +878,24 @@ and a .I size . However, this is not possible with the indexes, thus when querying for example a partition the API provides the entire list of matching values. This is not acceptable for databases with large partitions and tags: memory will be over-used and requests will be slow. +. +.SS DODB and security +Right now, security isn't managed in DODB, at all. +Sure, DODB isn't vulnerable to SQL injections, but an internet-facing application may encounter a few other problems including, but not limited to, code injection, buffer overflows, etc. +Of course, DODB isn't a mechanism to protect applications from any possible attack, so most of the vulnerabilities cannot be countered by the library. +However, a few security mechanisms exist to prevent data leak or data modification from an outsider, and the DODB library should implement some of them in the future. + +.B "Preventing data leak" . +Since DODB is a library, any attack on the application using it can lead to a data leak. +For the moment, any part of the application can access data stored in memory. +Operating systems provide system calls to protect parts of the allocated memory; +.FUNCTION_CALL mlock , +.FUNCTION_CALL mprotect +prevents a region of memory from being put in the swap. + +.B "Discussion on security, not related to DODB" . +No authorization mechanism prevents the application to access un-authorized data, including, but not limited to, any file on the file-system. +. +. .SECTION Conclusion .TBD diff --git a/src/cached.cr b/src/cached.cr index 987410b..9bdd945 100644 --- a/src/cached.cr +++ b/src/cached.cr @@ -1,5 +1,6 @@ require "file_utils" require "json" +require "./dodb.cr" class Hash(K,V) def reverse diff --git a/src/dodb.cr b/src/dodb.cr index 27ad9db..9b1161c 100644 --- a/src/dodb.cr +++ b/src/dodb.cr @@ -111,37 +111,19 @@ abstract class DODB::Storage(V) ## # name is the name that will be used on the file system. - def new_index(name : String, &block : Proc(V, String)) + def new_index(name : String, &block : Proc(V, String) | Proc(V, String | DODB::NoIndex)) CachedIndex(V).new(self, @directory_name, name, block).tap do |indexer| @indexers << indexer end end - def new_nilable_index(name : String, &block : Proc(V, String | DODB::NoIndex)) - CachedIndex(V).new(self, @directory_name, name, block).tap do |indexer| - @indexers << indexer - end - end - - def new_uncached_index(name : String, &block : Proc(V, String)) + def new_uncached_index(name : String, &block : Proc(V, String) | Proc(V, String | DODB::NoIndex)) Index(V).new(self, @directory_name, name, block).tap do |indexer| @indexers << indexer end end - def new_nilable_uncached_index(name : String, &block : Proc(V, String | DODB::NoIndex)) - Index(V).new(self, @directory_name, name, block).tap do |indexer| - @indexers << indexer - end - end - - def new_RAM_index(name : String, &block : Proc(V, String)) - RAMOnlyIndex(V).new(self, @directory_name, name, block).tap do |indexer| - @indexers << indexer - end - end - - def new_nilable_RAM_index(name : String, &block : Proc(V, String | DODB::NoIndex)) + def new_RAM_index(name : String, &block : Proc(V, String) | Proc(V, String | DODB::NoIndex)) RAMOnlyIndex(V).new(self, @directory_name, name, block).tap do |indexer| @indexers << indexer end @@ -155,37 +137,19 @@ abstract class DODB::Storage(V) ## # name is the name that will be used on the file system. - def new_partition(name : String, &block : Proc(V, String)) + def new_partition(name : String, &block : Proc(V, String) | Proc(V, String | DODB::NoIndex)) CachedPartition(V).new(self, @directory_name, name, block).tap do |table| @indexers << table end end - def new_uncached_partition(name : String, &block : Proc(V, String)) + def new_uncached_partition(name : String, &block : Proc(V, String) | Proc(V, String | DODB::NoIndex)) Partition(V).new(self, @directory_name, name, block).tap do |table| @indexers << table end end - def new_RAM_partition(name : String, &block : Proc(V, String)) - RAMOnlyPartition(V).new(self, @directory_name, name, block).tap do |table| - @indexers << table - end - end - - def new_nilable_partition(name : String, &block : Proc(V, String | DODB::NoIndex)) - CachedPartition(V).new(self, @directory_name, name, block).tap do |table| - @indexers << table - end - end - - def new_nilable_uncached_partition(name : String, &block : Proc(V, String | DODB::NoIndex)) - Partition(V).new(self, @directory_name, name, block).tap do |table| - @indexers << table - end - end - - def new_nilable_RAM_partition(name : String, &block : Proc(V, String | DODB::NoIndex)) + def new_RAM_partition(name : String, &block : Proc(V, String) | Proc(V, String | DODB::NoIndex)) RAMOnlyPartition(V).new(self, @directory_name, name, block).tap do |table| @indexers << table end @@ -201,37 +165,19 @@ abstract class DODB::Storage(V) @indexers.each &.index(stringify_key(key), value) end - def new_tags(name : String, &block : Proc(V, Array(String))) + def new_tags(name : String, &block : Proc(V, Array(String)) | Proc(V, Array(String) | DODB::NoIndex)) CachedTags(V).new(self, @directory_name, name, block).tap do |tags| @indexers << tags end end - def new_uncached_tags(name : String, &block : Proc(V, Array(String))) + def new_uncached_tags(name : String, &block : Proc(V, Array(String)) | Proc(V, Array(String) | DODB::NoIndex)) Tags(V).new(self, @directory_name, name, block).tap do |tags| @indexers << tags end end - def new_RAM_tags(name : String, &block : Proc(V, Array(String))) - RAMOnlyTags(V).new(self, @directory_name, name, block).tap do |tags| - @indexers << tags - end - end - - def new_nilable_tags(name : String, &block : Proc(V, Array(String) | DODB::NoIndex)) - CachedTags(V).new(self, @directory_name, name, block).tap do |tags| - @indexers << tags - end - end - - def new_nilable_uncached_tags(name : String, &block : Proc(V, Array(String) | DODB::NoIndex)) - Tags(V).new(self, @directory_name, name, block).tap do |tags| - @indexers << tags - end - end - - def new_nilable_RAM_tags(name : String, &block : Proc(V, Array(String) | DODB::NoIndex)) + def new_RAM_tags(name : String, &block : Proc(V, Array(String)) | Proc(V, Array(String) | DODB::NoIndex)) RAMOnlyTags(V).new(self, @directory_name, name, block).tap do |tags| @indexers << tags end diff --git a/src/dodb/index.cr b/src/dodb/index.cr index ac937b9..3842305 100644 --- a/src/dodb/index.cr +++ b/src/dodb/index.cr @@ -119,6 +119,12 @@ class DODB::Index(V) < DODB::Indexer(V) @storage[key] = new_value end + def update_or_create(new_value : V) + update new_value + rescue MissingEntry + @storage << new_value + end + def update_or_create(index : String, new_value : V) update index, new_value rescue MissingEntry