Remove nilable indexes.

Makefile

@@ -15,3 +15,13 @@ wipe-db:
 
 release:
 	make build OPTS="--release --progress"
+
+doc:
+	crystal docs
+
+HTTPD_ACCESS_LOGS ?= /tmp/access-dodb-docs.log
+HTTPD_ADDR        ?= 127.0.0.1
+HTTPD_PORT        ?= 9000
+DIR               ?= docs
+serve-doc:
+	darkhttpd $(DIR) --addr $(HTTPD_ADDR) --port $(HTTPD_PORT) --log $(HTTPD_ACCESS_LOGS)

graphs/graphs.ms

@@ -850,25 +850,27 @@ T}:400 to 500x slower
 as expected, retrieving a single value is fast and the size of the database doesn't matter much.
 Each deserialization and, more importantly, each disk access is a pain point.
 Caching the value enables a massive performance gain, data can be retrieved several hundred times quicker.
-.bp
 .SS Partitions (1 to n relations)
 .LP
 
 .so graph_query_partition.grap
 
-.bp
 .SS Tags (n to n relations)
 .LP
 .so graph_query_tag.grap
 .
+.
+.
 .SECTION Future work
 This section presents all the features I want to see in a future version of the DODB library.
+.
 .SS Cached database and indexes with selective memory
 Right now, both cached database and cached indexes will store any cached value indefinitively.
 Giving the cache the ability to select the values to keep in memory would enable a massive speed-up even in memory-constrained environments.
 The policy could be as simple as keeping in memory only the most recently requested values.
 
 These new versions of cached database and indexes will become the standard, default DODB behavior.
+.
 .SS Pagination via the indexes: offset and limit
 Right now, browsing the entire database by requesting a limited list at a time is possible, thanks to some functions accepting an
 .I offset
@@ -876,5 +878,24 @@ and a
 .I size .
 However, this is not possible with the indexes, thus when querying for example a partition the API provides the entire list of matching values.
 This is not acceptable for databases with large partitions and tags: memory will be over-used and requests will be slow.
+.
+.SS DODB and security
+Right now, security isn't managed in DODB, at all.
+Sure, DODB isn't vulnerable to SQL injections, but an internet-facing application may encounter a few other problems including, but not limited to, code injection, buffer overflows, etc.
+Of course, DODB isn't a mechanism to protect applications from any possible attack, so most of the vulnerabilities cannot be countered by the library.
+However, a few security mechanisms exist to prevent data leak or data modification from an outsider, and the DODB library should implement some of them in the future.
+
+.B "Preventing data leak" .
+Since DODB is a library, any attack on the application using it can lead to a data leak.
+For the moment, any part of the application can access data stored in memory.
+Operating systems provide system calls to protect parts of the allocated memory;
+.FUNCTION_CALL mlock ,
+.FUNCTION_CALL mprotect
+prevents a region of memory from being put in the swap.
+
+.B "Discussion on security, not related to DODB" .
+No authorization mechanism prevents the application to access un-authorized data, including, but not limited to, any file on the file-system.
+.
+.
 .SECTION Conclusion
 .TBD

src/cached.cr

@@ -1,5 +1,6 @@
 require "file_utils"
 require "json"
+require "./dodb.cr"
 
 class Hash(K,V)
 	def reverse

src/dodb.cr

@@ -111,37 +111,19 @@ abstract class DODB::Storage(V)
 
 	##
 	# name is the name that will be used on the file system.
-	def new_index(name : String, &block : Proc(V, String))
+	def new_index(name : String, &block : Proc(V, String) | Proc(V, String | DODB::NoIndex))
 		CachedIndex(V).new(self, @directory_name, name, block).tap do |indexer|
 			@indexers << indexer
 		end
 	end
 
-	def new_nilable_index(name : String, &block : Proc(V, String | DODB::NoIndex))
+	def new_uncached_index(name : String, &block : Proc(V, String) | Proc(V, String | DODB::NoIndex))
-		CachedIndex(V).new(self, @directory_name, name, block).tap do |indexer|
-			@indexers << indexer
-		end
-	end
-
-	def new_uncached_index(name : String, &block : Proc(V, String))
 		Index(V).new(self, @directory_name, name, block).tap do |indexer|
 			@indexers << indexer
 		end
 	end
 
-	def new_nilable_uncached_index(name : String, &block : Proc(V, String | DODB::NoIndex))
+	def new_RAM_index(name : String, &block : Proc(V, String) | Proc(V, String | DODB::NoIndex))
-		Index(V).new(self, @directory_name, name, block).tap do |indexer|
-			@indexers << indexer
-		end
-	end
-
-	def new_RAM_index(name : String, &block : Proc(V, String))
-		RAMOnlyIndex(V).new(self, @directory_name, name, block).tap do |indexer|
-			@indexers << indexer
-		end
-	end
-
-	def new_nilable_RAM_index(name : String, &block : Proc(V, String | DODB::NoIndex))
 		RAMOnlyIndex(V).new(self, @directory_name, name, block).tap do |indexer|
 			@indexers << indexer
 		end
@@ -155,37 +137,19 @@ abstract class DODB::Storage(V)
 
 	##
 	# name is the name that will be used on the file system.
-	def new_partition(name : String, &block : Proc(V, String))
+	def new_partition(name : String, &block : Proc(V, String) | Proc(V, String | DODB::NoIndex))
 		CachedPartition(V).new(self, @directory_name, name, block).tap do |table|
 			@indexers << table
 		end
 	end
 
-	def new_uncached_partition(name : String, &block : Proc(V, String))
+	def new_uncached_partition(name : String, &block : Proc(V, String) | Proc(V, String | DODB::NoIndex))
 		Partition(V).new(self, @directory_name, name, block).tap do |table|
 			@indexers << table
 		end
 	end
 
-	def new_RAM_partition(name : String, &block : Proc(V, String))
+	def new_RAM_partition(name : String, &block : Proc(V, String) | Proc(V, String | DODB::NoIndex))
-		RAMOnlyPartition(V).new(self, @directory_name, name, block).tap do |table|
-			@indexers << table
-		end
-	end
-
-	def new_nilable_partition(name : String, &block : Proc(V, String | DODB::NoIndex))
-		CachedPartition(V).new(self, @directory_name, name, block).tap do |table|
-			@indexers << table
-		end
-	end
-
-	def new_nilable_uncached_partition(name : String, &block : Proc(V, String | DODB::NoIndex))
-		Partition(V).new(self, @directory_name, name, block).tap do |table|
-			@indexers << table
-		end
-	end
-
-	def new_nilable_RAM_partition(name : String, &block : Proc(V, String | DODB::NoIndex))
 		RAMOnlyPartition(V).new(self, @directory_name, name, block).tap do |table|
 			@indexers << table
 		end
@@ -201,37 +165,19 @@ abstract class DODB::Storage(V)
 		@indexers.each &.index(stringify_key(key), value)
 	end
 
-	def new_tags(name : String, &block : Proc(V, Array(String)))
+	def new_tags(name : String, &block : Proc(V, Array(String)) | Proc(V, Array(String) | DODB::NoIndex))
 		CachedTags(V).new(self, @directory_name, name, block).tap do |tags|
 			@indexers << tags
 		end
 	end
 
-	def new_uncached_tags(name : String, &block : Proc(V, Array(String)))
+	def new_uncached_tags(name : String, &block : Proc(V, Array(String)) | Proc(V, Array(String) | DODB::NoIndex))
 		Tags(V).new(self, @directory_name, name, block).tap do |tags|
 			@indexers << tags
 		end
 	end
 
-	def new_RAM_tags(name : String, &block : Proc(V, Array(String)))
+	def new_RAM_tags(name : String, &block : Proc(V, Array(String)) | Proc(V, Array(String) | DODB::NoIndex))
-		RAMOnlyTags(V).new(self, @directory_name, name, block).tap do |tags|
-			@indexers << tags
-		end
-	end
-
-	def new_nilable_tags(name : String, &block : Proc(V, Array(String) | DODB::NoIndex))
-		CachedTags(V).new(self, @directory_name, name, block).tap do |tags|
-			@indexers << tags
-		end
-	end
-
-	def new_nilable_uncached_tags(name : String, &block : Proc(V, Array(String) | DODB::NoIndex))
-		Tags(V).new(self, @directory_name, name, block).tap do |tags|
-			@indexers << tags
-		end
-	end
-
-	def new_nilable_RAM_tags(name : String, &block : Proc(V, Array(String) | DODB::NoIndex))
 		RAMOnlyTags(V).new(self, @directory_name, name, block).tap do |tags|
 			@indexers << tags
 		end

src/dodb/index.cr

@@ -119,6 +119,12 @@ class DODB::Index(V) < DODB::Indexer(V)
 		@storage[key] = new_value
 	end
 
+	def update_or_create(new_value : V)
+		update new_value
+	rescue MissingEntry
+		@storage << new_value
+	end
+
 	def update_or_create(index : String, new_value : V)
 		update index, new_value
 	rescue MissingEntry