From 814baced0506944b27769d5b3c2e48972a07ff5b Mon Sep 17 00:00:00 2001 From: Philippe PITTOLI Date: Wed, 29 May 2024 16:47:56 +0200 Subject: [PATCH] `last_key` is now cached. --- paper/graphs/query_partition.grap | 20 ++++++++++++++++---- paper/legend.grap | 11 +++++++++++ paper/paper.ms | 8 ++++++++ spec/benchmark-cars.cr | 2 +- src/dodb/storage.cr | 27 ++++++++++++++++----------- src/dodb/storage/cached.cr | 13 ++----------- src/dodb/storage/common.cr | 6 +----- src/dodb/storage/ramonly.cr | 11 +++-------- 8 files changed, 58 insertions(+), 40 deletions(-) diff --git a/paper/graphs/query_partition.grap b/paper/graphs/query_partition.grap index 815cdc1..85d4258 100644 --- a/paper/graphs/query_partition.grap +++ b/paper/graphs/query_partition.grap @@ -20,11 +20,17 @@ legendydown = 220 boite(legendxleft,legendxright,legendyup,legendydown) legend(legendxleft,legendxright,legendyup,legendydown) +y_scale = 1000000 + +linear_growth(red, 110, 1000, 10000) +linear_growth(black, 110, 1000, 10000) +linear_growth(pink, 160, 1000, 10000) +linear_growth(blue, 13982, 1000, 10000) +linear_growth(green, 17350, 1000, 10000) + copy "../data/partitions.d" thru X cx = $1*2 - y_scale = 1000000 - line_no_mustache(cbram, obram, $3, red) line_no_mustache(cbcache, obcache, $6, black) line_no_mustache(cbfifo, obfifo, $9, pink) @@ -52,11 +58,17 @@ label right "\s+2Logarithmic" unaligned "scale\s0" obram = obuncache = obfifo = obcache = obsemi = 0 cbram = cbuncache = cbfifo = cbcache = cbsemi = 0 +y_scale = 1000000 + +linear_growth(red, 85, 1000, 10000) +linear_growth(black, 85, 1000, 10000) +linear_growth(pink, 160, 1000, 10000) +linear_growth(blue, 13982, 1000, 10000) +linear_growth(green, 17350, 1000, 10000) + copy "../data/partitions.d" thru X cx = $1*2 - y_scale = 1000000 - line_no_mustache(cbram, obram, $3, red) line_no_mustache(cbcache, obcache, $6, black) line_no_mustache(cbfifo, obfifo, $9, pink) diff --git a/paper/legend.grap b/paper/legend.grap index 9cbe433..4c98209 100644 --- a/paper/legend.grap +++ b/paper/legend.grap @@ -132,3 +132,14 @@ define line_with_mustache { line_no_mustache($1,$2,$3,$4) line from cx,$5/y_scale to cx,$6/y_scale } + +# WARNING: this macro y_scale. +define linear_growth { + # $1 = color + # $2 = duration of the retrieval for a single value + # $3 = min number of retrievals + # $4 = max number of retrievals +.gcolor $1 +line dashed from $3,($2*$3)/y_scale to $4,($2*$4)/y_scale +.gcolor +} diff --git a/paper/paper.ms b/paper/paper.ms index 280f47c..69a1cd5 100644 --- a/paper/paper.ms +++ b/paper/paper.ms @@ -993,12 +993,20 @@ Performance is analyzed based the partition size (the number of red cars) and th .ps \n[PS] .QP This figure shows the retrieval of cars based on a partition (their color), with both a linear and a logarithmic scale. +The number of cars retrieved scales from 2000 to 10000. .QE In this example, both the linear and the logarithmic scales are represented to better grasp the difference between all databases. The linear scale shows the linearity of the request time for uncached databases. Respectively, the logarithmically scaled figure does the same for cached databases, which are flattened in the linear scale since they all are hundreds of time quicker than the uncached ones. +The duration of a retrieval grows linearly with the number of matched entries. +On both figures, a dashed line is drawn representing a linear growth based on the quickest retrieval observed from basic indexes for each database. +This dashed line and the observed results differ slightly; observed results grow more than what has been calculated. +This difference comes, at least partially, from the additional process of putting all the results in an array (which may also include some memory management) and the accumulated random delays for the retrieval of each value (due to processus scheduling on the machine). + +Further analysis of the results may be interesting but are far beyond the scope of this document. + .SS Tags (n to n relations) .LP .ps -2 diff --git a/spec/benchmark-cars.cr b/spec/benchmark-cars.cr index 12454b1..a3c06c9 100644 --- a/spec/benchmark-cars.cr +++ b/spec/benchmark-cars.cr @@ -128,7 +128,7 @@ def perform_add(storage : DODB::Storage(Car)) perform_benchmark_average Context.nb_run, do corvet = corvet0.clone corvet.name = "Corvet-#{i}" - storage << corvet + storage.unsafe_add corvet i += 1 end end diff --git a/src/dodb/storage.cr b/src/dodb/storage.cr index 1614251..41e9d6c 100644 --- a/src/dodb/storage.cr +++ b/src/dodb/storage.cr @@ -4,6 +4,8 @@ abstract class DODB::Storage(V) # List of triggers (basic indexes, partitions, tags, etc.). @triggers = [] of Trigger(V) + property cached_last_key : Int32 + # Directory where data and triggers will be written. property directory_name : String @@ -16,11 +18,7 @@ abstract class DODB::Storage(V) Dir.mkdir_p data_path Dir.mkdir_p locks_directory - begin - self.last_key - rescue - self.last_key = -1 - end + @cached_last_key = init_last_key end # Requests a (named) lock. @@ -53,19 +51,26 @@ abstract class DODB::Storage(V) "#{@directory_name}/last-key" end - # Reads the last *key* in the database. - def last_key : Int32 + # Reads the last database *key* from the storage device. + def init_last_key : Int32 File.read(key_file).to_i + rescue + -1 + end + + # Reads the (cached) last key. + def last_key : Int32 + @cached_last_key end # Changes the last *key* in the database. def last_key=(x : Int32) file = File.open(key_file, "w") - file << x.to_s - file.close + @cached_last_key = x + x rescue raise Exception.new "could not update last-key file" @@ -104,7 +109,7 @@ abstract class DODB::Storage(V) # For single-thread applications, use the `#unsafe_add` operation instead. def <<(item : V) request_lock "key" - key = last_key + 1 + key = init_last_key + 1 self[key] = item self.last_key = key @@ -398,7 +403,7 @@ abstract class DODB::Storage(V) run_triggers key, value - if key > last_key + if key > @cached_last_key self.last_key = key end end diff --git a/src/dodb/storage/cached.cr b/src/dodb/storage/cached.cr index b0257d6..cd688c5 100644 --- a/src/dodb/storage/cached.cr +++ b/src/dodb/storage/cached.cr @@ -43,11 +43,7 @@ class DODB::Storage::Cached(V) < DODB::Storage(V) Dir.mkdir_p data_path Dir.mkdir_p locks_directory - begin - self.last_key - rescue - self.last_key = -1 - end + @cached_last_key = init_last_key # Load the database in RAM at start-up. DODB::Storage::Uncached(V).new(@directory_name).each_with_key do |v, key| @@ -56,11 +52,6 @@ class DODB::Storage::Cached(V) < DODB::Storage(V) end end - # Gets data from the hash in RAM. - def []?(key : Int32) : V? - self[key] rescue nil - end - # Gets the data with the *key*. # In case the data is missing, returns an exception `DODB::MissingEntry`. # @@ -91,7 +82,7 @@ class DODB::Storage::Cached(V) < DODB::Storage(V) run_triggers key, value - if key > last_key + if key > @cached_last_key self.last_key = key end diff --git a/src/dodb/storage/common.cr b/src/dodb/storage/common.cr index 744a25d..fe67ac1 100644 --- a/src/dodb/storage/common.cr +++ b/src/dodb/storage/common.cr @@ -43,11 +43,7 @@ class DODB::Storage::Common(V) < DODB::Storage::Cached(V) Dir.mkdir_p data_path Dir.mkdir_p locks_directory - begin - self.last_key - rescue - self.last_key = -1 - end + @cached_last_key = init_last_key end # Verifies that the value is in cache, or read it on disk. diff --git a/src/dodb/storage/ramonly.cr b/src/dodb/storage/ramonly.cr index 72d6ef6..58b5000 100644 --- a/src/dodb/storage/ramonly.cr +++ b/src/dodb/storage/ramonly.cr @@ -27,17 +27,12 @@ class DODB::Storage::RAMOnly(V) < DODB::Storage::Cached(V) def initialize(@directory_name : String) Dir.mkdir_p data_path Dir.mkdir_p locks_directory - @last_key = -1 - end - - # The `last_key` function doesn't read a file in the `DODB::Storage::RAMOnly` database. - def last_key - @last_key + @cached_last_key = -1 end # The `last_key=` function doesn't write to a file in the `DODB::Storage::RAMOnly` database. def last_key=(key : Int32) - @last_key = key + @cached_last_key = key end # WARNING: takes `[]?` and `[]` implementations from `CachedDataBase`. @@ -54,7 +49,7 @@ class DODB::Storage::RAMOnly(V) < DODB::Storage::Cached(V) run_triggers key, value - if key > last_key + if key > @cached_last_key self.last_key = key end