Baguette/dodb.cr
3
0
Fork 1
Code Issues Pull Requests Releases Wiki Activity

EfficientFIFO: now graphed.

Browse Source
This commit is contained in:
Philippe PITTOLI 2024-05-26 19:03:37 +02:00
parent b0e3d8c5e8
commit c01ec614ae
8 changed files with 259 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
data/fifo_100000_100000_redux.d Normal file
View File

@ -0,0 +1,100 @@
1000 6523.0 117.0
2000 12848.0 82.0
3000 7311.0 81.0
4000 5784.0 101.0
5000 9036.0 83.0
6000 11619.0 103.0
7000 10136.0 107.0
8000 11722.0 131.0
9000 13009.0 82.0
10000 14422.0 103.0
11000 15850.0 82.0
12000 17281.0 201.0
13000 18703.0 103.0
14000 25187.0 81.0
15000 27016.0 107.0
16000 23010.0 124.0
17000 24443.0 120.0
18000 25873.0 86.0
19000 34139.0 308.0
20000 28742.0 82.0
21000 30178.0 84.0
22000 31610.0 182.0
23000 33018.0 200.0
24000 74792.0 83.0
25000 45018.0 209.0
26000 46648.0 102.0
27000 64619.0 172.0
28000 40205.0 107.0
29000 41642.0 195.0
30000 43070.0 96.0
31000 55635.0 85.0
32000 45938.0 94.0
33000 47375.0 87.0
34000 48828.0 102.0
35000 50296.0 180.0
36000 51671.0 306.0
37000 53103.0 181.0
38000 54538.0 308.0
39000 55969.0 179.0
40000 57423.0 181.0
41000 104576.0 180.0
42000 60306.0 179.0
43000 61808.0 180.0
44000 117821.0 218.0
45000 64677.0 176.0
46000 66173.0 293.0
47000 67613.0 182.0
48000 69053.0 180.0
49000 73815.0 189.0
50000 71914.0 185.0
51000 73359.0 289.0
52000 99921.0 116.0
53000 76256.0 201.0
54000 104055.0 179.0
55000 79128.0 277.0
56000 80567.0 118.0
57000 82006.0 455.0
58000 83440.0 530.0
59000 84879.0 314.0
60000 86307.0 86.0
61000 87758.0 253.0
62000 89186.0 292.0
63000 90633.0 649.0
64000 92076.0 204.0
65000 94432.0 300.0
66000 94951.0 456.0
67000 96386.0 180.0
68000 97825.0 180.0
69000 128849.0 327.0
70000 100731.0 201.0
71000 102131.0 169.0
72000 103574.0 173.0
73000 105009.0 201.0
74000 106444.0 116.0
75000 107878.0 190.0
76000 109317.0 177.0
77000 110746.0 176.0
78000 112184.0 292.0
79000 142068.0 170.0
80000 115060.0 87.0
81000 116492.0 176.0
82000 117935.0 404.0
83000 122138.0 325.0
84000 120799.0 292.0
85000 122269.0 308.0
86000 123688.0 297.0
87000 125124.0 333.0
88000 126556.0 351.0
89000 128761.0 214.0
90000 129438.0 182.0
91000 130871.0 455.0
92000 177811.0 181.0
93000 134150.0 105.0
94000 135181.0 105.0
95000 206680.0 181.0
96000 138050.0 255.0
97000 182654.0 198.0
98000 140931.0 289.0
99000 142364.0 383.0
100000 143808.0 297.0

46
paper/graph_addition_fifo.grap Normal file
View File

@ -0,0 +1,46 @@
.G1
copy "legend.grap"
frame invis ht 3 wid 4 left solid bot solid
coord x 0,100000
label left "Cost of adding" unaligned "a new entry (ns)" left 0.8
label bot "Number of entries already in the structure" down 0.1
obfifo = obefficientfifo = 0
cbfifo = cbefficientfifo = 0
legendxleft = 1000
legendxright = 50000
legendyup = 190000
legendydown = 140000
boite(legendxleft,legendxright,legendyup,legendydown)
legend_fifo_addition(legendxleft,legendxright,legendyup,legendydown)
copy "../data/fifo_100000_100000_redux.d" thru X
cx = $1
cbfifo = $2
cbefficientfifo = $3
# fifo efficientfifo
.gcolor orange
if (obfifo > 0) then {line from cx,cbfifo to ox,obfifo}
.gcolor
.gcolor green
if (obefficientfifo > 0) then {line from cx,cbefficientfifo to ox,obefficientfifo}
.gcolor
obfifo = cbfifo
obefficientfifo = cbefficientfifo
ox = cx
# fifo efficientfifo
.gcolor orange
bullet at cx,cbfifo
.gcolor
.gcolor green
bullet at cx,cbefficientfifo
.gcolor
X
.G2

31
paper/legend.grap
View File

@ -52,3 +52,34 @@ define legend {
"Uncached db and index" ljust at tstartx,cy
.ps +2
}
define legend_fifo_addition {
xleft = $1
xright = $2
yup = $3
ydown = $4
diffx = xright - xleft
diffy = yup - ydown
hdiff = diffy/3.5
cy = yup - (diffy/10)
cx = (diffx/20) + xleft
lstartx = cx
lendx = cx + diffx/8
tstartx = lendx + diffx/20
.ps -2
cy = cy - hdiff
.gcolor orange
line from lstartx,cy to lendx,cy
.gcolor
"basic FIFO" ljust at tstartx,cy
cy = cy - hdiff
.gcolor green
line from lstartx,cy to lendx,cy
.gcolor
"Efficient FIFO" ljust at tstartx,cy
.ps +2
}

5
paper/paper.ms
View File

@ -967,3 +967,8 @@ Since this implementation of DODB is related to the Crystal language (which isn'
.
.SECTION Conclusion
.TBD
.APPENDIX FIFO vs Efficient FIFO
.ps -2
.so graph_addition_fifo.grap
.ps \n[PS]

6
spec/benchmark-cars.cr
View File

@ -46,12 +46,6 @@ def report(storage, name, &block)
avr
end
def long_operation(text)
STDOUT.write "#{text}\r".to_slice
yield
STDOUT.write " \r".to_slice
end
def verbose_add_cars(storage, nbcars, name, max_indexes)
long_operation "add #{nbcars} values to #{name}" do
add_cars storage, nbcars, max_indexes: max_indexes

37
spec/benchmark-fifo.cr
View File

@ -1,4 +1,5 @@
require "benchmark"
require "./utilities.cr"
require "../src/fifo.cr"
def add(fifo : FIFO(Int32) | EfficientFIFO(Int32), nb : UInt32)
@ -9,9 +10,21 @@ def add(fifo : FIFO(Int32) | EfficientFIFO(Int32), nb : UInt32)
end
end
def report_add(fifo : FIFO(Int32) | EfficientFIFO(Int32), nb : UInt32, fname : String)
File.open("#{Context.report_dir}/#{fname}.raw", "w") do |file|
i = 0
while i < nb
elapsed_time = perform_something { fifo << i }
i += 1
file.puts "#{i} #{elapsed_time.total_nanoseconds}"
end
end
end
class Context
class_property nb_values : UInt32 = 100_000
class_property fifo_size : UInt32 = 10_000
class_property report_dir = "results"
end
if nb_values = ENV["NBVAL"]?
@ -22,7 +35,28 @@ if fifo_size = ENV["FIFOSIZE"]?
Context.fifo_size = fifo_size.to_u32
end
Benchmark.ips do |x|
if ARGV.size > 0
puts "Usage: benchmark-fifo"
puts ""
puts "envvar: REPORT_DIR=<directory> where to put the results"
puts "envvar: REPORT_EACH_ADD=<any> to report the duration of each addition of a value in the structure"
puts "envvar: NBVAL=<nb> (default: 100_000) nb of values to add to the structure"
puts "envvar: FIFOSIZE=<nb> (default: 10_000) max number of values in the structure"
exit 0
end
ENV["REPORT_DIR"]?.try { |report_dir| Context.report_dir = report_dir }
Dir.mkdir_p Context.report_dir
if ENV["REPORT_EACH_ADD"]?
FIFO(Int32).new(Context.fifo_size).tap do |fifo|
report_add fifo, Context.nb_values, "fifo_#{Context.fifo_size}_#{Context.nb_values}"
end
EfficientFIFO(Int32).new(Context.fifo_size).tap do |fifo|
report_add fifo, Context.nb_values, "efficientfifo_#{Context.fifo_size}_#{Context.nb_values}"
end
else
Benchmark.ips do |x|
x.report("adding #{Context.nb_values} values, FIFO limited to #{Context.fifo_size}") do
fifo = FIFO(Int32).new Context.fifo_size
add fifo, Context.nb_values
@ -32,4 +66,5 @@ Benchmark.ips do |x|
fifo = EfficientFIFO(Int32).new Context.fifo_size
add fifo, Context.nb_values
end
end
end

6
spec/utilities.cr
View File

@ -30,3 +30,9 @@ end
def should_nb_files(path : String, expected_nb_files : UInt32)
raise Exception.new "should_nb_files: not implemented yet"
end
def long_operation(text)
STDOUT.write "#{text}\r".to_slice
yield
STDOUT.write " \r".to_slice
end

29
src/fifo.cr
View File

@ -20,6 +20,7 @@ require "./list.cr"
# ```
#
# The number of entries in the FIFO structure is configurable.
# WARNING: this implementation becomes slow very fast, but doesn't cost much memory.
class FIFO(V)
# This array is used as the *fifo structure*.
property data : Array(V)
@ -46,7 +47,33 @@ class FIFO(V)
end
end
# TODO: this is a draft.
# This class enables to keep track of used data.
#
# **Implementation details.**
# Contrary to the `FIFO` class, this implementation is time-efficient.
# However, this efficiency is a memory tradeoff: all the entries are added to a double-linked list to keep
# track of the order **and** to a hash to perform efficient searches of the values in the double-linked list.
# Thus, all the nodes are added twice, once in the list, once in the hash.
#
# Each time a value is added, it is put in a FIFO structure.
# Adding a value several times is considered as "using the value",
# so it is pushed back at the entry of the FIFO (as a new value).
# In case the number of entries exceeds what is allowed,
# the least recently used value is removed.
# ```
# fifo = EfficientFIFO(Int32).new 3 # Only 3 allowed entries.
#
# pp! fifo << 1 # -> nil (there is still room in the FIFO structure)
# pp! fifo << 2 # -> nil (there is still room in the FIFO structure)
# pp! fifo << 3 # -> nil (last entry without exceeding the allowed size)
# pp! fifo << 4 # -> 1 (least recently used data)
# pp! fifo << 4 # -> nil (already in the structure)
# pp! fifo << 2 # -> nil (already in the structure)
# pp! fifo << 5 # -> 3 (least recently used data)
# ```
#
# The number of entries in the FIFO structure is configurable.
# NOTE: this implementation is time-efficient, but costs some memory.
class EfficientFIFO(V)
# This array is used as the *fifo structure*.
property list : DoubleLinkedList(V)