diff --git a/data/fifo_100000_100000.d b/data/lru_100000_100000.d similarity index 100% rename from data/fifo_100000_100000.d rename to data/lru_100000_100000.d diff --git a/data/fifo_100000_100000_redux.d b/data/lru_100000_100000_redux.d similarity index 100% rename from data/fifo_100000_100000_redux.d rename to data/lru_100000_100000_redux.d diff --git a/data/fifo_index.d b/data/lru_index.d similarity index 100% rename from data/fifo_index.d rename to data/lru_index.d diff --git a/data/fifo_partitions.d b/data/lru_partitions.d similarity index 100% rename from data/fifo_partitions.d rename to data/lru_partitions.d diff --git a/data/fifo_tags.d b/data/lru_tags.d similarity index 100% rename from data/fifo_tags.d rename to data/lru_tags.d diff --git a/paper/bibliography b/paper/bibliography index 4005067..0c562e9 100644 --- a/paper/bibliography +++ b/paper/bibliography @@ -16,3 +16,9 @@ %T DarkHTTPd, when you need a webserver in a hurry. %D 2017 %I https://unix4lyfe.org/darkhttpd/ + +%K netlibre +%A Philippe PITTOLI +%T netlib.re, a free DNS registrar +%D 2013 +%I https://www.netlib.re diff --git a/paper/graphs/addition_fifo.grap b/paper/graphs/addition_fifo.grap deleted file mode 100644 index a9b9545..0000000 --- a/paper/graphs/addition_fifo.grap +++ /dev/null @@ -1,46 +0,0 @@ -.G1 -copy "legend.grap" -frame invis ht 3 wid 4 left solid bot solid -coord x 0,100000 - -label left "Cost of adding" unaligned "a new entry (ns)" left 0.8 -label bot "Number of entries already in the structure" down 0.1 - -obfifo = obefficientfifo = 0 -cbfifo = cbefficientfifo = 0 - -legendxleft = 1000 -legendxright = 50000 -legendyup = 190000 -legendydown = 140000 - -boite(legendxleft,legendxright,legendyup,legendydown) -legend_fifo_addition(legendxleft,legendxright,legendyup,legendydown) - -copy "../data/fifo_100000_100000_redux.d" thru X - cx = $1 - - cbfifo = $2 - cbefficientfifo = $3 - - # fifo efficientfifo -.gcolor orange - if (obfifo > 0) then {line from cx,cbfifo to ox,obfifo} -.gcolor -.gcolor green - if (obefficientfifo > 0) then {line from cx,cbefficientfifo to ox,obefficientfifo} -.gcolor - - obfifo = cbfifo - obefficientfifo = cbefficientfifo - ox = cx - - # fifo efficientfifo -.gcolor orange - bullet at cx,cbfifo -.gcolor -.gcolor green - bullet at cx,cbefficientfifo -.gcolor -X -.G2 diff --git a/paper/graphs/addition_lru.grap b/paper/graphs/addition_lru.grap new file mode 100644 index 0000000..f7256bd --- /dev/null +++ b/paper/graphs/addition_lru.grap @@ -0,0 +1,46 @@ +.G1 +copy "legend.grap" +frame invis ht 3 wid 4 left solid bot solid +coord x 0,100000 + +label left "Cost of adding" unaligned "a new entry (ns)" left 0.8 +label bot "Number of entries already in the structure" down 0.1 + +oblru = obefficientlru = 0 +cblru = cbefficientlru = 0 + +legendxleft = 1000 +legendxright = 50000 +legendyup = 190000 +legendydown = 140000 + +boite(legendxleft,legendxright,legendyup,legendydown) +legend_lru_addition(legendxleft,legendxright,legendyup,legendydown) + +copy "../data/lru_100000_100000_redux.d" thru X + cx = $1 + + cblru = $2 + cbefficientlru = $3 + + # lru efficientlru +.gcolor orange + if (oblru > 0) then {line from cx,cblru to ox,oblru} +.gcolor +.gcolor green + if (obefficientlru > 0) then {line from cx,cbefficientlru to ox,obefficientlru} +.gcolor + + oblru = cblru + obefficientlru = cbefficientlru + ox = cx + + # lru efficientlru +.gcolor orange + bullet at cx,cblru +.gcolor +.gcolor green + bullet at cx,cbefficientlru +.gcolor +X +.G2 diff --git a/paper/graphs/fifo_query_index.grap b/paper/graphs/lru_query_index.grap similarity index 56% rename from paper/graphs/fifo_query_index.grap rename to paper/graphs/lru_query_index.grap index 73d60a1..769cf41 100644 --- a/paper/graphs/fifo_query_index.grap +++ b/paper/graphs/lru_query_index.grap @@ -8,8 +8,8 @@ ticks bot out at 50000 "50,000", 100000 "100,000", 150000 "150,000", 200000 "200 label left "Request duration with" unaligned "an index (ns)" "(Median)" left 0.8 label bot "Number of cars in the database" down 0.1 -obfifo1 = obfifo5 = obfifo10 = obfifo20 = 0 # old bullets -cbfifo1 = cbfifo5 = cbfifo10 = cbfifo20 = 0 # current bullets +oblru1 = oblru5 = oblru10 = oblru20 = 0 # old bullets +cblru1 = cblru5 = cblru10 = cblru20 = 0 # current bullets legendxleft = 120000 legendxright = 180000 @@ -19,12 +19,12 @@ legendydown = 20 boite(legendxleft,legendxright,legendyup,legendydown) legend_common(legendxleft,legendxright,legendyup,legendydown) -copy "../data/fifo_index.d" thru X +copy "../data/lru_index.d" thru X cx = $1*5 y_scale = 1 - ## fifo1 fifo5 fifo10 fifo20 + ## lru1 lru5 lru10 lru20 #line from cx,$2/y_scale to cx,$4/y_scale #line from cx,$5/y_scale to cx,$7/y_scale #line from cx,$8/y_scale to cx,$10/y_scale @@ -34,38 +34,38 @@ copy "../data/fifo_index.d" thru X cx = $1*5 - cbfifo1 = $3/y_scale - cbfifo5 = $6/y_scale - cbfifo10 = $9/y_scale - cbfifo20 = $12/y_scale + cblru1 = $3/y_scale + cblru5 = $6/y_scale + cblru10 = $9/y_scale + cblru20 = $12/y_scale .gcolor red - if (obfifo1 > 0) then {line from cx,cbfifo1 to ox,obfifo1} + if (oblru1 > 0) then {line from cx,cblru1 to ox,oblru1} .gcolor .gcolor green - if (obfifo5 > 0) then {line from cx,cbfifo5 to ox,obfifo5} + if (oblru5 > 0) then {line from cx,cblru5 to ox,oblru5} .gcolor - if (obfifo10 > 0) then {line from cx,cbfifo10 to ox,obfifo10} + if (oblru10 > 0) then {line from cx,cblru10 to ox,oblru10} .gcolor pink - if (obfifo20 > 0) then {line from cx,cbfifo20 to ox,obfifo20} + if (oblru20 > 0) then {line from cx,cblru20 to ox,oblru20} .gcolor - obfifo1 = cbfifo1 - obfifo5 = cbfifo5 - obfifo10 = cbfifo10 - obfifo20 = cbfifo20 + oblru1 = cblru1 + oblru5 = cblru5 + oblru10 = cblru10 + oblru20 = cblru20 ox = cx - # fifo1 fifo5 fifo10 fifo20 + # lru1 lru5 lru10 lru20 .gcolor red - bullet at cx,cbfifo1 + bullet at cx,cblru1 .gcolor .gcolor green - bullet at cx,cbfifo5 + bullet at cx,cblru5 .gcolor - bullet at cx,cbfifo10 + bullet at cx,cblru10 .gcolor pink - bullet at cx,cbfifo20 + bullet at cx,cblru20 .gcolor X .G2 diff --git a/paper/graphs/fifo_query_partition.grap b/paper/graphs/lru_query_partition.grap similarity index 68% rename from paper/graphs/fifo_query_partition.grap rename to paper/graphs/lru_query_partition.grap index 7e53ff2..2cfd322 100644 --- a/paper/graphs/fifo_query_partition.grap +++ b/paper/graphs/lru_query_partition.grap @@ -8,8 +8,8 @@ ticks bot out from 1000 to 10000 by 1000 label left "Request duration" unaligned "for a partition (µs)" "(Median)" left 0.8 label bot "Number of cars matching the partition" down 0.1 -obfifo1 = obfifo5 = obfifo10 = obfifo20 = 0 -cbfifo1 = cbfifo5 = cbfifo10 = cbfifo20 = 0 +oblru1 = oblru5 = oblru10 = oblru20 = 0 +cblru1 = cblru5 = cblru10 = cblru20 = 0 legendxleft = 1000 legendxright = 2500 @@ -19,7 +19,7 @@ legendydown = 2500 boite(legendxleft,legendxright,legendyup,legendydown) legend_common(legendxleft,legendxright,legendyup,legendydown) -copy "../data/fifo_partitions.d" thru X +copy "../data/lru_partitions.d" thru X cx = $1*2 y_scale = 1000 @@ -28,10 +28,10 @@ copy "../data/fifo_partitions.d" thru X # following retrivials, the mustache doesn't provide a better # understanding of the data. - line_no_mustache(cbfifo1, obfifo1, $3, red) - line_no_mustache(cbfifo5, obfifo5, $6, green) - line_no_mustache(cbfifo10, obfifo10, $9, black) - line_no_mustache(cbfifo20, obfifo20, $12, pink) + line_no_mustache(cblru1, oblru1, $3, red) + line_no_mustache(cblru5, oblru5, $6, green) + line_no_mustache(cblru10, oblru10, $9, black) + line_no_mustache(cblru20, oblru20, $12, pink) ox = cx X diff --git a/paper/graphs/fifo_query_tag.grap b/paper/graphs/lru_query_tag.grap similarity index 68% rename from paper/graphs/fifo_query_tag.grap rename to paper/graphs/lru_query_tag.grap index 64aec98..740dc2a 100644 --- a/paper/graphs/fifo_query_tag.grap +++ b/paper/graphs/lru_query_tag.grap @@ -7,8 +7,8 @@ ticks bot out from 1000 to 5000 by 1000 label left "Request duration" unaligned "for a tag (µs)" "(Median)" left 0.8 label bot "Number of cars matching the tag" down 0.1 -obfifo1 = obfifo5 = obfifo10 = obfifo20 = 0 -cbfifo1 = cbfifo5 = cbfifo10 = cbfifo20 = 0 +oblru1 = oblru5 = oblru10 = oblru20 = 0 +cblru1 = cblru5 = cblru10 = cblru20 = 0 legendxleft = 2000 legendxright = 3500 @@ -18,7 +18,7 @@ legendydown = 1500 boite(legendxleft,legendxright,legendyup,legendydown) legend_common(legendxleft,legendxright,legendyup,legendydown) -copy "../data/fifo_tags.d" thru X +copy "../data/lru_tags.d" thru X cx = $1 y_scale = 1000 @@ -27,10 +27,10 @@ copy "../data/fifo_tags.d" thru X # following retrivials, the mustache doesn't provide a better # understanding of the data. - line_no_mustache(cbfifo1, obfifo1, $3, red) - line_no_mustache(cbfifo5, obfifo5, $6, green) - line_no_mustache(cbfifo10, obfifo10, $9, black) - line_no_mustache(cbfifo20, obfifo20, $12, pink) + line_no_mustache(cblru1, oblru1, $3, red) + line_no_mustache(cblru5, oblru5, $6, green) + line_no_mustache(cblru10, oblru10, $9, black) + line_no_mustache(cblru20, oblru20, $12, pink) ox = cx X diff --git a/paper/graphs/query_index.grap b/paper/graphs/query_index.grap index 4de8b29..ce9d34b 100644 --- a/paper/graphs/query_index.grap +++ b/paper/graphs/query_index.grap @@ -10,8 +10,8 @@ ticks bot off label left "Request duration with" unaligned "an index (ns)" "(Median)" left 0.8 label right "\s+2Linear" unaligned "scale\s0" -obram = obuncache = obcache = obfifo = obsemi = 0 # old bullets -cbram = cbuncache = cbcache = cbfifo = cbsemi = 0 # current bullets +obram = obuncache = obcache = oblru = obsemi = 0 # old bullets +cbram = cbuncache = cbcache = cblru = cbsemi = 0 # current bullets legendxleft = 100000 legendxright = 250000 @@ -28,7 +28,7 @@ copy "../data/index.d" thru X line_no_mustache(cbram, obram, $3, red) line_no_mustache(cbcache, obcache, $6, black) - line_no_mustache(cbfifo, obfifo, $9, pink) + line_no_mustache(cblru, oblru, $9, pink) line_no_mustache(cbsemi, obsemi, $12, blue) line_no_mustache(cbuncache, obuncache, $15, green) @@ -45,8 +45,8 @@ label left "Request duration with" unaligned "an index (ns)" "(Median)" left 0.8 label bot "Number of cars in the database" down 0.1 label right "\s+2Logarithmic" unaligned "scale\s0" -obram = obuncache = obcache = obfifo = obsemi = 0 # old bullets -cbram = cbuncache = cbcache = cbfifo = cbsemi = 0 # current bullets +obram = obuncache = obcache = oblru = obsemi = 0 # old bullets +cbram = cbuncache = cbcache = cblru = cbsemi = 0 # current bullets copy "../data/index.d" thru X cx = $1*5 @@ -55,7 +55,7 @@ copy "../data/index.d" thru X line_no_mustache(cbram, obram, $3, red) line_no_mustache(cbcache, obcache, $6, black) - line_no_mustache(cbfifo, obfifo, $9, pink) + line_no_mustache(cblru, oblru, $9, pink) line_no_mustache(cbsemi, obsemi, $12, blue) line_no_mustache(cbuncache, obuncache, $15, green) diff --git a/paper/graphs/query_partition.grap b/paper/graphs/query_partition.grap index 85d4258..33baa3e 100644 --- a/paper/graphs/query_partition.grap +++ b/paper/graphs/query_partition.grap @@ -9,8 +9,8 @@ ticks bot off label left "Request duration" unaligned "for a partition (ms)" "(Median)" left 0.8 label right "\s+2Linear" unaligned "scale\s0" -obram = obuncache = obfifo = obcache = obsemi = 0 -cbram = cbuncache = cbfifo = cbcache = cbsemi = 0 +obram = obuncache = oblru = obcache = obsemi = 0 +cbram = cbuncache = cblru = cbcache = cbsemi = 0 legendxleft = 1000 legendxright = 6500 @@ -33,13 +33,13 @@ copy "../data/partitions.d" thru X line_no_mustache(cbram, obram, $3, red) line_no_mustache(cbcache, obcache, $6, black) - line_no_mustache(cbfifo, obfifo, $9, pink) + line_no_mustache(cblru, oblru, $9, pink) line_no_mustache(cbsemi, obsemi, $12, blue) line_no_mustache(cbuncache, obuncache, $15, green) #line_with_mustache(cbram, obram, $3, red, $2, $4) #line_with_mustache(cbcache, obcache, $6, black, $5, $7) - #line_with_mustache(cbfifo, obfifo, $9, pink, $8, $10) + #line_with_mustache(cblru, oblru, $9, pink, $8, $10) #line_with_mustache(cbsemi, obsemi, $12,blue, $11, $13) #line_with_mustache(cbuncache,obuncache,$15,green, $14, $16) @@ -55,8 +55,8 @@ label left "Request duration" unaligned "for a partition (ms)" "(Median)" left 0 label bot "Number of cars matching the partition" down 0.1 label right "\s+2Logarithmic" unaligned "scale\s0" -obram = obuncache = obfifo = obcache = obsemi = 0 -cbram = cbuncache = cbfifo = cbcache = cbsemi = 0 +obram = obuncache = oblru = obcache = obsemi = 0 +cbram = cbuncache = cblru = cbcache = cbsemi = 0 y_scale = 1000000 @@ -71,13 +71,13 @@ copy "../data/partitions.d" thru X line_no_mustache(cbram, obram, $3, red) line_no_mustache(cbcache, obcache, $6, black) - line_no_mustache(cbfifo, obfifo, $9, pink) + line_no_mustache(cblru, oblru, $9, pink) line_no_mustache(cbsemi, obsemi, $12, blue) line_no_mustache(cbuncache, obuncache, $15, green) #line_with_mustache(cbram, obram, $3, red, $2, $4) #line_with_mustache(cbcache, obcache, $6, black, $5, $7) - #line_with_mustache(cbfifo, obfifo, $9, pink, $8, $10) + #line_with_mustache(cblru, oblru, $9, pink, $8, $10) #line_with_mustache(cbsemi, obsemi, $12,blue, $11, $13) #line_with_mustache(cbuncache,obuncache,$15,green, $14, $16) diff --git a/paper/graphs/query_tag.grap b/paper/graphs/query_tag.grap index d34470e..6841e2d 100644 --- a/paper/graphs/query_tag.grap +++ b/paper/graphs/query_tag.grap @@ -10,8 +10,8 @@ label left "Request duration" unaligned "for a tag (ms)" "(Median)" left 0.8 #label bot "Number of cars matching the tag" down 0.1 label right "\s+2Linear" unaligned "scale\s0" -obram = obuncache = obfifo = obcache = obsemi = 0 -cbram = cbuncache = cbfifo = cbcache = cbsemi = 0 +obram = obuncache = oblru = obcache = obsemi = 0 +cbram = cbuncache = cblru = cbcache = cbsemi = 0 legendxleft = 200 legendxright = 3000 @@ -28,13 +28,13 @@ copy "../data/tags.d" thru X line_no_mustache(cbram, obram, $3, red) line_no_mustache(cbcache, obcache, $6, black) - line_no_mustache(cbfifo, obfifo, $9, pink) + line_no_mustache(cblru, oblru, $9, pink) line_no_mustache(cbsemi, obsemi, $12, blue) line_no_mustache(cbuncache, obuncache, $15, green) #line_with_mustache(cbram, obram, $3, red, $2, $4) #line_with_mustache(cbcache, obcache, $6, black, $5, $7) - #line_with_mustache(cbfifo, obfifo, $9, pink, $8, $10) + #line_with_mustache(cblru, oblru, $9, pink, $8, $10) #line_with_mustache(cbsemi, obsemi, $12,blue, $11, $13) #line_with_mustache(cbuncache,obuncache,$15,green, $14, $16) @@ -50,8 +50,8 @@ label left "Request duration" unaligned "for a tag (ms)" "(Median)" left 0.8 label bot "Number of cars matching the tag" down 0.1 label right "\s+2Logarithmic" unaligned "scale\s0" -obram = obuncache = obfifo = obcache = obsemi = 0 -cbram = cbuncache = cbfifo = cbcache = cbsemi = 0 +obram = obuncache = oblru = obcache = obsemi = 0 +cbram = cbuncache = cblru = cbcache = cbsemi = 0 copy "../data/tags.d" thru X cx = $1 @@ -60,13 +60,13 @@ copy "../data/tags.d" thru X line_no_mustache(cbram, obram, $3, red) line_no_mustache(cbcache, obcache, $6, black) - line_no_mustache(cbfifo, obfifo, $9, pink) + line_no_mustache(cblru, oblru, $9, pink) line_no_mustache(cbsemi, obsemi, $12, blue) line_no_mustache(cbuncache, obuncache, $15, green) #line_with_mustache(cbram, obram, $3, red, $2, $4) #line_with_mustache(cbcache, obcache, $6, black, $5, $7) - #line_with_mustache(cbfifo, obfifo, $9, pink, $8, $10) + #line_with_mustache(cblru, oblru, $9, pink, $8, $10) #line_with_mustache(cbsemi, obsemi, $12,blue, $11, $13) #line_with_mustache(cbuncache,obuncache,$15,green, $14, $16) diff --git a/paper/legend.grap b/paper/legend.grap index 4c98209..21fb1f4 100644 --- a/paper/legend.grap +++ b/paper/legend.grap @@ -80,7 +80,7 @@ define legend_common { .ps +2 } -define legend_fifo_addition { +define legend_lru_addition { xleft = $1 xright = $2 yup = $3 @@ -99,9 +99,9 @@ define legend_fifo_addition { .ps -2 cy = cy - hdiff - legend_line(cy,lstartx,lendx,tstartx,orange,"Basic FIFO") + legend_line(cy,lstartx,lendx,tstartx,orange,"Basic LRU") cy = cy - hdiff - legend_line(cy,lstartx,lendx,tstartx,green,"Efficient FIFO") + legend_line(cy,lstartx,lendx,tstartx,green,"Efficient LRU") .ps +2 } diff --git a/paper/paper.ms b/paper/paper.ms index 1bc5d2b..6fe29a2 100644 --- a/paper/paper.ms +++ b/paper/paper.ms @@ -1022,7 +1022,11 @@ the idea is to read (or write) memory only once you ask for it via a syscall. Thus, you cannot access data from anywhere (by mistake or after an attack). These mechanisms could be used internally by DODB to prevent a data leak since memory is handled by the library. -However, the Crystal language doesn't provide a way to manage memory manually and this may be a problem for mlock and mprotect. +However, the Crystal language doesn't provide a way to manage memory manually and this may be a problem for mlock and mprotect\*[*]. +.FOOTNOTE1 +But again, the Crystal language is an implementation detail. +The working principle of DODB can be translated in basically any other language, including some enabling access to low-level memory operations. +.FOOTNOTE2 Depending on the plateform (the operating system), these syscalls may require the memory to be aligned with the memory pages. Thus, the implementation won't be easy. @@ -1033,7 +1037,11 @@ Since this implementation of DODB is related to the Crystal language (which isn' . .SECTION Real-world usage: netlib.re -Netlibre is a service providing free domain names and a website to manage DNS zones. +Netlibre +.[ +netlibre +.] +is a service providing free domain names and a website to manage DNS zones. Domains can be shared and transfered between users, so organizations do not have to rely on a single person. Resource records are managed with dedicated interfaces, users are helped as much as possible through many automatic zone verifications. Resource records can be automatically updated via @@ -1041,7 +1049,7 @@ Resource records can be automatically updated via enabling users to host a service despite having a dynamic IP address. The resource can be updated with a trivial command: .SOURCE Ruby ps=9 vs=10 -wget "https://netlib.re/token-update/" +wget "https://www.netlib.re/token-update/" .SOURCE .B "The technical parts" . @@ -1059,7 +1067,9 @@ Several "databases" are maintained: Performance-wise, netlibre handles between 2 to 3k req/s with a single core, without any optimization. Code is written in an almost naive\*[*] way and still performing fine. .FOOTNOTE1 -Keep in mind that netlibre uses poll(2), a very old syscall to handle its event loop (from the 80's!); not newest and way faster event facilities such as epoll(2) and the like. +Keep in mind that netlibre (through +.B libipc ) +uses poll(2), a very old syscall to handle its event loop (from the 80's!); not newest and way faster event facilities such as epoll(2) and the like. .FOOTNOTE2 Indexes with file-system representation enables quick debugging sessions and to perform a few basic tasks such as listing all the domains of a user which, in practice, is great to have at our fingertips with simple unix tools. @@ -1078,9 +1088,9 @@ database should be an acceptable choice for most applications. .TBD -.APPENDIX FIFO vs Efficient FIFO +.APPENDIX LRU vs Efficient LRU .ps -2 -.so graphs/addition_fifo.grap +.so graphs/addition_lru.grap .ps \n[PS] .APPENDIX Common database performance @@ -1090,7 +1100,7 @@ database enables to configure the number of allowed entries in the cache. The following figures show the performance of the common database depending on the cache size. .ps -2 -.so graphs/fifo_query_index.grap +.so graphs/lru_query_index.grap .ps \n[PS] .QP This figure shows the request durations to retrieve data based on a basic index with a database containing up to 250k entries. @@ -1108,7 +1118,7 @@ $+-$ 170 ns. delim off .EN .ps -2 -.so graphs/fifo_query_partition.grap +.so graphs/lru_query_partition.grap .ps \n[PS] .QP This figure shows the request durations to retrieve data based on a partition containing up to 10k entries. @@ -1116,7 +1126,7 @@ This figure shows the request durations to retrieve data based on a partition co As we see in the figure, the duration for data retrieval grows almost linearly for databases with a sufficient cache size (starting with 10k entries). When the cache size is not sufficient, the requests are hundred times slower, which explain why the database with a cache size of one thousand entries isn't even represented in the graph, and why the 5k database has great results up to 5k partitions. .ps -2 -.so graphs/fifo_query_tag.grap +.so graphs/lru_query_tag.grap .ps \n[PS] .QP This figure shows the request durations to retrieve data based on a tag containing up to 5k entries.