From d02de92d241dedcd72948eff9269489018d20ddd Mon Sep 17 00:00:00 2001 From: Philippe PITTOLI Date: Wed, 29 Jan 2020 15:56:49 +0100 Subject: [PATCH] rewrite: include JSON::Serializable --- src/common/filestorage.cr | 131 +++++++++++++++++-------------------- src/server/context.cr | 33 ++++++---- src/server/handlers.cr | 11 ++-- src/tests/context-tests.cr | 21 ++++++ 4 files changed, 105 insertions(+), 91 deletions(-) create mode 100644 src/tests/context-tests.cr diff --git a/src/common/filestorage.cr b/src/common/filestorage.cr index e442a5c..b115208 100644 --- a/src/common/filestorage.cr +++ b/src/common/filestorage.cr @@ -31,14 +31,11 @@ module FileStorage end class Chunk - JSON.mapping({ - # chunk's number - n: Int32, - # number of chunks - on: Int32, - # digest of the current chunk - digest: String - }) + include JSON::Serializable + + property n : Int32 # chunk's number + property on : Int32 # number of chunks + property digest : String # digest of the current chunk def initialize(@n, @on, data) @digest = FileStorage.data_digest data.to_slice @@ -50,10 +47,10 @@ module FileStorage # arbitrary parts of each file. class Token - JSON.mapping({ - uid: Int32, - login: String - }) + include JSON::Serializable + + property uid : Int32 + property login : String def initialize(@uid, @login) end @@ -69,23 +66,23 @@ module FileStorage # A file has a name, a size and tags. class FileInfo - JSON.mapping({ - name: String, - size: UInt64, - nb_chunks: Int32, - # SHA256 file digest - digest: String, + include JSON::Serializable - # list of SHA256, if we are on UDP - # chunks: Array(SHA256), - tags: Array(String)? - }) + property name : String + property size : UInt64 + property nb_chunks : Int32 + property digest : String - def initialize(file : File, @tags = nil) + # list of SHA256, if we are on UDP + # chunks: Array(SHA256), + property tags : Array(String) + + def initialize(file : File, tags = nil) @name = File.basename file.path @size = file.size @digest = FileStorage.file_digest file @nb_chunks = (@size / FileStorage.message_buffer_size).ceil.to_i + @tags = tags || [] of String end end @@ -95,11 +92,10 @@ module FileStorage alias Request = UploadRequest | DownloadRequest class UploadRequest < Message - JSON.mapping({ - # autogenerated - mid: String, - file: FileInfo - }) + include JSON::Serializable + + property mid : String # autogenerated + property file : FileInfo def initialize(@file) @mid = UUID.random.to_s @@ -109,28 +105,25 @@ module FileStorage # WIP class DownloadRequest < Message - JSON.mapping({ - # autogenerated - mid: String, - # SHA256 digest of the file, used as ID - uuid: String?, - name: String?, - tags: Array(String)? - }) + include JSON::Serializable - def initialize(@uuid = nil, @name = nil, @tags = nil) + property mid : String # autogenerated + property filedigest : String? # SHA256 digest of the file, used as ID + property name : String? + property tags : Array(String)? + + def initialize(@filedigest = nil, @name = nil, @tags = nil) @mid = UUID.random.to_s end end class Authentication < Message - JSON.mapping({ - # autogenerated - mid: String, - token: Token, - uploads: Array(UploadRequest), - downloads: Array(DownloadRequest) - }) + include JSON::Serializable + + property mid : String # autogenerated + property token : Token + property uploads : Array(UploadRequest) + property downloads : Array(DownloadRequest) def initialize(@token, @uploads = Array(UploadRequest).new, @downloads = Array(DownloadRequest).new) @mid = UUID.random.to_s @@ -138,52 +131,46 @@ module FileStorage end class Response < Message - JSON.mapping({ - mid: String, - response: String, - reason: String? - }) + include JSON::Serializable + + property mid : String + property response : String + property reason : String? def initialize(@mid, @response, @reason = nil) end end class Error < Message - JSON.mapping({ - mid: String, - # a response for each request - response: String, - reason: String? - }) + include JSON::Serializable + + property mid : String + property response : String # a response for each request + property reason : String? def initialize(@mid, @response, @reason = nil) end end class Responses < Message - JSON.mapping({ - mid: String, - # a response for each request - responses: Array(Response), - response: String, - reason: String? - }) + include JSON::Serializable + + property mid : String + property responses : Array(Response) # a response for each request + property response : String + property reason : String? def initialize(@mid, @response, @responses, @reason = nil) end end class Transfer < Message - JSON.mapping({ - # autogenerated - mid: String, - # SHA256 digest of the entire file - filedigest: String, - # For now, just the counter in a string - chunk: Chunk, - # base64 slice - data: String, - }) + include JSON::Serializable + + property mid : String # autogenerated + property filedigest : String # SHA256 digest of the entire file + property chunk : Chunk # For now, just the counter in a string + property data : String # base64 slice def initialize(file_info : FileInfo, count, bindata) # count: chunk number diff --git a/src/server/context.cr b/src/server/context.cr index a855023..d547d28 100644 --- a/src/server/context.cr +++ b/src/server/context.cr @@ -1,4 +1,5 @@ +require "dodb" require "json" # keep track of connected users and their requests @@ -20,31 +21,35 @@ class TransferInfo include JSON::Serializable property owner : Int32 - property file_info : FileInfo - property chunks : Hash(Int32, Bool) + property file_info : FileStorage::FileInfo + property chunks : Array(Int32) def initialize(@owner, @file_info) - @chunks = Hash(Int32, Bool).new - @file_info.nb_chunks.times do |n| - @chunks[n] = false - end + @chunks = [0...@file_info.nb_chunks] end end class Context - class_property service_name = "filestorage" - class_property storage_directory = "./storage" + class_property service_name = "filestorage" + class_property storage_directory = "./storage" class_property file_info_directory = "./file-infos" - class_property db : DODB::DataBase(TransferInfo) = self.init_db + class_property db = DODB::DataBase(TransferInfo).new @@file_info_directory + + # search file informations by their index, owner and tags + class_property db_by_filedigest : DODB::Index(TransferInfo) = @@db.new_index "filedigest", &.file_info.digest + class_property db_by_owner : DODB::Partition(TransferInfo) = @@db.new_partition "owner", &.owner.to_s + class_property db_by_tags : DODB::Tags(TransferInfo) = @@db.new_tags "tags", &.file_info.tags + + def self.db_reconnect + # In case file_info_directory changes: database reinstanciation - def init_db @@db = DODB::DataBase(TransferInfo).new @@file_info_directory - # init index, partitions and tags - Context.db.new_index "filedigest", &.file_info.digest - Context.db.new_partition "owner", &.owner - Context.db.new_tags "tags", &.tags + # recreate indexes, partitions and tags objects, too + @@db_by_filedigest = @@db.new_index "filedigest", &.file_info.digest + @@db_by_owner = @@db.new_partition "owner", &.owner.to_s + @@db_by_tags = @@db.new_tags "tags", &.file_info.tags end # list of connected users (fd => uid) diff --git a/src/server/handlers.cr b/src/server/handlers.cr index eeb6a06..10cf503 100644 --- a/src/server/handlers.cr +++ b/src/server/handlers.cr @@ -3,10 +3,9 @@ require "dodb" require "base64" # reception of a file chunk -def hdl_transfer(message : FileStorage::Transfer, - user : User, - event : IPC::Event::Message) : FileStorage::Response - puts "receiving a file" +def hdl_transfer(message : FileStorage::Transfer, user : User) : FileStorage::Response + + # We received a message containing a chunk of file. mid = message.mid mid ||= "no message id" @@ -30,7 +29,9 @@ def hdl_transfer(message : FileStorage::Transfer, # Get the transfer info from the db # is the file info recorded? by_digest = Context.db.get_index "filedigest" - transfer_info = by_digest.get? message.filedigest + transfer_info = by_digest.get message.filedigest + + if by_owner = Context.db.get_partition "owner" pp! by_owner diff --git a/src/tests/context-tests.cr b/src/tests/context-tests.cr new file mode 100644 index 0000000..65c692a --- /dev/null +++ b/src/tests/context-tests.cr @@ -0,0 +1,21 @@ +require "../common/filestorage.cr" +require "ipc" +require "option_parser" + +filename = "./README.md" + +OptionParser.parse do |parser| + parser.on "-f file-to-transfer", + "--file to-transfer", + "File to transfer (simulation)." do |opt| + filename = opt + end + + parser.unknown_args do |args| + pp! args + end +end + +require "../server/context.cr" + +pp! Context