From ff74b4f2173c8da9296706ef232f9f94be12044b Mon Sep 17 00:00:00 2001 From: Karchnu Date: Sat, 16 May 2020 01:40:09 +0200 Subject: [PATCH] Compiling again. --- shard.yml | 4 ++ src/common/requests/download.cr | 4 +- src/common/requests/login.cr | 2 +- src/common/requests/transfer.cr | 8 +++- src/common/requests/upload.cr | 24 +++++------ src/server/main.cr | 6 ++- src/server/storage.cr | 74 ++++++++++++++++++++++++++++----- src/server/storage/user-data.cr | 5 +-- 8 files changed, 94 insertions(+), 33 deletions(-) diff --git a/shard.yml b/shard.yml index 742cf0c..ea686c1 100644 --- a/shard.yml +++ b/shard.yml @@ -14,6 +14,10 @@ dependencies: git: https://git.baguette.netlib.re/Baguette/dodb.cr authd: git: https://github.com/Lukc/authd + weird-crystal-base: + git: https://git.baguette.netlib.re/Baguette/weird-crystal-base + branch: master + targets: common-tests: diff --git a/src/common/requests/download.cr b/src/common/requests/download.cr index e200a0e..faf2415 100644 --- a/src/common/requests/download.cr +++ b/src/common/requests/download.cr @@ -10,7 +10,7 @@ class FileStorage::Request end def handle(filestoraged : FileStorage::Service, event : IPC::Event::Events) - user = altideald.get_logged_user event + user = filestoraged.get_logged_user event raise Exception.new "unauthorized" if user.nil? @@ -19,7 +19,7 @@ class FileStorage::Request user_data = filestoraged.get_user_data user.uid - filestoraged.storage.download self, event + filestoraged.storage.download self, user_data rescue e return Response::Error.new @mid, "unauthorized" end diff --git a/src/common/requests/login.cr b/src/common/requests/login.cr index c1d1e98..76f476c 100644 --- a/src/common/requests/login.cr +++ b/src/common/requests/login.cr @@ -26,7 +26,7 @@ class FileStorage::Request return Response::Login.new @mid rescue e - return Response::Error.new "unauthorized" + return Response::Error.new @mid, "unauthorized" end end FileStorage.requests << Login diff --git a/src/common/requests/transfer.cr b/src/common/requests/transfer.cr index a309aba..46a7fbf 100644 --- a/src/common/requests/transfer.cr +++ b/src/common/requests/transfer.cr @@ -2,6 +2,10 @@ class FileStorage::Request JSONIPC.request Transfer, 40 do property mid : String # autogenerated property filedigest : String # SHA256 digest of the entire file + # Chunk: + # - n : Int32 => chunk number + # - on : Int32 => number of chunks + # - digest : String => 1024-byte data in base64 format property chunk : Chunk # For now, just the counter in a string property data : String # base64 slice def initialize(file_info : FileInfo, count, bindata) @@ -14,7 +18,7 @@ class FileStorage::Request end def handle(filestoraged : FileStorage::Service, event : IPC::Event::Events) - user = altideald.get_logged_user event + user = filestoraged.get_logged_user event raise Exception.new "unauthorized" if user.nil? @@ -23,7 +27,7 @@ class FileStorage::Request user_data = filestoraged.get_user_data user.uid - filestoraged.storage.transfer self, event + filestoraged.storage.transfer self, user_data rescue e return Response::Error.new @mid, "unauthorized" end diff --git a/src/common/requests/upload.cr b/src/common/requests/upload.cr index 8ac2758..eff012f 100644 --- a/src/common/requests/upload.cr +++ b/src/common/requests/upload.cr @@ -8,7 +8,7 @@ class FileStorage::Request end def handle(filestoraged : FileStorage::Service, event : IPC::Event::Events) - user = altideald.get_logged_user event + user = filestoraged.get_logged_user event raise Exception.new "unauthorized" if user.nil? @@ -17,7 +17,7 @@ class FileStorage::Request user_data = filestoraged.get_user_data user.uid - filestoraged.storage.upload self, event + filestoraged.storage.upload self, user_data rescue e return Response::Error.new @mid, "unauthorized" end @@ -49,15 +49,13 @@ class FileStorage::Response end end - JSONIPC.request Responses, 100 do - include JSON::Serializable - - property mid : String - property responses : Array(Response) # a response for each request - property response : String - property reason : String? - - def initialize(@mid, @response, @responses, @reason = nil) - end - end +# JSONIPC.request Responses, 100 do +# property mid : String +# property responses : Array(Response) # a response for each request +# property response : String +# property reason : String? +# +# def initialize(@mid, @response, @responses, @reason = nil) +# end +# end end diff --git a/src/server/main.cr b/src/server/main.cr index 81052e4..74b1d6c 100644 --- a/src/server/main.cr +++ b/src/server/main.cr @@ -3,6 +3,10 @@ require "ipc" require "json" require "authd" +require "colorize" + +require "weird-crystal-base" + require "../common/colors" # require "../common/filestorage.cr" @@ -42,7 +46,7 @@ class FileStorage::Service < IPC::Service # users_status: keep track of the users' status even if they are disconnected, # allowing the application to handle connection problems. - property users_status = Hash(Int32, User).new + property users_status : Hash(Int32, UserData) = Hash(Int32, UserData).new # Actual storage. getter storage : FileStorage::Storage diff --git a/src/server/storage.cr b/src/server/storage.cr index afea7b9..cb49cc3 100644 --- a/src/server/storage.cr +++ b/src/server/storage.cr @@ -6,8 +6,6 @@ require "openssl" require "dodb" require "base64" -require "../common/utils" - require "./storage/*" # private function @@ -35,17 +33,26 @@ class FileStorage::Storage property db_by_owner : DODB::Partition(TransferInfo) property db_by_tags : DODB::Tags(TransferInfo) - def initialize(storage_directory, file_info_directory) - @db = DODB::DataBase(TransferInfo).new @file_info_directory + # Where to store the data. + property root : String + + getter user_data : DODB::DataBase(UserData) + getter user_data_per_user : DODB::Index(UserData) + + def initialize(@root, file_info_directory) + @db = DODB::DataBase(TransferInfo).new file_info_directory # Create indexes, partitions and tags objects. @db_by_filedigest = @db.new_index "filedigest", &.file_info.digest @db_by_owner = @db.new_partition "owner", &.owner.to_s @db_by_tags = @db.new_tags "tags", &.file_info.tags + + @user_data = DODB::DataBase(UserData).new "#{@root}/user-data" + @user_data_per_user = @user_data.new_index "uid", &.uid.to_s end # Reception of a file chunk. - def transfer(message : FileStorage::Transfer, user : User) : FileStorage::Response + def transfer(message : FileStorage::Request::Transfer, user : UserData) # We received a message containing a chunk of file. mid = message.mid @@ -78,11 +85,11 @@ class FileStorage::Storage FileStorage::Response::Transfer.new mid rescue e puts "Error handling transfer: #{e.message}" - FileStorage::Response.new mid.not_nil!, "Not Ok", "Unexpected error: #{e.message}" + FileStorage::Response::Error.new mid.not_nil!, "Unexpected error: #{e.message}" end # the client sent an upload request - def upload(request : FileStorage::Request::Upload, user : User) : FileStorage::Response + def upload(request : FileStorage::Request::Upload, user : UserData) mid = request.mid mid ||= "no message id" @@ -108,12 +115,12 @@ class FileStorage::Storage FileStorage::Response::Upload.new request.mid rescue e puts "Error handling transfer: #{e.message}" - FileStorage::Response.new mid.not_nil!, "Not Ok", "Unexpected error: #{e.message}" + FileStorage::Response::Error.new mid.not_nil!, "Unexpected error: #{e.message}" end # TODO # The client sent a download request. - def download(request : FileStorage::DownloadRequest, user : User) : FileStorage::Response + def download(request : FileStorage::Request::Download, user : UserData) puts "hdl download: mid=#{request.mid}" pp! request @@ -121,12 +128,11 @@ class FileStorage::Storage FileStorage::Response::Download.new request.mid end - # Entry point for request management # Each request should have a response. # Then, responses are sent in a single message. def requests(requests : Array(FileStorage::Request), - user : User, + user : UserData, event : IPC::Event::Message) : Array(FileStorage::Response) puts "hdl request" @@ -147,4 +153,50 @@ class FileStorage::Storage responses end + + def remove_chunk_from_db(transfer_info : TransferInfo, chunk_number : Int32) + transfer_info.chunks.delete chunk_number + @db_by_filedigest.update transfer_info.file_info.digest, transfer_info + end + + def write_a_chunk(userid : String, + file_info : FileStorage::FileInfo, + chunk_number : Int32, + data : Bytes) + + # storage: @root/files/userid/fileuuid.bin + dir = "#{@root}/files/#{userid}" + + FileUtils.mkdir_p dir + + path = "#{dir}/#{file_info.digest}.bin" + # Create file if non existant + File.open(path, "a+") do |file| + end + + # Write in it + File.open(path, "ab") do |file| + offset = chunk_number * FileStorage.message_buffer_size + file.seek(offset, IO::Seek::Set) + file.write data + end + end end + +### # TODO: +### # why getting the file_info here? We could check for the transfer_info right away +### # it has more info, and we'll get it later eventually +### +### file_info = nil +### begin +### file_info = user.uploads.select do |v| +### v.file.digest == message.filedigest +### end.first.file +### +### pp! file_info +### rescue e : IndexError +### puts "No recorded upload request for file #{message.filedigest}" +### +### rescue e +### puts "Unexpected error: #{e}" +### end diff --git a/src/server/storage/user-data.cr b/src/server/storage/user-data.cr index 5d8d9ca..6233eba 100644 --- a/src/server/storage/user-data.cr +++ b/src/server/storage/user-data.cr @@ -9,13 +9,12 @@ class FileStorage::UserData include JSON::Serializable property uid : Int32 - property token : AuthD::User::Public + # property token : AuthD::User::Public? property uploads : Array(FileStorage::Request::Upload) property downloads : Array(FileStorage::Request::Download) - def initialize(@token, + def initialize(@uid, @uploads = Array(FileStorage::Request::Upload).new, @downloads = Array(FileStorage::Request::Download).new) - @uid = token.uid end end