2020-05-14 17:03:09 +02:00
|
|
|
|
require "json"
|
|
|
|
|
require "uuid"
|
|
|
|
|
require "uuid/json"
|
|
|
|
|
require "openssl"
|
|
|
|
|
|
|
|
|
|
require "dodb"
|
|
|
|
|
require "base64"
|
|
|
|
|
|
|
|
|
|
require "./storage/*"
|
|
|
|
|
|
|
|
|
|
# private function
|
|
|
|
|
def digest(value : String)
|
|
|
|
|
|
|
|
|
|
underlying_io = IO::Memory.new value
|
|
|
|
|
buffer = Bytes.new(4096)
|
|
|
|
|
|
|
|
|
|
io = OpenSSL::DigestIO.new underlying_io, "SHA256"
|
|
|
|
|
io.read buffer
|
|
|
|
|
|
|
|
|
|
io.digest.hexstring
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
# XXX TODO FIXME: architectural questions
|
|
|
|
|
# Why keeping upload and download requests?
|
2020-06-06 20:43:14 +02:00
|
|
|
|
# The server can be just for uploads, delegating downloads to HTTP,
|
|
|
|
|
# but in environment without HTTP integration, this could still be relevant.
|
2020-05-14 17:03:09 +02:00
|
|
|
|
|
|
|
|
|
class FileStorage::Storage
|
2020-10-23 02:51:18 +02:00
|
|
|
|
property db : DODB::CachedDataBase(TransferInfo)
|
2020-05-14 17:03:09 +02:00
|
|
|
|
|
|
|
|
|
# Search file informations by their index, owner and tags.
|
|
|
|
|
property db_by_filedigest : DODB::Index(TransferInfo)
|
|
|
|
|
property db_by_owner : DODB::Partition(TransferInfo)
|
|
|
|
|
property db_by_tags : DODB::Tags(TransferInfo)
|
|
|
|
|
|
2020-06-06 20:43:14 +02:00
|
|
|
|
# Where to store data: files, users informations, files metadata.
|
2020-05-16 01:40:09 +02:00
|
|
|
|
property root : String
|
|
|
|
|
|
|
|
|
|
getter user_data : DODB::DataBase(UserData)
|
|
|
|
|
getter user_data_per_user : DODB::Index(UserData)
|
|
|
|
|
|
2020-06-06 20:43:14 +02:00
|
|
|
|
# FileStorage::Storage constructor takes a `root directory` as parameter
|
|
|
|
|
# which is used to create 3 sub-dirs:
|
|
|
|
|
# - files/ : actual files stored on the file-system
|
|
|
|
|
# - meta/ : DODB TransferInfo
|
|
|
|
|
# - users/ : DODB UserData (for later use: quotas, rights)
|
|
|
|
|
|
2020-10-21 19:59:57 +02:00
|
|
|
|
def initialize(@root, reindex : Bool = false)
|
2020-10-23 02:51:18 +02:00
|
|
|
|
@db = DODB::CachedDataBase(TransferInfo).new "#{@root}/meta"
|
2020-06-06 20:43:14 +02:00
|
|
|
|
|
|
|
|
|
# Where to store uploaded files.
|
|
|
|
|
FileUtils.mkdir_p "#{@root}/files"
|
2020-05-14 17:03:09 +02:00
|
|
|
|
|
|
|
|
|
# Create indexes, partitions and tags objects.
|
|
|
|
|
@db_by_filedigest = @db.new_index "filedigest", &.file_info.digest
|
|
|
|
|
@db_by_owner = @db.new_partition "owner", &.owner.to_s
|
|
|
|
|
@db_by_tags = @db.new_tags "tags", &.file_info.tags
|
2020-05-16 01:40:09 +02:00
|
|
|
|
|
2020-06-06 20:43:14 +02:00
|
|
|
|
@user_data = DODB::DataBase(UserData).new "#{@root}/users"
|
2020-05-16 01:40:09 +02:00
|
|
|
|
@user_data_per_user = @user_data.new_index "uid", &.uid.to_s
|
2020-10-21 19:59:57 +02:00
|
|
|
|
|
|
|
|
|
if reindex
|
|
|
|
|
@db.reindex_everything!
|
|
|
|
|
@user_data.reindex_everything!
|
|
|
|
|
end
|
2020-05-14 17:03:09 +02:00
|
|
|
|
end
|
|
|
|
|
|
2020-06-06 20:43:14 +02:00
|
|
|
|
# Path part of the URL.
|
|
|
|
|
def get_path(file_digest : String)
|
|
|
|
|
"/files/#{file_digest}"
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
# Path on the file-system.
|
|
|
|
|
def get_fs_path(file_digest : String)
|
|
|
|
|
"#{@root}#{get_path file_digest}"
|
|
|
|
|
end
|
|
|
|
|
|
2020-05-14 17:03:09 +02:00
|
|
|
|
# Reception of a file chunk.
|
2020-06-06 20:43:14 +02:00
|
|
|
|
def write_chunk(message : FileStorage::Request::PutChunk, user : UserData)
|
2020-05-14 17:03:09 +02:00
|
|
|
|
|
|
|
|
|
# We received a message containing a chunk of file.
|
|
|
|
|
mid = message.mid
|
|
|
|
|
mid ||= "no message id"
|
|
|
|
|
|
2020-06-06 20:43:14 +02:00
|
|
|
|
# Get the transfer info from the db.
|
2020-05-14 17:03:09 +02:00
|
|
|
|
transfer_info = @db_by_filedigest.get message.filedigest
|
|
|
|
|
|
2020-06-06 20:43:14 +02:00
|
|
|
|
file_digest = transfer_info.file_info.digest
|
|
|
|
|
|
2020-05-14 17:03:09 +02:00
|
|
|
|
if transfer_info.nil?
|
|
|
|
|
raise "file not recorded"
|
|
|
|
|
end
|
|
|
|
|
|
2020-06-06 20:43:14 +02:00
|
|
|
|
if transfer_info.nil?
|
|
|
|
|
# The user did not ask permission to upload the file (upload request).
|
|
|
|
|
return FileStorage::Errors::FileDoesNotExist.new mid, file_digest
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
# Verify the user had a granted upload request.
|
|
|
|
|
if transfer_info.owner != user.uid
|
|
|
|
|
return FileStorage::Errors::ChunkUploadDenied.new mid, file_digest
|
|
|
|
|
end
|
2020-05-14 17:03:09 +02:00
|
|
|
|
|
2020-06-06 20:43:14 +02:00
|
|
|
|
# TODO: this should be dynamic (per file) in the future.
|
|
|
|
|
chunk_size = FileStorage.message_buffer_size
|
|
|
|
|
chunk_number = message.chunk.n
|
2020-05-14 17:03:09 +02:00
|
|
|
|
data = Base64.decode message.data
|
2020-06-14 18:16:01 +02:00
|
|
|
|
path = get_path file_digest
|
2020-05-14 17:03:09 +02:00
|
|
|
|
|
2020-05-29 18:36:51 +02:00
|
|
|
|
# Verify that the chunk sent was really missing.
|
|
|
|
|
if transfer_info.chunks.select do |v| v == chunk_number end.size == 1
|
2020-06-06 20:43:14 +02:00
|
|
|
|
write_a_chunk file_digest, chunk_size, chunk_number, data
|
2020-05-14 17:03:09 +02:00
|
|
|
|
else
|
2020-06-08 23:02:31 +02:00
|
|
|
|
begin
|
|
|
|
|
# Send the next remaining chunk to upload.
|
2020-06-14 20:23:10 +02:00
|
|
|
|
chunks = transfer_info.chunks
|
|
|
|
|
if chunks.size != 0
|
|
|
|
|
next_chunk = transfer_info.chunks.sort.first
|
|
|
|
|
return FileStorage::Errors::ChunkAlreadyUploaded.new mid, file_digest, next_chunk
|
|
|
|
|
end
|
2020-06-08 23:02:31 +02:00
|
|
|
|
# In case the file was completely uploaded already.
|
2020-06-14 18:16:01 +02:00
|
|
|
|
return FileStorage::Errors::FileFullyUploaded.new mid, path
|
2020-06-14 18:11:36 +02:00
|
|
|
|
rescue e
|
2020-08-28 01:53:52 +02:00
|
|
|
|
Baguette::Log.error "error during transfer_info.chunks.sort.first"
|
2020-06-14 18:11:36 +02:00
|
|
|
|
raise e
|
2020-06-08 23:02:31 +02:00
|
|
|
|
end
|
2020-05-14 17:03:09 +02:00
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
remove_chunk_from_db transfer_info, chunk_number
|
|
|
|
|
|
|
|
|
|
# TODO: verify the digest, if no more chunks.
|
|
|
|
|
|
2020-05-20 09:49:16 +02:00
|
|
|
|
digest = transfer_info.file_info.digest
|
2020-06-06 20:43:14 +02:00
|
|
|
|
FileStorage::Response::PutChunk.new mid, digest, chunk_number
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
# Provide a file chunk to the client.
|
|
|
|
|
def read_chunk(message : FileStorage::Request::GetChunk, user : UserData)
|
|
|
|
|
|
|
|
|
|
# We received a message containing a chunk of file.
|
|
|
|
|
mid = message.mid
|
|
|
|
|
mid ||= "no message id"
|
|
|
|
|
|
|
|
|
|
file_digest = message.filedigest
|
|
|
|
|
# TODO: this should be dynamic (per file) in the future.
|
|
|
|
|
chunk_size = FileStorage.message_buffer_size
|
|
|
|
|
chunk_number = message.n
|
|
|
|
|
transfer_info = @db_by_filedigest.get file_digest
|
|
|
|
|
|
|
|
|
|
if transfer_info.nil?
|
|
|
|
|
# The user is asking for an inexistant file.
|
|
|
|
|
return FileStorage::Errors::FileDoesNotExist.new mid, file_digest
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
# Verify that the chunk is already present.
|
|
|
|
|
if transfer_info.chunks.select do |v| v == chunk_number end.size != 0
|
|
|
|
|
raise "non existent chunk or not yet uploaded"
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
# b64 data
|
|
|
|
|
data = read_a_chunk file_digest, chunk_size, chunk_number
|
|
|
|
|
b64_encoded_data = Base64.encode data
|
|
|
|
|
|
|
|
|
|
# whole file digest
|
|
|
|
|
digest = transfer_info.file_info.digest
|
|
|
|
|
|
|
|
|
|
# about the transfered chunk
|
|
|
|
|
chunk = Chunk.new chunk_number, transfer_info.file_info.nb_chunks, b64_encoded_data
|
|
|
|
|
|
|
|
|
|
FileStorage::Response::GetChunk.new mid, digest, chunk, b64_encoded_data
|
2020-05-14 17:03:09 +02:00
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
# the client sent an upload request
|
2020-05-16 01:40:09 +02:00
|
|
|
|
def upload(request : FileStorage::Request::Upload, user : UserData)
|
2020-05-14 17:03:09 +02:00
|
|
|
|
|
|
|
|
|
mid = request.mid
|
|
|
|
|
mid ||= "no message id"
|
|
|
|
|
|
2020-08-28 01:53:52 +02:00
|
|
|
|
Baguette::Log.debug "hdl upload: mid=#{request.mid}"
|
2020-05-14 17:03:09 +02:00
|
|
|
|
pp! request
|
|
|
|
|
|
2020-06-06 20:43:14 +02:00
|
|
|
|
# The final path of the file.
|
|
|
|
|
file_digest = request.file.digest
|
|
|
|
|
path = get_path file_digest
|
|
|
|
|
|
2020-05-14 17:03:09 +02:00
|
|
|
|
# TODO: verify the rights and quotas of the user
|
|
|
|
|
# file_info attributes: name, size, nb_chunks, digest, tags
|
|
|
|
|
|
2020-06-06 20:43:14 +02:00
|
|
|
|
# First: check if the file already exists.
|
|
|
|
|
transfer_info = @db_by_filedigest.get? file_digest
|
2020-05-14 17:03:09 +02:00
|
|
|
|
if transfer_info.nil?
|
2020-08-28 01:53:52 +02:00
|
|
|
|
Baguette::Log.debug "new file: #{file_digest}"
|
2020-06-14 20:23:10 +02:00
|
|
|
|
|
2020-05-14 17:03:09 +02:00
|
|
|
|
# In case file informations aren't already registered
|
2020-06-06 20:43:14 +02:00
|
|
|
|
# which is normal at this point.
|
|
|
|
|
@db << TransferInfo.new user.uid, request.file
|
2020-05-14 17:03:09 +02:00
|
|
|
|
else
|
2020-08-28 01:53:52 +02:00
|
|
|
|
Baguette::Log.debug "file already upload (at least partially): #{file_digest}"
|
2020-06-06 20:43:14 +02:00
|
|
|
|
# File information already exists, request may be duplicated,
|
|
|
|
|
# in this case: ignore the upload request.
|
2020-06-08 23:02:31 +02:00
|
|
|
|
begin
|
2020-06-14 20:23:10 +02:00
|
|
|
|
chunks = transfer_info.chunks
|
|
|
|
|
if chunks.size != 0
|
|
|
|
|
next_chunk = chunks.sort.first
|
|
|
|
|
return FileStorage::Errors::FileExists.new mid, path, next_chunk
|
|
|
|
|
end
|
2020-06-08 23:02:31 +02:00
|
|
|
|
# In case the file was completely uploaded already.
|
2020-06-14 18:16:01 +02:00
|
|
|
|
return FileStorage::Errors::FileFullyUploaded.new mid, path
|
|
|
|
|
rescue e
|
2020-08-28 01:53:52 +02:00
|
|
|
|
Baguette::Log.error "error at transfer_info.chunks.sort.first in upload"
|
2020-06-14 18:16:01 +02:00
|
|
|
|
raise e
|
2020-06-08 23:02:31 +02:00
|
|
|
|
end
|
2020-05-14 17:03:09 +02:00
|
|
|
|
end
|
|
|
|
|
|
2020-06-06 20:43:14 +02:00
|
|
|
|
# TODO: store upload request in UserData?
|
|
|
|
|
|
2020-05-20 09:20:52 +02:00
|
|
|
|
FileStorage::Response::Upload.new request.mid, path
|
2020-05-14 17:03:09 +02:00
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
# The client sent a download request.
|
2020-05-16 01:40:09 +02:00
|
|
|
|
def download(request : FileStorage::Request::Download, user : UserData)
|
2020-05-14 17:03:09 +02:00
|
|
|
|
|
2020-06-06 20:43:14 +02:00
|
|
|
|
mid = request.mid
|
|
|
|
|
mid ||= "no message id"
|
2020-08-28 01:53:52 +02:00
|
|
|
|
Baguette::Log.debug "hdl download: mid=#{mid}"
|
2020-05-14 17:03:09 +02:00
|
|
|
|
|
2020-06-06 20:43:14 +02:00
|
|
|
|
unless (file_digest = request.filedigest).nil?
|
|
|
|
|
unless (file_transfer = @db_by_filedigest.get? file_digest).nil?
|
|
|
|
|
# The file exists.
|
|
|
|
|
# TODO: verify rights here.
|
|
|
|
|
|
|
|
|
|
# This is acceptation.
|
|
|
|
|
# Return some useful values: number of chunks.
|
2020-10-21 19:59:57 +02:00
|
|
|
|
return FileStorage::Response::Download.new mid, file_transfer.file_info
|
2020-06-06 20:43:14 +02:00
|
|
|
|
else
|
|
|
|
|
return FileStorage::Errors::GenericError.new mid, "Unknown file digest: #{file_digest}"
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
# TODO: search a file by its name and tags
|
|
|
|
|
|
|
|
|
|
# TODO: store download request in UserData?
|
|
|
|
|
|
|
|
|
|
# Should have returned by now: file wasn't found.
|
|
|
|
|
FileStorage::Errors::GenericError.new mid, "File not found with provided parameters."
|
2020-05-14 17:03:09 +02:00
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
# Entry point for request management
|
|
|
|
|
# Each request should have a response.
|
|
|
|
|
# Then, responses are sent in a single message.
|
2020-06-06 20:43:14 +02:00
|
|
|
|
# def requests(requests : Array(FileStorage::Request),
|
|
|
|
|
# user : UserData,
|
|
|
|
|
# event : IPC::Event::Message) : Array(FileStorage::Response)
|
|
|
|
|
#
|
2020-08-28 01:53:52 +02:00
|
|
|
|
# Baguette::Log.debug "hdl request"
|
2020-06-06 20:43:14 +02:00
|
|
|
|
# responses = Array(FileStorage::Response | FileStorage::Errors).new
|
|
|
|
|
#
|
|
|
|
|
# requests.each do |request|
|
|
|
|
|
# case request
|
|
|
|
|
# when FileStorage::DownloadRequest
|
|
|
|
|
# responses << download request, user
|
|
|
|
|
# when FileStorage::UploadRequest
|
|
|
|
|
# responses << upload request, user
|
|
|
|
|
# else
|
|
|
|
|
# raise "request not understood"
|
|
|
|
|
# end
|
|
|
|
|
#
|
|
|
|
|
# end
|
|
|
|
|
#
|
|
|
|
|
# responses
|
|
|
|
|
# end
|
|
|
|
|
|
|
|
|
|
def read_a_chunk(file_digest : String, chunk_size : Int32, chunk_number : Int32)
|
|
|
|
|
offset = chunk_number * chunk_size
|
|
|
|
|
buffer_data = Bytes.new chunk_size
|
|
|
|
|
|
|
|
|
|
path = get_fs_path file_digest
|
|
|
|
|
real_size = 0
|
2020-10-21 19:59:57 +02:00
|
|
|
|
File.open(path, "rb") do |file|
|
|
|
|
|
file.seek offset
|
|
|
|
|
real_size = file.read buffer_data
|
2020-05-14 17:03:09 +02:00
|
|
|
|
end
|
|
|
|
|
|
2020-06-06 20:43:14 +02:00
|
|
|
|
buffer_data[0..real_size-1]
|
2020-05-14 17:03:09 +02:00
|
|
|
|
end
|
2020-05-16 01:40:09 +02:00
|
|
|
|
|
|
|
|
|
def remove_chunk_from_db(transfer_info : TransferInfo, chunk_number : Int32)
|
|
|
|
|
transfer_info.chunks.delete chunk_number
|
|
|
|
|
@db_by_filedigest.update transfer_info.file_info.digest, transfer_info
|
|
|
|
|
end
|
|
|
|
|
|
2020-06-06 20:43:14 +02:00
|
|
|
|
def write_a_chunk(digest : String,
|
|
|
|
|
chunk_size : Int32,
|
2020-05-16 01:40:09 +02:00
|
|
|
|
chunk_number : Int32,
|
|
|
|
|
data : Bytes)
|
|
|
|
|
|
2020-06-06 20:43:14 +02:00
|
|
|
|
# storage: @root/files/digest
|
|
|
|
|
path = get_fs_path digest
|
2020-05-16 01:40:09 +02:00
|
|
|
|
|
|
|
|
|
# Create file if non existant
|
|
|
|
|
File.open(path, "a+") do |file|
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
# Write in it
|
|
|
|
|
File.open(path, "ab") do |file|
|
2020-06-06 20:43:14 +02:00
|
|
|
|
offset = chunk_number * chunk_size
|
2020-05-16 01:40:09 +02:00
|
|
|
|
file.seek(offset, IO::Seek::Set)
|
|
|
|
|
file.write data
|
|
|
|
|
end
|
|
|
|
|
end
|
2020-05-14 17:03:09 +02:00
|
|
|
|
end
|