Naming, download, errors, consistency… pretty much rewrite of the whole thing.
parent
52409bdfec
commit
b4f09f2efd
|
@ -0,0 +1,64 @@
|
||||||
|
|
||||||
|
class FileStorage::Client
|
||||||
|
|
||||||
|
def login(token : String)
|
||||||
|
request = FileStorage::Request::Login.new token
|
||||||
|
send request
|
||||||
|
|
||||||
|
response = parse_message [ FileStorage::Response::Login, FileStorage::Response::Error ], read
|
||||||
|
|
||||||
|
case response
|
||||||
|
when FileStorage::Response::Login
|
||||||
|
# Received favorites, likes, etc.
|
||||||
|
when FileStorage::Response::Error
|
||||||
|
raise "user was not logged in: #{response.reason}"
|
||||||
|
end
|
||||||
|
|
||||||
|
response
|
||||||
|
end
|
||||||
|
|
||||||
|
def transfer(file_info : FileInfo, count, bindata)
|
||||||
|
request = FileStorage::Request::PutChunk.new file_info, count, bindata
|
||||||
|
send request
|
||||||
|
|
||||||
|
response = parse_message [ FileStorage::Response::PutChunk, FileStorage::Response::Error ], read
|
||||||
|
|
||||||
|
case response
|
||||||
|
when FileStorage::Response::PutChunk
|
||||||
|
when FileStorage::Response::Error
|
||||||
|
raise "File chunk was not transfered: #{response.reason}"
|
||||||
|
end
|
||||||
|
|
||||||
|
response
|
||||||
|
end
|
||||||
|
|
||||||
|
def download(filedigest = nil, name = nil, tags = nil)
|
||||||
|
request = FileStorage::Request::Download.new filedigest, name, tags
|
||||||
|
send request
|
||||||
|
|
||||||
|
response = parse_message [ FileStorage::Response::Download, FileStorage::Response::Error ], read
|
||||||
|
|
||||||
|
case response
|
||||||
|
when FileStorage::Response::Download
|
||||||
|
when FileStorage::Response::Error
|
||||||
|
raise "Download request denied: #{response.reason}"
|
||||||
|
end
|
||||||
|
|
||||||
|
response
|
||||||
|
end
|
||||||
|
|
||||||
|
def upload(token : String)
|
||||||
|
request = FileStorage::Request::Upload.new token
|
||||||
|
send request
|
||||||
|
|
||||||
|
response = parse_message [ FileStorage::Response::Upload, FileStorage::Response::Error ], read
|
||||||
|
|
||||||
|
case response
|
||||||
|
when FileStorage::Response::Upload
|
||||||
|
when FileStorage::Response::Error
|
||||||
|
raise "Upload request failed: #{response.reason}"
|
||||||
|
end
|
||||||
|
|
||||||
|
response
|
||||||
|
end
|
||||||
|
end
|
|
@ -27,28 +27,11 @@ class FileStorage::Request
|
||||||
FileStorage.requests << Download
|
FileStorage.requests << Download
|
||||||
end
|
end
|
||||||
|
|
||||||
class FileStorage::Client
|
|
||||||
def download(filedigest = nil, name = nil, tags = nil)
|
|
||||||
request = FileStorage::Request::Download.new filedigest, name, tags
|
|
||||||
send request
|
|
||||||
|
|
||||||
response = parse_message [ FileStorage::Response::Download, FileStorage::Response::Error ], read
|
|
||||||
|
|
||||||
case response
|
|
||||||
when FileStorage::Response::Download
|
|
||||||
when FileStorage::Response::Error
|
|
||||||
raise "Download request denied: #{response.reason}"
|
|
||||||
end
|
|
||||||
|
|
||||||
response
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
|
|
||||||
class FileStorage::Response
|
class FileStorage::Response
|
||||||
JSONIPC.request Download, 30 do
|
JSONIPC.request Download, 30 do
|
||||||
property mid : String
|
property mid : String
|
||||||
def initialize(@mid)
|
property nb_chunks : Int32
|
||||||
|
def initialize(@mid, @nb_chunks)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -0,0 +1,54 @@
|
||||||
|
class FileStorage::Errors
|
||||||
|
JSONIPC.request GenericError, 200 do
|
||||||
|
property mid : String
|
||||||
|
property reason : String
|
||||||
|
def initialize(@mid, @reason)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
FileStorage.errors << GenericError
|
||||||
|
|
||||||
|
# When uploading a chunk already present in the DB.
|
||||||
|
JSONIPC.request ChunkAlreadyUploaded, 201 do
|
||||||
|
property mid : String
|
||||||
|
property reason = "Chunk already present"
|
||||||
|
property filedigest : String
|
||||||
|
property chunk : Chunk
|
||||||
|
|
||||||
|
def initialize(@mid, @filedigest, @chunk)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
FileStorage.errors << ChunkAlreadyUploaded
|
||||||
|
|
||||||
|
# You upload a chunk, but you are not the owner of the file.
|
||||||
|
JSONIPC.request ChunkUploadDenied, 202 do
|
||||||
|
property mid : String
|
||||||
|
property reason = "This file is not yours"
|
||||||
|
property filedigest : String
|
||||||
|
|
||||||
|
def initialize(@mid, @filedigest)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
FileStorage.errors << ChunkUploadDenied
|
||||||
|
|
||||||
|
# When uploading a file already present in the DB.
|
||||||
|
JSONIPC.request FileExists, 203 do
|
||||||
|
property mid : String
|
||||||
|
property reason = "file already present"
|
||||||
|
property filedigest : String
|
||||||
|
|
||||||
|
def initialize(@mid, @filedigest)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
FileStorage.errors << FileExists
|
||||||
|
|
||||||
|
# When transfering a chunk for an inexistent file.
|
||||||
|
JSONIPC.request FileDoesNotExist, 204 do
|
||||||
|
property mid : String
|
||||||
|
property reason = "file does not exist"
|
||||||
|
property filedigest : String
|
||||||
|
|
||||||
|
def initialize(@mid, @filedigest)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
FileStorage.errors << FileDoesNotExist
|
||||||
|
end
|
|
@ -34,24 +34,6 @@ class FileStorage::Request
|
||||||
FileStorage.requests << Login
|
FileStorage.requests << Login
|
||||||
end
|
end
|
||||||
|
|
||||||
class FileStorage::Client
|
|
||||||
def login(token : String)
|
|
||||||
request = FileStorage::Request::Login.new token
|
|
||||||
send request
|
|
||||||
|
|
||||||
response = parse_message [ FileStorage::Response::Login, FileStorage::Response::Error ], read
|
|
||||||
|
|
||||||
case response
|
|
||||||
when FileStorage::Response::Login
|
|
||||||
# Received favorites, likes, etc.
|
|
||||||
when FileStorage::Response::Error
|
|
||||||
raise "user was not logged in: #{response.reason}"
|
|
||||||
end
|
|
||||||
|
|
||||||
response
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
class FileStorage::Response
|
class FileStorage::Response
|
||||||
JSONIPC.request Login, 5 do
|
JSONIPC.request Login, 5 do
|
||||||
property mid : String
|
property mid : String
|
||||||
|
|
|
@ -1,11 +1,11 @@
|
||||||
class FileStorage::Request
|
class FileStorage::Request
|
||||||
JSONIPC.request Transfer, 40 do
|
JSONIPC.request PutChunk, 40 do
|
||||||
property mid : String # autogenerated
|
property mid : String # autogenerated
|
||||||
property filedigest : String # SHA256 digest of the entire file
|
property filedigest : String # SHA256 digest of the entire file
|
||||||
# Chunk:
|
# Chunk:
|
||||||
# - n : Int32 => chunk number
|
# - n : Int32 => chunk number
|
||||||
# - on : Int32 => number of chunks
|
# - on : Int32 => number of chunks
|
||||||
# - digest : String => 1024-byte data in base64 format
|
# - digest : String => digest of the chunk
|
||||||
property chunk : Chunk # For now, just the counter in a string
|
property chunk : Chunk # For now, just the counter in a string
|
||||||
property data : String # base64 slice
|
property data : String # base64 slice
|
||||||
def initialize(file_info : FileInfo, count, bindata)
|
def initialize(file_info : FileInfo, count, bindata)
|
||||||
|
@ -27,38 +27,59 @@ class FileStorage::Request
|
||||||
|
|
||||||
user_data = filestoraged.get_user_data user.uid
|
user_data = filestoraged.get_user_data user.uid
|
||||||
|
|
||||||
filestoraged.storage.transfer self, user_data
|
filestoraged.storage.write_chunk self, user_data
|
||||||
rescue e
|
rescue e
|
||||||
return Response::Error.new @mid, "unauthorized"
|
return Response::Error.new @mid, "unauthorized"
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
FileStorage.requests << Transfer
|
FileStorage.requests << PutChunk
|
||||||
end
|
|
||||||
|
|
||||||
class FileStorage::Client
|
JSONIPC.request GetChunk, 41 do
|
||||||
def transfer(file_info : FileInfo, count, bindata)
|
property mid : String # autogenerated
|
||||||
request = FileStorage::Request::Transfer.new file_info, count, bindata
|
property filedigest : String # SHA256 digest of the entire file
|
||||||
send request
|
property n : Int32 # chunk number
|
||||||
|
|
||||||
response = parse_message [ FileStorage::Response::Transfer, FileStorage::Response::Error ], read
|
def initialize(@filedigest, @n)
|
||||||
|
@mid = UUID.random.to_s
|
||||||
case response
|
|
||||||
when FileStorage::Response::Transfer
|
|
||||||
when FileStorage::Response::Error
|
|
||||||
raise "File chunk was not transfered: #{response.reason}"
|
|
||||||
end
|
end
|
||||||
|
|
||||||
response
|
def handle(filestoraged : FileStorage::Service, event : IPC::Event::Events)
|
||||||
|
user = filestoraged.get_logged_user event
|
||||||
|
|
||||||
|
raise Exception.new "unauthorized" if user.nil?
|
||||||
|
|
||||||
|
# FIXME: Maybe this should be moved to FileStorage::Service
|
||||||
|
fd = event.connection.fd
|
||||||
|
|
||||||
|
user_data = filestoraged.get_user_data user.uid
|
||||||
|
|
||||||
|
filestoraged.storage.read_chunk self, user_data
|
||||||
|
rescue e
|
||||||
|
return Response::Error.new @mid, "unauthorized"
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
FileStorage.requests << GetChunk
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
class FileStorage::Response
|
class FileStorage::Response
|
||||||
JSONIPC.request Transfer, 40 do
|
JSONIPC.request PutChunk, 40 do
|
||||||
property mid : String
|
property mid : String
|
||||||
property file_digest : String
|
property file_digest : String
|
||||||
property n : Int32 # chunk number
|
property n : Int32 # chunk number
|
||||||
def initialize(@mid, @file_digest, @n)
|
def initialize(@mid, @file_digest, @n)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
JSONIPC.request GetChunk, 41 do
|
||||||
|
property mid : String
|
||||||
|
property file_digest : String
|
||||||
|
# Chunk:
|
||||||
|
# - n : Int32 => chunk number
|
||||||
|
# - on : Int32 => number of chunks
|
||||||
|
# - digest : String => digest of the chunk
|
||||||
|
property chunk : Chunk # Currently: info about the chunk
|
||||||
|
property data : String # base64 slice
|
||||||
|
def initialize(@mid, @file_digest, @chunk, @data)
|
||||||
|
end
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -25,23 +25,6 @@ class FileStorage::Request
|
||||||
FileStorage.requests << Upload
|
FileStorage.requests << Upload
|
||||||
end
|
end
|
||||||
|
|
||||||
class FileStorage::Client
|
|
||||||
def upload(token : String)
|
|
||||||
request = FileStorage::Request::Upload.new token
|
|
||||||
send request
|
|
||||||
|
|
||||||
response = parse_message [ FileStorage::Response::Upload, FileStorage::Response::Error ], read
|
|
||||||
|
|
||||||
case response
|
|
||||||
when FileStorage::Response::Upload
|
|
||||||
when FileStorage::Response::Error
|
|
||||||
raise "Upload request failed: #{response.reason}"
|
|
||||||
end
|
|
||||||
|
|
||||||
response
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
class FileStorage::Response
|
class FileStorage::Response
|
||||||
JSONIPC.request Upload, 20 do
|
JSONIPC.request Upload, 20 do
|
||||||
property mid : String
|
property mid : String
|
||||||
|
@ -49,10 +32,11 @@ class FileStorage::Response
|
||||||
def initialize(@mid, @path)
|
def initialize(@mid, @path)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
FileStorage.responses << Upload
|
||||||
|
|
||||||
# JSONIPC.request Responses, 100 do
|
# JSONIPC.request Responses, 100 do
|
||||||
# property mid : String
|
# property mid : String
|
||||||
# property responses : Array(Response) # a response for each request
|
# property responses : Array(Response | Errors) # a response for each request
|
||||||
# property response : String
|
# property response : String
|
||||||
# property reason : String?
|
# property reason : String?
|
||||||
#
|
#
|
||||||
|
|
|
@ -1,43 +0,0 @@
|
||||||
|
|
||||||
def remove_chunk_from_db(transfer_info : TransferInfo, chunk_number : Int32)
|
|
||||||
transfer_info.chunks.delete chunk_number
|
|
||||||
Context.db_by_filedigest.update transfer_info.file_info.digest, transfer_info
|
|
||||||
end
|
|
||||||
|
|
||||||
def write_a_chunk(userid : String, file_info : FileStorage::FileInfo, chunk_number : Int32, data : Bytes)
|
|
||||||
|
|
||||||
# storage: Context.storage_directory/userid/fileuuid.bin
|
|
||||||
dir = "#{Context.storage_directory}/#{userid}"
|
|
||||||
|
|
||||||
FileUtils.mkdir_p dir
|
|
||||||
|
|
||||||
path = "#{dir}/#{file_info.digest}.bin"
|
|
||||||
# Create file if non existant
|
|
||||||
File.open(path, "a+") do |file|
|
|
||||||
end
|
|
||||||
|
|
||||||
# Write in it
|
|
||||||
File.open(path, "ab") do |file|
|
|
||||||
offset = chunk_number * FileStorage.message_buffer_size
|
|
||||||
file.seek(offset, IO::Seek::Set)
|
|
||||||
file.write data
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
### # TODO:
|
|
||||||
### # why getting the file_info here? We could check for the transfer_info right away
|
|
||||||
### # it has more info, and we'll get it later eventually
|
|
||||||
###
|
|
||||||
### file_info = nil
|
|
||||||
### begin
|
|
||||||
### file_info = user.uploads.select do |v|
|
|
||||||
### v.file.digest == message.filedigest
|
|
||||||
### end.first.file
|
|
||||||
###
|
|
||||||
### pp! file_info
|
|
||||||
### rescue e : IndexError
|
|
||||||
### puts "No recorded upload request for file #{message.filedigest}"
|
|
||||||
###
|
|
||||||
### rescue e
|
|
||||||
### puts "Unexpected error: #{e}"
|
|
||||||
### end
|
|
|
@ -19,23 +19,14 @@ require "../common/colors"
|
||||||
# upload or download its files.
|
# upload or download its files.
|
||||||
|
|
||||||
# TODO:
|
# TODO:
|
||||||
# * Authd integration.
|
|
||||||
# * Elegantly handling errors.
|
# * Elegantly handling errors.
|
||||||
# * Store the file, /files/userid/UID.bin for example: /files/1002/UID.bin.
|
# * Store the file, @root/files/UID for example: ./files/UID.
|
||||||
# * Metadata should be in a dodb.
|
|
||||||
# /storage/partitions/by_uid/UID.json -> content:
|
|
||||||
# data: /files/uid/UID.bin (storing raw files)
|
|
||||||
# uid: 1002
|
|
||||||
# name: "The File About Things"
|
|
||||||
# size: 1500
|
|
||||||
# tags: thing1 thing2
|
|
||||||
# * Knowing which parts of the files are still to be sent.
|
# * Knowing which parts of the files are still to be sent.
|
||||||
# * Rights.
|
# * Rights.
|
||||||
# * Quotas.
|
# * Quotas.
|
||||||
|
|
||||||
require "./storage.cr"
|
require "./storage.cr"
|
||||||
require "./network.cr"
|
require "./network.cr"
|
||||||
#### require "./sample.cr"
|
|
||||||
|
|
||||||
require "dodb"
|
require "dodb"
|
||||||
require "json"
|
require "json"
|
||||||
|
@ -58,12 +49,9 @@ class FileStorage::Service < IPC::Service
|
||||||
@auth : AuthD::Client
|
@auth : AuthD::Client
|
||||||
@auth_key : String
|
@auth_key : String
|
||||||
|
|
||||||
def initialize(storage_directory, file_info_directory, @auth_key)
|
def initialize(storage_directory, @auth_key)
|
||||||
# Data and metadata storage directories.
|
# Data and metadata storage directory.
|
||||||
# storage_directory : String
|
@storage = FileStorage::Storage.new storage_directory
|
||||||
# file_info_directory : String
|
|
||||||
|
|
||||||
@storage = FileStorage::Storage.new storage_directory, file_info_directory
|
|
||||||
|
|
||||||
@logged_users = Hash(Int32, AuthD::User::Public).new
|
@logged_users = Hash(Int32, AuthD::User::Public).new
|
||||||
@logged_connections = Hash(Int32, IPC::Connection).new
|
@logged_connections = Hash(Int32, IPC::Connection).new
|
||||||
|
@ -164,14 +152,14 @@ class FileStorage::Service < IPC::Service
|
||||||
info "<< #{request.class.name.sub /^FileStorage::Request::/, ""}"
|
info "<< #{request.class.name.sub /^FileStorage::Request::/, ""}"
|
||||||
|
|
||||||
response = request.handle self, event
|
response = request.handle self, event
|
||||||
|
response_type = response.class.name
|
||||||
|
|
||||||
if response.is_a? FileStorage::Response::Error
|
if response.responds_to?(:reason)
|
||||||
warning ">> #{response.class.name.sub /^FileStorage::Response::/, ""} (#{response.reason})"
|
warning ">> #{response_type.sub /^FileStorage::Errors::/, ""} (#{response.reason})"
|
||||||
else
|
else
|
||||||
info ">> #{response.class.name.sub /^FileStorage::Response::/, ""}"
|
info ">> #{response.class.name.sub /^FileStorage::Response::/, ""}"
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
#################################################################
|
#################################################################
|
||||||
# THERE START
|
# THERE START
|
||||||
#################################################################
|
#################################################################
|
||||||
|
@ -229,7 +217,7 @@ class FileStorage::Service < IPC::Service
|
||||||
# raise "The user isn't recorded in the users_status structure"
|
# raise "The user isn't recorded in the users_status structure"
|
||||||
# end
|
# end
|
||||||
#
|
#
|
||||||
# transfer = FileStorage::Transfer.from_json(
|
# transfer = FileStorage::PutChunk.from_json(
|
||||||
# String.new event.message.payload
|
# String.new event.message.payload
|
||||||
# )
|
# )
|
||||||
# response = hdl_transfer transfer, Context.users_status[userid]
|
# response = hdl_transfer transfer, Context.users_status[userid]
|
||||||
|
@ -254,10 +242,6 @@ class FileStorage::Service < IPC::Service
|
||||||
warning "unhandled IPC event: #{event.class}"
|
warning "unhandled IPC event: #{event.class}"
|
||||||
end
|
end
|
||||||
|
|
||||||
# TODOOOOOOOOOO TODO FIXME
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
rescue exception
|
rescue exception
|
||||||
error "exception: #{typeof(exception)} - #{exception.message}"
|
error "exception: #{typeof(exception)} - #{exception.message}"
|
||||||
end
|
end
|
||||||
|
@ -265,25 +249,18 @@ class FileStorage::Service < IPC::Service
|
||||||
end
|
end
|
||||||
|
|
||||||
def self.from_cli
|
def self.from_cli
|
||||||
storage_directory = "file-data-storage"
|
storage_directory = "files/"
|
||||||
file_info_directory = "file-info-storage"
|
|
||||||
key = "nico-nico-nii" # Default authd key, as per the specs. :eyes:
|
key = "nico-nico-nii" # Default authd key, as per the specs. :eyes:
|
||||||
|
|
||||||
OptionParser.parse do |parser|
|
OptionParser.parse do |parser|
|
||||||
parser.banner = "usage: filestoraged [options]"
|
parser.banner = "usage: filestoraged [options]"
|
||||||
|
|
||||||
parser.on "-d storage-directory",
|
parser.on "-r root-directory",
|
||||||
"--storage-directory storage-directory",
|
"--root-directory dir",
|
||||||
"The directory where to put uploaded files." do |opt|
|
"The root directory for FileStoraged." do |opt|
|
||||||
storage_directory = opt
|
storage_directory = opt
|
||||||
end
|
end
|
||||||
|
|
||||||
parser.on "-m metadata-directory",
|
|
||||||
"--metadata-directory storage-directory",
|
|
||||||
"The directory where to put metadata of uploaded files." do |opt|
|
|
||||||
file_info_directory = opt
|
|
||||||
end
|
|
||||||
|
|
||||||
parser.on "-h",
|
parser.on "-h",
|
||||||
"--help",
|
"--help",
|
||||||
"Displays this help and exits." do
|
"Displays this help and exits." do
|
||||||
|
@ -299,25 +276,8 @@ class FileStorage::Service < IPC::Service
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
::FileStorage::Service.new storage_directory, file_info_directory, key
|
::FileStorage::Service.new storage_directory, key
|
||||||
end
|
end
|
||||||
|
|
||||||
# TODO: this will probably be dropped at some point.
|
|
||||||
# def do_response(event : IPC::Event::Message,
|
|
||||||
# response : FileStorage::Message)
|
|
||||||
#
|
|
||||||
# case response
|
|
||||||
# when FileStorage::Response
|
|
||||||
# event.connection.send FileStorage::MessageType::Response.to_u8, response.to_json
|
|
||||||
# when FileStorage::Responses
|
|
||||||
# event.connection.send FileStorage::MessageType::Responses.to_u8, response.to_json
|
|
||||||
# when FileStorage::Error
|
|
||||||
# event.connection.send FileStorage::MessageType::Error.to_u8, response.to_json
|
|
||||||
# else
|
|
||||||
# puts "response should not happen: #{response}"
|
|
||||||
# pp! response
|
|
||||||
# end
|
|
||||||
# end
|
|
||||||
end
|
end
|
||||||
|
|
||||||
FileStorage::Service.from_cli.run
|
FileStorage::Service.from_cli.run
|
||||||
|
|
|
@ -36,6 +36,7 @@ end
|
||||||
class FileStorage
|
class FileStorage
|
||||||
class_getter requests = [] of JSONIPC.class
|
class_getter requests = [] of JSONIPC.class
|
||||||
class_getter responses = [] of JSONIPC.class
|
class_getter responses = [] of JSONIPC.class
|
||||||
|
class_getter errors = [] of JSONIPC.class
|
||||||
end
|
end
|
||||||
|
|
||||||
class FileStorage::Client < IPC::Client
|
class FileStorage::Client < IPC::Client
|
||||||
|
|
|
@ -22,8 +22,8 @@ end
|
||||||
|
|
||||||
# XXX TODO FIXME: architectural questions
|
# XXX TODO FIXME: architectural questions
|
||||||
# Why keeping upload and download requests?
|
# Why keeping upload and download requests?
|
||||||
# The server can be just for uploads, delegating downloads to HTTP.
|
# The server can be just for uploads, delegating downloads to HTTP,
|
||||||
# In environment without HTTP integration, this could still be pertinent.
|
# but in environment without HTTP integration, this could still be relevant.
|
||||||
|
|
||||||
class FileStorage::Storage
|
class FileStorage::Storage
|
||||||
property db : DODB::DataBase(TransferInfo)
|
property db : DODB::DataBase(TransferInfo)
|
||||||
|
@ -33,47 +33,77 @@ class FileStorage::Storage
|
||||||
property db_by_owner : DODB::Partition(TransferInfo)
|
property db_by_owner : DODB::Partition(TransferInfo)
|
||||||
property db_by_tags : DODB::Tags(TransferInfo)
|
property db_by_tags : DODB::Tags(TransferInfo)
|
||||||
|
|
||||||
# Where to store the data.
|
# Where to store data: files, users informations, files metadata.
|
||||||
property root : String
|
property root : String
|
||||||
|
|
||||||
getter user_data : DODB::DataBase(UserData)
|
getter user_data : DODB::DataBase(UserData)
|
||||||
getter user_data_per_user : DODB::Index(UserData)
|
getter user_data_per_user : DODB::Index(UserData)
|
||||||
|
|
||||||
def initialize(@root, file_info_directory)
|
# FileStorage::Storage constructor takes a `root directory` as parameter
|
||||||
@db = DODB::DataBase(TransferInfo).new file_info_directory
|
# which is used to create 3 sub-dirs:
|
||||||
|
# - files/ : actual files stored on the file-system
|
||||||
|
# - meta/ : DODB TransferInfo
|
||||||
|
# - users/ : DODB UserData (for later use: quotas, rights)
|
||||||
|
|
||||||
|
def initialize(@root)
|
||||||
|
@db = DODB::DataBase(TransferInfo).new "'#{@root}/meta"
|
||||||
|
|
||||||
|
# Where to store uploaded files.
|
||||||
|
FileUtils.mkdir_p "#{@root}/files"
|
||||||
|
|
||||||
# Create indexes, partitions and tags objects.
|
# Create indexes, partitions and tags objects.
|
||||||
@db_by_filedigest = @db.new_index "filedigest", &.file_info.digest
|
@db_by_filedigest = @db.new_index "filedigest", &.file_info.digest
|
||||||
@db_by_owner = @db.new_partition "owner", &.owner.to_s
|
@db_by_owner = @db.new_partition "owner", &.owner.to_s
|
||||||
@db_by_tags = @db.new_tags "tags", &.file_info.tags
|
@db_by_tags = @db.new_tags "tags", &.file_info.tags
|
||||||
|
|
||||||
@user_data = DODB::DataBase(UserData).new "#{@root}/user-data"
|
@user_data = DODB::DataBase(UserData).new "#{@root}/users"
|
||||||
@user_data_per_user = @user_data.new_index "uid", &.uid.to_s
|
@user_data_per_user = @user_data.new_index "uid", &.uid.to_s
|
||||||
end
|
end
|
||||||
|
|
||||||
|
# Path part of the URL.
|
||||||
|
def get_path(file_digest : String)
|
||||||
|
"/files/#{file_digest}"
|
||||||
|
end
|
||||||
|
|
||||||
|
# Path on the file-system.
|
||||||
|
def get_fs_path(file_digest : String)
|
||||||
|
"#{@root}#{get_path file_digest}"
|
||||||
|
end
|
||||||
|
|
||||||
# Reception of a file chunk.
|
# Reception of a file chunk.
|
||||||
def transfer(message : FileStorage::Request::Transfer, user : UserData)
|
def write_chunk(message : FileStorage::Request::PutChunk, user : UserData)
|
||||||
|
|
||||||
# We received a message containing a chunk of file.
|
# We received a message containing a chunk of file.
|
||||||
mid = message.mid
|
mid = message.mid
|
||||||
mid ||= "no message id"
|
mid ||= "no message id"
|
||||||
|
|
||||||
# Get the transfer info from the db
|
# Get the transfer info from the db.
|
||||||
transfer_info = @db_by_filedigest.get message.filedigest
|
transfer_info = @db_by_filedigest.get message.filedigest
|
||||||
|
|
||||||
|
file_digest = transfer_info.file_info.digest
|
||||||
|
|
||||||
if transfer_info.nil?
|
if transfer_info.nil?
|
||||||
# The user has to send an upload request before sending anything.
|
|
||||||
# If not the case, it should be discarded.
|
|
||||||
raise "file not recorded"
|
raise "file not recorded"
|
||||||
end
|
end
|
||||||
|
|
||||||
chunk_number = message.chunk.n
|
if transfer_info.nil?
|
||||||
|
# The user did not ask permission to upload the file (upload request).
|
||||||
|
return FileStorage::Errors::FileDoesNotExist.new mid, file_digest
|
||||||
|
end
|
||||||
|
|
||||||
|
# Verify the user had a granted upload request.
|
||||||
|
if transfer_info.owner != user.uid
|
||||||
|
return FileStorage::Errors::ChunkUploadDenied.new mid, file_digest
|
||||||
|
end
|
||||||
|
|
||||||
|
# TODO: this should be dynamic (per file) in the future.
|
||||||
|
chunk_size = FileStorage.message_buffer_size
|
||||||
|
chunk_number = message.chunk.n
|
||||||
data = Base64.decode message.data
|
data = Base64.decode message.data
|
||||||
|
|
||||||
# Verify that the chunk sent was really missing.
|
# Verify that the chunk sent was really missing.
|
||||||
if transfer_info.chunks.select do |v| v == chunk_number end.size == 1
|
if transfer_info.chunks.select do |v| v == chunk_number end.size == 1
|
||||||
write_a_chunk user.uid.to_s, transfer_info.file_info, chunk_number, data
|
write_a_chunk file_digest, chunk_size, chunk_number, data
|
||||||
else
|
else
|
||||||
# TODO: send the remaining chunks to upload.
|
# TODO: send the remaining chunks to upload.
|
||||||
raise "non existent chunk or already uploaded"
|
raise "non existent chunk or already uploaded"
|
||||||
|
@ -84,10 +114,49 @@ class FileStorage::Storage
|
||||||
# TODO: verify the digest, if no more chunks.
|
# TODO: verify the digest, if no more chunks.
|
||||||
|
|
||||||
digest = transfer_info.file_info.digest
|
digest = transfer_info.file_info.digest
|
||||||
FileStorage::Response::Transfer.new mid, digest, chunk_number
|
FileStorage::Response::PutChunk.new mid, digest, chunk_number
|
||||||
rescue e
|
rescue e
|
||||||
puts "Error handling transfer: #{e.message}"
|
puts "Error handling write_chunk: #{e.message}"
|
||||||
FileStorage::Response::Error.new mid.not_nil!, "Unexpected error: #{e.message}"
|
FileStorage::Errors::GenericError.new mid.not_nil!, "Unexpected error: #{e.message}"
|
||||||
|
end
|
||||||
|
|
||||||
|
# Provide a file chunk to the client.
|
||||||
|
def read_chunk(message : FileStorage::Request::GetChunk, user : UserData)
|
||||||
|
|
||||||
|
# We received a message containing a chunk of file.
|
||||||
|
mid = message.mid
|
||||||
|
mid ||= "no message id"
|
||||||
|
|
||||||
|
file_digest = message.filedigest
|
||||||
|
# TODO: this should be dynamic (per file) in the future.
|
||||||
|
chunk_size = FileStorage.message_buffer_size
|
||||||
|
chunk_number = message.n
|
||||||
|
transfer_info = @db_by_filedigest.get file_digest
|
||||||
|
|
||||||
|
if transfer_info.nil?
|
||||||
|
# The user is asking for an inexistant file.
|
||||||
|
return FileStorage::Errors::FileDoesNotExist.new mid, file_digest
|
||||||
|
end
|
||||||
|
|
||||||
|
# Verify that the chunk is already present.
|
||||||
|
if transfer_info.chunks.select do |v| v == chunk_number end.size != 0
|
||||||
|
raise "non existent chunk or not yet uploaded"
|
||||||
|
end
|
||||||
|
|
||||||
|
# b64 data
|
||||||
|
data = read_a_chunk file_digest, chunk_size, chunk_number
|
||||||
|
b64_encoded_data = Base64.encode data
|
||||||
|
|
||||||
|
# whole file digest
|
||||||
|
digest = transfer_info.file_info.digest
|
||||||
|
|
||||||
|
# about the transfered chunk
|
||||||
|
chunk = Chunk.new chunk_number, transfer_info.file_info.nb_chunks, b64_encoded_data
|
||||||
|
|
||||||
|
FileStorage::Response::GetChunk.new mid, digest, chunk, b64_encoded_data
|
||||||
|
rescue e
|
||||||
|
puts "Error handling read_chunk: #{e.message}"
|
||||||
|
FileStorage::Errors::GenericError.new mid.not_nil!, "Unexpected error: #{e.message}"
|
||||||
end
|
end
|
||||||
|
|
||||||
# the client sent an upload request
|
# the client sent an upload request
|
||||||
|
@ -99,62 +168,101 @@ class FileStorage::Storage
|
||||||
puts "hdl upload: mid=#{request.mid}"
|
puts "hdl upload: mid=#{request.mid}"
|
||||||
pp! request
|
pp! request
|
||||||
|
|
||||||
|
# The final path of the file.
|
||||||
|
file_digest = request.file.digest
|
||||||
|
path = get_path file_digest
|
||||||
|
|
||||||
# TODO: verify the rights and quotas of the user
|
# TODO: verify the rights and quotas of the user
|
||||||
# file_info attributes: name, size, nb_chunks, digest, tags
|
# file_info attributes: name, size, nb_chunks, digest, tags
|
||||||
|
|
||||||
# First: check if the file already exists
|
# First: check if the file already exists.
|
||||||
transfer_info = @db_by_filedigest.get? request.file.digest
|
transfer_info = @db_by_filedigest.get? file_digest
|
||||||
if transfer_info.nil?
|
if transfer_info.nil?
|
||||||
# In case file informations aren't already registered
|
# In case file informations aren't already registered
|
||||||
# which is normal at this point
|
# which is normal at this point.
|
||||||
transfer_info = TransferInfo.new user.uid, request.file
|
@db << TransferInfo.new user.uid, request.file
|
||||||
@db << transfer_info
|
|
||||||
else
|
else
|
||||||
# File information already exists, request may be duplicated
|
# File information already exists, request may be duplicated,
|
||||||
# In this case: ignore the upload request
|
# in this case: ignore the upload request.
|
||||||
|
return FileStorage::Errors::FileExists.new mid, file_digest
|
||||||
end
|
end
|
||||||
|
|
||||||
path = "/files/#{user.uid}/#{request.file.digest}.bin"
|
# TODO: store upload request in UserData?
|
||||||
|
|
||||||
FileStorage::Response::Upload.new request.mid, path
|
FileStorage::Response::Upload.new request.mid, path
|
||||||
rescue e
|
rescue e
|
||||||
puts "Error handling upload: #{e.message}"
|
puts "Error handling upload request: #{e.message}"
|
||||||
FileStorage::Response::Error.new mid.not_nil!, "Unexpected error: #{e.message}"
|
FileStorage::Errors::GenericError.new mid.not_nil!, "Unexpected error in upload request: #{e.message}"
|
||||||
end
|
end
|
||||||
|
|
||||||
# TODO
|
|
||||||
# The client sent a download request.
|
# The client sent a download request.
|
||||||
def download(request : FileStorage::Request::Download, user : UserData)
|
def download(request : FileStorage::Request::Download, user : UserData)
|
||||||
|
|
||||||
puts "hdl download: mid=#{request.mid}"
|
mid = request.mid
|
||||||
pp! request
|
mid ||= "no message id"
|
||||||
|
puts "hdl download: mid=#{mid}"
|
||||||
|
|
||||||
FileStorage::Response::Download.new request.mid
|
unless (file_digest = request.filedigest).nil?
|
||||||
|
unless (file_transfer = @db_by_filedigest.get? file_digest).nil?
|
||||||
|
# The file exists.
|
||||||
|
# TODO: verify rights here.
|
||||||
|
|
||||||
|
# This is acceptation.
|
||||||
|
# Return some useful values: number of chunks.
|
||||||
|
return FileStorage::Response::Download.new mid, file_transfer.file_info.nb_chunks
|
||||||
|
else
|
||||||
|
return FileStorage::Errors::GenericError.new mid, "Unknown file digest: #{file_digest}"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
# TODO: search a file by its name and tags
|
||||||
|
|
||||||
|
# TODO: store download request in UserData?
|
||||||
|
|
||||||
|
# Should have returned by now: file wasn't found.
|
||||||
|
FileStorage::Errors::GenericError.new mid, "File not found with provided parameters."
|
||||||
|
rescue e
|
||||||
|
puts "Error handling download request: #{e.message}"
|
||||||
|
FileStorage::Errors::GenericError.new mid.not_nil!, "Unexpected error in download request: #{e.message}"
|
||||||
end
|
end
|
||||||
|
|
||||||
# Entry point for request management
|
# Entry point for request management
|
||||||
# Each request should have a response.
|
# Each request should have a response.
|
||||||
# Then, responses are sent in a single message.
|
# Then, responses are sent in a single message.
|
||||||
def requests(requests : Array(FileStorage::Request),
|
# def requests(requests : Array(FileStorage::Request),
|
||||||
user : UserData,
|
# user : UserData,
|
||||||
event : IPC::Event::Message) : Array(FileStorage::Response)
|
# event : IPC::Event::Message) : Array(FileStorage::Response)
|
||||||
|
#
|
||||||
|
# puts "hdl request"
|
||||||
|
# responses = Array(FileStorage::Response | FileStorage::Errors).new
|
||||||
|
#
|
||||||
|
# requests.each do |request|
|
||||||
|
# case request
|
||||||
|
# when FileStorage::DownloadRequest
|
||||||
|
# responses << download request, user
|
||||||
|
# when FileStorage::UploadRequest
|
||||||
|
# responses << upload request, user
|
||||||
|
# else
|
||||||
|
# raise "request not understood"
|
||||||
|
# end
|
||||||
|
#
|
||||||
|
# puts
|
||||||
|
# end
|
||||||
|
#
|
||||||
|
# responses
|
||||||
|
# end
|
||||||
|
|
||||||
puts "hdl request"
|
def read_a_chunk(file_digest : String, chunk_size : Int32, chunk_number : Int32)
|
||||||
responses = Array(FileStorage::Response).new
|
offset = chunk_number * chunk_size
|
||||||
|
buffer_data = Bytes.new chunk_size
|
||||||
|
|
||||||
requests.each do |request|
|
path = get_fs_path file_digest
|
||||||
case request
|
real_size = 0
|
||||||
when FileStorage::DownloadRequest
|
File.open(path, "rb").read_at offset, chunk_size do |buffer|
|
||||||
responses << download request, user
|
real_size = buffer.read buffer_data
|
||||||
when FileStorage::UploadRequest
|
|
||||||
responses << upload request, user
|
|
||||||
else
|
|
||||||
raise "request not understood"
|
|
||||||
end
|
|
||||||
|
|
||||||
puts
|
|
||||||
end
|
end
|
||||||
|
|
||||||
responses
|
buffer_data[0..real_size-1]
|
||||||
end
|
end
|
||||||
|
|
||||||
def remove_chunk_from_db(transfer_info : TransferInfo, chunk_number : Int32)
|
def remove_chunk_from_db(transfer_info : TransferInfo, chunk_number : Int32)
|
||||||
|
@ -162,24 +270,21 @@ class FileStorage::Storage
|
||||||
@db_by_filedigest.update transfer_info.file_info.digest, transfer_info
|
@db_by_filedigest.update transfer_info.file_info.digest, transfer_info
|
||||||
end
|
end
|
||||||
|
|
||||||
def write_a_chunk(userid : String,
|
def write_a_chunk(digest : String,
|
||||||
file_info : FileStorage::FileInfo,
|
chunk_size : Int32,
|
||||||
chunk_number : Int32,
|
chunk_number : Int32,
|
||||||
data : Bytes)
|
data : Bytes)
|
||||||
|
|
||||||
# storage: @root/files/userid/fileuuid.bin
|
# storage: @root/files/digest
|
||||||
dir = "#{@root}/files/#{userid}"
|
path = get_fs_path digest
|
||||||
|
|
||||||
FileUtils.mkdir_p dir
|
|
||||||
|
|
||||||
path = "#{dir}/#{file_info.digest}.bin"
|
|
||||||
# Create file if non existant
|
# Create file if non existant
|
||||||
File.open(path, "a+") do |file|
|
File.open(path, "a+") do |file|
|
||||||
end
|
end
|
||||||
|
|
||||||
# Write in it
|
# Write in it
|
||||||
File.open(path, "ab") do |file|
|
File.open(path, "ab") do |file|
|
||||||
offset = chunk_number * FileStorage.message_buffer_size
|
offset = chunk_number * chunk_size
|
||||||
file.seek(offset, IO::Seek::Set)
|
file.seek(offset, IO::Seek::Set)
|
||||||
file.write data
|
file.write data
|
||||||
end
|
end
|
||||||
|
|
|
@ -24,7 +24,6 @@ class FileStorage
|
||||||
|
|
||||||
# private function
|
# private function
|
||||||
def self.data_digest(data : Bytes)
|
def self.data_digest(data : Bytes)
|
||||||
|
|
||||||
iodata = IO::Memory.new data, false
|
iodata = IO::Memory.new data, false
|
||||||
buffer = Bytes.new FileStorage.file_reading_buffer_size
|
buffer = Bytes.new FileStorage.file_reading_buffer_size
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,45 @@
|
||||||
|
class FileStorage
|
||||||
|
def self.message_buffer_size
|
||||||
|
def self.file_reading_buffer_size
|
||||||
|
def self.data_digest(data : Bytes)
|
||||||
|
def self.file_digest(file : File)
|
||||||
|
end
|
||||||
|
|
||||||
|
class FileStorage::Chunk
|
||||||
|
n : Int32 # chunk's number
|
||||||
|
on : Int32 # number of chunks
|
||||||
|
digest : String # digest of the current chunk
|
||||||
|
|
||||||
|
initialize(@n, @on, data)
|
||||||
|
end
|
||||||
|
|
||||||
|
class FileStorage::FileInfo
|
||||||
|
name : String
|
||||||
|
size : UInt64
|
||||||
|
nb_chunks : Int32
|
||||||
|
digest : String
|
||||||
|
tags : Array(String)
|
||||||
|
end
|
||||||
|
|
||||||
|
class TransferInfo
|
||||||
|
owner : Int32
|
||||||
|
file_info : FileStorage::FileInfo
|
||||||
|
chunks : Array(Int32)
|
||||||
|
|
||||||
|
initialize(@owner, @file_info)
|
||||||
|
end
|
||||||
|
|
||||||
|
# Keep track of connected users and their requests.
|
||||||
|
class FileStorage::UserData
|
||||||
|
property uid : Int32
|
||||||
|
property uploads : Array(Upload) # NOT USED.
|
||||||
|
property downloads : Array(Download) # NOT USED.
|
||||||
|
|
||||||
|
initialize(@uid, @uploads = Array(Upload).new, @downloads = Array(Download).new)
|
||||||
|
end
|
||||||
|
|
||||||
|
root/
|
||||||
|
files/ : actual files
|
||||||
|
meta/ : DODB TransferInfo
|
||||||
|
users/ : DODP UserData (for later use: quotas, rights)
|
||||||
|
|
Loading…
Reference in New Issue