Sources split.

master
Luka Vandervelden 2019-06-09 16:59:05 +02:00
parent 21f8d2d7d2
commit 61c9f33f89
4 changed files with 503 additions and 491 deletions

76
src/environment.cr Normal file
View File

@ -0,0 +1,76 @@
require "yaml"
class Environment
enum Type
Prefix
RootFileSystem
end
YAML.mapping({
name: String,
type: {
type: Type,
default: Type::Prefix
},
domain_name: {
type: String?,
key: "domain-name"
},
checks: {
# FIXME: Would probably need a more neutral namespace.
type: Array(ServiceDefinition::Checks),
default: Array(ServiceDefinition::Checks).new
}
})
def initialize()
@name = "root"
@type = Type::Prefix
@checks = Array(ServiceDefinition::Checks).new
# FIXME: Should this *really* be here?
@checks << ServiceDefinition::Checks.from_yaml <<-EOF
name: Creating data directory
directory: /srv/%{ENVIRONMENT}
command: mkdir -p /srv/%{ENVIRONMENT} && chmod a+rwt /srv/%{ENVIRONMENT}
EOF
end
class_getter root = Environment.new
class_getter all = [@@root] of Environment
def self.load(path)
Dir.each_child path do |child|
unless child.match /\.yaml$/
next
end
file_path = "#{path}/#{child}"
begin
environment = Environment.from_yaml File.read file_path
rescue e
STDERR << "error loading #{file_path}: " << e << "\n"
# FIXME: Print stacktrace? Debug mode?
next
end
@@all << environment
end
end
def self.get(name)
_def = @@all.find &.name.==(name)
if _def.nil?
raise Exception.new "Environment '#{name}' does not exist."
end
_def
end
def to_s
"#{name} (#{type.to_s.downcase})"
end
end

View File

@ -12,497 +12,9 @@ ENVIRONMENTS_DIRECTORY = "./environments"
# TODO:
# - Be more declarative about the definition of commands.
def split_command(string)
args = string.split /\ (?=([^\"]*\"[^\"]*\")*[^\"]*$)/
command = args[0]
args.delete_at 0
return command, args
end
class ServiceDefinition
struct Consumes
YAML.mapping({
token: String,
optional: {
type: Bool,
default: false
}
})
end
struct Provides
YAML.mapping({
token: String
})
end
struct Checks
YAML.mapping({
name: String,
file: String?,
directory: String?,
command: String
})
end
YAML.mapping({
name: String,
command: String,
stop_command: {
type: String?,
key: "stop-command"
},
directory: String?, # Place to chdir to before running @command.
environment: {
type: Environment,
default: Environment.root
},
provides: {
type: Array(Provides),
default: [] of Provides
},
consumes: {
type: Array(Consumes),
default: [] of Consumes
},
checks: {
type: Array(Checks),
default: [] of Checks
},
environment_variables: {
type: Array(String),
key: "environment-variables",
default: [] of String
}
})
def self.new(name)
Service.from_yaml File.read "#{name}.yaml"
end
class_getter all = [] of ServiceDefinition
def self.load(path)
Dir.each_child path do |child|
unless child.match /\.yaml$/
next
end
@@all << ServiceDefinition.from_yaml File.read "#{path}/#{child}"
end
end
def self.get(name) : ServiceDefinition
_def = @@all.find &.name.==(name)
if _def.nil?
raise Exception.new "Service '#{name}' does not exist."
end
_def
end
def to_s
name
end
end
class Environment
enum Type
Prefix
RootFileSystem
end
YAML.mapping({
name: String,
type: {
type: Type,
default: Type::Prefix
},
domain_name: {
type: String?,
key: "domain-name"
},
checks: {
# FIXME: Would probably need a more neutral namespace.
type: Array(ServiceDefinition::Checks),
default: Array(ServiceDefinition::Checks).new
}
})
def initialize()
@name = "root"
@type = Type::Prefix
@checks = Array(ServiceDefinition::Checks).new
# FIXME: Should this *really* be here?
@checks << ServiceDefinition::Checks.from_yaml <<-EOF
name: Creating data directory
directory: /srv/%{ENVIRONMENT}
command: mkdir -p /srv/%{ENVIRONMENT} && chmod a+rwt /srv/%{ENVIRONMENT}
EOF
end
class_getter root = Environment.new
class_getter all = [@@root] of Environment
def self.load(path)
Dir.each_child path do |child|
unless child.match /\.yaml$/
next
end
file_path = "#{path}/#{child}"
begin
environment = Environment.from_yaml File.read file_path
rescue e
STDERR << "error loading #{file_path}: " << e << "\n"
# FIXME: Print stacktrace? Debug mode?
next
end
@@all << environment
end
end
def self.get(name)
_def = @@all.find &.name.==(name)
if _def.nil?
raise Exception.new "Environment '#{name}' does not exist."
end
_def
end
def to_s
"#{name} (#{type.to_s.downcase})"
end
end
class Service
getter environment : Environment
getter providers = ProvidersList.new
class Exception < ::Exception
end
class ProvidersList < Hash(String, String)
def []?(name)
super(name).try { |x| Service.get_by_id x}
end
end
struct AsYAML
YAML.mapping({
name: String,
environment: String,
consumes: {
type: Array(YAMLConsumes),
default: [] of YAMLConsumes
}
})
end
struct YAMLConsumes
YAML.mapping({
token: String,
from: String
})
end
def initialize(name, environment_name : String?, @consumes = [] of YAMLConsumes)
@reference = ServiceDefinition.get name
@environment = if environment_name.nil? || environment_name == ""
Environment.root
else
Environment.get environment_name
end
@consumes.each do |consume|
@providers[consume.token] = consume.from
end
end
def self.from_yaml(yaml)
yaml = AsYAML.from_yaml yaml
self.new yaml.name, yaml.environment, yaml.consumes
end
def to_yaml
{
name: name,
environment: @environment.name
}.to_yaml
end
def id
if @environment.name == "root"
name
else
"#{@environment.name}/#{name}"
end
end
# FIXME: At this point, macros would be both more readable and shorter.
def name
@reference.name
end
def type
@reference.type
end
def command
@reference.command
end
def stop_command
@reference.stop_command
end
def provides
@reference.provides
end
def export_environment_variables
ENV["SERVICE_ENVIRONMENT"] = @environment.name
ENV["SERVICE_ENVIRONMENT_TYPE"] = @environment.type.to_s
# FIXME: Parsing should probably be done… when parsing the file.
# FIXME: Parsing is probably a bit primitive. Maybe this isnt the right way of defining this.
@reference.environment_variables.each do |string|
# FIXME: Should probably deserve a warning.
variable = string.match(/^[^=]*=/).not_nil![0]
value = string[variable.size..string.size]
variable = variable[0..variable.size-2]
ENV[variable] = value
end
end
private def evaluate(string)
string.gsub /%{[a-zA-Z]+}/ do |match|
match = match[2..match.size-2]
if match.downcase == "environment"
@environment.name
else
""
end
end
end
def start(pid_dir : String, log_dir : String)
export_environment_variables
(@environment.checks + @reference.checks).each do |check|
run_check = false
check.file.try do |file|
run_check = true if ! File.exists? evaluate file
end
check.directory.try do |directory|
run_check = true if ! Dir.exists? evaluate directory
end
unless run_check
next
end
puts " - #{check.name}"
# FIXME: Output? Only in debug mode?
child = Process.run "sh", ["-c", evaluate check.command], output: Process::Redirect::Inherit, error: Process::Redirect::Inherit
if child.exit_status != 0
raise Service::Exception.new "Child process exited with status “#{child.exit_status}”."
break
end
end
# FIXME: Should evaluate be used in split_command? What namespace should split_command use?
command, args = split_command command
args.map! do |arg| evaluate arg end
process = Process.fork do
base_log_name = "#{log_dir}/#{name}.#{@environment.name}"
stdout_file = File.open "#{base_log_name}.out", "w"
stderr_file = File.open "#{base_log_name}.err", "w"
LibC.dup2 stdout_file.fd, 1
LibC.dup2 stderr_file.fd, 2
Process.exec command, args, chdir: @reference.directory
end
self.save_pid pid_dir, process.pid
end
# TODO:
# - Custom shutdown commands.
# - Should we wait for the process to die?
# - Shouldnt we remove the pid file?
def stop(pid_dir : String)
_pid = pid pid_dir
if _pid
command = stop_command
if command
command, args = split_command command
Process.run(command, args)
else
Process.kill Signal::TERM, _pid
end
else
# Already stopped or dead, nothing to be done here.
end
end
def get_pid_file(pid_dir)
"#{pid_dir}/#{name}.pid"
end
def pid(pid_dir)
File.read(get_pid_file pid_dir).to_i
rescue e # pid file missing, corrupted or otherwise not readable
nil
end
def save_pid(pid_dir, new_pid)
File.write get_pid_file(pid_dir), new_pid
end
enum Status
Running
Dead
Stopped
end
def status(pid_dir)
_pid = pid pid_dir
if _pid
if Process.exists? _pid
Status::Running
else
Status::Dead
end
else
Status::Stopped
end
end
def running?(pid_dir)
status(pid_dir) == Service::Status::Running
end
def to_s
"#{name} (in #{@environment.name})"
end
def summary
"Name: #{name}\n" +
"Environment: #{environment.name} (#{environment.type.to_s.downcase})\n" +
(
if provides.size > 0
"Provides:\n" +
(provides.map { |x| " - " + x.token + "\n" }).join
else
""
end
) +
(
if @reference.consumes.size > 0
"Consumes:\n" +
(@reference.consumes.map { |x|
provider = @providers[x.token]?
if provider
" - #{x.token} (from #{provider.id})\n"
else
" - #{x.token} " +
"(NOT CURRENTLY PROVIDED)".colorize(:red).to_s +
"\n"
end
}).join
else
""
end
)
end
class_getter all = [] of Service
def self.load(path)
Dir.each_child path do |child|
unless child.match /\.yaml$/
next
end
@@all << Service.from_yaml File.read "#{path}/#{child}"
end
end
def write(path)
File.write "#{path}/#{name}.#{@environment.name}.yaml", to_yaml
end
def remove(path)
File.delete "#{path}/#{name}.#{@environment.name}.yaml"
end
def self.get_by_id(id)
matches = id.match /[^\/]*/
unless matches # Should not happen, above regex would always match.
raise Exception.new "FIXME"
end
environment_name = matches[0]
service_name = if environment_name == id
environment_name = "root"
id
else
id.sub 0..matches[0].size, ""
end
@@all.find do |service|
service.name == service_name
end
end
def is_id?(id)
id == self.id || (@environment.name == "root" && id == "root/#{name}")
end
alias ServiceTree = Array(ServiceTree) | Service
# Returns a dependency tree.
# Youll probably want to flatten and reverse it afterwards.
def dependency_tree
tree = [self] of ServiceTree
@consumes.each do |token|
service = Service.get_by_id token.from
unless service
# FIXME: Does it make the dep tree invalid?
# FIXME: Only optional deps should be fine.
next
end
tree << service.dependency_tree
end
tree
end
def reverse_dependency_tree
tree = [self] of ServiceTree
@@all.each do |service|
service.providers.any? do |token, id|
if self.is_id?(id) && ! tree.any? service
tree << service
end
end
end
tree
end
end
require "./service_definition.cr"
require "./environment.cr"
require "./service.cr"
args = [] of String

337
src/service.cr Normal file
View File

@ -0,0 +1,337 @@
require "yaml"
require "colorize"
def split_command(string)
args = string.split /\ (?=([^\"]*\"[^\"]*\")*[^\"]*$)/
command = args[0]
args.delete_at 0
return command, args
end
class Service
getter environment : Environment
getter providers = ProvidersList.new
class Exception < ::Exception
end
class ProvidersList < Hash(String, String)
def []?(name)
super(name).try { |x| Service.get_by_id x}
end
end
struct AsYAML
YAML.mapping({
name: String,
environment: String,
consumes: {
type: Array(YAMLConsumes),
default: [] of YAMLConsumes
}
})
end
struct YAMLConsumes
YAML.mapping({
token: String,
from: String
})
end
def initialize(name, environment_name : String?, @consumes = [] of YAMLConsumes)
@reference = ServiceDefinition.get name
@environment = if environment_name.nil? || environment_name == ""
Environment.root
else
Environment.get environment_name
end
@consumes.each do |consume|
@providers[consume.token] = consume.from
end
end
def self.from_yaml(yaml)
yaml = AsYAML.from_yaml yaml
self.new yaml.name, yaml.environment, yaml.consumes
end
def to_yaml
{
name: name,
environment: @environment.name
}.to_yaml
end
def id
if @environment.name == "root"
name
else
"#{@environment.name}/#{name}"
end
end
# FIXME: At this point, macros would be both more readable and shorter.
def name
@reference.name
end
def type
@reference.type
end
def command
@reference.command
end
def stop_command
@reference.stop_command
end
def provides
@reference.provides
end
def export_environment_variables
ENV["SERVICE_ENVIRONMENT"] = @environment.name
ENV["SERVICE_ENVIRONMENT_TYPE"] = @environment.type.to_s
# FIXME: Parsing should probably be done… when parsing the file.
# FIXME: Parsing is probably a bit primitive. Maybe this isnt the right way of defining this.
@reference.environment_variables.each do |string|
# FIXME: Should probably deserve a warning.
variable = string.match(/^[^=]*=/).not_nil![0]
value = string[variable.size..string.size]
variable = variable[0..variable.size-2]
ENV[variable] = value
end
end
private def evaluate(string)
string.gsub /%{[a-zA-Z]+}/ do |match|
match = match[2..match.size-2]
if match.downcase == "environment"
@environment.name
else
""
end
end
end
def start(pid_dir : String, log_dir : String)
export_environment_variables
(@environment.checks + @reference.checks).each do |check|
run_check = false
check.file.try do |file|
run_check = true if ! File.exists? evaluate file
end
check.directory.try do |directory|
run_check = true if ! Dir.exists? evaluate directory
end
unless run_check
next
end
puts " - #{check.name}"
# FIXME: Output? Only in debug mode?
child = Process.run "sh", ["-c", evaluate check.command], output: Process::Redirect::Inherit, error: Process::Redirect::Inherit
if child.exit_status != 0
raise Service::Exception.new "Child process exited with status “#{child.exit_status}”."
break
end
end
# FIXME: Should evaluate be used in split_command? What namespace should split_command use?
command, args = split_command command
args.map! do |arg| evaluate arg end
process = Process.fork do
base_log_name = "#{log_dir}/#{name}.#{@environment.name}"
stdout_file = File.open "#{base_log_name}.out", "w"
stderr_file = File.open "#{base_log_name}.err", "w"
LibC.dup2 stdout_file.fd, 1
LibC.dup2 stderr_file.fd, 2
Process.exec command, args, chdir: @reference.directory
end
self.save_pid pid_dir, process.pid
end
# TODO:
# - Custom shutdown commands.
# - Should we wait for the process to die?
# - Shouldnt we remove the pid file?
def stop(pid_dir : String)
_pid = pid pid_dir
if _pid
command = stop_command
if command
command, args = split_command command
Process.run(command, args)
else
Process.kill Signal::TERM, _pid
end
else
# Already stopped or dead, nothing to be done here.
end
end
def get_pid_file(pid_dir)
"#{pid_dir}/#{name}.pid"
end
def pid(pid_dir)
File.read(get_pid_file pid_dir).to_i
rescue e # pid file missing, corrupted or otherwise not readable
nil
end
def save_pid(pid_dir, new_pid)
File.write get_pid_file(pid_dir), new_pid
end
enum Status
Running
Dead
Stopped
end
def status(pid_dir)
_pid = pid pid_dir
if _pid
if Process.exists? _pid
Status::Running
else
Status::Dead
end
else
Status::Stopped
end
end
def running?(pid_dir)
status(pid_dir) == Service::Status::Running
end
def to_s
"#{name} (in #{@environment.name})"
end
def summary
"Name: #{name}\n" +
"Environment: #{environment.name} (#{environment.type.to_s.downcase})\n" +
(
if provides.size > 0
"Provides:\n" +
(provides.map { |x| " - " + x.token + "\n" }).join
else
""
end
) +
(
if @reference.consumes.size > 0
"Consumes:\n" +
(@reference.consumes.map { |x|
provider = @providers[x.token]?
if provider
" - #{x.token} (from #{provider.id})\n"
else
" - #{x.token} " +
"(NOT CURRENTLY PROVIDED)".colorize(:red).to_s +
"\n"
end
}).join
else
""
end
)
end
class_getter all = [] of Service
def self.load(path)
Dir.each_child path do |child|
unless child.match /\.yaml$/
next
end
@@all << Service.from_yaml File.read "#{path}/#{child}"
end
end
def write(path)
File.write "#{path}/#{name}.#{@environment.name}.yaml", to_yaml
end
def remove(path)
File.delete "#{path}/#{name}.#{@environment.name}.yaml"
end
def self.get_by_id(id)
matches = id.match /[^\/]*/
unless matches # Should not happen, above regex would always match.
raise Exception.new "FIXME"
end
environment_name = matches[0]
service_name = if environment_name == id
environment_name = "root"
id
else
id.sub 0..matches[0].size, ""
end
@@all.find do |service|
service.name == service_name
end
end
def is_id?(id)
id == self.id || (@environment.name == "root" && id == "root/#{name}")
end
alias ServiceTree = Array(ServiceTree) | Service
# Returns a dependency tree.
# Youll probably want to flatten and reverse it afterwards.
def dependency_tree
tree = [self] of ServiceTree
@consumes.each do |token|
service = Service.get_by_id token.from
unless service
# FIXME: Does it make the dep tree invalid?
# FIXME: Only optional deps should be fine.
next
end
tree << service.dependency_tree
end
tree
end
def reverse_dependency_tree
tree = [self] of ServiceTree
@@all.each do |service|
service.providers.any? do |token, id|
if self.is_id?(id) && ! tree.any? service
tree << service
end
end
end
tree
end
end

87
src/service_definition.cr Normal file
View File

@ -0,0 +1,87 @@
require "yaml"
class ServiceDefinition
struct Consumes
YAML.mapping({
token: String,
optional: {
type: Bool,
default: false
}
})
end
struct Provides
YAML.mapping({
token: String
})
end
struct Checks
YAML.mapping({
name: String,
file: String?,
directory: String?,
command: String
})
end
YAML.mapping({
name: String,
command: String,
stop_command: {
type: String?,
key: "stop-command"
},
directory: String?, # Place to chdir to before running @command.
environment: {
type: Environment,
default: Environment.root
},
provides: {
type: Array(Provides),
default: [] of Provides
},
consumes: {
type: Array(Consumes),
default: [] of Consumes
},
checks: {
type: Array(Checks),
default: [] of Checks
},
environment_variables: {
type: Array(String),
key: "environment-variables",
default: [] of String
}
})
def self.new(name)
Service.from_yaml File.read "#{name}.yaml"
end
class_getter all = [] of ServiceDefinition
def self.load(path)
Dir.each_child path do |child|
unless child.match /\.yaml$/
next
end
@@all << ServiceDefinition.from_yaml File.read "#{path}/#{child}"
end
end
def self.get(name) : ServiceDefinition
_def = @@all.find &.name.==(name)
if _def.nil?
raise Exception.new "Service '#{name}' does not exist."
end
_def
end
def to_s
name
end
end