From 61c9f33f8918840c61a17fd78d7d79a4ecce5bba Mon Sep 17 00:00:00 2001 From: Luka Vandervelden Date: Sun, 9 Jun 2019 16:59:05 +0200 Subject: [PATCH] Sources split. --- src/environment.cr | 76 ++++++ src/main.cr | 494 +------------------------------------- src/service.cr | 337 ++++++++++++++++++++++++++ src/service_definition.cr | 87 +++++++ 4 files changed, 503 insertions(+), 491 deletions(-) create mode 100644 src/environment.cr create mode 100644 src/service.cr create mode 100644 src/service_definition.cr diff --git a/src/environment.cr b/src/environment.cr new file mode 100644 index 0000000..a9c7887 --- /dev/null +++ b/src/environment.cr @@ -0,0 +1,76 @@ +require "yaml" + +class Environment + enum Type + Prefix + RootFileSystem + end + + YAML.mapping({ + name: String, + type: { + type: Type, + default: Type::Prefix + }, + domain_name: { + type: String?, + key: "domain-name" + }, + checks: { + # FIXME: Would probably need a more neutral namespace. + type: Array(ServiceDefinition::Checks), + default: Array(ServiceDefinition::Checks).new + } + }) + + def initialize() + @name = "root" + @type = Type::Prefix + @checks = Array(ServiceDefinition::Checks).new + + # FIXME: Should this *really* be here? + @checks << ServiceDefinition::Checks.from_yaml <<-EOF + name: Creating data directory + directory: /srv/%{ENVIRONMENT} + command: mkdir -p /srv/%{ENVIRONMENT} && chmod a+rwt /srv/%{ENVIRONMENT} + EOF + end + + class_getter root = Environment.new + class_getter all = [@@root] of Environment + + def self.load(path) + Dir.each_child path do |child| + unless child.match /\.yaml$/ + next + end + + file_path = "#{path}/#{child}" + + begin + environment = Environment.from_yaml File.read file_path + rescue e + STDERR << "error loading #{file_path}: " << e << "\n" + # FIXME: Print stacktrace? Debug mode? + next + end + + @@all << environment + end + end + + def self.get(name) + _def = @@all.find &.name.==(name) + + if _def.nil? + raise Exception.new "Environment '#{name}' does not exist." + end + + _def + end + + def to_s + "#{name} (#{type.to_s.downcase})" + end +end + diff --git a/src/main.cr b/src/main.cr index 4231e6f..7828d28 100644 --- a/src/main.cr +++ b/src/main.cr @@ -12,497 +12,9 @@ ENVIRONMENTS_DIRECTORY = "./environments" # TODO: # - Be more declarative about the definition of commands. -def split_command(string) - args = string.split /\ (?=([^\"]*\"[^\"]*\")*[^\"]*$)/ - - command = args[0] - args.delete_at 0 - - return command, args -end - -class ServiceDefinition - struct Consumes - YAML.mapping({ - token: String, - optional: { - type: Bool, - default: false - } - }) - end - struct Provides - YAML.mapping({ - token: String - }) - end - struct Checks - YAML.mapping({ - name: String, - file: String?, - directory: String?, - command: String - }) - end - YAML.mapping({ - name: String, - command: String, - stop_command: { - type: String?, - key: "stop-command" - }, - directory: String?, # Place to chdir to before running @command. - environment: { - type: Environment, - default: Environment.root - }, - provides: { - type: Array(Provides), - default: [] of Provides - }, - consumes: { - type: Array(Consumes), - default: [] of Consumes - }, - checks: { - type: Array(Checks), - default: [] of Checks - }, - environment_variables: { - type: Array(String), - key: "environment-variables", - default: [] of String - } - }) - - def self.new(name) - Service.from_yaml File.read "#{name}.yaml" - end - - class_getter all = [] of ServiceDefinition - - def self.load(path) - Dir.each_child path do |child| - unless child.match /\.yaml$/ - next - end - - @@all << ServiceDefinition.from_yaml File.read "#{path}/#{child}" - end - end - - def self.get(name) : ServiceDefinition - _def = @@all.find &.name.==(name) - - if _def.nil? - raise Exception.new "Service '#{name}' does not exist." - end - - _def - end - - def to_s - name - end -end - -class Environment - enum Type - Prefix - RootFileSystem - end - - YAML.mapping({ - name: String, - type: { - type: Type, - default: Type::Prefix - }, - domain_name: { - type: String?, - key: "domain-name" - }, - checks: { - # FIXME: Would probably need a more neutral namespace. - type: Array(ServiceDefinition::Checks), - default: Array(ServiceDefinition::Checks).new - } - }) - - def initialize() - @name = "root" - @type = Type::Prefix - @checks = Array(ServiceDefinition::Checks).new - - # FIXME: Should this *really* be here? - @checks << ServiceDefinition::Checks.from_yaml <<-EOF - name: Creating data directory - directory: /srv/%{ENVIRONMENT} - command: mkdir -p /srv/%{ENVIRONMENT} && chmod a+rwt /srv/%{ENVIRONMENT} - EOF - end - - class_getter root = Environment.new - class_getter all = [@@root] of Environment - - def self.load(path) - Dir.each_child path do |child| - unless child.match /\.yaml$/ - next - end - - file_path = "#{path}/#{child}" - - begin - environment = Environment.from_yaml File.read file_path - rescue e - STDERR << "error loading #{file_path}: " << e << "\n" - # FIXME: Print stacktrace? Debug mode? - next - end - - @@all << environment - end - end - - def self.get(name) - _def = @@all.find &.name.==(name) - - if _def.nil? - raise Exception.new "Environment '#{name}' does not exist." - end - - _def - end - - def to_s - "#{name} (#{type.to_s.downcase})" - end -end - -class Service - getter environment : Environment - getter providers = ProvidersList.new - - class Exception < ::Exception - end - - class ProvidersList < Hash(String, String) - def []?(name) - super(name).try { |x| Service.get_by_id x} - end - end - - struct AsYAML - YAML.mapping({ - name: String, - environment: String, - consumes: { - type: Array(YAMLConsumes), - default: [] of YAMLConsumes - } - }) - end - struct YAMLConsumes - YAML.mapping({ - token: String, - from: String - }) - end - - def initialize(name, environment_name : String?, @consumes = [] of YAMLConsumes) - @reference = ServiceDefinition.get name - @environment = if environment_name.nil? || environment_name == "" - Environment.root - else - Environment.get environment_name - end - - @consumes.each do |consume| - @providers[consume.token] = consume.from - end - end - - def self.from_yaml(yaml) - yaml = AsYAML.from_yaml yaml - - - self.new yaml.name, yaml.environment, yaml.consumes - end - def to_yaml - { - name: name, - environment: @environment.name - }.to_yaml - end - - def id - if @environment.name == "root" - name - else - "#{@environment.name}/#{name}" - end - end - - # FIXME: At this point, macros would be both more readable and shorter. - def name - @reference.name - end - def type - @reference.type - end - def command - @reference.command - end - def stop_command - @reference.stop_command - end - def provides - @reference.provides - end - - def export_environment_variables - ENV["SERVICE_ENVIRONMENT"] = @environment.name - ENV["SERVICE_ENVIRONMENT_TYPE"] = @environment.type.to_s - - # FIXME: Parsing should probably be done… when parsing the file. - # FIXME: Parsing is probably a bit primitive. Maybe this isn’t the right way of defining this. - @reference.environment_variables.each do |string| - # FIXME: Should probably deserve a warning. - variable = string.match(/^[^=]*=/).not_nil![0] - value = string[variable.size..string.size] - variable = variable[0..variable.size-2] - - ENV[variable] = value - end - end - private def evaluate(string) - string.gsub /%{[a-zA-Z]+}/ do |match| - match = match[2..match.size-2] - - if match.downcase == "environment" - @environment.name - else - "" - end - end - end - - def start(pid_dir : String, log_dir : String) - export_environment_variables - - (@environment.checks + @reference.checks).each do |check| - run_check = false - - check.file.try do |file| - run_check = true if ! File.exists? evaluate file - end - - check.directory.try do |directory| - run_check = true if ! Dir.exists? evaluate directory - end - - unless run_check - next - end - - puts " - #{check.name}" - - # FIXME: Output? Only in debug mode? - child = Process.run "sh", ["-c", evaluate check.command], output: Process::Redirect::Inherit, error: Process::Redirect::Inherit - - if child.exit_status != 0 - raise Service::Exception.new "Child process exited with status “#{child.exit_status}”." - break - end - end - - # FIXME: Should evaluate be used in split_command? What namespace should split_command use? - command, args = split_command command - args.map! do |arg| evaluate arg end - - process = Process.fork do - base_log_name = "#{log_dir}/#{name}.#{@environment.name}" - stdout_file = File.open "#{base_log_name}.out", "w" - stderr_file = File.open "#{base_log_name}.err", "w" - - LibC.dup2 stdout_file.fd, 1 - LibC.dup2 stderr_file.fd, 2 - - Process.exec command, args, chdir: @reference.directory - end - - self.save_pid pid_dir, process.pid - end - - # TODO: - # - Custom shutdown commands. - # - Should we wait for the process to die? - # - Shouldn’t we remove the pid file? - def stop(pid_dir : String) - _pid = pid pid_dir - - if _pid - command = stop_command - if command - command, args = split_command command - - Process.run(command, args) - else - Process.kill Signal::TERM, _pid - end - else - # Already stopped or dead, nothing to be done here. - end - end - - def get_pid_file(pid_dir) - "#{pid_dir}/#{name}.pid" - end - - def pid(pid_dir) - File.read(get_pid_file pid_dir).to_i - rescue e # pid file missing, corrupted or otherwise not readable - nil - end - - def save_pid(pid_dir, new_pid) - File.write get_pid_file(pid_dir), new_pid - end - - enum Status - Running - Dead - Stopped - end - - def status(pid_dir) - _pid = pid pid_dir - - if _pid - if Process.exists? _pid - Status::Running - else - Status::Dead - end - else - Status::Stopped - end - end - def running?(pid_dir) - status(pid_dir) == Service::Status::Running - end - - def to_s - "#{name} (in #{@environment.name})" - end - - def summary - "Name: #{name}\n" + - "Environment: #{environment.name} (#{environment.type.to_s.downcase})\n" + - ( - if provides.size > 0 - "Provides:\n" + - (provides.map { |x| " - " + x.token + "\n" }).join - else - "" - end - ) + - ( - if @reference.consumes.size > 0 - "Consumes:\n" + - (@reference.consumes.map { |x| - provider = @providers[x.token]? - if provider - " - #{x.token} (from #{provider.id})\n" - else - " - #{x.token} " + - "(NOT CURRENTLY PROVIDED)".colorize(:red).to_s + - "\n" - end - }).join - else - "" - end - ) - end - - class_getter all = [] of Service - def self.load(path) - Dir.each_child path do |child| - unless child.match /\.yaml$/ - next - end - - @@all << Service.from_yaml File.read "#{path}/#{child}" - end - end - - def write(path) - File.write "#{path}/#{name}.#{@environment.name}.yaml", to_yaml - end - - def remove(path) - File.delete "#{path}/#{name}.#{@environment.name}.yaml" - end - - def self.get_by_id(id) - matches = id.match /[^\/]*/ - unless matches # Should not happen, above regex would always match. - raise Exception.new "FIXME" - end - environment_name = matches[0] - service_name = if environment_name == id - environment_name = "root" - id - else - id.sub 0..matches[0].size, "" - end - - @@all.find do |service| - service.name == service_name - end - end - def is_id?(id) - id == self.id || (@environment.name == "root" && id == "root/#{name}") - end - - alias ServiceTree = Array(ServiceTree) | Service - - # Returns a dependency tree. - # You’ll probably want to flatten and reverse it afterwards. - def dependency_tree - tree = [self] of ServiceTree - - @consumes.each do |token| - service = Service.get_by_id token.from - - unless service - # FIXME: Does it make the dep tree invalid? - # FIXME: Only optional deps should be fine. - next - end - - tree << service.dependency_tree - end - - tree - end - - def reverse_dependency_tree - tree = [self] of ServiceTree - - @@all.each do |service| - service.providers.any? do |token, id| - if self.is_id?(id) && ! tree.any? service - tree << service - end - end - end - - tree - end -end +require "./service_definition.cr" +require "./environment.cr" +require "./service.cr" args = [] of String diff --git a/src/service.cr b/src/service.cr new file mode 100644 index 0000000..a4eae56 --- /dev/null +++ b/src/service.cr @@ -0,0 +1,337 @@ +require "yaml" +require "colorize" + +def split_command(string) + args = string.split /\ (?=([^\"]*\"[^\"]*\")*[^\"]*$)/ + + command = args[0] + args.delete_at 0 + + return command, args +end + +class Service + getter environment : Environment + getter providers = ProvidersList.new + + class Exception < ::Exception + end + + class ProvidersList < Hash(String, String) + def []?(name) + super(name).try { |x| Service.get_by_id x} + end + end + + struct AsYAML + YAML.mapping({ + name: String, + environment: String, + consumes: { + type: Array(YAMLConsumes), + default: [] of YAMLConsumes + } + }) + end + struct YAMLConsumes + YAML.mapping({ + token: String, + from: String + }) + end + + def initialize(name, environment_name : String?, @consumes = [] of YAMLConsumes) + @reference = ServiceDefinition.get name + @environment = if environment_name.nil? || environment_name == "" + Environment.root + else + Environment.get environment_name + end + + @consumes.each do |consume| + @providers[consume.token] = consume.from + end + end + + def self.from_yaml(yaml) + yaml = AsYAML.from_yaml yaml + + + self.new yaml.name, yaml.environment, yaml.consumes + end + def to_yaml + { + name: name, + environment: @environment.name + }.to_yaml + end + + def id + if @environment.name == "root" + name + else + "#{@environment.name}/#{name}" + end + end + + # FIXME: At this point, macros would be both more readable and shorter. + def name + @reference.name + end + def type + @reference.type + end + def command + @reference.command + end + def stop_command + @reference.stop_command + end + def provides + @reference.provides + end + + def export_environment_variables + ENV["SERVICE_ENVIRONMENT"] = @environment.name + ENV["SERVICE_ENVIRONMENT_TYPE"] = @environment.type.to_s + + # FIXME: Parsing should probably be done… when parsing the file. + # FIXME: Parsing is probably a bit primitive. Maybe this isn’t the right way of defining this. + @reference.environment_variables.each do |string| + # FIXME: Should probably deserve a warning. + variable = string.match(/^[^=]*=/).not_nil![0] + value = string[variable.size..string.size] + variable = variable[0..variable.size-2] + + ENV[variable] = value + end + end + private def evaluate(string) + string.gsub /%{[a-zA-Z]+}/ do |match| + match = match[2..match.size-2] + + if match.downcase == "environment" + @environment.name + else + "" + end + end + end + + def start(pid_dir : String, log_dir : String) + export_environment_variables + + (@environment.checks + @reference.checks).each do |check| + run_check = false + + check.file.try do |file| + run_check = true if ! File.exists? evaluate file + end + + check.directory.try do |directory| + run_check = true if ! Dir.exists? evaluate directory + end + + unless run_check + next + end + + puts " - #{check.name}" + + # FIXME: Output? Only in debug mode? + child = Process.run "sh", ["-c", evaluate check.command], output: Process::Redirect::Inherit, error: Process::Redirect::Inherit + + if child.exit_status != 0 + raise Service::Exception.new "Child process exited with status “#{child.exit_status}”." + break + end + end + + # FIXME: Should evaluate be used in split_command? What namespace should split_command use? + command, args = split_command command + args.map! do |arg| evaluate arg end + + process = Process.fork do + base_log_name = "#{log_dir}/#{name}.#{@environment.name}" + stdout_file = File.open "#{base_log_name}.out", "w" + stderr_file = File.open "#{base_log_name}.err", "w" + + LibC.dup2 stdout_file.fd, 1 + LibC.dup2 stderr_file.fd, 2 + + Process.exec command, args, chdir: @reference.directory + end + + self.save_pid pid_dir, process.pid + end + + # TODO: + # - Custom shutdown commands. + # - Should we wait for the process to die? + # - Shouldn’t we remove the pid file? + def stop(pid_dir : String) + _pid = pid pid_dir + + if _pid + command = stop_command + if command + command, args = split_command command + + Process.run(command, args) + else + Process.kill Signal::TERM, _pid + end + else + # Already stopped or dead, nothing to be done here. + end + end + + def get_pid_file(pid_dir) + "#{pid_dir}/#{name}.pid" + end + + def pid(pid_dir) + File.read(get_pid_file pid_dir).to_i + rescue e # pid file missing, corrupted or otherwise not readable + nil + end + + def save_pid(pid_dir, new_pid) + File.write get_pid_file(pid_dir), new_pid + end + + enum Status + Running + Dead + Stopped + end + + def status(pid_dir) + _pid = pid pid_dir + + if _pid + if Process.exists? _pid + Status::Running + else + Status::Dead + end + else + Status::Stopped + end + end + def running?(pid_dir) + status(pid_dir) == Service::Status::Running + end + + def to_s + "#{name} (in #{@environment.name})" + end + + def summary + "Name: #{name}\n" + + "Environment: #{environment.name} (#{environment.type.to_s.downcase})\n" + + ( + if provides.size > 0 + "Provides:\n" + + (provides.map { |x| " - " + x.token + "\n" }).join + else + "" + end + ) + + ( + if @reference.consumes.size > 0 + "Consumes:\n" + + (@reference.consumes.map { |x| + provider = @providers[x.token]? + if provider + " - #{x.token} (from #{provider.id})\n" + else + " - #{x.token} " + + "(NOT CURRENTLY PROVIDED)".colorize(:red).to_s + + "\n" + end + }).join + else + "" + end + ) + end + + class_getter all = [] of Service + def self.load(path) + Dir.each_child path do |child| + unless child.match /\.yaml$/ + next + end + + @@all << Service.from_yaml File.read "#{path}/#{child}" + end + end + + def write(path) + File.write "#{path}/#{name}.#{@environment.name}.yaml", to_yaml + end + + def remove(path) + File.delete "#{path}/#{name}.#{@environment.name}.yaml" + end + + def self.get_by_id(id) + matches = id.match /[^\/]*/ + unless matches # Should not happen, above regex would always match. + raise Exception.new "FIXME" + end + environment_name = matches[0] + service_name = if environment_name == id + environment_name = "root" + id + else + id.sub 0..matches[0].size, "" + end + + @@all.find do |service| + service.name == service_name + end + end + def is_id?(id) + id == self.id || (@environment.name == "root" && id == "root/#{name}") + end + + alias ServiceTree = Array(ServiceTree) | Service + + # Returns a dependency tree. + # You’ll probably want to flatten and reverse it afterwards. + def dependency_tree + tree = [self] of ServiceTree + + @consumes.each do |token| + service = Service.get_by_id token.from + + unless service + # FIXME: Does it make the dep tree invalid? + # FIXME: Only optional deps should be fine. + next + end + + tree << service.dependency_tree + end + + tree + end + + def reverse_dependency_tree + tree = [self] of ServiceTree + + @@all.each do |service| + service.providers.any? do |token, id| + if self.is_id?(id) && ! tree.any? service + tree << service + end + end + end + + tree + end +end + + diff --git a/src/service_definition.cr b/src/service_definition.cr new file mode 100644 index 0000000..0f3ffdb --- /dev/null +++ b/src/service_definition.cr @@ -0,0 +1,87 @@ +require "yaml" + +class ServiceDefinition + struct Consumes + YAML.mapping({ + token: String, + optional: { + type: Bool, + default: false + } + }) + end + struct Provides + YAML.mapping({ + token: String + }) + end + struct Checks + YAML.mapping({ + name: String, + file: String?, + directory: String?, + command: String + }) + end + YAML.mapping({ + name: String, + command: String, + stop_command: { + type: String?, + key: "stop-command" + }, + directory: String?, # Place to chdir to before running @command. + environment: { + type: Environment, + default: Environment.root + }, + provides: { + type: Array(Provides), + default: [] of Provides + }, + consumes: { + type: Array(Consumes), + default: [] of Consumes + }, + checks: { + type: Array(Checks), + default: [] of Checks + }, + environment_variables: { + type: Array(String), + key: "environment-variables", + default: [] of String + } + }) + + def self.new(name) + Service.from_yaml File.read "#{name}.yaml" + end + + class_getter all = [] of ServiceDefinition + + def self.load(path) + Dir.each_child path do |child| + unless child.match /\.yaml$/ + next + end + + @@all << ServiceDefinition.from_yaml File.read "#{path}/#{child}" + end + end + + def self.get(name) : ServiceDefinition + _def = @@all.find &.name.==(name) + + if _def.nil? + raise Exception.new "Service '#{name}' does not exist." + end + + _def + end + + def to_s + name + end +end +