Complete removal of YAML files.

master
Luka Vandervelden 2019-08-10 17:16:24 +02:00
parent 3769303ad5
commit efe2c0e964
9 changed files with 181 additions and 156 deletions

11
services/altideal.spec Normal file
View File

@ -0,0 +1,11 @@
name: altideal
command: ./bin/altideal -k /srv/${ENVIRONMENT}/jwt_key
environment-variables:
- LD_LIBRARY_PATH=/usr/local/lib
directory: /home/lukc/Sources/altideal
consumes: auth, pubsub, postgres
%check
name: database setup
file: /srv/${ENVIRONMENT}/altideal-is-setup
command: cd /home/lukc/Sources/altideal && PGHOST=/tmp/postgresql-${ENVIRONMENT} ./setup.zsh --database && touch /srv/${ENVIRONMENT}/altideal-is-setup

22
services/authd.spec Normal file
View File

@ -0,0 +1,22 @@
name: authd
environment-variables:
- LD_LIBRARY_PATH=/usr/local/lib
user: authd
command: /home/lukc/Sources/authd/bin/authd -K /srv/${ENVIRONMENT}/jwt_key -u /srv/${ENVIRONMENT}/passwd -g /srv/${ENVIRONMENT}/group
%check
name: Creating IPC directory
directory: /run/ipc
command: install -d -m6777 /run/ipc
%check
name: Creating JWT key
file: /srv/${ENVIRONMENT}/jwt_key
command: head -c 64 /dev/urandom | base64 > /srv/${ENVIRONMENT}/jwt_key
%check
name: passwd file
file: /srv/${ENVIRONMENT}/passwd
command: touch /srv/${ENVIRONMENT}/passwd
%check
name: group file
file: /srv/${ENVIRONMENT}/group
command: touch /srv/${ENVIRONMENT}/group
provides: auth

View File

@ -1,24 +0,0 @@
name: authd
environment-variables:
- LD_LIBRARY_PATH=/usr/local/lib
user: authd
command: >-
/home/lukc/Sources/authd/bin/authd
-K /srv/%{ENVIRONMENT}/jwt_key
-u /srv/%{ENVIRONMENT}/passwd
-g /srv/%{ENVIRONMENT}/group
checks:
- name: Creating IPC directory
directory: /run/ipc
command: install -d -m6777 /run/ipc
- name: Creating JWT key
file: /srv/%{ENVIRONMENT}/jwt_key
command: head -c 64 /dev/urandom | base64 > /srv/%{ENVIRONMENT}/jwt_key
- name: passwd file
file: /srv/%{ENVIRONMENT}/passwd
command: touch /srv/%{ENVIRONMENT}/passwd
- name: group file
file: /srv/%{ENVIRONMENT}/group
command: touch /srv/%{ENVIRONMENT}/group
provides:
- token: auth

24
services/postgresql.spec Normal file
View File

@ -0,0 +1,24 @@
name: postgresql
user: postgres
command: /usr/bin/postgres -D /srv/${ENVIRONMENT}/postgresql -k /tmp/postgresql-${ENVIRONMENT}
#stop-command: kill -HUP ${PID}
environment-variables:
- PGROOT=/srv/${ENVIRONMENT}/postgresql
provides: postgres
%check
name: database directory creation
directory: /srv/${ENVIRONMENT}/postgresql
command: mkdir -p /srv/${ENVIRONMENT}/postgresql && chown postgres:postgres /srv/${ENVIRONMENT}/postgresql
%check
name: database creation
file: /srv/${ENVIRONMENT}/postgresql/base
command: su - postgres -c "initdb --locale en_US.UTF-8 -D '/srv/${ENVIRONMENT}/postgresql'"
%check
name: sockets directory
directory: /tmp/postgres-${ENVIRONMENT}
# FIXME: impose permissions
command: mkdir -p /tmp/postgresql-${ENVIRONMENT} && chown postgres:postgres /tmp/postgresql-${ENVIRONMENT}
# FIXME: add postgresql-check-db-dir around here

View File

@ -1,18 +0,0 @@
name: postgresql
user: postgres
command: /usr/bin/postgres -D /srv/%{ENVIRONMENT}/postgresql -k /tmp/postgresql-%{ENVIRONMENT}
#stop-command: kill -HUP %{PID}
environment-variables:
- PGROOT=/srv/%{ENVIRONMENT}/postgresql
checks:
- name: database directory creation
directory: /srv/%{ENVIRONMENT}/postgresql
command: mkdir -p /srv/%{ENVIRONMENT}/postgresql && chown postgres:postgres /srv/%{ENVIRONMENT}/postgresql
- name: database creation
file: /srv/%{ENVIRONMENT}/postgresql/base
command: su - postgres -c "initdb --locale en_US.UTF-8 -D '/srv/%{ENVIRONMENT}/postgresql'"
- name: sockets directory
directory: /tmp/postgres-%{ENVIRONMENT}
# FIXME: impose permissions
command: mkdir -p /tmp/postgresql-%{ENVIRONMENT} && chown postgres:postgres /tmp/postgresql-%{ENVIRONMENT}
# FIXME: add postgresql-check-db-dir around here

10
services/pubsubd.spec Normal file
View File

@ -0,0 +1,10 @@
name: pubsubd
environment-variables:
- LD_LIBRARY_PATH=/usr/local/lib
command: pubsubd -s /srv/${ENVIRONMENT}/pubsub
provides: pubsub
%check
name: Creating storage directory
directory: /srv/${ENVIRONMENT}/pubsub
command: install -d -m6777 /srv/${ENVIRONMENT}/pubsub

View File

@ -1,4 +1,4 @@
require "yaml" require "spec"
class Environment class Environment
enum Type enum Type
@ -6,34 +6,32 @@ class Environment
RootFileSystem RootFileSystem
end end
YAML.mapping({ getter name : String
name: String, getter type : Type = Type::Prefix
type: { getter domain_name : String?
type: Type, getter checks = Array(ServiceDefinition::Checks).new
default: Type::Prefix
},
domain_name: {
type: String?,
key: "domain-name"
},
checks: {
# FIXME: Would probably need a more neutral namespace.
type: Array(ServiceDefinition::Checks),
default: Array(ServiceDefinition::Checks).new
}
})
def initialize() def initialize()
@name = "root" @name = "root"
@type = Type::Prefix
@checks = Array(ServiceDefinition::Checks).new @checks = Array(ServiceDefinition::Checks).new
# FIXME: Should this *really* be here? # FIXME: Should this *really* be here?
@checks << ServiceDefinition::Checks.from_yaml <<-EOF @checks << ServiceDefinition::Checks.new "Creating data directory",
name: Creating data directory "mkdir -p /srv/${ENVIRONMENT} && chmod a+rwt /srv/${ENVIRONMENT}",
directory: /srv/%{ENVIRONMENT} directory: "/srv/${ENVIRONMENT}"
command: mkdir -p /srv/%{ENVIRONMENT} && chmod a+rwt /srv/%{ENVIRONMENT} end
EOF
def initialize(specs : Specs)
assignments = specs.assignments
@name = assignments["name"].as_s
assignments["type"].try &.as_s.tap do |type|
@type = Type.parse type
end
specs.sections.select(&.name.==("check")).each do |check|
@checks << ServiceDefinition::Checks.new check
end
end end
class_getter root = Environment.new class_getter root = Environment.new
@ -41,14 +39,14 @@ class Environment
def self.load(path) def self.load(path)
Dir.each_child path do |child| Dir.each_child path do |child|
unless child.match /\.yaml$/ unless child.match /\.spec$/
next next
end end
file_path = "#{path}/#{child}" file_path = "#{path}/#{child}"
begin begin
environment = Environment.from_yaml File.read file_path environment = Environment.new Specs.parse(file_path).not_nil!
rescue e rescue e
STDERR << "error loading #{file_path}: " << e << "\n" STDERR << "error loading #{file_path}: " << e << "\n"
# FIXME: Print stacktrace? Debug mode? # FIXME: Print stacktrace? Debug mode?

View File

@ -17,6 +17,7 @@ end
class Service class Service
getter environment : Environment getter environment : Environment
getter providers = ProvidersList.new getter providers = ProvidersList.new
getter consumed_tokens = Array(Consumer).new
class Exception < ::Exception class Exception < ::Exception
end end
@ -27,24 +28,14 @@ class Service
end end
end end
struct AsYAML struct Consumer
YAML.mapping({ getter token : String
name: String, getter from : String
environment: String, def initialize(@token, @from)
consumes: { end
type: Array(YAMLConsumes),
default: [] of YAMLConsumes
}
})
end
struct YAMLConsumes
YAML.mapping({
token: String,
from: String
})
end end
def initialize(name, environment_name : String?, @consumes = [] of YAMLConsumes) def initialize(name, environment_name : String?, @consumed_tokens = [] of Consumer)
@reference = ServiceDefinition.get name @reference = ServiceDefinition.get name
@environment = if environment_name.nil? || environment_name == "" @environment = if environment_name.nil? || environment_name == ""
Environment.root Environment.root
@ -52,22 +43,41 @@ class Service
Environment.get environment_name Environment.get environment_name
end end
@consumes.each do |consume| @consumed_tokens.each do |consume|
@providers[consume.token] = consume.from @providers[consume.token] = consume.from
end end
end end
def self.from_yaml(yaml) def initialize(specs : Specs)
yaml = AsYAML.from_yaml yaml assignments = specs.assignments
@reference = ServiceDefinition.get assignments["name"].as_s
self.new yaml.name, yaml.environment, yaml.consumes env = assignments["environment"]?.try &.as_s
@environment = if env.nil? || env == ""
Environment.root
else
Environment.get env
end
specs.sections.select(&.name.==("consumes")).each do |section|
@consumed_tokens << Consumer.new section.options[0], section.content["from"].as_s
end
end end
def to_yaml
{ def to_spec
name: name, file = [
environment: @environment.name "name: #{@reference.name}",
}.to_yaml "environment: #{@environment.name}"
# FIXME: consumed tokens are missing.
]
@consumed_tokens.each do |token|
file << "%consumes #{token.token}"
file << " from: #{token.from}"
end
file.join("\n") + "\n"
end end
def id def id
@ -95,9 +105,11 @@ class Service
@reference.provides @reference.provides
end end
def export_environment_variables private def build_environment
ENV["SERVICE_ENVIRONMENT"] = @environment.name env = {} of String => String
ENV["SERVICE_ENVIRONMENT_TYPE"] = @environment.type.to_s
env["ENVIRONMENT"] = @environment.name
env["ENVIRONMENT_TYPE"] = @environment.type.to_s
# FIXME: Parsing should probably be done… when parsing the file. # FIXME: Parsing should probably be done… when parsing the file.
# FIXME: Parsing is probably a bit primitive. Maybe this isnt the right way of defining this. # FIXME: Parsing is probably a bit primitive. Maybe this isnt the right way of defining this.
@ -107,11 +119,15 @@ class Service
value = string[variable.size..string.size] value = string[variable.size..string.size]
variable = variable[0..variable.size-2] variable = variable[0..variable.size-2]
ENV[variable] = value env[variable] = value
end end
env
end end
# FIXME: Is working on ${} really a good idea?
private def evaluate(string) private def evaluate(string)
string.gsub /%{[a-zA-Z]+}/ do |match| string.gsub /\${[a-zA-Z]+}/ do |match|
match = match[2..match.size-2] match = match[2..match.size-2]
if match.downcase == "environment" if match.downcase == "environment"
@ -123,16 +139,16 @@ class Service
end end
def start(pid_dir : String, log_dir : String) def start(pid_dir : String, log_dir : String)
export_environment_variables
(@environment.checks + @reference.checks).each do |check| (@environment.checks + @reference.checks).each do |check|
run_check = false run_check = false
check.file.try do |file| check.file.try do |file|
file = evaluate file
run_check = true if ! File.exists? evaluate file run_check = true if ! File.exists? evaluate file
end end
check.directory.try do |directory| check.directory.try do |directory|
directory = evaluate directory
run_check = true if ! Dir.exists? evaluate directory run_check = true if ! Dir.exists? evaluate directory
end end
@ -143,7 +159,10 @@ class Service
puts " - #{check.name}" puts " - #{check.name}"
child = Process.fork do child = Process.fork do
Process.exec "sh", ["-c", evaluate check.command], output: Process::Redirect::Inherit, error: Process::Redirect::Inherit Process.exec "sh", ["-c", evaluate check.command],
output: Process::Redirect::Inherit,
error: Process::Redirect::Inherit,
env: build_environment
end.wait end.wait
if child.exit_status != 0 if child.exit_status != 0
@ -171,7 +190,9 @@ class Service
end end
end end
Process.exec command, args, chdir: @reference.directory Process.exec command, args,
chdir: @reference.directory,
env: build_environment
end end
self.save_pid pid_dir, process.pid self.save_pid pid_dir, process.pid
@ -272,20 +293,26 @@ class Service
class_getter all = [] of Service class_getter all = [] of Service
def self.load(path) def self.load(path)
Dir.each_child path do |child| Dir.each_child path do |child|
unless child.match /\.yaml$/ unless child.match /\.spec$/
next next
end end
@@all << Service.from_yaml File.read "#{path}/#{child}" begin
specs = Specs.parse("#{path}/#{child}").not_nil!
rescue
next
end
@@all << Service.new specs
end end
end end
def write(path) def write(path)
File.write "#{path}/#{name}.#{@environment.name}.yaml", to_yaml File.write "#{path}/#{name}.#{@environment.name}.spec", to_spec
end end
def remove(path) def remove(path)
File.delete "#{path}/#{name}.#{@environment.name}.yaml" File.delete "#{path}/#{name}.#{@environment.name}.spec"
end end
def self.get_by_id(id) def self.get_by_id(id)
@ -316,7 +343,7 @@ class Service
def dependency_tree def dependency_tree
tree = [self] of ServiceTree tree = [self] of ServiceTree
@consumes.each do |token| @consumed_tokens.each do |token|
service = Service.get_by_id token.from service = Service.get_by_id token.from
unless service unless service

View File

@ -3,7 +3,8 @@ require "spec"
class ServiceDefinition class ServiceDefinition
struct Consumes struct Consumes
def initialize(@token : String) getter token : String
def initialize(@token)
@optional = false @optional = false
if @token.match /\?$/ if @token.match /\?$/
@ -11,73 +12,49 @@ class ServiceDefinition
@optional = true @optional = true
end end
end end
YAML.mapping({
token: String,
optional: {
type: Bool,
default: false
}
})
end end
struct Provides struct Provides
def initialize(@token : String) getter token : String
def initialize(@token)
end end
YAML.mapping({
token: String
})
end end
struct Checks struct Checks
getter name : String
getter command : String
getter directory : String?
getter file : String?
def initialize(@name, @command, @file = nil, @directory = nil)
end
def initialize(section : Specs::Section) def initialize(section : Specs::Section)
@name = section.content["name"].as_s @name = section.content["name"].as_s
@file = section.content["file"]?.try &.as_s @file = section.content["file"]?.try &.as_s
@directory = section.content["directory"]?.try &.as_s @directory = section.content["directory"]?.try &.as_s
@command = section.content["command"].as_s @command = section.content["command"].as_s
end end
YAML.mapping({
name: String,
file: String?,
directory: String?,
command: String
})
end end
YAML.mapping({
name: String,
command: String,
stop_command: {
type: String?,
key: "stop-command"
},
directory: String?, # Place to chdir to before running @command.
user: String?, # User that should run the service.
provides: {
type: Array(Provides),
default: [] of Provides
},
consumes: {
type: Array(Consumes),
default: [] of Consumes
},
checks: {
type: Array(Checks),
default: [] of Checks
},
environment_variables: {
type: Array(String),
key: "environment-variables",
default: [] of String
}
})
class_getter all = [] of ServiceDefinition class_getter all = [] of ServiceDefinition
getter name : String
getter command : String
getter stop_command : String?
getter directory : String?
getter user : String?
getter provides : String?
getter consumes : Array(Consumes)
getter environment_variables : Array(String)
getter checks : Array(Checks)
getter provides : Array(Provides)
def initialize(specs : Specs) def initialize(specs : Specs)
sections = specs.sections sections = specs.sections
pp! specs
specs = specs.assignments specs = specs.assignments
@name = specs["name"].as_s @name = specs["name"].as_s
@command = specs["command"].as_s @command = specs["command"].as_s
@stop_command = specs["stop-command"]?.try &.as_s @stop_command = specs["stop-command"]?.try &.as_s
@directory = specs["directory"]?.try &.as_s @directory = specs["directory"]?.try &.as_s
@user = specs["user"]?.try &.as_s
@provides = specs["provides"]?.try &.as_a_or_s.map { |x| Provides.new x } || Array(Provides).new @provides = specs["provides"]?.try &.as_a_or_s.map { |x| Provides.new x } || Array(Provides).new
@consumes = specs["consumes"]?.try &.as_a_or_s.map { |x| Consumes.new x } || Array(Consumes).new @consumes = specs["consumes"]?.try &.as_a_or_s.map { |x| Consumes.new x } || Array(Consumes).new
@checks = sections.select(&.name.== "check").map { |x| Checks.new x } || Array(Checks).new @checks = sections.select(&.name.== "check").map { |x| Checks.new x } || Array(Checks).new
@ -86,9 +63,7 @@ class ServiceDefinition
def self.load(path) def self.load(path)
Dir.each_child path do |child| Dir.each_child path do |child|
if child.match /\.yaml$/ if child.match /\.spec$/
@@all << ServiceDefinition.from_yaml File.read "#{path}/#{child}"
elsif child.match /\.spec$/
@@all << ServiceDefinition.new Specs.parse("#{path}/#{child}").not_nil! @@all << ServiceDefinition.new Specs.parse("#{path}/#{child}").not_nil!
else else
next next