Ports management overhaul.

master
Luka Vandervelden 2019-10-29 12:32:49 +01:00
parent 5c6198a023
commit 6ee52f5baa
12 changed files with 140 additions and 81 deletions

View File

@ -9,9 +9,9 @@ variables+=(
DATADIR '/srv'
)
targets=(service status get-port gen-config)
targets=(service status gen-config get-port)
for target in service status get-port gen-config; do
for target in service status gen-config get-port; do
type[$target]=crystal
sources[$target]=src/${target}.cr
depends[$target]=src/config.cr

View File

@ -1,6 +1,7 @@
command: gitea -C . -w . -c ./custom/conf/app.ini
consumes: postgresql
requires-domain: true
ports: http
%directory ${SERVICE_ROOT}/custom/conf
name: working directory

View File

@ -1,5 +1,7 @@
command: nginx -c ${SERVICE_ROOT}/nginx.conf
provides: www
consumes: http?
provides: www, http
ports: http=80, https=443, test
%directory ${SERVICE_ROOT}/
name: working directory

View File

@ -5,6 +5,7 @@ command: postgres -D ${SERVICE_ROOT} -k /tmp/postgresql-${ENVIRONMENT}
environment-variables:
- PGROOT=${SERVICE_ROOT}
provides: postgresql
ports: postgresql
%pre-start
name: database directory creation

View File

@ -8,13 +8,15 @@ def sanitize_path(path)
end
class Service
alias CrinjaHash = Hash(String, Hash(String, Int32) | String | Nil)
def to_genconfig
Hash(String, String?).new.tap do |entry|
CrinjaHash.new.tap do |entry|
entry["name"] = name
entry["id"] = full_id
entry["environment"] = environment.name
entry["root"] = root
entry["domain"] = domain
entry["ports"] = ports
end
end
end
@ -24,10 +26,11 @@ module GenConfig
alias Variables =
String |
Array(String) | Array(Variables) |
Hash(String, String) | Hash(String, Variables) |
Array(Hash(String, String?)) |
Hash(String, Array(Hash(String, String?))) |
Hash(String, String?) |
Hash(String, String) |
Service::CrinjaHash |
Array(Service::CrinjaHash) |
Hash(String, Array(Service::CrinjaHash)) |
Hash(String, Service::CrinjaHash) |
Crinja::Callable::Instance
def self.parse_options(unparsed : Array(String))
@ -102,8 +105,8 @@ class GenConfig::Context
next unless provider
provider.to_genconfig
end
{token, provider.to_genconfig}
end.to_h
options["consumers"] = service.provides
.map(&.token)
@ -114,10 +117,10 @@ class GenConfig::Context
end
end
options["port"] = Crinja.function do
service = (arguments.varargs[0]? || "").to_s.gsub /\//, ':'
`get-port #{service}`.chomp
end
#options["port"] = Crinja.function do
# service = (arguments.varargs[0]? || "").to_s.gsub /\//, ':'
# `get-port #{service}`.chomp
#end
# FIXME: Move this to a separate binary?
options["random_password"] = Crinja.function do
@ -141,12 +144,6 @@ class GenConfig::Context
end
end
providers = {} of String => String
ENV["SERVICE_TOKENS"]?.try &.split(':').each do |token|
providers[token] = ENV["#{token.upcase}_PROVIDER"]? || ""
end
options["providers"] = providers
template = File.read source
target_file << Crinja.render(template, options)

View File

@ -1,63 +1,19 @@
require "file_utils"
require "option_parser"
require "./config.cr"
require "./service/environment.cr"
require "./service/service_definition.cr"
require "./service/service.cr"
START_PORT = 49152
PORTS_CACHE_DIRECTORY = "#{CACHE_DIRECTORY}/ports/"
Environment.load ENVIRONMENTS_DIRECTORY
ServiceDefinition.load SERVICES_DIRECTORY
Service.load RC_DIRECTORY
service = ""
wanted_default_port : String? = nil
parser = OptionParser.parse do |parser|
parser.banner = "usage: get-post <id> [default-port] [options]\n" +
"options:\n"
parser.on "-h", "--help", "Prints this help message." do
puts parser
exit 0
end
parser.unknown_args do |arg|
if arg.size < 1 || arg.size > 2
puts parser
exit 1
end
service = arg[0]
wanted_default_port = arg[1]?
end
end
service_port_file = "#{PORTS_CACHE_DIRECTORY}/#{service.gsub /\//, ":"}"
begin
if File.exists? service_port_file
puts File.read service_port_file
exit 0
end
FileUtils.mkdir_p PORTS_CACHE_DIRECTORY
used_ports = Dir.children(PORTS_CACHE_DIRECTORY)
.map { |x| "#{PORTS_CACHE_DIRECTORY}/#{x}" }
.map { |x| File.read(x).to_i }
.sort
port = START_PORT
if wanted_default_port && ! used_ports.any? &.==(wanted_default_port)
port = wanted_default_port
else
while used_ports.any? &.==(port)
port = port + 1
end
end
File.write service_port_file, port
puts port
rescue e
STDERR.puts "error: #{e.message}"
if ARGV.size != 2
STDERR.puts "usage: get-port <service-id> <port-id>"
exit 1
end
service_id, port_id = ARGV
puts Service.get_by_id(service_id).not_nil!.ports[port_id]

View File

@ -45,6 +45,12 @@ commands = CommandsList.new
commands.push "add", "Adds a service to an environment." do |args|
providers = Hash(String, String).new
domain = nil
ports = Hash(String, Int32).new
if Service.get_by_id args[0]
raise ::Service::Exception.new "'#{args[0]}' already exists. You have to remove it before re-adding it."
next
end
environment, service = Service.parse_id args[0]
@ -62,11 +68,14 @@ commands.push "add", "Adds a service to an environment." do |args|
if key == "domain"
domain = value
elsif match = key.match /^port:(.*)/
ports[match[1]] = value.to_i
else
providers[key] = value
end
end
# FIXME: Some of that code is ugly and should not even be here.
Service.new(service, environment).tap do |service|
if domain
service.domain = domain
@ -90,6 +99,32 @@ commands.push "add", "Adds a service to an environment." do |args|
service.providers[token.token] = provider
end
service.port_definitions.each do |definition|
name = definition.name
number = ports[name]?
if number.nil?
default_port = definition.default_value
if default_port && ! Service.is_port_used default_port, ports.map { |k, v| v }
number = default_port
ports[name] = number
end
elsif Service.is_port_used number
raise Service::Exception.new "Port #{number} is already reserved by another service."
next
end
if number.nil?
number = Service.get_free_port ports.map { |k, v| v }
ports[name] = number
end
# FIXME: Check the port is not duplicated here.
service.ports[definition.name] = number
end
if service.requires_domain && !service.domain
raise Service::Exception.new "'#{service.name}' requires a domain= parameter to be provided!"
end

View File

@ -20,6 +20,7 @@ class Service
getter providers = ProvidersList.new
property domain : String?
getter ports = Hash(String, Int32).new
# The place well store configuration and data.
@root : String?
@ -65,6 +66,21 @@ class Service
@root = assignments["root"]?.try &.as_s
assignments["ports"]?.try &.as_a_or_s.each do |port_string|
match = port_string.match /(.*)=(.*)/
if match.nil?
STDERR.puts "warning: '#{id}' has invalid port strings"
next
end
_, name, number = match
number = number.to_i
@ports[name] = number
end
specs.sections.select(&.name.==("consumes")).each do |section|
env, provider = Service.parse_id section.content["from"].as_s
@providers[section.options[0]] = "#{env}/#{provider}"
@ -77,6 +93,14 @@ class Service
"environment: #{@environment.name}"
]
if @ports.size > 0
ports_list = @ports.map do |name, number|
"#{name}=#{number}"
end.join ", "
file << "ports: #{ports_list}"
end
if @root
file << "root: #{@root}"
end
@ -127,6 +151,9 @@ class Service
def requires_domain
@reference.requires_domain
end
def port_definitions
@reference.port_definitions
end
def root
@root || "#{@environment.root}/#{name}"
@ -402,6 +429,31 @@ class Service
{environment, service}
end
def self.get_used_ports(other_reservations = Array(Int32).new)
Service.all.map(&.ports.to_a)
.flatten
.map { |k, v| v }
.+(other_reservations)
end
def self.get_free_port(other_reservations = Array(Int32).new)
port = 49152
used_ports = Service.get_used_ports other_reservations
while used_ports.any? &.==(port)
port = port + 1
end
port
end
def self.is_port_used(port, other_reservations = Array(Int32).new)
used_ports = Service.get_used_ports other_reservations
used_ports.any? &.==(port)
end
alias ServiceTree = Array(ServiceTree) | Service
# Returns a dependency tree.

View File

@ -37,6 +37,18 @@ class ServiceDefinition
end
end
struct PortDefinition
getter name : String
getter default_value : Int32?
def initialize(string : String)
match = string.match(/([^?=]*)(=([^?]*))?/).not_nil!
@name = match[1]
@default_value = match[3]?.try &.to_i
end
end
class_getter all = [] of ServiceDefinition
getter name : String
@ -49,6 +61,7 @@ class ServiceDefinition
getter environment_variables : Array(String)
getter pre_start_hooks : Array(Hook)
getter provides : Array(Provides)
getter port_definitions : Array(PortDefinition)
getter requires_domain = false
@ -63,6 +76,8 @@ class ServiceDefinition
@consumes = specs["consumes"]?.try &.as_a_or_s.map { |x| Consumes.new x } || Array(Consumes).new
@environment_variables = specs["environment-variables"]?.try &.as_a_or_s || Array(String).new
@port_definitions = specs["ports"]?.try &.as_a_or_s.map { |x| PortDefinition.new x } || Array(PortDefinition).new
# FIXME: as_b?
requires_domain = specs["requires-domain"]?.try &.as_s
case requires_domain

View File

@ -15,7 +15,7 @@ SECRET_KEY = vPFgSqRMIe7Dzk4frRM4UA3CETedL8agK7x6IQFQt9YfRPiQGhQbYAGfyan71iU
[database]
DB_TYPE = postgres
HOST = 127.0.0.1:{{ port(providers.postgresql) }}
HOST = 127.0.0.1:{{ providers.postgresql.ports.postgresql }}
NAME = {{ service.id | replace("/", "_") }}_db
USER = {{ service.id | replace("/", "_") }}
PASSWD = {{ random_password( service.id ) }}
@ -29,8 +29,8 @@ ROOT = {{ service.root }}/repositories
[server]
SSH_DOMAIN = {{ service.domain }}
DOMAIN = {{ service.domain }}
HTTP_PORT = {{ port(service.id) }}
ROOT_URL = http://{{ service.domain }}:{{ port(service.id) }}/
HTTP_PORT = {{ service.ports.http }}
ROOT_URL = http://{{ service.domain }}:{{ service.ports.http }}/
DISABLE_SSH = false
SSH_PORT = 22
LFS_START_SERVER = true

View File

@ -60,7 +60,7 @@
# comma-separated list of addresses;
# defaults to 'localhost'; use '*' for all
# (change requires restart)
port = {{port(service.id)}}
port = {{service.ports.postgresql}}
max_connections = 100 # (change requires restart)
#superuser_reserved_connections = 3 # (change requires restart)
#unix_socket_directories = '/run/postgresql' # comma-separated list of directories

View File

@ -4,7 +4,7 @@
: ${PGUSER:=postgres}
# TODO: add default postgresql port
: ${pgport:=$(get-port $POSTGRESQL_PROVIDER)}
: ${pgport:=$(get-port $POSTGRESQL_PROVIDER postgresql)}
: ${PGDATA:=$POSTGRESQL_ROOT}
: ${dbuser:=${SERVICE_ID//\//_}}