Ports management overhaul.

project.zsh

@@ -9,9 +9,9 @@ variables+=(
 	DATADIR     '/srv'
 )
 
-targets=(service status get-port gen-config)
+targets=(service status gen-config get-port)
 
-for target in service status get-port gen-config; do
+for target in service status gen-config get-port; do
 	type[$target]=crystal
 	sources[$target]=src/${target}.cr
 	depends[$target]=src/config.cr

services/gitea.spec

@@ -1,6 +1,7 @@
 command: gitea -C . -w . -c ./custom/conf/app.ini
 consumes: postgresql
 requires-domain: true
+ports: http
 
 %directory ${SERVICE_ROOT}/custom/conf
 	name: working directory

services/nginx.spec

@@ -1,5 +1,7 @@
 command: nginx -c ${SERVICE_ROOT}/nginx.conf
-provides: www
+consumes: http?
+provides: www, http
+ports: http=80, https=443, test
 
 %directory ${SERVICE_ROOT}/
 	name: working directory

services/postgresql.spec

@@ -5,6 +5,7 @@ command: postgres -D ${SERVICE_ROOT} -k /tmp/postgresql-${ENVIRONMENT}
 environment-variables:
     - PGROOT=${SERVICE_ROOT}
 provides: postgresql
+ports: postgresql
 
 %pre-start
     name: database directory creation

src/gen-config.cr

@@ -8,13 +8,15 @@ def sanitize_path(path)
 end
 
 class Service
+	alias CrinjaHash = Hash(String, Hash(String, Int32) | String | Nil)
 	def to_genconfig
-		Hash(String, String?).new.tap do |entry|
+		CrinjaHash.new.tap do |entry|
 			entry["name"] = name
 			entry["id"] = full_id
 			entry["environment"] = environment.name
 			entry["root"] = root
 			entry["domain"] = domain
+			entry["ports"] = ports
 		end
 	end
 end
@@ -24,10 +26,11 @@ module GenConfig
 	alias Variables =
 		String |
 		Array(String) | Array(Variables) |
-		Hash(String, String) | Hash(String, Variables) |
-		Array(Hash(String, String?)) |
-		Hash(String, Array(Hash(String, String?))) |
-		Hash(String, String?) |
+		Hash(String, String) |
+		Service::CrinjaHash |
+		Array(Service::CrinjaHash) |
+		Hash(String, Array(Service::CrinjaHash)) |
+		Hash(String, Service::CrinjaHash) |
 		Crinja::Callable::Instance
 
 	def self.parse_options(unparsed : Array(String))
@@ -102,8 +105,8 @@ class GenConfig::Context
 
 					next unless provider
 
-					provider.to_genconfig
-				end
+					{token, provider.to_genconfig}
+				end.to_h
 
 				options["consumers"] = service.provides
 					.map(&.token)
@@ -114,10 +117,10 @@ class GenConfig::Context
 			end
 		end
 
-		options["port"] = Crinja.function do
-			service = (arguments.varargs[0]? || "").to_s.gsub /\//, ':'
-			`get-port #{service}`.chomp
-		end
+		#options["port"] = Crinja.function do
+		#	service = (arguments.varargs[0]? || "").to_s.gsub /\//, ':'
+		#	`get-port #{service}`.chomp
+		#end
 
 		# FIXME: Move this to a separate binary?
 		options["random_password"] = Crinja.function do
@@ -141,12 +144,6 @@ class GenConfig::Context
 			end
 		end
 
-		providers = {} of String => String
-		ENV["SERVICE_TOKENS"]?.try &.split(':').each do |token|
-			providers[token] = ENV["#{token.upcase}_PROVIDER"]? || ""
-		end
-		options["providers"] = providers
-
 		template = File.read source
 
 		target_file << Crinja.render(template, options)

src/get-port.cr

@@ -1,63 +1,19 @@
-require "file_utils"
-require "option_parser"
 
 require "./config.cr"
+require "./service/environment.cr"
+require "./service/service_definition.cr"
+require "./service/service.cr"
 
-START_PORT = 49152
-PORTS_CACHE_DIRECTORY = "#{CACHE_DIRECTORY}/ports/"
+Environment.load ENVIRONMENTS_DIRECTORY
+ServiceDefinition.load SERVICES_DIRECTORY
+Service.load RC_DIRECTORY
 
-service = ""
-wanted_default_port : String? = nil
-
-parser = OptionParser.parse do |parser|
-	parser.banner = "usage: get-post <id> [default-port] [options]\n" +
-		"options:\n"
-
-	parser.on "-h", "--help", "Prints this help message." do
-		puts parser
-		exit 0
-	end
-
-	parser.unknown_args do |arg|
-		if arg.size < 1 || arg.size > 2
-			puts parser
-			exit 1
-		end
-
-		service = arg[0]
-		wanted_default_port = arg[1]?
-	end
+if ARGV.size != 2
+	STDERR.puts "usage: get-port <service-id> <port-id>"
+	exit 1
 end
 
-service_port_file = "#{PORTS_CACHE_DIRECTORY}/#{service.gsub /\//, ":"}"
-
-begin
-	if File.exists? service_port_file
-		puts File.read service_port_file
-		exit 0
-	end
+service_id, port_id = ARGV
 
-	FileUtils.mkdir_p PORTS_CACHE_DIRECTORY
-
-	used_ports = Dir.children(PORTS_CACHE_DIRECTORY)
-		.map { |x| "#{PORTS_CACHE_DIRECTORY}/#{x}" }
-		.map { |x| File.read(x).to_i }
-		.sort
-
-	port = START_PORT
-	if wanted_default_port && ! used_ports.any? &.==(wanted_default_port)
-		port = wanted_default_port
-	else
-		while used_ports.any? &.==(port)
-			port = port + 1
-		end
-	end
-
-	File.write service_port_file, port
-
-	puts port
-rescue e
-	STDERR.puts "error: #{e.message}"
-	exit 1
-end
+puts Service.get_by_id(service_id).not_nil!.ports[port_id]
 

src/service.cr

@@ -45,6 +45,12 @@ commands = CommandsList.new
 commands.push "add", "Adds a service to an environment." do |args|
 	providers = Hash(String, String).new
 	domain = nil
+	ports = Hash(String, Int32).new
+
+	if Service.get_by_id args[0]
+		raise ::Service::Exception.new "'#{args[0]}' already exists. You have to remove it before re-adding it."
+		next
+	end
 
 	environment, service = Service.parse_id args[0]
 
@@ -62,11 +68,14 @@ commands.push "add", "Adds a service to an environment." do |args|
 
 		if key == "domain"
 			domain = value
+		elsif match = key.match /^port:(.*)/
+			ports[match[1]] = value.to_i
 		else
 			providers[key] = value
 		end
 	end
 
+	# FIXME: Some of that code is ugly and should not even be here.
 	Service.new(service, environment).tap do |service|
 		if domain
 			service.domain = domain
@@ -90,6 +99,32 @@ commands.push "add", "Adds a service to an environment." do |args|
 			service.providers[token.token] = provider
 		end
 
+		service.port_definitions.each do |definition|
+			name = definition.name
+			number = ports[name]?
+
+			if number.nil?
+				default_port = definition.default_value
+
+				if default_port && ! Service.is_port_used default_port, ports.map { |k, v| v }
+					number = default_port
+					ports[name] = number
+				end
+			elsif Service.is_port_used number
+				raise Service::Exception.new "Port #{number} is already reserved by another service."
+				next
+			end
+
+			if number.nil?
+					number = Service.get_free_port ports.map { |k, v| v }
+					ports[name] = number
+			end
+
+			# FIXME: Check the port is not duplicated here.
+
+			service.ports[definition.name] = number
+		end
+
 		if service.requires_domain && !service.domain
 			raise Service::Exception.new "'#{service.name}' requires a domain= parameter to be provided!"
 		end

src/service/service.cr

@@ -20,6 +20,7 @@ class Service
 	getter providers = ProvidersList.new
 
 	property domain : String?
+	getter ports = Hash(String, Int32).new
 
 	# The place we’ll store configuration and data.
 	@root : String?
@@ -65,6 +66,21 @@ class Service
 
 		@root = assignments["root"]?.try &.as_s
 
+		assignments["ports"]?.try &.as_a_or_s.each do |port_string|
+			match = port_string.match /(.*)=(.*)/
+
+			if match.nil?
+				STDERR.puts "warning: '#{id}' has invalid port strings"
+				next
+			end
+
+			_, name, number = match
+
+			number = number.to_i
+
+			@ports[name] = number
+		end
+
 		specs.sections.select(&.name.==("consumes")).each do |section|
 			env, provider = Service.parse_id section.content["from"].as_s
 			@providers[section.options[0]] = "#{env}/#{provider}"
@@ -77,6 +93,14 @@ class Service
 			"environment: #{@environment.name}"
 		]
 
+		if @ports.size > 0
+			ports_list = @ports.map do |name, number|
+				"#{name}=#{number}"
+			end.join ", "
+
+			file << "ports: #{ports_list}"
+		end
+
 		if @root
 			file << "root: #{@root}"
 		end
@@ -127,6 +151,9 @@ class Service
 	def requires_domain
 		@reference.requires_domain
 	end
+	def port_definitions
+		@reference.port_definitions
+	end
 
 	def root
 		@root || "#{@environment.root}/#{name}"
@@ -402,6 +429,31 @@ class Service
 		{environment, service}
 	end
 
+	def self.get_used_ports(other_reservations = Array(Int32).new)
+		Service.all.map(&.ports.to_a)
+			.flatten
+			.map { |k, v| v }
+			.+(other_reservations)
+	end
+
+	def self.get_free_port(other_reservations = Array(Int32).new)
+		port = 49152
+
+		used_ports = Service.get_used_ports other_reservations
+
+		while used_ports.any? &.==(port)
+			port = port + 1
+		end
+
+		port
+	end
+
+	def self.is_port_used(port, other_reservations = Array(Int32).new)
+		used_ports = Service.get_used_ports other_reservations
+
+		used_ports.any? &.==(port)
+	end
+
 	alias ServiceTree = Array(ServiceTree) | Service
 
 	# Returns a dependency tree.

src/service/service_definition.cr

@@ -37,6 +37,18 @@ class ServiceDefinition
 		end
 	end
 
+	struct PortDefinition
+		getter name : String
+		getter default_value : Int32?
+
+		def initialize(string : String)
+			match = string.match(/([^?=]*)(=([^?]*))?/).not_nil!
+
+			@name = match[1]
+			@default_value = match[3]?.try &.to_i
+		end
+	end
+
 	class_getter all = [] of ServiceDefinition
 
 	getter name : String
@@ -49,6 +61,7 @@ class ServiceDefinition
 	getter environment_variables : Array(String)
 	getter pre_start_hooks : Array(Hook)
 	getter provides : Array(Provides)
+	getter port_definitions : Array(PortDefinition)
 
 	getter requires_domain = false
 
@@ -63,6 +76,8 @@ class ServiceDefinition
 		@consumes      = specs["consumes"]?.try &.as_a_or_s.map { |x| Consumes.new x } || Array(Consumes).new
 		@environment_variables = specs["environment-variables"]?.try &.as_a_or_s || Array(String).new
 
+		@port_definitions = specs["ports"]?.try &.as_a_or_s.map { |x| PortDefinition.new x } || Array(PortDefinition).new
+
 		# FIXME: as_b?
 		requires_domain = specs["requires-domain"]?.try &.as_s
 		case requires_domain

templates/gitea.cfg.j2

@@ -15,7 +15,7 @@ SECRET_KEY     = vPFgSqRMIe7Dzk4frRM4UA3CETedL8agK7x6IQFQt9YfRPiQGhQbYAGfyan71iU
 
 [database]
 DB_TYPE  = postgres
-HOST     = 127.0.0.1:{{ port(providers.postgresql) }}
+HOST     = 127.0.0.1:{{ providers.postgresql.ports.postgresql }}
 NAME     = {{ service.id | replace("/", "_") }}_db
 USER     = {{ service.id | replace("/", "_") }}
 PASSWD   = {{ random_password( service.id ) }}
@@ -29,8 +29,8 @@ ROOT = {{ service.root }}/repositories
 [server]
 SSH_DOMAIN       = {{ service.domain }}
 DOMAIN           = {{ service.domain }}
-HTTP_PORT        = {{ port(service.id) }}
-ROOT_URL         = http://{{ service.domain }}:{{ port(service.id) }}/
+HTTP_PORT        = {{ service.ports.http }}
+ROOT_URL         = http://{{ service.domain }}:{{ service.ports.http }}/
 DISABLE_SSH      = false
 SSH_PORT         = 22
 LFS_START_SERVER = true

templates/postgresql.conf.j2

@@ -60,7 +60,7 @@
 					# comma-separated list of addresses;
 					# defaults to 'localhost'; use '*' for all
 					# (change requires restart)
-port = {{port(service.id)}}
+port = {{service.ports.postgresql}}
 max_connections = 100			# (change requires restart)
 #superuser_reserved_connections = 3	# (change requires restart)
 #unix_socket_directories = '/run/postgresql'	# comma-separated list of directories

utils/pg_create_user.sh

@@ -4,7 +4,7 @@
 : ${PGUSER:=postgres}
 
 # TODO: add default postgresql port
-: ${pgport:=$(get-port $POSTGRESQL_PROVIDER)}
+: ${pgport:=$(get-port $POSTGRESQL_PROVIDER postgresql)}
 
 : ${PGDATA:=$POSTGRESQL_ROOT}
 : ${dbuser:=${SERVICE_ID//\//_}}