From c8ff92b5080bc25b5489657dd0d531aa918dbcd1 Mon Sep 17 00:00:00 2001 From: Philippe Pittoli Date: Tue, 9 Mar 2021 17:51:51 +0100 Subject: [PATCH] New package-create version. See more in git log. This new version has a new code structure: - context now has (configure|building|install)_backends attributes - Package::Instructions now only is a wrapper around Array(String) with a phase name and a 'run' method - new phases, with total user control through recipes: source-split, (pre-)?(configure|build|install) and post-install - Do.cp Misc: - baguette backend removes the data.tar before processing the manifest - new '-doc' split - check for `bsdtar` in the PATH before doing anything - logs are way more verbose --- src/backends.cr | 18 +++++++ src/backends/baguette.cr | 3 ++ src/backends/splitter.cr | 36 ++++++++------ src/context.cr | 48 +++++++++++------- src/do.cr | 9 ++++ src/instructions.cr | 86 ++++++++------------------------ src/main.cr | 7 ++- src/recipe.cr | 103 ++++++++++++++++++++++++++++----------- 8 files changed, 180 insertions(+), 130 deletions(-) diff --git a/src/backends.cr b/src/backends.cr index a71b71b..b7db1d5 100644 --- a/src/backends.cr +++ b/src/backends.cr @@ -37,4 +37,22 @@ abstract class Package::Backend::Packaging end end +# Package::Backend::Splitter = create new package from a recipe +# takes (then stores) the given block +# this block takes a recipe as a parameter and create a new package +# the new package: +# keep prefixes +# new name = split name (-man, -src, ...) +# split files + +class Package::Backend::Splitter + def initialize(&block : Proc(Recipe, Package)) + @callback = block + end + + def create_split(recipe : Recipe) : Package + @callback.call recipe + end +end + require "./backends/*" \ No newline at end of file diff --git a/src/backends/baguette.cr b/src/backends/baguette.cr index 018d6b1..f9e71a9 100644 --- a/src/backends/baguette.cr +++ b/src/backends/baguette.cr @@ -40,6 +40,9 @@ class Package::Backend::Packaging::Baguette < Package::Backend::Packaging # produces data.tar.zst Do.run fake_root, "zstd", ["--ultra", data_archive_path] + ::Baguette::Log.detail "Removing the uncompressed archive" + Do.run fake_root, "rm", [data_archive_path] + ::Baguette::Log.detail "Generating control.spec" generate_spec package, control_spec_file_path diff --git a/src/backends/splitter.cr b/src/backends/splitter.cr index ba6fad9..e279dae 100644 --- a/src/backends/splitter.cr +++ b/src/backends/splitter.cr @@ -1,20 +1,4 @@ - class Package::Backend::Splitter - def initialize(&block : Proc(Recipe, Package)) - @callback = block - end - - def create_split(recipe : Recipe) : Package - @callback.call recipe - end - - # Package::Backend::Splitter = create new package - # takes (then stores) the given block - # this block takes a recipe as a parameter and create a new package - # the new package: - # keep prefixes - # new name = split name (-man, -src, ...) - # split files # Man-pages and documentation def self.man(prefixes : Array(String)) : Splitter @@ -53,6 +37,26 @@ class Package::Backend::Splitter end end + # Documentation files: /share/doc/, *.html + def self.doc(prefixes : Array(String)) : Splitter + Backend::Splitter.new do |recipe| + Package.new(recipe, true).tap do |split| + prefixes = (prefixes + [recipe.prefix]).uniq + + split.name = "#{recipe.name}-doc" + split.files = prefixes.map do |prefix| + [ + "#{prefix}/share/doc", + ] + end.flatten + split.file_patterns = prefixes.map do |prefix| + Regex.new("^" + prefix + ".*\\.html$") + end + end + end + end + + # NOTE: src is not a regular split, _all_ files should be included, # as well as the patches used for the build. # Files are copied before build. diff --git a/src/context.cr b/src/context.cr index 58cfb35..a38a96f 100644 --- a/src/context.cr +++ b/src/context.cr @@ -10,9 +10,22 @@ class Package::Context property sources_directory = Dir.current property packages_directory = Dir.current - getter packaging_backends = [] of Backend::Packaging - getter building_backends = [] of Backend::Building - getter splitter_backends = [] of Backend::Splitter + #alias Backends = Backend::Packaging | Backend::Building | Backend::Splitter + property user_instructions = Hash(String, Instructions).new + + property configure_backends = Hash(String, Backend::Building).new + property building_backends = Hash(String, Backend::Building).new + property install_backends = Hash(String, Backend::Building).new + + getter packaging_backends = Hash(String, Backend::Packaging).new + getter splitter_backends = Hash(String, Backend::Splitter).new + + getter all_phase_names = [ + "sources-split", + "pre-configure", "configure", + "pre-build", "build", + "pre-install", "install", "post-install", + ] # Directories where ports can be found. getter repositories = [] of String @@ -35,27 +48,28 @@ class Package::Context # Add package backends: baguette (package-tools) and apk. # They implement the Backend::Packaging abstract class: # init (@name) + package (context, package) method - @packaging_backends << Backend::Packaging::Baguette.new - @packaging_backends << Backend::Packaging::APK.new - @packaging_backends << Backend::Packaging::Pkgutils.new + @packaging_backends["baguette"] = Backend::Packaging::Baguette.new + @packaging_backends["apk"] = Backend::Packaging::APK.new + @packaging_backends["pkgutils"] = Backend::Packaging::Pkgutils.new - @selected_packaging_backend = @packaging_backends[0] + @selected_packaging_backend = @packaging_backends["baguette"] # Add building backends: configuration, build and install. - @building_backends << Backend::Configure.autotools - @building_backends << Backend::Configure.cmake - @building_backends << Backend::Build.make - @building_backends << Backend::Install.make + @configure_backends["autotools"] = Backend::Configure.autotools + @configure_backends["cmake"] = Backend::Configure.cmake + @building_backends["make"] = Backend::Build.make + @install_backends["make"] = Backend::Install.make # Split = new package from a recipe, each one with different files being included. - @splitter_backends << Backend::Splitter.man @prefixes - @splitter_backends << Backend::Splitter.dev @prefixes + @splitter_backends["man"] = Backend::Splitter.man @prefixes + @splitter_backends["dev"] = Backend::Splitter.dev @prefixes + @splitter_backends["doc"] = Backend::Splitter.doc @prefixes # src isn't really a split, files already are copied before build. - @splitter_backends << Backend::Splitter.src @prefixes + #@splitter_backends << Backend::Splitter.src @prefixes end def packaging_backend=(name : String) - @selected_packaging_backend = @packaging_backends.find(&.name.==(name)).not_nil! + @selected_packaging_backend = @packaging_backends[name].not_nil! end def packaging_backend=(backend : Backend::Packaging) @@ -130,8 +144,8 @@ class Package::Context STDERR.puts "Error during selecting packaging backend: #{e}" STDERR.puts "#{value.as_s} seems not to be a valid backend" STDERR.puts "valid backends:" - @packaging_backends.each do |backend| - STDERR.puts "- #{backend.name}" + @packaging_backends.each do |name, backend| + STDERR.puts "- #{name}" end exit 1 end diff --git a/src/do.cr b/src/do.cr index d14efad..aca76bc 100644 --- a/src/do.cr +++ b/src/do.cr @@ -20,6 +20,13 @@ class Do < Process Dir.cd chdir end + def self.cp(origin : String, destination : String) + corig = origin.colorize(:light_magenta).to_s + cdest = destination.colorize(:light_orange).to_s + Baguette::Log.detail "cp #{corig} #{cdest}" + FileUtils.cp origin, destination + end + def self.run(chdir, command, args) output = Process::Redirect::Inherit @@ -37,6 +44,8 @@ class Do < Process file.puts "logging command $ #{command}" file.puts " parameters $ #{args}" end + + output = File.open Baguette::Context.logfile_path, "a" end c = Process.run command, args, chdir: chdir, output: output, error: output, env: @@environment diff --git a/src/instructions.cr b/src/instructions.cr index b9912a1..34991fc 100644 --- a/src/instructions.cr +++ b/src/instructions.cr @@ -3,76 +3,30 @@ # Simple array of strings with a name. # # Methods: -# - run(context, recipe) : BuildStatus +# - run(recipe) : BuildStatus # execute each instruction -# instructions are provided by the recipe or by a backend if not specified +# instructions are provided by the recipe -class Package::Instructions - class Set < Array(String) - getter phase : String - def initialize(@phase) - super() - end - # FIXME: def execute - - def run(context : Context, recipe : Recipe) : BuildStatus - # Does the recipe provided instructions for this phase? - if size > 0 - each do |command| - child = Do.run recipe.building_directory, "sh", ["-x", "-c", command] - - if child.exit_status != 0 - return BuildStatus::Failed - end - end - - return BuildStatus::Success - end - - last_backend = "not-known" - # In case the recipe didn't provide instructions: checking backends. - context.building_backends.select(&.phase.==(@phase)).each do |backend| - last_backend = backend.name - Baguette::Log.info "Doing phase '#{@phase}' :: '" + - backend.name.colorize(:light_magenta).to_s + - "'" - Do.cd recipe.building_directory - - rvalue = backend.build context, recipe - - if rvalue == BuildStatus::Pass - next - end - - return rvalue - end - - BuildStatus::Pass - rescue e - # Possible TODO: print the origin of the exception (backend, user-provided code, other/unknown). - Baguette::Log.error "Exception during phase '#{@phase}', backend '#{last_backend}'" - Baguette::Log.error "Exception caught: #{e.message}" - BuildStatus::Failed - end +class Package::Instructions < Array(String) + getter phase : String + def initialize(@phase) + super() end - # FIXME: Switch to an enum at some point? - getter src_split = Set.new "src-split" - getter configure = Set.new "configure" - getter pre_configure = Set.new "pre-configure" - getter build = Set.new "build" - getter pre_build = Set.new "pre-build" - getter install = Set.new "install" - getter pre_install = Set.new "pre-install" - getter post_install = Set.new "post-install" + def run(directory : String) : BuildStatus + if size == 0 + Baguette::Log.error "Empty instructions for phase #{@phase}" + return BuildStatus::Pass + end - def to_a - [ - src_split, - pre_configure, configure, - pre_build, build, - pre_install, install, post_install - ] + each do |command| + child = Do.run directory, "sh", ["-x", "-c", command] + + if child.exit_status != 0 + return BuildStatus::Failed + end + end + + BuildStatus::Success end end - diff --git a/src/main.cr b/src/main.cr index 86497a7..3b52186 100644 --- a/src/main.cr +++ b/src/main.cr @@ -99,7 +99,7 @@ end # Verify package-create dependencies: tar, strip -Do.require_cmd "tar" +Do.require_cmd "bsdtar" Do.require_cmd "strip" if File.exists? CLIOpts.configuration_file @@ -184,7 +184,10 @@ begin recipe.extract - recipe.build + # Perform every building operation: + # sources-split, (pre-)(configure|build|install) and post-install + recipe.run + recipe.package recipe.clean unless CLIOpts.do_not_clean diff --git a/src/recipe.cr b/src/recipe.cr index 60e61fb..1d3f5d0 100644 --- a/src/recipe.cr +++ b/src/recipe.cr @@ -65,7 +65,6 @@ class Package::Recipe # Build instructions, helpers and other build-only data. setter dirname : String? # matching getter defined manually later. - getter instructions = Instructions.new getter options = Hash(String, String).new getter watch_script : String? @@ -122,16 +121,17 @@ class Package::Recipe value.as_a_or_s.each do |source| @sources << source end - when "configure" - @instructions.configure << value.as_s_or_ls - when "build" - @instructions.build << value.as_s_or_ls - when "install" - @instructions.install << value.as_s_or_ls + + # User instructions. + when /(?(pre-)?(configure|build|install)|post-install)/ + phase = $~["phase"] + @context.user_instructions[phase] = Instructions.new(phase).<<(value.as_s_or_ls) + when "dirname" @dirname = value.as_s when "prefix" @prefix = value.as_s + when "dependencies" value.as_a_or_s.each do |atom| @run_dependencies << atom @@ -145,6 +145,7 @@ class Package::Recipe value.as_a_or_s.each do |atom| @run_dependencies << atom end + when "conflicts" value.as_a_or_s.each do |atom| @conflicts << atom @@ -156,8 +157,7 @@ class Package::Recipe when "options" value.as_a_or_s.each do |option| - match = option.split(':').map( - &.gsub(/^[ \t]*/, "").gsub(/[ \t]*$/, "")) + match = option.split(':').map(&.strip) if match.size != 2 STDERR.puts "WARNING: misformed option: #{option}" @@ -221,7 +221,7 @@ class Package::Recipe if url.scheme == "file" Baguette::Log.info "Copying '#{url.filename}'" - FileUtils.cp "#{recipe_directory}/#{url.filename}", filename + Do.cp "#{recipe_directory}/#{url.filename}", filename else Baguette::Log.info "Downloading '#{url.filename}'" @@ -271,17 +271,44 @@ class Package::Recipe @context.sources_directory end - FileUtils.cp "#{directory}/#{url.filename}", + Do.cp "#{directory}/#{url.filename}", "#{building_directory}/#{url.filename}" end end end + def execute_backend(backends : Hash(String, Backend::Building)) : BuildStatus + last_backend = "not-known" + # In case the recipe didn't provide instructions: checking backends. + backends.each do |name, backend| + last_backend = name + cname = name.colorize(:light_green).to_s + Baguette::Log.info "Backend :: '#{cname}'" + Do.cd building_directory + + rvalue = backend.build @context, self + + if rvalue == BuildStatus::Pass + Baguette::Log.info "Pass…" + next + end + + return rvalue + end + + BuildStatus::Pass + + rescue e + Baguette::Log.error "Exception backend '#{last_backend}'" + Baguette::Log.error "Exception caught: #{e.message}" + BuildStatus::Failed + end + # TODO: # - Have some instructions be non-critical, like the (pre|post) ones. # - Be careful about return values, flee from everything if something # goes somehow wrong. - def build + def run Do.mkdir_p fake_root_directory ENV["PKG"] = fake_root_directory @@ -290,16 +317,35 @@ class Package::Recipe old_dir = Dir.current # TODO: skip sequences - instructions.to_a.each do |instruction| - Baguette::Log.info "Building ('#{instruction.phase}' phase)" - # TODO: finish this following lines - #if Package::Context.any? &.phase==instruction.phase - # next - #end + @context.all_phase_names.each do |phase| + cphase = phase.colorize(:light_magenta).to_s + Baguette::Log.info "Building ('#{cphase}')" - if instruction.run(@context, self).failed? - raise BuildError.new self, "Building (#{instruction.phase} phase) failed." - break + ret = if instructions = @context.user_instructions[phase]? + instructions.run building_directory + else + # Some phases have available backends. + case phase + when "configure" + cphase = "configure".colorize(:light_blue).to_s + Baguette::Log.info "Executing phase :: #{cphase}" + execute_backend @context.configure_backends + when "build" + cphase = "build".colorize(:light_blue).to_s + Baguette::Log.info "Executing phase :: #{cphase}" + execute_backend @context.building_backends + when "install" + cphase = "install".colorize(:light_blue).to_s + Baguette::Log.info "Executing phase :: #{cphase}" + execute_backend @context.install_backends + else + # Not available backend and no user instructions. + BuildStatus::Pass + end + end + + if ret.failed? + raise BuildError.new self, "Building ('#{phase}') failed." end end @@ -383,7 +429,9 @@ class Package::Recipe end def auto_splits : Array(Package) - @context.splitter_backends.compact_map do |backend| + @context.splitter_backends.compact_map do |name, backend| + csplit = name.colorize(:light_green).to_s + Baguette::Log.info "Creating split for #{csplit}" backend.create_split self end end @@ -473,13 +521,10 @@ class Package::Recipe spec += "dirname: #{@dirname}\n" end - [ - {"configure", @instructions.configure}, - {"build", @instructions.build}, - {"install", @instructions.install} - ].each do |name, instructions| - if instructions.size > 0 - spec += "@#{name}\n" + @context.all_phase_names.each do |phase| + if instructions = @context.user_instructions[phase] + next unless instructions.size > 0 + spec += "@#{phase}\n" instructions.each do |line| line = line.gsub /^\t* *\n/, "" line = line.gsub /\n *\t*\n/, "\n"