From 3a9deb234038a1d401a5aa136f1b7981564cfaf8 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Fri, 22 Oct 2021 06:53:48 +0200 Subject: [PATCH] WIP: refactor packages.py to use Chroot --- chroot.py | 87 ++++++++++++++++++---------- packages.py | 161 +++++++++++++++++++++++----------------------------- 2 files changed, 127 insertions(+), 121 deletions(-) diff --git a/chroot.py b/chroot.py index 3316a2f..19bfee2 100644 --- a/chroot.py +++ b/chroot.py @@ -50,6 +50,11 @@ BASIC_MOUNTS = { 'type': 'tmpfs', 'options': ['bind'], }, + '/etc/resolv.conf': { + 'src': '/etc/resolv.conf', + 'type': None, + 'options': ['bind'], + }, } Chroot = None @@ -93,22 +98,21 @@ def get_chroot( def get_base_chroot(arch: Arch, **kwargs) -> Chroot: name = base_chroot_name(arch) default = Chroot(name, arch, initialize=False, copy_base=False) - return get_chroot(**kwargs, default=default) + if kwargs.pop('initialize', False): + logging.debug('get_base_chroot: Had to remove "initialize" from args. This indicates a bug.') + return get_chroot(name, **kwargs, initialize=False, default=default) def get_build_chroot(arch: Arch, extra_repos=None, **kwargs) -> Chroot: - name = base_chroot_name(arch) + name = build_chroot_name(arch) extra_repos = get_kupfer_local(arch).repos if extra_repos is None else extra_repos - args = {'extra_repos': extra_repos} - if kwargs: - args |= kwargs - default = Chroot(name, arch, initialize=False) - return get_chroot(**args, default=default) + default = Chroot(name, arch, initialize=False, extra_repos=extra_repos) + return get_chroot(name, **kwargs, default=default) def get_device_chroot(name: str, arch: Arch, **kwargs) -> Chroot: default = Chroot(name, arch, initialize=False) - return get_chroot(**kwargs, default=default) + return get_chroot(name, **kwargs, default=default) class Chroot: @@ -118,7 +122,7 @@ class Chroot: arch: Arch initialized: bool = False active: bool = False - active_mounts = [] + active_mounts: list[str] = [] copy_base: bool = True extra_repos: dict[str, RepoInfo] = {} base_packages: list[str] = ['base'] @@ -134,12 +138,14 @@ class Chroot: initialize: bool = False, extra_repos: dict[str, RepoInfo] = {}, base_packages: list[str] = ['base', 'base-devel', 'git'], + path_override: str = None, ): if copy_base is None: + logging.debug(f'{name}: copy_base is none!') copy_base = (name == base_chroot_name(arch)) self.name = name self.arch = arch - self.path = os.path.join(config.get_path('chroots'), name) + self.path = os.path.join(config.get_path('chroots'), name) if not path_override else path_override self.copy_base = copy_base self.extra_repos |= extra_repos if initialize: @@ -149,7 +155,8 @@ class Chroot: def get_path(self, *joins) -> str: if joins: - joins[0] = joins[0].lstrip('/') + joins = (joins[0].lstrip('/'),) + joins[1:] + return os.path.join(self.path, *joins) def initialize( @@ -165,8 +172,12 @@ class Chroot: raise Exception(f"Chroot {self.name} is already initialized, this seems like a bug") return + self.deactivate_core() + if self.copy_base: base_chroot = get_base_chroot(self.arch, initialize=True) + if base_chroot == self: + raise Exception('base_chroot == self, bailing out. this is a bug') logging.info(f'Copying {base_chroot.name} chroot to {self.name}') result = subprocess.run([ 'rsync', @@ -234,17 +245,17 @@ class Chroot: """returns the absolute path `relative_target` was mounted at""" relative_destination = relative_destination.lstrip('/') absolute_destination = self.get_path(relative_destination) - if relative_destination in self.active_mounts or os.path.ismount(absolute_destination): + if os.path.ismount(absolute_destination): if fail_if_mounted: - raise Exception(f'{self.name}: {relative_destination} is already mounted') - logging.warning(f'{self.name}: {relative_destination} already mounted. Skipping.') + raise Exception(f'{self.name}: {absolute_destination} is already mounted') + logging.warning(f'{self.name}: {absolute_destination} already mounted. Skipping.') else: if makedir and os.path.isdir(absolute_source): os.makedirs(absolute_destination, exist_ok=True) result = mount(absolute_source, absolute_destination, options=options, fs_type=fs_type, register_unmount=False) if result.returncode != 0: raise Exception(f'{self.name}: failed to mount {absolute_source} to {relative_destination}') - self.active_mounts += relative_destination + self.active_mounts += [relative_destination] atexit.register(self.deactivate) return absolute_destination @@ -253,7 +264,7 @@ class Chroot: return path = self.get_path(relative_path) result = umount(path) - if result.returncode == 0: + if result.returncode == 0 and relative_path in self.active_mounts: self.active_mounts.remove(relative_path) return result @@ -263,12 +274,16 @@ class Chroot: if fail_if_active: raise Exception(f'chroot {self.name} already active!') return - if not self.initialised: - self.init(fail_if_active=False) + if not self.initialized: + self.initialize(fail_if_initialized=False) for dst, opts in BASIC_MOUNTS.items(): - self.mount(opts['src'], dst, fs_type=opts['type'], options=opts['options']) + self.mount(opts['src'], dst, fs_type=opts['type'], options=opts['options'], fail_if_mounted=fail_if_active) self.active = True + def deactivate_core(self): + for dst in BASIC_MOUNTS.keys(): + self.umount(dst) + def deactivate(self, fail_if_inactive: bool = False): if not self.active: if fail_if_inactive: @@ -288,17 +303,28 @@ class Chroot: script: str, inner_env: dict[str, str] = {}, outer_env: dict[str, str] = os.environ.copy() | {'QEMU_LD_PREFIX': '/usr/aarch64-linux-gnu'}, - attach_tty=False) -> subprocess.CompletedProcess: + attach_tty: str = False, + capture_output: str = False) -> subprocess.CompletedProcess: self.activate() if outer_env is None: outer_env = os.environ.copy() env_cmd = ['/usr/bin/env'] + [f'{shell_quote(key)}={shell_quote(value)}' for key, value in inner_env.items()] run_func = subprocess.call if attach_tty else subprocess.run - result = run_func(['chroot', self.path] + env_cmd + [ + kwargs = { + 'env': outer_env, + } + if not attach_tty: + kwargs |= {'capture_output': capture_output} + + if not isinstance(script, str) and isinstance(script, list): + script = ' '.join(script) + cmd = ['chroot', self.path] + env_cmd + [ '/bin/bash', '-c', script, - ], env=outer_env) + ] + logging.debug(f'{self.name}: Running cmd: "{cmd}"') + result = run_func(cmd, **kwargs) return result def create_user( @@ -325,11 +351,12 @@ class Chroot: def try_install_packages(self, packages: list[str], refresh: bool = False, allow_fail: bool = True) -> dict[str, subprocess.CompletedProcess]: """Try installing packages, fall back to installing one by one""" + results = {} if refresh: - self.run_cmd('pacman -Syy --noconfirm') + results['refresh'] = self.run_cmd('pacman -Syy --noconfirm') cmd = 'pacman -S --noconfirm --needed' result = self.run_cmd(f'{cmd} {" ".join(packages)}') - results = {package: result for package in packages} + results |= {package: result for package in packages} if result.returncode != 0 and allow_fail: results = {} logging.debug('Falling back to serial installation') @@ -386,16 +413,16 @@ class Chroot: raise Exception(f'Failed to mount native chroot {native_chroot.name} to {native_mount}') return native_mount - def mount_pkgbuilds(self) -> str: + def mount_pkgbuilds(self, fail_if_mounted: bool = False) -> str: packages = config.get_path('pkgbuilds') - return self.mount(absolute_source=packages, relative_destination=packages.lstrip('/')) + return self.mount(absolute_source=packages, relative_destination=packages.lstrip('/'), fail_if_mounted=fail_if_mounted) - def mount_pacman_cache(self) -> str: - return self.mount(config.get_path('pacman'), '/var/cache/pacman') + def mount_pacman_cache(self, fail_if_mounted: bool = False) -> str: + return self.mount(os.path.join(config.get_path('pacman'), self.arch), 'var/cache/pacman', fail_if_mounted=fail_if_mounted) - def mount_packages(self) -> str: + def mount_packages(self, fail_if_mounted: bool = False) -> str: packages = config.get_path('packages') - return self.mount(absolute_source=packages, relative_destination=packages.lstrip('/')) + return self.mount(absolute_source=packages, relative_destination=packages.lstrip('/'), fail_if_mounted=fail_if_mounted) def mount_crosscompile(self, foreign_chroot: Chroot): mount_dest = os.path.join('chroot', os.path.basename(foreign_chroot.path)) diff --git a/packages.py b/packages.py index 27d4f85..683d035 100644 --- a/packages.py +++ b/packages.py @@ -7,13 +7,13 @@ import subprocess from copy import deepcopy from joblib import Parallel, delayed -from constants import REPOSITORIES, CROSSDIRECT_PKGS, GCC_HOSTSPECS, ARCHES +from constants import REPOSITORIES, CROSSDIRECT_PKGS, GCC_HOSTSPECS, ARCHES, Arch from config import config -from chroot import create_chroot, run_chroot_cmd, try_install_packages, mount_crossdirect, write_cross_makepkg_conf, mount_packages, mount_pacman_cache +from chroot import get_build_chroot, Chroot from distro import get_kupfer_local from ssh import run_ssh_command, scp_put_files from wrapper import enforce_wrap -from utils import mount, umount, git +from utils import git from binfmt import register as binfmt_register makepkg_env = os.environ.copy() | { @@ -26,8 +26,6 @@ makepkg_cross_env = makepkg_env | {'PACMAN': os.path.join(config.runtime['script makepkg_cmd = [ 'makepkg', - '--config', - os.path.join(config.runtime['script_source_dir'], 'local/etc/makepkg.conf'), '--noconfirm', '--ignorearch', '--needed', @@ -131,7 +129,7 @@ def init_pkgbuilds(interactive=False): clone_pkbuilds(pkgbuilds_dir, repo_url, branch, interactive=interactive) -def init_prebuilts(arch: str, dir: str = None): +def init_prebuilts(arch: Arch, dir: str = None): """Ensure that all `constants.REPOSITORIES` inside `dir` exist""" prebuilts_dir = dir if dir else config.get_package_dir(arch) os.makedirs(prebuilts_dir, exist_ok=True) @@ -305,9 +303,9 @@ def generate_dependency_chain(package_repo: dict[str, Package], to_build: list[P return list([lvl for lvl in dep_levels[::-1] if lvl]) -def add_file_to_repo(file_path: str, repo_name: str, arch: str): +def add_file_to_repo(file_path: str, repo_name: str, arch: Arch): repo_dir = os.path.join(config.get_package_dir(arch), repo_name) - pacman_cache_dir = os.path.join(config.get_path('pacman'), arch) + pacman_cache_dir = os.path.join(config.get_path('pacman')) file_name = os.path.basename(file_path) target_file = os.path.join(repo_dir, file_name) @@ -346,7 +344,7 @@ def add_file_to_repo(file_path: str, repo_name: str, arch: str): os.unlink(old) -def add_package_to_repo(package: Package, arch: str): +def add_package_to_repo(package: Package, arch: Arch): logging.info(f'Adding {package.path} to repo {package.repo}') pkgbuild_dir = os.path.join(config.get_path('pkgbuilds'), package.path) @@ -360,20 +358,25 @@ def add_package_to_repo(package: Package, arch: str): return files -def check_package_version_built(package: Package, arch) -> bool: +def check_package_version_built(package: Package, arch: Arch) -> bool: + #chroot = Chroot(name='rootfs', arch=config.runtime['arch'], copy_base=False, initialize=False) + native_chroot = setup_build_chroot(config.runtime['arch']) + #config_path = os.path.join(native_chroot.path, + config_path = '/' + native_chroot.write_makepkg_conf( + target_arch=arch, + cross_chroot_relative=os.path.join('chroot', arch), + cross=True, + ) - config_path = '/' + write_cross_makepkg_conf(native_chroot='/', arch=arch, target_chroot_relative=None, cross=False) - - result = subprocess.run( - makepkg_cmd + [ - '--config', - config_path, - '--nobuild', - '--noprepare', - '--packagelist', - ], - env=makepkg_cross_env, - cwd=package.path, + cmd = ['cd', package.path, '&&'] + makepkg_cmd + [ + '--config', + config_path, + '--nobuild', + '--noprepare', + '--packagelist', + ] + result = native_chroot.run_cmd( + cmd, capture_output=True, ) if result.returncode != 0: @@ -392,49 +395,38 @@ def check_package_version_built(package: Package, arch) -> bool: return not missing -def setup_build_chroot(arch: str, extra_packages=[]) -> str: - chroot_name = f'build_{arch}' - logging.info(f'Initializing {arch} build chroot') - chroot_path = create_chroot( - chroot_name, - arch=arch, - packages=list(set(['base', 'base-devel', 'git'] + extra_packages)), +def setup_build_chroot(arch: Arch, extra_packages: list[str] = [], clean_chroot: bool = False) -> Chroot: + chroot = get_build_chroot( + arch, extra_repos=get_kupfer_local(arch).repos, ) - pacman_cache = mount_pacman_cache(chroot_path, arch) - - logging.info(f'Updating chroot {chroot_name}') - result = subprocess.run( - pacman_cmd + [ - '--root', - chroot_path, - '--arch', - arch, - '--config', - chroot_path + '/etc/pacman.conf', - ], - capture_output=True, - ) - if result.returncode != 0: - logging.fatal(result.stdout) - logging.fatal(result.stderr) - raise Exception(f'Failed to update chroot {chroot_name}') - umount(pacman_cache) - return chroot_path + logging.info(f'Initializing {arch} build chroot') + if clean_chroot: + chroot.reset() + chroot.initialize() + chroot.activate() + chroot.mount_pacman_cache() + chroot.mount_pkgbuilds() + chroot.mount_packages() + if extra_packages: + chroot.try_install_packages(extra_packages, allow_fail=False) + return chroot -def setup_sources(package: Package, chroot: str, arch: str, pkgbuilds_dir: str = None): +def setup_sources(package: Package, chroot: Chroot, pkgbuilds_dir: str = None): pkgbuilds_dir = pkgbuilds_dir if pkgbuilds_dir else config.get_path('pkgbuilds') makepkg_setup_args = [ + '--config', + os.path.join(chroot.path, 'etc/makepkg.conf'), '--nobuild', '--holdver', '--nodeps', ] - logging.info(f'Setting up sources for {package.path} in {chroot}') + logging.info(f'Setting up sources for {package.path} in {chroot.name}') result = subprocess.run( - [os.path.join(chroot, 'usr/bin/makepkg')] + makepkg_cmd[1:] + makepkg_setup_args, - env=makepkg_cross_env | {'PACMAN_CHROOT': chroot}, + [os.path.join(chroot.path, 'usr/bin/makepkg')] + makepkg_cmd[1:] + makepkg_setup_args, + env=makepkg_cross_env | {'PACMAN_CHROOT': chroot.path}, cwd=os.path.join(pkgbuilds_dir, package.path), ) if result.returncode != 0: @@ -443,31 +435,30 @@ def setup_sources(package: Package, chroot: str, arch: str, pkgbuilds_dir: str = def build_package( package: Package, - arch: str, + arch: Arch, repo_dir: str = None, enable_crosscompile: bool = True, enable_crossdirect: bool = True, enable_ccache: bool = True, + clean_chroot: bool = False, ): makepkg_compile_opts = ['--holdver'] makepkg_conf_path = 'etc/makepkg.conf' repo_dir = repo_dir if repo_dir else config.get_path('pkgbuilds') foreign_arch = config.runtime['arch'] != arch - target_chroot = setup_build_chroot(arch=arch, extra_packages=(list(set(package.depends) - set(package.names)))) + target_chroot = setup_build_chroot( + arch=arch, + extra_packages=(list(set(package.depends) - set(package.names))), + clean_chroot=clean_chroot, + ) native_chroot = target_chroot if not foreign_arch else setup_build_chroot( arch=config.runtime['arch'], extra_packages=['base-devel'] + CROSSDIRECT_PKGS, + clean_chroot=clean_chroot, ) cross = foreign_arch and package.mode == 'cross' and enable_crosscompile - umount_dirs = [] - set([target_chroot, native_chroot]) - - # eliminate target_chroot == native_chroot with set() - for chroot, _arch in set([(native_chroot, config.runtime['arch']), (target_chroot, arch)]): - logging.debug(f'Mounting packages to {chroot}') - dir = mount_packages(chroot, _arch) - umount_dirs += [dir] + chroots = set([target_chroot, native_chroot]) if cross: logging.info(f'Cross-compiling {package.path}') @@ -479,17 +470,14 @@ def build_package( env['PATH'] = f"/usr/lib/ccache:{env['PATH']}" logging.info('Setting up dependencies for cross-compilation') # include crossdirect for ccache symlinks and qemu-user - results = try_install_packages(package.depends + CROSSDIRECT_PKGS + [f"{GCC_HOSTSPECS[config.runtime['arch']][arch]}-gcc"], native_chroot) + results = native_chroot.try_install_packages(package.depends + CROSSDIRECT_PKGS + [f"{GCC_HOSTSPECS[native_chroot.arch][arch]}-gcc"]) if results['crossdirect'].returncode != 0: raise Exception('Unable to install crossdirect') # mount foreign arch chroot inside native chroot - chroot_relative = os.path.join('chroot', os.path.basename(target_chroot)) - chroot_mount_path = os.path.join(native_chroot, chroot_relative) - makepkg_relative = write_cross_makepkg_conf(native_chroot=native_chroot, arch=arch, target_chroot_relative=chroot_relative) - makepkg_conf_path = os.path.join('/', makepkg_relative) - os.makedirs(chroot_mount_path) - mount(target_chroot, chroot_mount_path) - umount_dirs += [chroot_mount_path] + chroot_relative = os.path.join('chroot', target_chroot.name) + makepkg_path_absolute = native_chroot.write_makepkg_conf(target_arch=arch, cross_chroot_relative=chroot_relative, cross=True) + makepkg_conf_path = os.path.join('etc', os.path.basename(makepkg_path_absolute)) + native_chroot.mount_crosscompile(target_chroot) else: logging.info(f'Host-compiling {package.path}') build_root = target_chroot @@ -497,38 +485,29 @@ def build_package( env = deepcopy(makepkg_env) if foreign_arch and enable_crossdirect and package.name not in CROSSDIRECT_PKGS: env['PATH'] = f"/native/usr/lib/crossdirect/{arch}:{env['PATH']}" - umount_dirs += [mount_crossdirect(native_chroot=native_chroot, target_chroot=target_chroot, target_arch=arch)] + target_chroot.mount_crossdirect(native_chroot) else: if enable_ccache: logging.debug('ccache enabled') env['PATH'] = f"/usr/lib/ccache:{env['PATH']}" logging.debug(('Building for native arch. ' if not foreign_arch else '') + 'Skipping crossdirect.') - src_dir = os.path.join(build_root, 'pkgbuilds') - os.makedirs(src_dir, exist_ok=True) - #setup_sources(package, build_root, enable_crosscompile=enable_crosscompile) - - result = mount(config.get_path('pkgbuilds'), src_dir) - if result.returncode != 0: - raise Exception(f'Failed to bind mount pkgbuilds to {src_dir}') - umount_dirs += [src_dir] + setup_sources(package, build_root) + for chroot in chroots: + chroot.mount_pkgbuilds() + chroot.mount_packages() + chroot.activate() makepkg_conf_absolute = os.path.join('/', makepkg_conf_path) build_cmd = f'cd {package.path} && makepkg --config {makepkg_conf_absolute} --needed --noconfirm --ignorearch {" ".join(makepkg_compile_opts)}' logging.debug(f'Building: Running {build_cmd}') - result = run_chroot_cmd(build_cmd, chroot_path=build_root, inner_env=env) + result = build_root.run_cmd(build_cmd, inner_env=env) if result.returncode != 0: raise Exception(f'Failed to compile package {package.path}') - # cleanup - for dir in umount_dirs: - umount_result = umount(dir) - if umount_result != 0: - logging.warning(f'Failed to unmount {dir}') - -def get_unbuilt_package_levels(repo: dict[str, Package], packages: list[Package], arch: str, force: bool = False) -> list[set[Package]]: +def get_unbuilt_package_levels(repo: dict[str, Package], packages: list[Package], arch: Arch, force: bool = False) -> list[set[Package]]: package_levels = generate_dependency_chain(repo, packages) build_names = set[str]() build_levels = list[set[Package]]() @@ -550,7 +529,7 @@ def get_unbuilt_package_levels(repo: dict[str, Package], packages: list[Package] def build_packages( repo: dict[str, Package], packages: list[Package], - arch: str, + arch: Arch, force: bool = False, enable_crosscompile: bool = True, enable_crossdirect: bool = True, @@ -579,7 +558,7 @@ def build_packages( def build_packages_by_paths( paths: list[str], - arch: str, + arch: Arch, repo: dict[str, Package], force=False, enable_crosscompile: bool = True, @@ -623,7 +602,7 @@ def cmd_build(paths: list[str], force=False, arch=None): build(paths, force, arch) -def build(paths: list[str], force: bool, arch: str): +def build(paths: list[str], force: bool, arch: Arch): if arch is None: # TODO: arch = config.get_profile()... arch = 'aarch64'