diff --git a/chroot.py b/chroot.py index 7e1e037..7a3bc0c 100644 --- a/chroot.py +++ b/chroot.py @@ -139,23 +139,34 @@ def create_chroot_user( raise Exception('Failed to setup user') -def try_install_packages(packages: list[str], chroot: str) -> dict[str, subprocess.CompletedProcess]: - """Try installing packages one by one""" - results = {} - for pkg in set(packages): - # Don't check for errors here because there might be packages that are listed as dependencies but are not available on x86_64 - results[pkg] = run_chroot_cmd(f'pacman -Syy --noconfirm --needed {pkg}', chroot) +def try_install_packages(packages: list[str], chroot: str, refresh: bool = False, allow_fail: bool = True) -> dict[str, subprocess.CompletedProcess]: + """Try installing packages, fall back to installing one by one""" + if refresh: + run_chroot_cmd('pacman -Syy --noconfirm', chroot) + cmd = 'pacman -S --noconfirm --needed' + result = run_chroot_cmd(f'{cmd} {" ".join(packages)}', chroot) + results = {package: result for package in packages} + if result.returncode != 0 and allow_fail: + results = {} + logging.debug('Falling back to serial installation') + for pkg in set(packages): + # Don't check for errors here because there might be packages that are listed as dependencies but are not available on x86_64 + results[pkg] = run_chroot_cmd(f'{cmd} {pkg}', chroot) return results def mount_crossdirect(native_chroot: str, target_chroot: str, target_arch: str, host_arch: str = None): + """ + mount `native_chroot` at `target_chroot`/native + returns the absolute path that `native_chroot` has been mounted at. + """ if host_arch is None: host_arch = config.runtime['arch'] gcc = f'{GCC_HOSTSPECS[host_arch][target_arch]}-gcc' native_mount = os.path.join(target_chroot, 'native') logging.debug(f'Activating crossdirect in {native_mount}') - results = try_install_packages(CROSSDIRECT_PKGS + [gcc], native_chroot) + results = try_install_packages(CROSSDIRECT_PKGS + [gcc], native_chroot, refresh=True, allow_fail=False) if results[gcc].returncode != 0: logging.debug('Failed to install cross-compiler package {gcc}') if results['crossdirect'].returncode != 0: @@ -170,18 +181,45 @@ def mount_crossdirect(native_chroot: str, target_chroot: str, target_arch: str, result = mount(native_chroot, native_mount) if result.returncode != 0: raise Exception(f'Failed to mount native chroot {native_chroot} to {native_mount}') + return native_mount + + +def mount_relative(chroot_path: str, absolute_source: str, relative_target: str) -> tuple[subprocess.CompletedProcess, str]: + """returns the absolute path `relative_target` was mounted at""" + target = os.path.join(chroot_path, relative_target.lstrip('/')) + os.makedirs(target, exist_ok=True) + result = mount(absolute_source, target) + return result, target + + +def mount_pacman_cache(chroot_path: str, arch: str) -> str: + global_cache = os.path.join(config.get_path('pacman'), arch) + relative = os.path.join('var', 'cache', 'pacman', arch) + result, absolute = mount_relative(chroot_path, global_cache, relative) + if result.returncode != 0: + raise Exception(f'Failed to mount {global_cache} to {absolute}') + return absolute + + +def mount_packages(chroot_path, arch) -> str: + packages = config.get_package_dir(arch) + result, absolute = mount_relative(chroot_path, absolute_source=packages, relative_target=packages.lstrip('/')) + if result.returncode != 0: + raise Exception(f'Failed to mount {packages} to {absolute}: {result.returncode}') + return absolute def write_cross_makepkg_conf(native_chroot: str, arch: str, target_chroot_relative: str, cross: bool = True) -> str: """ Generate a makepkg_cross_$arch.conf file in `native_chroot`/etc, building for `target_chroot_relative` - Returns the absolute (host) path to the makepkg config file. + Returns the relative (to `native_chroot`) path to written file, e.g. `etc/makepkg_cross_aarch64.conf`. """ makepkg_cross_conf = generate_makepkg_conf(arch, cross=cross, chroot=target_chroot_relative) - makepkg_conf_path = os.path.join(native_chroot, 'etc', f'makepkg_cross_{arch}.conf') + makepkg_conf_path_relative = os.path.join('etc', f'makepkg_cross_{arch}.conf') + makepkg_conf_path = os.path.join(native_chroot, makepkg_conf_path_relative) with open(makepkg_conf_path, 'w') as f: f.write(makepkg_cross_conf) - return makepkg_conf_path + return makepkg_conf_path_relative @click.command('chroot') @@ -226,5 +264,5 @@ def cmd_chroot(type: str = 'build', arch: str = None, enable_crossdirect=True): mount_crossdirect(native_chroot=native_chroot, target_chroot=chroot_path, target_arch=arch) cmd = ['arch-chroot', chroot_path, '/bin/bash'] - logging.debug('Starting chroot: ' + repr(cmd)) + logging.debug('Starting chroot shell: ' + repr(cmd)) subprocess.call(cmd) diff --git a/config.py b/config.py index 7b26f9e..9b687c8 100644 --- a/config.py +++ b/config.py @@ -280,6 +280,9 @@ class ConfigStateHolder: paths = self.file['paths'] return resolve_path_template(paths[path_name], paths) + def get_package_dir(self, arch: str): + return os.path.join(self.get_path('packages'), arch) + def dump(self) -> str: dump_toml(self.file) diff --git a/distro.py b/distro.py index 72f97a2..e77432a 100644 --- a/distro.py +++ b/distro.py @@ -201,4 +201,4 @@ def get_kupfer_https(arch: str) -> Distro: def get_kupfer_local(arch: str) -> Distro: - return get_kupfer(arch, f"file://{config.get_path('packages')}/$repo") + return get_kupfer(arch, f"file://{config.get_path('packages')}/$arch/$repo") diff --git a/image.py b/image.py index da8e4d3..b64a013 100644 --- a/image.py +++ b/image.py @@ -161,7 +161,7 @@ def cmd_build(): rootfs_mount = get_chroot_path(chroot_name) mount_rootfs_image(image_name, rootfs_mount) - packages_dir = config.get_path('packages') + packages_dir = config.get_packages(arch) if os.path.exists(os.path.join(packages_dir, 'main')): extra_repos = get_kupfer_local(arch).repos else: diff --git a/packages.py b/packages.py index a40a8ad..3f7ff8b 100644 --- a/packages.py +++ b/packages.py @@ -9,7 +9,7 @@ from joblib import Parallel, delayed from constants import REPOSITORIES, CROSSDIRECT_PKGS, GCC_HOSTSPECS from config import config -from chroot import create_chroot, run_chroot_cmd, try_install_packages, mount_crossdirect, write_cross_makepkg_conf +from chroot import create_chroot, run_chroot_cmd, try_install_packages, mount_crossdirect, write_cross_makepkg_conf, mount_packages, mount_pacman_cache from distro import get_kupfer_local from wrapper import enforce_wrap, check_programs_wrap from utils import mount, umount @@ -98,8 +98,8 @@ class Package: return f'package({self.name},{repr(self.names)})' -def check_prebuilts(dir: str = None): - prebuilts_dir = dir if dir else config.get_path('packages') +def check_prebuilts(arch: str, dir: str = None): + prebuilts_dir = dir if dir else config.get_package_dir(arch) os.makedirs(prebuilts_dir, exist_ok=True) for repo in REPOSITORIES: os.makedirs(os.path.join(prebuilts_dir, repo), exist_ok=True) @@ -271,10 +271,60 @@ def generate_dependency_chain(package_repo: dict[str, Package], to_build: list[P return list([lvl for lvl in dep_levels[::-1] if lvl]) -def check_package_version_built(package: Package, arch) -> bool: - built = True +def add_file_to_repo(file_path: str, repo_name: str, arch: str): + repo_dir = os.path.join(config.get_package_dir(arch), repo_name) + pacman_cache_dir = os.path.join(config.get_path('pacman'), arch) + file_name = os.path.basename(file_path) + target_file = os.path.join(repo_dir, file_name) - config_path = write_cross_makepkg_conf(native_chroot='/', arch=arch, target_chroot_relative=None, cross=False) + os.makedirs(repo_dir, exist_ok=True) + if file_path != target_file: + logging.debug(f'moving {file_path} to {target_file} ({repo_dir})') + shutil.copy( + file_path, + repo_dir, + ) + os.unlink(file_path) + + # clean up same name package from pacman cache + cache_file = os.path.join(pacman_cache_dir, file_name) + if os.path.exists(cache_file): + os.unlink(cache_file) + result = subprocess.run([ + 'repo-add', + '--remove', + '--prevent-downgrade', + os.path.join( + repo_dir, + f'{repo_name}.db.tar.xz', + ), + target_file, + ]) + if result.returncode != 0: + raise Exception(f'Failed add package {target_file} to repo {repo_name}') + for ext in ['db', 'files']: + file = os.path.join(repo_dir, f'{repo_name}.{ext}') + if os.path.exists(file + '.tar.xz'): + os.unlink(file) + shutil.copyfile(file + '.tar.xz', file) + old = file + '.tar.xz.old' + if os.path.exists(old): + os.unlink(old) + + +def add_package_to_repo(package: Package, arch: str): + logging.info(f'Adding {package.path} to repo {package.repo}') + pkgbuild_dir = os.path.join(config.get_path('pkgbuilds'), package.path) + + for file in os.listdir(pkgbuild_dir): + # Forced extension by makepkg.conf + if file.endswith('.pkg.tar.xz') or file.endswith('.pkg.tar.zst'): + return add_file_to_repo(os.path.join(pkgbuild_dir, file), package.repo, arch) + + +def check_package_version_built(package: Package, arch) -> bool: + + config_path = '/' + write_cross_makepkg_conf(native_chroot='/', arch=arch, target_chroot_relative=None, cross=False) result = subprocess.run( makepkg_cmd + [ @@ -289,17 +339,17 @@ def check_package_version_built(package: Package, arch) -> bool: capture_output=True, ) if result.returncode != 0: - logging.fatal(f'Failed to get package list for {package.path}:' + '\n' + result.stdout.decode() + '\n' + result.stderr.decode()) - exit(1) + raise Exception(f'Failed to get package list for {package.path}:' + '\n' + result.stdout.decode() + '\n' + result.stderr.decode()) for line in result.stdout.decode('utf-8').split('\n'): if line != "": - file = os.path.join(config.get_path('packages'), package.repo, os.path.basename(line)) + file = os.path.join(config.get_package_dir(arch), package.repo, os.path.basename(line)) logging.debug(f'Checking if {file} is built') - if not os.path.exists(file): - built = False + if os.path.exists(file): + add_file_to_repo(file, repo_name=package.repo, arch=arch) + return True - return built + return False def setup_build_chroot(arch: str, extra_packages=[]) -> str: @@ -311,6 +361,7 @@ def setup_build_chroot(arch: str, extra_packages=[]) -> str: packages=list(set(['base-devel', 'git', 'ccache'] + extra_packages)), extra_repos=get_kupfer_local(arch).repos, ) + pacman_cache = mount_pacman_cache(chroot_path, arch) logging.info(f'Updating chroot {chroot_name}') result = subprocess.run( @@ -328,11 +379,12 @@ def setup_build_chroot(arch: str, extra_packages=[]) -> str: logging.fatal(result.stdout) logging.fatal(result.stderr) raise Exception(f'Failed to update chroot {chroot_name}') + umount(pacman_cache) return chroot_path -def setup_sources(package: Package, chroot: str, arch: str, repo_dir: str = None): - repo_dir = repo_dir if repo_dir else config.get_path('pkgbuilds') +def setup_sources(package: Package, chroot: str, arch: str, pkgbuilds_dir: str = None): + pkgbuilds_dir = pkgbuilds_dir if pkgbuilds_dir else config.get_path('pkgbuilds') makepkg_setup_args = [ '--nobuild', '--holdver', @@ -343,7 +395,7 @@ def setup_sources(package: Package, chroot: str, arch: str, repo_dir: str = None result = subprocess.run( [os.path.join(chroot, 'usr/bin/makepkg')] + makepkg_cmd[1:] + makepkg_setup_args, env=makepkg_cross_env | {'PACMAN_CHROOT': chroot}, - cwd=os.path.join(repo_dir, package.path), + cwd=os.path.join(pkgbuilds_dir, package.path), ) if result.returncode != 0: raise Exception(f'Failed to check sources for {package.path}') @@ -366,16 +418,14 @@ def build_package( target_chroot = setup_build_chroot(arch=arch, extra_packages=package.depends) native_chroot = setup_build_chroot(arch=config.runtime['arch'], extra_packages=['base-devel']) if foreign_arch else target_chroot cross = foreign_arch and package.mode == 'cross' and enable_crosscompile + umount_dirs = [] + chroots = set([target_chroot, native_chroot]) # eliminate target_chroot == native_chroot with set() - for chroot in set([target_chroot, native_chroot]): - pkgs_path = config.get_path('packages') - chroot_pkgs = chroot + pkgs_path # NOTE: DO NOT USE PATH.JOIN, pkgs_path starts with / - os.makedirs(chroot_pkgs, exist_ok=True) - logging.debug(f'Mounting packages to {chroot_pkgs}') - result = mount(pkgs_path, chroot_pkgs) - if result.returncode != 0: - raise Exception(f'Unable to mount packages to {chroot_pkgs}') + for chroot, _arch in [(native_chroot, config.runtime['arch']), (target_chroot, arch)]: + logging.debug(f'Mounting packages to {chroot}') + dir = mount_packages(chroot, _arch) + umount_dirs += [dir] if cross: logging.info(f'Cross-compiling {package.path}') @@ -386,13 +436,18 @@ def build_package( if enable_ccache: env['PATH'] = f"/usr/lib/ccache:{env['PATH']}" logging.info('Setting up dependencies for cross-compilation') - try_install_packages(package.depends + ['crossdirect', f"{GCC_HOSTSPECS[config.runtime['arch']][arch]}-gcc"], native_chroot) + # include crossdirect for ccache symlinks. + results = try_install_packages(package.depends + ['crossdirect', f"{GCC_HOSTSPECS[config.runtime['arch']][arch]}-gcc"], native_chroot) + if results['crossdirect'].returncode != 0: + raise Exception('Unable to install crossdirect') # mount foreign arch chroot inside native chroot chroot_relative = os.path.join('chroot', os.path.basename(target_chroot)) chroot_mount_path = os.path.join(native_chroot, chroot_relative) - write_cross_makepkg_conf(native_chroot=native_chroot, arch=arch, target_chroot_relative=chroot_relative) + makepkg_relative = write_cross_makepkg_conf(native_chroot=native_chroot, arch=arch, target_chroot_relative=chroot_relative) + makepkg_conf_path = os.path.join('/', makepkg_relative) os.makedirs(chroot_mount_path) mount(target_chroot, chroot_mount_path) + umount_dirs += [chroot_mount_path] else: logging.info(f'Host-compiling {package.path}') build_root = target_chroot @@ -402,9 +457,9 @@ def build_package( env['PATH'] = f"/usr/lib/ccache:{env['PATH']}" if not foreign_arch: logging.debug('Building for native arch, skipping crossdirect.') - elif enable_crossdirect and not package.name in CROSSDIRECT_PKGS: + elif enable_crossdirect and package.name not in CROSSDIRECT_PKGS: env['PATH'] = f"/native/usr/lib/crossdirect/{arch}:{env['PATH']}" - mount_crossdirect(native_chroot=native_chroot, target_chroot=target_chroot, target_arch=arch) + umount_dirs += [mount_crossdirect(native_chroot=native_chroot, target_chroot=target_chroot, target_arch=arch)] else: logging.debug('Skipping crossdirect.') @@ -414,60 +469,22 @@ def build_package( result = mount(config.get_path('pkgbuilds'), src_dir) if result.returncode != 0: - raise Exception(f'Failed to bind mount pkgdirs to {build_root}/src') + raise Exception(f'Failed to bind mount pkgbuilds to {build_root}/src') + umount_dirs += [src_dir] makepkg_conf_absolute = os.path.join('/', makepkg_conf_path) build_cmd = f'cd /src/{package.path} && makepkg --config {makepkg_conf_absolute} --needed --noconfirm --ignorearch {" ".join(makepkg_compile_opts)}' + logging.debug(f'Building: Running {build_cmd}') result = run_chroot_cmd(build_cmd, chroot_path=build_root, inner_env=env) - umount_result = umount(src_dir) - if umount_result != 0: - logging.warning(f'Failed to unmount {src_dir}') + if result.returncode != 0: raise Exception(f'Failed to compile package {package.path}') - -def add_package_to_repo(package: Package, arch: str): - logging.info(f'Adding {package.path} to repo') - binary_dir = os.path.join(config.get_path('packages'), package.repo) - pkgbuild_dir = os.path.join(config.get_path('pkgbuilds'), package.path) - pacman_cache_dir = os.path.join(config.get_path('pacman'), arch) - os.makedirs(binary_dir, exist_ok=True) - - for file in os.listdir(pkgbuild_dir): - # Forced extension by makepkg.conf - if file.endswith('.pkg.tar.xz') or file.endswith('.pkg.tar.zst'): - shutil.move( - os.path.join(pkgbuild_dir, file), - os.path.join(binary_dir, file), - ) - # clean up same name package from pacman cache - cache_file = os.path.join(pacman_cache_dir, file) - if os.path.exists(cache_file): - os.unlink(cache_file) - result = subprocess.run([ - 'repo-add', - '--remove', - '--new', - '--prevent-downgrade', - os.path.join( - binary_dir, - f'{package.repo}.db.tar.xz', - ), - os.path.join(binary_dir, file), - ]) - if result.returncode != 0: - logging.fatal(f'Failed add package {package.path} to repo') - exit(1) - for repo in REPOSITORIES: - for ext in ['db', 'files']: - if os.path.exists(os.path.join(binary_dir, f'{repo}.{ext}.tar.xz')): - os.unlink(os.path.join(binary_dir, f'{repo}.{ext}')) - shutil.copyfile( - os.path.join(binary_dir, f'{repo}.{ext}.tar.xz'), - os.path.join(binary_dir, f'{repo}.{ext}'), - ) - if os.path.exists(os.path.join(binary_dir, f'{repo}.{ext}.tar.xz.old')): - os.unlink(os.path.join(binary_dir, f'{repo}.{ext}.tar.xz.old')) + # cleanup + for dir in umount_dirs: + umount_result = umount(dir) + if umount_result != 0: + logging.warning(f'Failed to unmount {dir}') @click.group(name='packages') @@ -485,7 +502,8 @@ def cmd_build(paths: list[str], force=False, arch=None): # arch = config.get_profile()... arch = 'aarch64' - check_prebuilts() + for _arch in set([arch, config.runtime['arch']]): + check_prebuilts(_arch) paths = list(paths) repo = discover_packages() @@ -522,7 +540,7 @@ def cmd_build(paths: list[str], force=False, arch=None): @cmd_packages.command(name='clean') def cmd_clean(): - check_programs_wrap('git') + enforce_wrap() result = subprocess.run([ 'git', 'clean', diff --git a/utils.py b/utils.py index c2eac62..6f68f01 100644 --- a/utils.py +++ b/utils.py @@ -33,13 +33,13 @@ def mount(src: str, dest: str, options=['bind'], type=None) -> subprocess.Comple if type: type = ['-t', type] - result = subprocess.run(['mount'] + type + opts + [ - src, - dest, - ]) - if result.returncode == 0: - atexit.register(umount, dest) - return result + result = subprocess.run( + ['mount'] + type + opts + [ + src, + dest, + ], + capture_output=False, + ) if result.returncode == 0: atexit.register(umount, dest) return result