From 3b69c2235bec6eb114cb5a44034e7617a91a9517 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Sat, 18 Sep 2021 17:28:27 +0200 Subject: [PATCH] Crosscompile is borked, host-compile seems working Signed-off-by: InsanePrawn --- Dockerfile | 2 ++ config.py | 12 ++++++----- image.py | 27 ++++++++----------------- packages.py | 58 ++++++++++++++++++++++++++++++++--------------------- 4 files changed, 52 insertions(+), 47 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1ad012a..e3e9453 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,4 +28,6 @@ RUN pip install -r requirements.txt COPY . . +RUN python -c "import constants; repos='\n'.join(['\n'.join([f'[{repo}]', f'Server = file:///prebuilts/\$repo', '']) for repo in constants.REPOSITORIES]); print(repos)" | tee /etc/pacman.d/kupfer-local + WORKDIR /src diff --git a/config.py b/config.py index bdda24e..2a32d6b 100644 --- a/config.py +++ b/config.py @@ -7,6 +7,7 @@ import click CONFIG_DEFAULT_PATH = os.path.join(appdirs.user_config_dir('kupfer'), 'kupferbootstrap.toml') + PROFILE_DEFAULTS = { 'device': '', 'flavour': '', @@ -34,6 +35,11 @@ CONFIG_DEFAULTS = { }, } +CONFIG_RUNTIME_DEFAULTS = { + 'verbose': False, + 'config_file': None, + 'arch': None, +} def sanitize_config(conf: dict, warn_missing_defaultprofile=True) -> dict: """checks the input config dict for unknown keys and returns only the known parts""" @@ -152,11 +158,7 @@ class ConfigStateHolder: # config options that are persisted to file file: dict = {} # runtime config not persisted anywhere - runtime: dict = { - 'verbose': False, - 'config_file': None, - 'arch': None, - } + runtime: dict = CONFIG_RUNTIME_DEFAULTS def __init__(self, runtime_conf={}, file_conf_path: str = None, file_conf_base: dict = {}): """init a stateholder, optionally loading `file_conf_path`""" diff --git a/image.py b/image.py index 3a388ca..b051eda 100644 --- a/image.py +++ b/image.py @@ -4,7 +4,8 @@ import subprocess import click from logger import logging from chroot import create_chroot, create_chroot_user, get_chroot_path -from constants import DEVICES, FLAVOURS +from constants import DEVICES, FLAVOURS, REPOSITORIES +from config import config def get_device_and_flavour() -> tuple[str, str]: @@ -165,24 +166,12 @@ def cmd_build(): rootfs_mount = get_chroot_path(chroot_name) mount_rootfs_image(image_name, rootfs_mount) - extra_repos = { - 'main': { - 'Server': 'https://gitlab.com/kupfer/packages/prebuilts/-/raw/main/$repo', - }, - 'device': { - 'Server': 'https://gitlab.com/kupfer/packages/prebuilts/-/raw/main/$repo', - }, - } - - if os.path.exists('/prebuilts'): - extra_repos = { - 'main': { - 'Server': 'file:///prebuilts/$repo', - }, - 'device': { - 'Server': 'file:///prebuilts/$repo', - }, - } + packages_dir = config.file['paths']['packages'] + if os.path.exists(packages_dir): + url = f'file://{packages_dir}/$repo', + else: + url = 'https://gitlab.com/kupfer/packages/prebuilts/-/raw/main/$repo' + extra_repos = {repo: {'Server': url} for repo in REPOSITORIES} create_chroot( chroot_name, diff --git a/packages.py b/packages.py index c4e8924..9859294 100644 --- a/packages.py +++ b/packages.py @@ -1,10 +1,11 @@ -from constants import REPOSITORIES import click +import atexit import logging import multiprocessing import os import shutil import subprocess +from constants import REPOSITORIES from config import config from chroot import create_chroot from joblib import Parallel, delayed @@ -151,11 +152,11 @@ def discover_packages(package_paths: list[str] = ['all'], dir: str = None) -> di return packages -def filter_packages_by_paths(repo: list[Package], paths: list[str]) -> list[Package]: +def filter_packages_by_paths(repo: dict[str, Package], paths: list[str]) -> list[Package]: if 'all' in paths: return repo.values() result = [] - for pkg in repo: + for pkg in repo.values(): if pkg.path in paths: result += [pkg] return result @@ -211,7 +212,6 @@ def generate_dependency_chain(package_repo: dict[str, Package], to_build: list[P 1. Moving packages that are dependencies of other packages up to `level`+1 2. Adding yet unadded local dependencies of all pkgs on `level` to `level`+1 3. increment level - 4. repeat until """ level = 0 # protect against dependency cycles @@ -295,7 +295,7 @@ def check_package_version_built(package: Package) -> bool: return built -def setup_build_chroot(arch='aarch64') -> str: +def setup_build_chroot(arch='aarch64', extra_packages=[]) -> str: chroot_name = f'build_{arch}' logging.info(f'Initializing {arch} build chroot') extra_repos = {} @@ -305,7 +305,7 @@ def setup_build_chroot(arch='aarch64') -> str: } chroot_path = create_chroot( chroot_name, - packages=['base-devel'], + packages=['base-devel', 'git'] + extra_packages, pacman_conf='/app/local/etc/pacman.conf', extra_repos=extra_repos, ) @@ -350,7 +350,15 @@ def setup_dependencies_and_sources(package: Package, chroot: str, repo_dir: str so that the build tools can be used """ repo_dir = repo_dir if repo_dir else config.file['paths']['pkgbuilds'] - if package.mode == 'cross' and enable_crosscompile: + makepkg_setup_args = [ + '--nobuild', + '--holdver', + ] + if not package.mode == 'cross' and enable_crosscompile: + makepkg_setup_args += ['--syncdeps'] + else: + makepkg_setup_args += ['--nodeps'] + logging.info('Setting up dependencies for cross-compilation') for p in package.depends: # Don't check for errors here because there might be packages that are listed as dependencies but are not available on x86_64 subprocess.run( @@ -359,11 +367,7 @@ def setup_dependencies_and_sources(package: Package, chroot: str, repo_dir: str ) result = subprocess.run( - [os.path.join(chroot, 'usr/bin/makepkg')] + makepkg_cmd[1:] + [ - '--nobuild', - '--holdver', - '--nodeps', - ], + [os.path.join(chroot, 'usr/bin/makepkg')] + makepkg_cmd[1:] + makepkg_setup_args, env=makepkg_cross_env | {'PACMAN_CHROOT': chroot}, cwd=os.path.join(repo_dir, package.path), ) @@ -376,10 +380,9 @@ def build_package(package: Package, repo_dir: str = None, arch='aarch64', enable '--noextract', '--skipinteg', '--holdver', - '--syncdeps', ] repo_dir = repo_dir if repo_dir else config.file['paths']['pkgbuilds'] - chroot = setup_build_chroot(arch=arch) + chroot = setup_build_chroot(arch=arch, extra_packages=package.depends) setup_dependencies_and_sources(package, chroot, enable_crosscompile=enable_crosscompile) if package.mode == 'cross' and enable_crosscompile: @@ -408,7 +411,7 @@ def build_package(package: Package, repo_dir: str = None, arch='aarch64', enable exit(1) result = subprocess.run( - [os.path.join(chroot, 'usr/bin/makepkg')] + makepkg_cmd[1:] + makepkg_compile_opts, + [os.path.join(chroot, 'usr/bin/makepkg')] + makepkg_cmd[1:] + ['--nodeps'] + makepkg_compile_opts, env=makepkg_cross_env | {'QEMU_LD_PREFIX': '/usr/aarch64-linux-gnu'}, cwd=os.path.join(repo_dir, package.path), ) @@ -417,7 +420,7 @@ def build_package(package: Package, repo_dir: str = None, arch='aarch64', enable exit(1) else: logging.info(f'Host-compiling {package.path}') - os.makedirs(f'{chroot}/src') + os.makedirs(f'{chroot}/src', exist_ok=True) result = subprocess.run([ 'mount', '-o', @@ -425,6 +428,19 @@ def build_package(package: Package, repo_dir: str = None, arch='aarch64', enable config.file['paths']['pkgbuilds'], f'{chroot}/src', ]) + + def umount(): + subprocess.run( + [ + 'umount', + '-lc', + f'/{chroot}/src', + ], + stderr=subprocess.DEVNULL, + ) + + atexit.register(umount) + if result.returncode != 0: logging.fatal(f'Failed to bind mount pkgdirs to {chroot}/src') exit(1) @@ -437,14 +453,13 @@ def build_package(package: Package, repo_dir: str = None, arch='aarch64', enable ] + env + [ '/bin/bash', '-c', - f'cd /src/{package.path} && makepkg --noconfirm --ignorearch {" ".join(makepkg_compile_opts)}', + f'cd /src/{package.path} && makepkg --syncdeps --needed --noconfirm --ignorearch {" ".join(makepkg_compile_opts)}', ]) + umount() if result.returncode != 0: logging.fatal(f'Failed to host-compile package {package.path}') exit(1) - umount() - def add_package_to_repo(package: Package): logging.info(f'Adding {package.path} to repo') @@ -492,7 +507,7 @@ def cmd_packages(): @click.command(name='build') @click.argument('paths', nargs=-1) -def cmd_build(paths, arch='aarch64'): +def cmd_build(paths: list[str], arch='aarch64'): check_prebuilts() paths = list(paths) @@ -683,6 +698,3 @@ def cmd_check(paths): cmd_packages.add_command(cmd_build) cmd_packages.add_command(cmd_clean) cmd_packages.add_command(cmd_check) -cmd_packages.add_command(cmd_build) -cmd_packages.add_command(cmd_clean) -cmd_packages.add_command(cmd_check)