From d85c00fa125fd2169ef42d1db2a62bf3873398ef Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Fri, 1 Oct 2021 18:25:42 +0200 Subject: [PATCH] Move mounting code to utils.py, move a lot of chroot-logic from packages to chroot.py, cmd_chroot also moar crossdirect Signed-off-by: InsanePrawn --- chroot.py | 125 ++++++++++++++++++++++++++++++++++++++++++++++----- config.py | 3 +- constants.py | 13 ++++++ image.py | 1 + main.py | 16 ++++--- packages.py | 125 ++++++++++++++------------------------------------- utils.py | 32 +++++++++++++ wrapper.py | 2 +- 8 files changed, 207 insertions(+), 110 deletions(-) diff --git a/chroot.py b/chroot.py index ff21943..d7192bc 100644 --- a/chroot.py +++ b/chroot.py @@ -1,9 +1,16 @@ +import click import logging import subprocess import os from config import config from distro import get_base_distros, RepoInfo from shlex import quote as shell_quote +from utils import mount +from distro import get_kupfer_local +from wrapper import enforce_wrap +from constants import GCC_HOSTSPECS + +BIND_BUILD_DIRS = 'BINDBUILDDIRS' def get_chroot_path(chroot_name, override_basepath: str = None) -> str: @@ -11,13 +18,12 @@ def get_chroot_path(chroot_name, override_basepath: str = None) -> str: return os.path.join(base_path, chroot_name) -def create_chroot( - chroot_name: str, - arch: str, - packages: list[str] = ['base'], - extra_repos: dict[str, RepoInfo] = {}, - chroot_base_path: str = None, -): +def create_chroot(chroot_name: str, + arch: str, + packages: list[str] = ['base'], + extra_repos: dict[str, RepoInfo] = {}, + chroot_base_path: str = None, + bind_mounts: dict[str, str] = BIND_BUILD_DIRS): base_chroot = f'base_{arch}' chroot_path = get_chroot_path(chroot_name, override_basepath=chroot_base_path) base_distro = get_base_distros()[arch] @@ -39,11 +45,34 @@ def create_chroot( f'{chroot_path}/', ]) if result.returncode != 0: - logging.fatal('Failed to sync chroot copy') - exit(1) + raise Exception(f'Failed to copy {base_chroot} to {chroot_name}') + + # patch makepkg + with open(f'{chroot_path}/usr/bin/makepkg', 'r') as file: + data = file.read() + data = data.replace('EUID == 0', 'EUID == -1') + with open(f'{chroot_path}/usr/bin/makepkg', 'w') as file: + file.write(data) + + # configure makepkg + with open(f'{chroot_path}/etc/makepkg.conf', 'r') as file: + data = file.read() + data = data.replace('xz -c', 'xz -T0 -c') + data = data.replace(' check ', ' !check ') + with open(f'{chroot_path}/etc/makepkg.conf', 'w') as file: + file.write(data) os.makedirs(chroot_path + '/etc', exist_ok=True) + if bind_mounts == BIND_BUILD_DIRS: + bind_mounts = {p: p for p in [config.get_path(path) for path in ['packages', 'pkgbuilds']]} + for src, _dest in bind_mounts.items(): + dest = os.path.join(chroot_path, _dest.lstrip('/')) + os.makedirs(dest, exist_ok=True) + result = mount(src, dest) + if result.returncode != 0: + raise Exception(f"Couldn't bind-mount {src} to {dest}") + conf_text = base_distro.get_pacman_conf(extra_repos) with open(pacman_conf_target, 'w') as file: file.write(conf_text) @@ -67,7 +96,7 @@ def create_chroot( return chroot_path -def run_chroot_cmd(script: str, chroot_path: str, env: dict[str, str] = {}): +def run_chroot_cmd(script: str, chroot_path: str, env: dict[str, str] = {}) -> subprocess.CompletedProcess: env_cmd = ['/usr/bin/env'] + [f'{shell_quote(key)}={shell_quote(value)}' for key, value in env.items()] result = subprocess.run(['arch-chroot', chroot_path] + env_cmd + [ @@ -99,3 +128,79 @@ def create_chroot_user( result = run_chroot_cmd([install_script], chroot_path=chroot_path) if result.returncode != 0: raise Exception('Failed to setup user') + + +def try_install_packages(packages: list[str], chroot: str) -> dict[str, subprocess.CompletedProcess]: + """Try installing packages one by one""" + results = {} + for pkg in set(packages): + # Don't check for errors here because there might be packages that are listed as dependencies but are not available on x86_64 + results[pkg] = run_chroot_cmd(f'pacman -Syy --noconfirm --needed {pkg}', chroot) + return results + + +def mount_crossdirect(native_chroot: str, target_chroot: str, target_arch: str, host_arch: str = None): + if host_arch is None: + host_arch = config.runtime['arch'] + gcc = f'{GCC_HOSTSPECS[host_arch][target_arch]}-gcc' + + native_mount = os.path.join(target_chroot, 'native') + logging.debug(f'Activating crossdirect in {native_mount}') + results = try_install_packages(['crossdirect', gcc], native_chroot) + if results[gcc].returncode != 0: + logging.debug('Failed to install cross-compiler package {gcc}') + if results['crossdirect'].returncode != 0: + raise Exception('Failed to install crossdirect') + + os.makedirs(native_mount, exist_ok=True) + + logging.debug(f'Mounting {native_chroot} to {native_mount}') + result = mount(native_chroot, native_mount) + if result.returncode != 0: + raise Exception(f'Failed to mount native chroot {native_chroot} to {native_mount}') + + +@click.command('chroot') +@click.argument('type', required=False, default='build') +@click.argument('arch', required=False, default=None) +def cmd_chroot(type: str = 'build', arch: str = None): + chroot_path = '' + if type not in ['base', 'build', 'rootfs']: + raise Exception('Unknown chroot type: ' + type) + + enforce_wrap() + if type == 'rootfs': + if arch: + name = 'rootfs_' + arch + else: + raise Exception('"rootfs" not yet implemented, sorry!') + # TODO: name = config.get_profile()[...] + chroot_path = os.path.join(config.get_path('chroots'), name) + if not os.path.exists(chroot_path): + raise Exception(f"rootfs {name} doesn't exist") + else: + if not arch: + #TODO: arch = config.get_profile()[...] + arch = 'aarch64' + chroot_name = type + '_' + arch + chroot_path = get_chroot_path(chroot_name) + if not os.path.exists(os.path.join(chroot_path, 'bin')): + create_chroot( + chroot_name, + arch=arch, + packages=['base-devel', 'git'], + extra_repos=get_kupfer_local(arch).repos, + ) + if type == 'build' and config.file['build']['crossdirect']: + native_arch = config.runtime['arch'] + native_chroot = create_chroot( + 'build_' + native_arch, + native_arch, + packages=['base-devel', 'crossdirect'], + extra_repos=get_kupfer_local(native_arch).repos, + ) + mount_crossdirect(native_chroot=native_chroot, target_chroot=chroot_path, target_arch=arch) + + cmd = ['arch-chroot', chroot_path, '/bin/bash'] + logging.debug('Starting chroot: ' + repr(cmd)) + subprocess.call(cmd) diff --git a/config.py b/config.py index 39bf9a5..9988599 100644 --- a/config.py +++ b/config.py @@ -25,7 +25,8 @@ PROFILE_DEFAULTS: Profile = { CONFIG_DEFAULTS = { 'build': { - 'crosscompile': True, + 'crosscompile': False, # currently broken + 'crossdirect': True, 'threads': 0, }, 'paths': { diff --git a/constants.py b/constants.py index 14d0ed2..11972ed 100644 --- a/constants.py +++ b/constants.py @@ -49,6 +49,7 @@ REPOSITORIES = [ 'main', ] +Arch = str ARCHES = [ 'x86_64', 'aarch64', @@ -74,3 +75,15 @@ BASE_DISTROS = { } KUPFER_HTTPS = 'https://gitlab.com/kupfer/packages/prebuilts/-/raw/main/$repo' + +DistroArch = TargetArch = Arch + +GCC_HOSTSPECS: dict[DistroArch, dict[TargetArch, str]] = { + 'x86_64': { + 'x86_64': 'x86_64-pc-linux-gnu', + 'aarch64': 'aarch64-linux-gnu', + }, + 'aarch64': { + 'aarch64': 'aarch64-unknown-linux-gnu', + } +} diff --git a/image.py b/image.py index 65568c4..da8e4d3 100644 --- a/image.py +++ b/image.py @@ -172,6 +172,7 @@ def cmd_build(): packages=packages, pacman_conf=os.path.join(config.runtime['script_source_dir'], 'local/etc/pacman.conf'), extra_repos=extra_repos, + bind_mounts={}, ) create_chroot_user(chroot_name, user=profile['username'], password=profile['password']) if post_cmds: diff --git a/main.py b/main.py index 8682ebe..acc4345 100755 --- a/main.py +++ b/main.py @@ -1,18 +1,19 @@ #!/usr/bin/env python3 +import click +from traceback import format_exc as get_trace +from logger import logging, setup_logging, verbose_option +from wrapper import nowrapper_option +from config import config, config_option, cmd_config +from forwarding import cmd_forwarding from packages import cmd_packages +from telnet import cmd_telnet +from chroot import cmd_chroot from cache import cmd_cache from image import cmd_image from boot import cmd_boot from flash import cmd_flash from ssh import cmd_ssh -from forwarding import cmd_forwarding -from telnet import cmd_telnet -from logger import logging, setup_logging, verbose_option -import click -from config import config, config_option, cmd_config -from wrapper import nowrapper_option -from traceback import format_exc as get_trace @click.group() @@ -43,6 +44,7 @@ cli.add_command(cmd_flash) cli.add_command(cmd_ssh) cli.add_command(cmd_forwarding) cli.add_command(cmd_telnet) +cli.add_command(cmd_chroot) if __name__ == '__main__': main() diff --git a/packages.py b/packages.py index 728b88c..7ef7a12 100644 --- a/packages.py +++ b/packages.py @@ -7,10 +7,11 @@ import shutil import subprocess from constants import REPOSITORIES from config import config -from chroot import create_chroot, run_chroot_cmd +from chroot import create_chroot, run_chroot_cmd, try_install_packages, mount_crossdirect from joblib import Parallel, delayed from distro import get_kupfer_local from wrapper import enforce_wrap, check_programs_wrap +from utils import mount, umount makepkg_env = os.environ.copy() | { 'LANG': 'C', @@ -301,7 +302,7 @@ def setup_build_chroot(arch: str, extra_packages=[]) -> str: chroot_path = create_chroot( chroot_name, arch=arch, - packages=['base-devel', 'git'] + extra_packages, + packages=list(set(['base-devel', 'git'] + extra_packages)), extra_repos=get_kupfer_local(arch).repos, ) @@ -318,23 +319,9 @@ def setup_build_chroot(arch: str, extra_packages=[]) -> str: capture_output=True, ) if result.returncode != 0: - logging.fatal(f'Failed to update chroot {chroot_name}:') logging.fatal(result.stdout) logging.fatal(result.stderr) - exit(1) - - with open(f'{chroot_path}/usr/bin/makepkg', 'r') as file: - data = file.read() - data = data.replace('EUID == 0', 'EUID == -1') - with open(f'{chroot_path}/usr/bin/makepkg', 'w') as file: - file.write(data) - - with open(f'{chroot_path}/etc/makepkg.conf', 'r') as file: - data = file.read() - data = data.replace('xz -c', 'xz -T0 -c') - data = data.replace(' check ', ' !check ') - with open(f'{chroot_path}/etc/makepkg.conf', 'w') as file: - file.write(data) + raise Exception(f'Failed to update chroot {chroot_name}') return chroot_path @@ -364,108 +351,64 @@ def build_package(package: Package, arch: str, repo_dir: str = None, enable_cros ] repo_dir = repo_dir if repo_dir else config.get_path('pkgbuilds') foreign_arch = config.runtime['arch'] != arch - chroot = setup_build_chroot(arch=arch, extra_packages=package.depends) - native_chroot = setup_build_chroot(arch=config.runtime['arch'], extra_packages=['base-devel']) if foreign_arch else chroot + target_chroot = setup_build_chroot(arch=arch, extra_packages=package.depends) + native_chroot = setup_build_chroot(arch=config.runtime['arch'], extra_packages=['base-devel']) if foreign_arch else target_chroot cross = foreign_arch and package.mode == 'cross' and enable_crosscompile - native_deps = [] env = {} + # eliminate target_chroot == native_chroot with set() + for chroot in set([target_chroot, native_chroot]): + pkgs_path = config.get_path('packages') + chroot_pkgs = chroot + pkgs_path # NOTE: DO NOT USE PATH.JOIN, pkgs_path starts with / + os.makedirs(chroot_pkgs, exist_ok=True) + logging.debug(f'Mounting packages to {chroot_pkgs}') + result = mount(pkgs_path, chroot_pkgs) + if result.returncode != 0: + raise Exception(f'Unable to mount packages to {chroot_pkgs}') + if cross: logging.info(f'Cross-compiling {package.path}') build_root = native_chroot makepkg_compile_opts += ['--nodeps'] - env = makepkg_cross_env | {'QEMU_LD_PREFIX': '/usr/aarch64-linux-gnu'} - - def umount(): - subprocess.run( - [ - 'umount', - '-lc', - f'{native_chroot}/usr/share/i18n/locales', - ], - stderr=subprocess.DEVNULL, - ) - - result = subprocess.run([ - 'mount', - '-o', - 'bind', - f"{chroot}/usr/share/i18n/locales", - f'{native_chroot}/usr/share/i18n/locales', - ]) - atexit.register(umount) + env = makepkg_env | {'QEMU_LD_PREFIX': '/usr/aarch64-unknown-linux-gnu'} logging.info('Setting up dependencies for cross-compilation') - native_deps += package.depends + try_install_packages(package.depends, native_chroot) else: logging.info(f'Host-compiling {package.path}') - build_root = chroot + build_root = target_chroot makepkg_compile_opts += ['--syncdeps'] env = makepkg_env - if foreign_arch and enable_crossdirect: - logging.debug('Activating crossdirect') - native_deps += ['crossdirect'] + if not foreign_arch: + logging.debug('Building for native arch, skipping crossdirect.') + elif enable_crossdirect and not package.name == 'crossdirect': env['PATH'] = f"/native/usr/lib/crossdirect/{arch}:{env['PATH']}" + mount_crossdirect(native_chroot=native_chroot, target_chroot=target_chroot, target_arch=arch) + else: + logging.debug('Skipping crossdirect.') - def umount(): - subprocess.run( - [ - 'umount', - '-lc', - f'{chroot}/native', - ], - stderr=subprocess.DEVNULL, - ) - - result = subprocess.run([ - 'mount', - '-o', - 'bind', - f"{native_chroot}", - f'{chroot}/native', - ]) - atexit.register(umount) - + src_dir = os.path.join(build_root, 'src') os.makedirs(f'{build_root}/src', exist_ok=True) setup_sources(package, build_root, enable_crosscompile=enable_crosscompile) - for dep in native_deps: - # Don't check for errors here because there might be packages that are listed as dependencies but are not available on x86_64 - run_chroot_cmd(f'pacman -Sy {dep}', native_chroot) - result = subprocess.run([ - 'mount', - '-o', - 'bind', - config.get_path('pkgbuilds'), - f'{build_root}/src', - ]) - - def umount(): - subprocess.run( - [ - 'umount', - '-lc', - f'/{build_root}/src', - ], - stderr=subprocess.DEVNULL, - ) - - atexit.register(umount) + result = mount(config.get_path('pkgbuilds'), src_dir) if result.returncode != 0: raise Exception(f'Failed to bind mount pkgdirs to {build_root}/src') - build_cmd = f'cd /src/{package.path} && makepkg --needed --noconfirm --ignorearch {" ".join(makepkg_compile_opts)}' + build_cmd = f'cd /src/{package.path} && echo $PATH && makepkg --needed --noconfirm --ignorearch {" ".join(makepkg_compile_opts)}' result = run_chroot_cmd(build_cmd, chroot_path=build_root, env=env) - umount() + umount_result = umount(src_dir) + if umount_result != 0: + logging.warning(f'Failed to unmount {src_dir}') if result.returncode != 0: raise Exception(f'Failed to compile package {package.path}') -def add_package_to_repo(package: Package): +def add_package_to_repo(package: Package, arch: str): logging.info(f'Adding {package.path} to repo') binary_dir = os.path.join(config.get_path('packages'), package.repo) pkgbuild_dir = os.path.join(config.get_path('pkgbuilds'), package.path) - pacman_cache_dir = config.get_path('pacman') + pacman_cache_dir = os.path.join(config.get_path('pacman'), arch) os.makedirs(binary_dir, exist_ok=True) for file in os.listdir(pkgbuild_dir): @@ -552,7 +495,7 @@ def cmd_build(paths: list[str], force=False, arch=None): crosscompile = config.file['build']['crosscompile'] for package in need_build: build_package(package, arch=arch, enable_crosscompile=crosscompile) - add_package_to_repo(package) + add_package_to_repo(package, arch) @cmd_packages.command(name='clean') diff --git a/utils.py b/utils.py index a30fbc0..b56e5a0 100644 --- a/utils.py +++ b/utils.py @@ -1,4 +1,6 @@ from shutil import which +import atexit +import subprocess def programs_available(programs) -> bool: @@ -8,3 +10,33 @@ def programs_available(programs) -> bool: if not which(program): return False return True + + +def umount(dest): + return subprocess.run( + [ + 'umount', + '-lc', + dest, + ], + stderr=subprocess.DEVNULL, + ) + + +def mount(src: str, dest: str, options=['bind'], type=None) -> subprocess.CompletedProcess: + opts = [] + type = [] + + for opt in options: + opts += ['-o', opt] + + if type: + type = ['-t', type] + + result = subprocess.run(['mount'] + type + opts + [ + src, + dest, + ]) + if result.returncode == 0: + atexit.register(umount, dest) + return result diff --git a/wrapper.py b/wrapper.py index af8a4da..17ac415 100644 --- a/wrapper.py +++ b/wrapper.py @@ -11,7 +11,7 @@ from utils import programs_available DOCKER_PATHS = { 'chroots': '/chroot', 'jumpdrive': '/var/cache/jumpdrive', - 'pacman': '/var/cache/pacman/pkg', + 'pacman': '/var/cache/pacman', 'packages': '/prebuilts', 'pkgbuilds': '/src', }