diff --git a/chroot.py b/chroot.py index 69f2c82..db9abca 100644 --- a/chroot.py +++ b/chroot.py @@ -5,10 +5,12 @@ import os from config import config from distro import get_base_distros, RepoInfo from shlex import quote as shell_quote +from shutil import copy from utils import mount from distro import get_kupfer_local from wrapper import enforce_wrap -from constants import GCC_HOSTSPECS +from constants import GCC_HOSTSPECS, CROSSDIRECT_PKGS +from glob import glob BIND_BUILD_DIRS = 'BINDBUILDDIRS' @@ -99,11 +101,13 @@ def create_chroot(chroot_name: str, def run_chroot_cmd(script: str, chroot_path: str, inner_env: dict[str, str] = {}, - outer_env: dict[str, str] = os.environ.copy() | {'QEMU_LD_PREFIX': '/usr/aarch64-linux-gnu'}) -> subprocess.CompletedProcess: + outer_env: dict[str, str] = os.environ.copy() | {'QEMU_LD_PREFIX': '/usr/aarch64-linux-gnu'}, + attach_tty=False) -> subprocess.CompletedProcess: if outer_env is None: outer_env = os.environ.copy() env_cmd = ['/usr/bin/env'] + [f'{shell_quote(key)}={shell_quote(value)}' for key, value in inner_env.items()] - result = subprocess.run(['arch-chroot', chroot_path] + env_cmd + [ + run_func = subprocess.call if attach_tty else subprocess.run + result = run_func(['arch-chroot', chroot_path] + env_cmd + [ '/bin/bash', '-c', script, @@ -150,7 +154,7 @@ def mount_crossdirect(native_chroot: str, target_chroot: str, target_arch: str, native_mount = os.path.join(target_chroot, 'native') logging.debug(f'Activating crossdirect in {native_mount}') - results = try_install_packages(['crossdirect', 'qemu-user-static-bin', 'binfmt-qemu-static-all-arch', gcc], native_chroot) + results = try_install_packages(CROSSDIRECT_PKGS + [gcc], native_chroot) if results[gcc].returncode != 0: logging.debug('Failed to install cross-compiler package {gcc}') if results['crossdirect'].returncode != 0: @@ -158,6 +162,9 @@ def mount_crossdirect(native_chroot: str, target_chroot: str, target_arch: str, os.makedirs(native_mount, exist_ok=True) + ld_so = glob(f"{os.path.join('native_chroot', 'usr', 'lib', 'ld-linux-')}*")[0] + copy(ld_so, os.path.join(target_chroot, 'usr', 'lib')) + logging.debug(f'Mounting {native_chroot} to {native_mount}') result = mount(native_chroot, native_mount) if result.returncode != 0: @@ -200,7 +207,7 @@ def cmd_chroot(type: str = 'build', arch: str = None, enable_crossdirect=True): native_chroot = create_chroot( 'build_' + native_arch, native_arch, - packages=['base-devel'] + (['crossdirect', 'qemu-user-static-bin', 'binfmt-qemu-static-all-arch'] if enable_crossdirect else []), + packages=['base-devel'] + (CROSSDIRECT_PKGS if enable_crossdirect else []), extra_repos=get_kupfer_local(native_arch).repos, ) mount_crossdirect(native_chroot=native_chroot, target_chroot=chroot_path, target_arch=arch) diff --git a/constants.py b/constants.py index 34c280d..27d1f7f 100644 --- a/constants.py +++ b/constants.py @@ -88,4 +88,4 @@ GCC_HOSTSPECS: dict[DistroArch, dict[TargetArch, str]] = { } } -CROSSDIRECT_PKGS = ['crossdirect', 'qemu-user-static-bin', 'binfmt-qemu-static-all-arch'] +CROSSDIRECT_PKGS = ['crossdirect', 'qemu-user-static-bin', 'binfmt-qemu-static'] diff --git a/packages.py b/packages.py index dcf4e04..ba2f0da 100644 --- a/packages.py +++ b/packages.py @@ -4,17 +4,19 @@ import multiprocessing import os import shutil import subprocess -from constants import REPOSITORIES +from constants import REPOSITORIES, CROSSDIRECT_PKGS, GCC_HOSTSPECS from config import config from chroot import create_chroot, run_chroot_cmd, try_install_packages, mount_crossdirect from joblib import Parallel, delayed from distro import get_kupfer_local from wrapper import enforce_wrap, check_programs_wrap from utils import mount, umount +from copy import deepcopy makepkg_env = os.environ.copy() | { 'LANG': 'C', 'MAKEFLAGS': f"-j{multiprocessing.cpu_count() if config.file['build']['threads'] < 1 else config.file['build']['threads']}", + 'QEMU_LD_PREFIX': '/usr/aarch64-unknown-linux-gnu' } makepkg_cross_env = makepkg_env | {'PACMAN': os.path.join(config.runtime['script_source_dir'], 'local/bin/pacman_aarch64')} @@ -324,7 +326,7 @@ def setup_build_chroot(arch: str, extra_packages=[]) -> str: return chroot_path -def setup_sources(package: Package, chroot: str, repo_dir: str = None, enable_crosscompile: bool = True): +def setup_sources(package: Package, chroot: str, arch: str, repo_dir: str = None): repo_dir = repo_dir if repo_dir else config.get_path('pkgbuilds') makepkg_setup_args = [ '--nobuild', @@ -342,7 +344,14 @@ def setup_sources(package: Package, chroot: str, repo_dir: str = None, enable_cr raise Exception(f'Failed to check sources for {package.path}') -def build_package(package: Package, arch: str, repo_dir: str = None, enable_crosscompile: bool = True, enable_crossdirect: bool = True): +def build_package( + package: Package, + arch: str, + repo_dir: str = None, + enable_crosscompile: bool = True, + enable_crossdirect: bool = True, + enable_ccache=True, +): makepkg_compile_opts = [ '--holdver', ] @@ -351,7 +360,6 @@ def build_package(package: Package, arch: str, repo_dir: str = None, enable_cros target_chroot = setup_build_chroot(arch=arch, extra_packages=package.depends) native_chroot = setup_build_chroot(arch=config.runtime['arch'], extra_packages=['base-devel']) if foreign_arch else target_chroot cross = foreign_arch and package.mode == 'cross' and enable_crosscompile - env = {} # eliminate target_chroot == native_chroot with set() for chroot in set([target_chroot, native_chroot]): @@ -367,26 +375,33 @@ def build_package(package: Package, arch: str, repo_dir: str = None, enable_cros logging.info(f'Cross-compiling {package.path}') build_root = native_chroot makepkg_compile_opts += ['--nodeps'] - env = makepkg_env - + env = makepkg_cross_env + if enable_ccache: + env['PATH'] = f"/usr/lib/ccache:{env['PATH']}" logging.info('Setting up dependencies for cross-compilation') - try_install_packages(package.depends, native_chroot) + try_install_packages(package.depends + ['crossdirect', f"{GCC_HOSTSPECS[config.runtime['arch']][arch]}-gcc"], native_chroot) + # mount foreign arch chroot inside native chroot + chroot_mount_path = os.path.join(native_chroot, 'chroot', os.path.basename(target_chroot)) + os.makedirs(chroot_mount_path) + mount(target_chroot, chroot_mount_path) else: logging.info(f'Host-compiling {package.path}') build_root = target_chroot makepkg_compile_opts += ['--syncdeps'] - env = makepkg_env + env = deepcopy(makepkg_env) + if enable_ccache: + env['PATH'] = f"/usr/lib/ccache:{env['PATH']}" if not foreign_arch: logging.debug('Building for native arch, skipping crossdirect.') - elif enable_crossdirect and not package.name in ['crossdirect', 'qemu-user-static-bin', 'binfmt-qemu-static-all-arch']: + elif enable_crossdirect and not package.name in CROSSDIRECT_PKGS: env['PATH'] = f"/native/usr/lib/crossdirect/{arch}:{env['PATH']}" mount_crossdirect(native_chroot=native_chroot, target_chroot=target_chroot, target_arch=arch) else: logging.debug('Skipping crossdirect.') src_dir = os.path.join(build_root, 'src') - os.makedirs(f'{build_root}/src', exist_ok=True) - setup_sources(package, build_root, enable_crosscompile=enable_crosscompile) + os.makedirs(src_dir, exist_ok=True) + #setup_sources(package, build_root, enable_crosscompile=enable_crosscompile) result = mount(config.get_path('pkgbuilds'), src_dir) if result.returncode != 0: diff --git a/utils.py b/utils.py index b56e5a0..c2eac62 100644 --- a/utils.py +++ b/utils.py @@ -19,7 +19,7 @@ def umount(dest): '-lc', dest, ], - stderr=subprocess.DEVNULL, + capture_output=True, ) @@ -40,3 +40,6 @@ def mount(src: str, dest: str, options=['bind'], type=None) -> subprocess.Comple if result.returncode == 0: atexit.register(umount, dest) return result + if result.returncode == 0: + atexit.register(umount, dest) + return result