diff --git a/Dockerfile b/Dockerfile index 1d82a34..2ae60bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,6 +28,6 @@ RUN pip install -r requirements.txt COPY . . -RUN python -c "import constants; repos='\n'.join(['\n'.join(['', f'[{repo}]', f'Server = file:///prebuilts/\$repo']) for repo in constants.REPOSITORIES]); print(repos)" | tee -a /etc/pacman.conf +RUN python -c "import constants; repos='\n'.join(['\n'.join(['', f'[{repo}]', f'Server = file:///prebuilts/\$arch/\$repo']) for repo in constants.REPOSITORIES]); print(repos)" | tee -a /etc/pacman.conf WORKDIR /src diff --git a/binfmt.py b/binfmt.py new file mode 100644 index 0000000..2497db4 --- /dev/null +++ b/binfmt.py @@ -0,0 +1,77 @@ +# modifed from pmbootstrap's binfmt.py, Copyright 2018 Oliver Smith, GPL-licensed + +import os +import logging +import subprocess + +from utils import mount + + +def binfmt_info(): + # Parse the info file + full = {} + info = "/usr/lib/binfmt.d/qemu-static.conf" + logging.debug("parsing: " + info) + with open(info, "r") as handle: + for line in handle: + if line.startswith('#') or ":" not in line: + continue + splitted = line.split(":") + result = { + # _ = splitted[0] # empty + 'name': splitted[1], + 'type': splitted[2], + 'offset': splitted[3], + 'magic': splitted[4], + 'mask': splitted[5], + 'interpreter': splitted[6], + 'flags': splitted[7], + 'line': line, + } + if not result['name'].startswith('qemu-'): + logging.fatal(f'Unknown binfmt handler "{result["name"]}"') + logging.debug(f'binfmt line: {line}') + continue + arch = ''.join(result['name'].split('-')[1:]) + full[arch] = result + + return full + + +def is_registered(arch: str) -> bool: + return os.path.exists("/proc/sys/fs/binfmt_misc/qemu-" + arch) + + +def register(arch): + if is_registered(arch): + return + + lines = binfmt_info() + + # Build registration string + # https://en.wikipedia.org/wiki/Binfmt_misc + # :name:type:offset:magic:mask:interpreter:flags + info = lines[arch] + code = info['line'] + binfmt = '/proc/sys/fs/binfmt_misc' + register = binfmt + '/register' + if not os.path.exists(register): + logging.info('mounting binfmt_misc') + result = mount('binfmt_misc', binfmt, options=[], fs_type='binfmt_misc') + if result.returncode != 0: + raise Exception(f'Failed mounting binfmt_misc to {binfmt}') + + # Register in binfmt_misc + logging.info(f"Registering qemu binfmt ({arch})") + subprocess.run(["sh", "-c", 'echo "' + code + '" > ' + register + ' 2>/dev/null']) + if not is_registered(arch): + logging.debug(f'binfmt line: {code}') + raise Exception(f'Failed to register qemu-user for {arch} with binfmt_misc, {binfmt}/{info["name"]} not found') + + +def unregister(args, arch): + binfmt_file = "/proc/sys/fs/binfmt_misc/qemu-" + arch + if not os.path.exists(binfmt_file): + return + logging.info(f"Unregistering qemu binfmt ({arch})") + subprocess.run(["sh", "-c", "echo -1 > " + binfmt_file]) diff --git a/chroot.py b/chroot.py index 5276846..dd1b342 100644 --- a/chroot.py +++ b/chroot.py @@ -57,8 +57,7 @@ def create_chroot(chroot_name: str, file.write(data) # configure makepkg - with open(f'{chroot_path}/etc/makepkg.conf', 'r') as file: - data = file.read() + data = generate_makepkg_conf(arch, cross=False) data = data.replace('xz -c', 'xz -T0 -c') data = data.replace(' check ', ' !check ') with open(f'{chroot_path}/etc/makepkg.conf', 'w') as file: diff --git a/config.py b/config.py index 58f0cdc..9f94462 100644 --- a/config.py +++ b/config.py @@ -366,9 +366,10 @@ def cmd_config_init(sections: list[str] = CONFIG_SECTIONS, non_interactive: bool results[section] = {} for key, current in config.file[section].items(): - result, changed = config_prompt(text=f'{section}.{key}', default=current, field_type=type(current)) + text = f'{section}.{key}' + result, changed = config_prompt(text=text, default=current, field_type=type(current)) if changed: - print(f'{key} = {result}') + print(f'{text} = {result}') results[section][key] = result config.update(results) @@ -398,7 +399,8 @@ def config_prompt(text: str, default: any, field_type: type = str, bold: bool = def true_or_zero(to_check) -> bool: """returns true if the value is truthy or int(0)""" - return to_check or to_check == 0 + zero = 0 # compiler complains about 'is with literal' otherwise + return to_check or to_check is zero # can't do == due to boolean<->int casting def list_to_comma_str(str_list: list[str], default='') -> str: if str_list is None: @@ -448,9 +450,10 @@ def cmd_profile_init(name: str = None, non_interactive: bool = False, noop: bool if not non_interactive: for key, current in profile.items(): current = profile[key] - result, changed = config_prompt(text=f'{name}.{key}', default=current, field_type=type(PROFILE_DEFAULTS[key])) + text = f'{name}.{key}' + result, changed = config_prompt(text=text, default=current, field_type=type(PROFILE_DEFAULTS[key])) if changed: - print(f'{key} = {result}') + print(f'{text} = {result}') profile[key] = result config.update_profile(name, profile) diff --git a/packages.py b/packages.py index 3afeab8..8d44d06 100644 --- a/packages.py +++ b/packages.py @@ -13,6 +13,7 @@ from chroot import create_chroot, run_chroot_cmd, try_install_packages, mount_cr from distro import get_kupfer_local from wrapper import enforce_wrap from utils import mount, umount +from binfmt import register as binfmt_register makepkg_env = os.environ.copy() | { 'LANG': 'C', @@ -98,7 +99,8 @@ class Package: return f'package({self.name},{repr(self.names)})' -def check_prebuilts(arch: str, dir: str = None): +def init_prebuilts(arch: str, dir: str = None): + """Ensure that all `constants.REPOSITORIES` inside `dir` exist""" prebuilts_dir = dir if dir else config.get_package_dir(arch) os.makedirs(prebuilts_dir, exist_ok=True) for repo in REPOSITORIES: @@ -409,16 +411,18 @@ def build_package( repo_dir: str = None, enable_crosscompile: bool = True, enable_crossdirect: bool = True, - enable_ccache=True, + enable_ccache: bool = True, ): - makepkg_compile_opts = [ - '--holdver', - ] + makepkg_compile_opts = ['--holdver'] makepkg_conf_path = 'etc/makepkg.conf' repo_dir = repo_dir if repo_dir else config.get_path('pkgbuilds') foreign_arch = config.runtime['arch'] != arch target_chroot = setup_build_chroot(arch=arch, extra_packages=(list(set(package.depends) - set(package.names)))) - native_chroot = setup_build_chroot(arch=config.runtime['arch'], extra_packages=['base-devel']) if foreign_arch else target_chroot + native_chroot = target_chroot if not foreign_arch else setup_build_chroot( + arch=config.runtime['arch'], + extra_packages=['base-devel'] + CROSSDIRECT_PKGS, + ) + cross = foreign_arch and package.mode == 'cross' and enable_crosscompile umount_dirs = [] set([target_chroot, native_chroot]) @@ -438,8 +442,8 @@ def build_package( if enable_ccache: env['PATH'] = f"/usr/lib/ccache:{env['PATH']}" logging.info('Setting up dependencies for cross-compilation') - # include crossdirect for ccache symlinks. - results = try_install_packages(package.depends + ['crossdirect', f"{GCC_HOSTSPECS[config.runtime['arch']][arch]}-gcc"], native_chroot) + # include crossdirect for ccache symlinks and qemu-user + results = try_install_packages(package.depends + CROSSDIRECT_PKGS + [f"{GCC_HOSTSPECS[config.runtime['arch']][arch]}-gcc"], native_chroot) if results['crossdirect'].returncode != 0: raise Exception('Unable to install crossdirect') # mount foreign arch chroot inside native chroot @@ -488,6 +492,78 @@ def build_package( logging.warning(f'Failed to unmount {dir}') +def get_unbuilt_package_levels(repo: dict[str, Package], packages: list[Package], arch: str, force: bool = False) -> list[set[Package]]: + package_levels = generate_dependency_chain(repo, packages) + build_names = set[str]() + build_levels = list[set[Package]]() + i = 0 + for level_packages in package_levels: + level = set[Package]() + for package in level_packages: + if ((not check_package_version_built(package, arch)) or set.intersection(set(package.depends), set(build_names)) or + (force and package in packages)): + level.add(package) + build_names.update(package.names) + if level: + build_levels.append(level) + logging.debug(f'Adding to level {i}:' + '\n' + ('\n'.join([p.name for p in level]))) + i += 1 + return build_levels + + +def build_packages( + repo: dict[str, Package], + packages: list[Package], + arch: str, + force: bool = False, + enable_crosscompile: bool = True, + enable_crossdirect: bool = True, + enable_ccache: bool = True, +): + build_levels = get_unbuilt_package_levels(repo, packages, arch, force=force) + + if not build_levels: + logging.info('Everything built already') + return + for level, need_build in enumerate(build_levels): + logging.info(f"(Level {level}) Building {', '.join([x.name for x in need_build])}") + for package in need_build: + build_package( + package, + arch=arch, + enable_crosscompile=enable_crosscompile, + enable_crossdirect=enable_crossdirect, + enable_ccache=enable_ccache, + ) + add_package_to_repo(package, arch) + + +def build_packages_by_paths( + paths: list[str], + arch: str, + force=False, + enable_crosscompile: bool = True, + enable_crossdirect: bool = True, + enable_ccache: bool = True, +): + if isinstance(paths, str): + paths = [paths] + + for _arch in set([arch, config.runtime['arch']]): + init_prebuilts(_arch) + repo: dict[str, Package] = discover_packages() + packages = filter_packages_by_paths(repo, paths) + build_packages( + repo, + packages, + arch, + force=force, + enable_crosscompile=enable_crosscompile, + enable_crossdirect=enable_crossdirect, + enable_ccache=enable_ccache, + ) + + @click.group(name='packages') def cmd_packages(): pass @@ -499,52 +575,34 @@ def cmd_packages(): @click.argument('paths', nargs=-1) def cmd_build(paths: list[str], force=False, arch=None): if arch is None: - # arch = config.get_profile()... + # TODO: arch = config.get_profile()... arch = 'aarch64' if arch not in ARCHES: raise Exception(f'Unknown architecture "{arch}". Choices: {", ".join(ARCHES)}') enforce_wrap() + native = config.runtime['arch'] + if arch != native: + # build qemu-user, binfmt, crossdirect + build_packages_by_paths( + ['main/' + pkg for pkg in CROSSDIRECT_PKGS], + native, + enable_crosscompile=False, + enable_crossdirect=False, + enable_ccache=False, + ) + for pkg in CROSSDIRECT_PKGS: + subprocess.run(['pacman', '-Syy', pkg, '--noconfirm', '--needed']) + binfmt_register(arch) - for _arch in set([arch, config.runtime['arch']]): - check_prebuilts(_arch) - - paths = list(paths) - repo = discover_packages() - - package_levels = generate_dependency_chain( - repo, - filter_packages_by_paths(repo, paths), + build_packages_by_paths( + paths, + arch, + force=force, + enable_crosscompile=config.file['build']['crosscompile'], + enable_crossdirect=config.file['build']['crossdirect'], + enable_ccache=config.file['build']['ccache'], ) - build_names = set[str]() - build_levels = list[set[Package]]() - i = 0 - for packages in package_levels: - level = set[Package]() - for package in packages: - if ((not check_package_version_built(package, arch)) or set.intersection(set(package.depends), set(build_names)) or - (force and package.path in paths)): - level.add(package) - build_names.update(package.names) - if level: - build_levels.append(level) - logging.debug(f'Adding to level {i}:' + '\n' + ('\n'.join([p.name for p in level]))) - i += 1 - - if not build_levels: - logging.info('Everything built already') - return - for level, need_build in enumerate(build_levels): - logging.info(f"(Level {level}) Building {', '.join([x.name for x in need_build])}") - for package in need_build: - build_package( - package, - arch=arch, - enable_crosscompile=config.file['build']['crosscompile'], - enable_crossdirect=config.file['build']['crossdirect'], - enable_ccache=config.file['build']['ccache'], - ) - add_package_to_repo(package, arch) @cmd_packages.command(name='clean') diff --git a/utils.py b/utils.py index 6f68f01..0755208 100644 --- a/utils.py +++ b/utils.py @@ -23,18 +23,16 @@ def umount(dest): ) -def mount(src: str, dest: str, options=['bind'], type=None) -> subprocess.CompletedProcess: +def mount(src: str, dest: str, options=['bind'], fs_type=None) -> subprocess.CompletedProcess: opts = [] - type = [] - for opt in options: opts += ['-o', opt] - if type: - type = ['-t', type] + if fs_type: + opts += ['-t', fs_type] result = subprocess.run( - ['mount'] + type + opts + [ + ['mount'] + opts + [ src, dest, ],