parchlinux-mobile/kupferbootstrap
1
0
Fork 1
mirror of https://gitlab.com/kupfer/kupferbootstrap.git synced 2025-02-23 13:45:45 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor packages.py, register qemu-binfmt manually

Browse source
This commit is contained in:
InsanePrawn 2021-10-10 18:24:22 +02:00
parent e6d084b513
commit 87345ab2da
2 changed files with 183 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

77
binfmt.py Normal file
View file

@ -0,0 +1,77 @@
# modifed from pmbootstrap's binfmt.py, Copyright 2018 Oliver Smith, GPL-licensed
import os
import logging
import subprocess
from utils import mount
def binfmt_info():
# Parse the info file
full = {}
info = "/usr/lib/binfmt.d/qemu-static.conf"
logging.debug("parsing: " + info)
with open(info, "r") as handle:
for line in handle:
if line.startswith('#') or ":" not in line:
continue
splitted = line.split(":")
result = {
# _ = splitted[0] # empty
'name': splitted[1],
'type': splitted[2],
'offset': splitted[3],
'magic': splitted[4],
'mask': splitted[5],
'interpreter': splitted[6],
'flags': splitted[7],
'line': line,
}
if not result['name'].startswith('qemu-'):
logging.fatal(f'Unknown binfmt handler "{result["name"]}"')
logging.debug(f'binfmt line: {line}')
continue
arch = ''.join(result['name'].split('-')[1:])
full[arch] = result
return full
def is_registered(arch: str) -> bool:
return os.path.exists("/proc/sys/fs/binfmt_misc/qemu-" + arch)
def register(arch):
if is_registered(arch):
return
lines = binfmt_info()
# Build registration string
# https://en.wikipedia.org/wiki/Binfmt_misc
# :name:type:offset:magic:mask:interpreter:flags
info = lines[arch]
code = info['line']
binfmt = '/proc/sys/fs/binfmt_misc'
register = binfmt + '/register'
if not os.path.exists(register):
logging.info('mounting binfmt_misc')
result = mount('binfmt_misc', binfmt, options=[], fs_type='binfmt_misc')
if result.returncode != 0:
raise Exception(f'Failed mounting binfmt_misc to {binfmt}')
# Register in binfmt_misc
logging.info(f"Registering qemu binfmt ({arch})")
subprocess.run(["sh", "-c", 'echo "' + code + '" > ' + register + ' 2>/dev/null'])
if not is_registered(arch):
logging.debug(f'binfmt line: {code}')
raise Exception(f'Failed to register qemu-user for {arch} with binfmt_misc, {binfmt}/{info["name"]} not found')
def unregister(args, arch):
binfmt_file = "/proc/sys/fs/binfmt_misc/qemu-" + arch
if not os.path.exists(binfmt_file):
return
logging.info(f"Unregistering qemu binfmt ({arch})")
subprocess.run(["sh", "-c", "echo -1 > " + binfmt_file])

153
packages.py
View file

@ -13,6 +13,7 @@ from chroot import create_chroot, run_chroot_cmd, try_install_packages, mount_cr
from distro import get_kupfer_local
from wrapper import enforce_wrap
from utils import mount, umount
from binfmt import register as binfmt_register
makepkg_env = os.environ.copy() | {
'LANG': 'C',
@ -98,7 +99,8 @@ class Package:
return f'package({self.name},{repr(self.names)})'
def check_prebuilts(arch: str, dir: str = None):
def init_prebuilts(arch: str, dir: str = None):
"""Ensure that all `constants.REPOSITORIES` inside `dir` exist"""
prebuilts_dir = dir if dir else config.get_package_dir(arch)
os.makedirs(prebuilts_dir, exist_ok=True)
for repo in REPOSITORIES:
@ -407,16 +409,18 @@ def build_package(
repo_dir: str = None,
enable_crosscompile: bool = True,
enable_crossdirect: bool = True,
enable_ccache=True,
enable_ccache: bool = True,
):
makepkg_compile_opts = [
'--holdver',
]
makepkg_compile_opts = ['--holdver']
makepkg_conf_path = 'etc/makepkg.conf'
repo_dir = repo_dir if repo_dir else config.get_path('pkgbuilds')
foreign_arch = config.runtime['arch'] != arch
target_chroot = setup_build_chroot(arch=arch, extra_packages=(list(set(package.depends) - set(package.names))))
native_chroot = setup_build_chroot(arch=config.runtime['arch'], extra_packages=['base-devel']) if foreign_arch else target_chroot
native_chroot = target_chroot if not foreign_arch else setup_build_chroot(
arch=config.runtime['arch'],
extra_packages=['base-devel'] + CROSSDIRECT_PKGS,
)
cross = foreign_arch and package.mode == 'cross' and enable_crosscompile
umount_dirs = []
set([target_chroot, native_chroot])
@ -436,8 +440,8 @@ def build_package(
if enable_ccache:
env['PATH'] = f"/usr/lib/ccache:{env['PATH']}"
logging.info('Setting up dependencies for cross-compilation')
# include crossdirect for ccache symlinks.
results = try_install_packages(package.depends + ['crossdirect', f"{GCC_HOSTSPECS[config.runtime['arch']][arch]}-gcc"], native_chroot)
# include crossdirect for ccache symlinks and qemu-user
results = try_install_packages(package.depends + CROSSDIRECT_PKGS + [f"{GCC_HOSTSPECS[config.runtime['arch']][arch]}-gcc"], native_chroot)
if results['crossdirect'].returncode != 0:
raise Exception('Unable to install crossdirect')
# mount foreign arch chroot inside native chroot
@ -486,6 +490,78 @@ def build_package(
logging.warning(f'Failed to unmount {dir}')
def get_unbuilt_package_levels(repo: dict[str, Package], packages: list[Package], arch: str, force: bool = False) -> list[set[Package]]:
package_levels = generate_dependency_chain(repo, packages)
build_names = set[str]()
build_levels = list[set[Package]]()
i = 0
for level_packages in package_levels:
level = set[Package]()
for package in level_packages:
if ((not check_package_version_built(package, arch)) or set.intersection(set(package.depends), set(build_names)) or
(force and package in packages)):
level.add(package)
build_names.update(package.names)
if level:
build_levels.append(level)
logging.debug(f'Adding to level {i}:' + '\n' + ('\n'.join([p.name for p in level])))
i += 1
return build_levels
def build_packages(
repo: dict[str, Package],
packages: list[Package],
arch: str,
force: bool = False,
enable_crosscompile: bool = True,
enable_crossdirect: bool = True,
enable_ccache: bool = True,
):
build_levels = get_unbuilt_package_levels(repo, packages, arch, force=force)
if not build_levels:
logging.info('Everything built already')
return
for level, need_build in enumerate(build_levels):
logging.info(f"(Level {level}) Building {', '.join([x.name for x in need_build])}")
for package in need_build:
build_package(
package,
arch=arch,
enable_crosscompile=enable_crosscompile,
enable_crossdirect=enable_crossdirect,
enable_ccache=enable_ccache,
)
add_package_to_repo(package, arch)
def build_packages_by_paths(
paths: list[str],
arch: str,
force=False,
enable_crosscompile: bool = True,
enable_crossdirect: bool = True,
enable_ccache: bool = True,
):
if isinstance(paths, str):
paths = [paths]
for _arch in set([arch, config.runtime['arch']]):
init_prebuilts(_arch)
repo: dict[str, Package] = discover_packages()
packages = filter_packages_by_paths(repo, paths)
build_packages(
repo,
packages,
arch,
force=force,
enable_crosscompile=enable_crosscompile,
enable_crossdirect=enable_crossdirect,
enable_ccache=enable_ccache,
)
@click.group(name='packages')
def cmd_packages():
pass
@ -497,52 +573,35 @@ def cmd_packages():
@click.argument('paths', nargs=-1)
def cmd_build(paths: list[str], force=False, arch=None):
if arch is None:
# arch = config.get_profile()...
# TODO: arch = config.get_profile()...
arch = 'aarch64'
if arch not in ARCHES:
raise Exception(f'Unknown architecture "{arch}". Choices: {", ".join(ARCHES)}')
enforce_wrap()
native = config.runtime['arch']
if arch != native:
# build qemu-user, binfmt, crossdirect
build_packages_by_paths(
['main/' + pkg for pkg in CROSSDIRECT_PKGS],
native,
enable_crosscompile=False,
enable_crossdirect=False,
enable_ccache=False,
)
for pkg in CROSSDIRECT_PKGS:
# do not add --needed so we reinstall qemu+binfmt always in order to register the binfmt handler
subprocess.run(['pacman', '-Syy', pkg, '--noconfirm'])
binfmt_register(arch)
for _arch in set([arch, config.runtime['arch']]):
check_prebuilts(_arch)
paths = list(paths)
repo = discover_packages()
package_levels = generate_dependency_chain(
repo,
filter_packages_by_paths(repo, paths),
build_packages_by_paths(
paths,
arch,
force=force,
enable_crosscompile=config.file['build']['crosscompile'],
enable_crossdirect=config.file['build']['crossdirect'],
enable_ccache=config.file['build']['ccache'],
)
build_names = set[str]()
build_levels = list[set[Package]]()
i = 0
for packages in package_levels:
level = set[Package]()
for package in packages:
if ((not check_package_version_built(package, arch)) or set.intersection(set(package.depends), set(build_names)) or
(force and package.path in paths)):
level.add(package)
build_names.update(package.names)
if level:
build_levels.append(level)
logging.debug(f'Adding to level {i}:' + '\n' + ('\n'.join([p.name for p in level])))
i += 1
if not build_levels:
logging.info('Everything built already')
return
for level, need_build in enumerate(build_levels):
logging.info(f"(Level {level}) Building {', '.join([x.name for x in need_build])}")
for package in need_build:
build_package(
package,
arch=arch,
enable_crosscompile=config.file['build']['crosscompile'],
enable_crossdirect=config.file['build']['crossdirect'],
enable_ccache=config.file['build']['ccache'],
)
add_package_to_repo(package, arch)
@cmd_packages.command(name='clean')