parchlinux-mobile/kupferbootstrap
1
0
Fork 1
mirror of https://gitlab.com/kupfer/kupferbootstrap.git synced 2025-02-23 05:35:44 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'config' of gitlab.com:kupfer/kupferbootstrap into config

Browse source
This commit is contained in:
jld3103 2021-10-10 21:23:49 +02:00
commit 949183b290
6 changed files with 196 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Dockerfile
View file

@ -28,6 +28,6 @@ RUN pip install -r requirements.txt
COPY . .
RUN python -c "import constants; repos='\n'.join(['\n'.join(['', f'[{repo}]', f'Server = file:///prebuilts/\$repo']) for repo in constants.REPOSITORIES]); print(repos)" | tee -a /etc/pacman.conf
RUN python -c "import constants; repos='\n'.join(['\n'.join(['', f'[{repo}]', f'Server = file:///prebuilts/\$arch/\$repo']) for repo in constants.REPOSITORIES]); print(repos)" | tee -a /etc/pacman.conf
WORKDIR /src

77
binfmt.py Normal file
View file

@ -0,0 +1,77 @@
# modifed from pmbootstrap's binfmt.py, Copyright 2018 Oliver Smith, GPL-licensed
import os
import logging
import subprocess
from utils import mount
def binfmt_info():
# Parse the info file
full = {}
info = "/usr/lib/binfmt.d/qemu-static.conf"
logging.debug("parsing: " + info)
with open(info, "r") as handle:
for line in handle:
if line.startswith('#') or ":" not in line:
continue
splitted = line.split(":")
result = {
# _ = splitted[0] # empty
'name': splitted[1],
'type': splitted[2],
'offset': splitted[3],
'magic': splitted[4],
'mask': splitted[5],
'interpreter': splitted[6],
'flags': splitted[7],
'line': line,
}
if not result['name'].startswith('qemu-'):
logging.fatal(f'Unknown binfmt handler "{result["name"]}"')
logging.debug(f'binfmt line: {line}')
continue
arch = ''.join(result['name'].split('-')[1:])
full[arch] = result
return full
def is_registered(arch: str) -> bool:
return os.path.exists("/proc/sys/fs/binfmt_misc/qemu-" + arch)
def register(arch):
if is_registered(arch):
return
lines = binfmt_info()
# Build registration string
# https://en.wikipedia.org/wiki/Binfmt_misc
# :name:type:offset:magic:mask:interpreter:flags
info = lines[arch]
code = info['line']
binfmt = '/proc/sys/fs/binfmt_misc'
register = binfmt + '/register'
if not os.path.exists(register):
logging.info('mounting binfmt_misc')
result = mount('binfmt_misc', binfmt, options=[], fs_type='binfmt_misc')
if result.returncode != 0:
raise Exception(f'Failed mounting binfmt_misc to {binfmt}')
# Register in binfmt_misc
logging.info(f"Registering qemu binfmt ({arch})")
subprocess.run(["sh", "-c", 'echo "' + code + '" > ' + register + ' 2>/dev/null'])
if not is_registered(arch):
logging.debug(f'binfmt line: {code}')
raise Exception(f'Failed to register qemu-user for {arch} with binfmt_misc, {binfmt}/{info["name"]} not found')
def unregister(args, arch):
binfmt_file = "/proc/sys/fs/binfmt_misc/qemu-" + arch
if not os.path.exists(binfmt_file):
return
logging.info(f"Unregistering qemu binfmt ({arch})")
subprocess.run(["sh", "-c", "echo -1 > " + binfmt_file])

3
chroot.py
View file

@ -57,8 +57,7 @@ def create_chroot(chroot_name: str,
file.write(data)
# configure makepkg
with open(f'{chroot_path}/etc/makepkg.conf', 'r') as file:
data = file.read()
data = generate_makepkg_conf(arch, cross=False)
data = data.replace('xz -c', 'xz -T0 -c')
data = data.replace(' check ', ' !check ')
with open(f'{chroot_path}/etc/makepkg.conf', 'w') as file:

13
config.py
View file

@ -366,9 +366,10 @@ def cmd_config_init(sections: list[str] = CONFIG_SECTIONS, non_interactive: bool
results[section] = {}
for key, current in config.file[section].items():
result, changed = config_prompt(text=f'{section}.{key}', default=current, field_type=type(current))
text = f'{section}.{key}'
result, changed = config_prompt(text=text, default=current, field_type=type(current))
if changed:
print(f'{key} = {result}')
print(f'{text} = {result}')
results[section][key] = result
config.update(results)
@ -398,7 +399,8 @@ def config_prompt(text: str, default: any, field_type: type = str, bold: bool =
def true_or_zero(to_check) -> bool:
"""returns true if the value is truthy or int(0)"""
return to_check or to_check == 0
zero = 0 # compiler complains about 'is with literal' otherwise
return to_check or to_check is zero # can't do == due to boolean<->int casting
def list_to_comma_str(str_list: list[str], default='') -> str:
if str_list is None:
@ -448,9 +450,10 @@ def cmd_profile_init(name: str = None, non_interactive: bool = False, noop: bool
if not non_interactive:
for key, current in profile.items():
current = profile[key]
result, changed = config_prompt(text=f'{name}.{key}', default=current, field_type=type(PROFILE_DEFAULTS[key]))
text = f'{name}.{key}'
result, changed = config_prompt(text=text, default=current, field_type=type(PROFILE_DEFAULTS[key]))
if changed:
print(f'{key} = {result}')
print(f'{text} = {result}')
profile[key] = result
config.update_profile(name, profile)

152
packages.py
View file

@ -13,6 +13,7 @@ from chroot import create_chroot, run_chroot_cmd, try_install_packages, mount_cr
from distro import get_kupfer_local
from wrapper import enforce_wrap
from utils import mount, umount
from binfmt import register as binfmt_register
makepkg_env = os.environ.copy() | {
'LANG': 'C',
@ -98,7 +99,8 @@ class Package:
return f'package({self.name},{repr(self.names)})'
def check_prebuilts(arch: str, dir: str = None):
def init_prebuilts(arch: str, dir: str = None):
"""Ensure that all `constants.REPOSITORIES` inside `dir` exist"""
prebuilts_dir = dir if dir else config.get_package_dir(arch)
os.makedirs(prebuilts_dir, exist_ok=True)
for repo in REPOSITORIES:
@ -409,16 +411,18 @@ def build_package(
repo_dir: str = None,
enable_crosscompile: bool = True,
enable_crossdirect: bool = True,
enable_ccache=True,
enable_ccache: bool = True,
):
makepkg_compile_opts = [
'--holdver',
]
makepkg_compile_opts = ['--holdver']
makepkg_conf_path = 'etc/makepkg.conf'
repo_dir = repo_dir if repo_dir else config.get_path('pkgbuilds')
foreign_arch = config.runtime['arch'] != arch
target_chroot = setup_build_chroot(arch=arch, extra_packages=(list(set(package.depends) - set(package.names))))
native_chroot = setup_build_chroot(arch=config.runtime['arch'], extra_packages=['base-devel']) if foreign_arch else target_chroot
native_chroot = target_chroot if not foreign_arch else setup_build_chroot(
arch=config.runtime['arch'],
extra_packages=['base-devel'] + CROSSDIRECT_PKGS,
)
cross = foreign_arch and package.mode == 'cross' and enable_crosscompile
umount_dirs = []
set([target_chroot, native_chroot])
@ -438,8 +442,8 @@ def build_package(
if enable_ccache:
env['PATH'] = f"/usr/lib/ccache:{env['PATH']}"
logging.info('Setting up dependencies for cross-compilation')
# include crossdirect for ccache symlinks.
results = try_install_packages(package.depends + ['crossdirect', f"{GCC_HOSTSPECS[config.runtime['arch']][arch]}-gcc"], native_chroot)
# include crossdirect for ccache symlinks and qemu-user
results = try_install_packages(package.depends + CROSSDIRECT_PKGS + [f"{GCC_HOSTSPECS[config.runtime['arch']][arch]}-gcc"], native_chroot)
if results['crossdirect'].returncode != 0:
raise Exception('Unable to install crossdirect')
# mount foreign arch chroot inside native chroot
@ -488,6 +492,78 @@ def build_package(
logging.warning(f'Failed to unmount {dir}')
def get_unbuilt_package_levels(repo: dict[str, Package], packages: list[Package], arch: str, force: bool = False) -> list[set[Package]]:
package_levels = generate_dependency_chain(repo, packages)
build_names = set[str]()
build_levels = list[set[Package]]()
i = 0
for level_packages in package_levels:
level = set[Package]()
for package in level_packages:
if ((not check_package_version_built(package, arch)) or set.intersection(set(package.depends), set(build_names)) or
(force and package in packages)):
level.add(package)
build_names.update(package.names)
if level:
build_levels.append(level)
logging.debug(f'Adding to level {i}:' + '\n' + ('\n'.join([p.name for p in level])))
i += 1
return build_levels
def build_packages(
repo: dict[str, Package],
packages: list[Package],
arch: str,
force: bool = False,
enable_crosscompile: bool = True,
enable_crossdirect: bool = True,
enable_ccache: bool = True,
):
build_levels = get_unbuilt_package_levels(repo, packages, arch, force=force)
if not build_levels:
logging.info('Everything built already')
return
for level, need_build in enumerate(build_levels):
logging.info(f"(Level {level}) Building {', '.join([x.name for x in need_build])}")
for package in need_build:
build_package(
package,
arch=arch,
enable_crosscompile=enable_crosscompile,
enable_crossdirect=enable_crossdirect,
enable_ccache=enable_ccache,
)
add_package_to_repo(package, arch)
def build_packages_by_paths(
paths: list[str],
arch: str,
force=False,
enable_crosscompile: bool = True,
enable_crossdirect: bool = True,
enable_ccache: bool = True,
):
if isinstance(paths, str):
paths = [paths]
for _arch in set([arch, config.runtime['arch']]):
init_prebuilts(_arch)
repo: dict[str, Package] = discover_packages()
packages = filter_packages_by_paths(repo, paths)
build_packages(
repo,
packages,
arch,
force=force,
enable_crosscompile=enable_crosscompile,
enable_crossdirect=enable_crossdirect,
enable_ccache=enable_ccache,
)
@click.group(name='packages')
def cmd_packages():
pass
@ -499,52 +575,34 @@ def cmd_packages():
@click.argument('paths', nargs=-1)
def cmd_build(paths: list[str], force=False, arch=None):
if arch is None:
# arch = config.get_profile()...
# TODO: arch = config.get_profile()...
arch = 'aarch64'
if arch not in ARCHES:
raise Exception(f'Unknown architecture "{arch}". Choices: {", ".join(ARCHES)}')
enforce_wrap()
native = config.runtime['arch']
if arch != native:
# build qemu-user, binfmt, crossdirect
build_packages_by_paths(
['main/' + pkg for pkg in CROSSDIRECT_PKGS],
native,
enable_crosscompile=False,
enable_crossdirect=False,
enable_ccache=False,
)
for pkg in CROSSDIRECT_PKGS:
subprocess.run(['pacman', '-Syy', pkg, '--noconfirm', '--needed'])
binfmt_register(arch)
for _arch in set([arch, config.runtime['arch']]):
check_prebuilts(_arch)
paths = list(paths)
repo = discover_packages()
package_levels = generate_dependency_chain(
repo,
filter_packages_by_paths(repo, paths),
build_packages_by_paths(
paths,
arch,
force=force,
enable_crosscompile=config.file['build']['crosscompile'],
enable_crossdirect=config.file['build']['crossdirect'],
enable_ccache=config.file['build']['ccache'],
)
build_names = set[str]()
build_levels = list[set[Package]]()
i = 0
for packages in package_levels:
level = set[Package]()
for package in packages:
if ((not check_package_version_built(package, arch)) or set.intersection(set(package.depends), set(build_names)) or
(force and package.path in paths)):
level.add(package)
build_names.update(package.names)
if level:
build_levels.append(level)
logging.debug(f'Adding to level {i}:' + '\n' + ('\n'.join([p.name for p in level])))
i += 1
if not build_levels:
logging.info('Everything built already')
return
for level, need_build in enumerate(build_levels):
logging.info(f"(Level {level}) Building {', '.join([x.name for x in need_build])}")
for package in need_build:
build_package(
package,
arch=arch,
enable_crosscompile=config.file['build']['crosscompile'],
enable_crossdirect=config.file['build']['crossdirect'],
enable_ccache=config.file['build']['ccache'],
)
add_package_to_repo(package, arch)
@cmd_packages.command(name='clean')

10
utils.py
View file

@ -23,18 +23,16 @@ def umount(dest):
)
def mount(src: str, dest: str, options=['bind'], type=None) -> subprocess.CompletedProcess:
def mount(src: str, dest: str, options=['bind'], fs_type=None) -> subprocess.CompletedProcess:
opts = []
type = []
for opt in options:
opts += ['-o', opt]
if type:
type = ['-t', type]
if fs_type:
opts += ['-t', fs_type]
result = subprocess.run(
['mount'] + type + opts + [
['mount'] + opts + [
src,
dest,
],