Split up repos by arches, always add already-built packages to repo file again, don't use repo-add --new, ...

Signed-off-by: InsanePrawn <insane.prawny@gmail.com>
This commit is contained in:
InsanePrawn 2021-10-04 20:16:27 +02:00
parent 1ba4dcfaec
commit 0b2caa02af
6 changed files with 155 additions and 96 deletions

View file

@ -139,23 +139,34 @@ def create_chroot_user(
raise Exception('Failed to setup user')
def try_install_packages(packages: list[str], chroot: str) -> dict[str, subprocess.CompletedProcess]:
"""Try installing packages one by one"""
results = {}
for pkg in set(packages):
# Don't check for errors here because there might be packages that are listed as dependencies but are not available on x86_64
results[pkg] = run_chroot_cmd(f'pacman -Syy --noconfirm --needed {pkg}', chroot)
def try_install_packages(packages: list[str], chroot: str, refresh: bool = False, allow_fail: bool = True) -> dict[str, subprocess.CompletedProcess]:
"""Try installing packages, fall back to installing one by one"""
if refresh:
run_chroot_cmd('pacman -Syy --noconfirm', chroot)
cmd = 'pacman -S --noconfirm --needed'
result = run_chroot_cmd(f'{cmd} {" ".join(packages)}', chroot)
results = {package: result for package in packages}
if result.returncode != 0 and allow_fail:
results = {}
logging.debug('Falling back to serial installation')
for pkg in set(packages):
# Don't check for errors here because there might be packages that are listed as dependencies but are not available on x86_64
results[pkg] = run_chroot_cmd(f'{cmd} {pkg}', chroot)
return results
def mount_crossdirect(native_chroot: str, target_chroot: str, target_arch: str, host_arch: str = None):
"""
mount `native_chroot` at `target_chroot`/native
returns the absolute path that `native_chroot` has been mounted at.
"""
if host_arch is None:
host_arch = config.runtime['arch']
gcc = f'{GCC_HOSTSPECS[host_arch][target_arch]}-gcc'
native_mount = os.path.join(target_chroot, 'native')
logging.debug(f'Activating crossdirect in {native_mount}')
results = try_install_packages(CROSSDIRECT_PKGS + [gcc], native_chroot)
results = try_install_packages(CROSSDIRECT_PKGS + [gcc], native_chroot, refresh=True, allow_fail=False)
if results[gcc].returncode != 0:
logging.debug('Failed to install cross-compiler package {gcc}')
if results['crossdirect'].returncode != 0:
@ -170,18 +181,45 @@ def mount_crossdirect(native_chroot: str, target_chroot: str, target_arch: str,
result = mount(native_chroot, native_mount)
if result.returncode != 0:
raise Exception(f'Failed to mount native chroot {native_chroot} to {native_mount}')
return native_mount
def mount_relative(chroot_path: str, absolute_source: str, relative_target: str) -> tuple[subprocess.CompletedProcess, str]:
"""returns the absolute path `relative_target` was mounted at"""
target = os.path.join(chroot_path, relative_target.lstrip('/'))
os.makedirs(target, exist_ok=True)
result = mount(absolute_source, target)
return result, target
def mount_pacman_cache(chroot_path: str, arch: str) -> str:
global_cache = os.path.join(config.get_path('pacman'), arch)
relative = os.path.join('var', 'cache', 'pacman', arch)
result, absolute = mount_relative(chroot_path, global_cache, relative)
if result.returncode != 0:
raise Exception(f'Failed to mount {global_cache} to {absolute}')
return absolute
def mount_packages(chroot_path, arch) -> str:
packages = config.get_package_dir(arch)
result, absolute = mount_relative(chroot_path, absolute_source=packages, relative_target=packages.lstrip('/'))
if result.returncode != 0:
raise Exception(f'Failed to mount {packages} to {absolute}: {result.returncode}')
return absolute
def write_cross_makepkg_conf(native_chroot: str, arch: str, target_chroot_relative: str, cross: bool = True) -> str:
"""
Generate a makepkg_cross_$arch.conf file in `native_chroot`/etc, building for `target_chroot_relative`
Returns the absolute (host) path to the makepkg config file.
Returns the relative (to `native_chroot`) path to written file, e.g. `etc/makepkg_cross_aarch64.conf`.
"""
makepkg_cross_conf = generate_makepkg_conf(arch, cross=cross, chroot=target_chroot_relative)
makepkg_conf_path = os.path.join(native_chroot, 'etc', f'makepkg_cross_{arch}.conf')
makepkg_conf_path_relative = os.path.join('etc', f'makepkg_cross_{arch}.conf')
makepkg_conf_path = os.path.join(native_chroot, makepkg_conf_path_relative)
with open(makepkg_conf_path, 'w') as f:
f.write(makepkg_cross_conf)
return makepkg_conf_path
return makepkg_conf_path_relative
@click.command('chroot')
@ -226,5 +264,5 @@ def cmd_chroot(type: str = 'build', arch: str = None, enable_crossdirect=True):
mount_crossdirect(native_chroot=native_chroot, target_chroot=chroot_path, target_arch=arch)
cmd = ['arch-chroot', chroot_path, '/bin/bash']
logging.debug('Starting chroot: ' + repr(cmd))
logging.debug('Starting chroot shell: ' + repr(cmd))
subprocess.call(cmd)

View file

@ -280,6 +280,9 @@ class ConfigStateHolder:
paths = self.file['paths']
return resolve_path_template(paths[path_name], paths)
def get_package_dir(self, arch: str):
return os.path.join(self.get_path('packages'), arch)
def dump(self) -> str:
dump_toml(self.file)

View file

@ -201,4 +201,4 @@ def get_kupfer_https(arch: str) -> Distro:
def get_kupfer_local(arch: str) -> Distro:
return get_kupfer(arch, f"file://{config.get_path('packages')}/$repo")
return get_kupfer(arch, f"file://{config.get_path('packages')}/$arch/$repo")

View file

@ -161,7 +161,7 @@ def cmd_build():
rootfs_mount = get_chroot_path(chroot_name)
mount_rootfs_image(image_name, rootfs_mount)
packages_dir = config.get_path('packages')
packages_dir = config.get_packages(arch)
if os.path.exists(os.path.join(packages_dir, 'main')):
extra_repos = get_kupfer_local(arch).repos
else:

View file

@ -9,7 +9,7 @@ from joblib import Parallel, delayed
from constants import REPOSITORIES, CROSSDIRECT_PKGS, GCC_HOSTSPECS
from config import config
from chroot import create_chroot, run_chroot_cmd, try_install_packages, mount_crossdirect, write_cross_makepkg_conf
from chroot import create_chroot, run_chroot_cmd, try_install_packages, mount_crossdirect, write_cross_makepkg_conf, mount_packages, mount_pacman_cache
from distro import get_kupfer_local
from wrapper import enforce_wrap, check_programs_wrap
from utils import mount, umount
@ -98,8 +98,8 @@ class Package:
return f'package({self.name},{repr(self.names)})'
def check_prebuilts(dir: str = None):
prebuilts_dir = dir if dir else config.get_path('packages')
def check_prebuilts(arch: str, dir: str = None):
prebuilts_dir = dir if dir else config.get_package_dir(arch)
os.makedirs(prebuilts_dir, exist_ok=True)
for repo in REPOSITORIES:
os.makedirs(os.path.join(prebuilts_dir, repo), exist_ok=True)
@ -271,10 +271,60 @@ def generate_dependency_chain(package_repo: dict[str, Package], to_build: list[P
return list([lvl for lvl in dep_levels[::-1] if lvl])
def check_package_version_built(package: Package, arch) -> bool:
built = True
def add_file_to_repo(file_path: str, repo_name: str, arch: str):
repo_dir = os.path.join(config.get_package_dir(arch), repo_name)
pacman_cache_dir = os.path.join(config.get_path('pacman'), arch)
file_name = os.path.basename(file_path)
target_file = os.path.join(repo_dir, file_name)
config_path = write_cross_makepkg_conf(native_chroot='/', arch=arch, target_chroot_relative=None, cross=False)
os.makedirs(repo_dir, exist_ok=True)
if file_path != target_file:
logging.debug(f'moving {file_path} to {target_file} ({repo_dir})')
shutil.copy(
file_path,
repo_dir,
)
os.unlink(file_path)
# clean up same name package from pacman cache
cache_file = os.path.join(pacman_cache_dir, file_name)
if os.path.exists(cache_file):
os.unlink(cache_file)
result = subprocess.run([
'repo-add',
'--remove',
'--prevent-downgrade',
os.path.join(
repo_dir,
f'{repo_name}.db.tar.xz',
),
target_file,
])
if result.returncode != 0:
raise Exception(f'Failed add package {target_file} to repo {repo_name}')
for ext in ['db', 'files']:
file = os.path.join(repo_dir, f'{repo_name}.{ext}')
if os.path.exists(file + '.tar.xz'):
os.unlink(file)
shutil.copyfile(file + '.tar.xz', file)
old = file + '.tar.xz.old'
if os.path.exists(old):
os.unlink(old)
def add_package_to_repo(package: Package, arch: str):
logging.info(f'Adding {package.path} to repo {package.repo}')
pkgbuild_dir = os.path.join(config.get_path('pkgbuilds'), package.path)
for file in os.listdir(pkgbuild_dir):
# Forced extension by makepkg.conf
if file.endswith('.pkg.tar.xz') or file.endswith('.pkg.tar.zst'):
return add_file_to_repo(os.path.join(pkgbuild_dir, file), package.repo, arch)
def check_package_version_built(package: Package, arch) -> bool:
config_path = '/' + write_cross_makepkg_conf(native_chroot='/', arch=arch, target_chroot_relative=None, cross=False)
result = subprocess.run(
makepkg_cmd + [
@ -289,17 +339,17 @@ def check_package_version_built(package: Package, arch) -> bool:
capture_output=True,
)
if result.returncode != 0:
logging.fatal(f'Failed to get package list for {package.path}:' + '\n' + result.stdout.decode() + '\n' + result.stderr.decode())
exit(1)
raise Exception(f'Failed to get package list for {package.path}:' + '\n' + result.stdout.decode() + '\n' + result.stderr.decode())
for line in result.stdout.decode('utf-8').split('\n'):
if line != "":
file = os.path.join(config.get_path('packages'), package.repo, os.path.basename(line))
file = os.path.join(config.get_package_dir(arch), package.repo, os.path.basename(line))
logging.debug(f'Checking if {file} is built')
if not os.path.exists(file):
built = False
if os.path.exists(file):
add_file_to_repo(file, repo_name=package.repo, arch=arch)
return True
return built
return False
def setup_build_chroot(arch: str, extra_packages=[]) -> str:
@ -311,6 +361,7 @@ def setup_build_chroot(arch: str, extra_packages=[]) -> str:
packages=list(set(['base-devel', 'git', 'ccache'] + extra_packages)),
extra_repos=get_kupfer_local(arch).repos,
)
pacman_cache = mount_pacman_cache(chroot_path, arch)
logging.info(f'Updating chroot {chroot_name}')
result = subprocess.run(
@ -328,11 +379,12 @@ def setup_build_chroot(arch: str, extra_packages=[]) -> str:
logging.fatal(result.stdout)
logging.fatal(result.stderr)
raise Exception(f'Failed to update chroot {chroot_name}')
umount(pacman_cache)
return chroot_path
def setup_sources(package: Package, chroot: str, arch: str, repo_dir: str = None):
repo_dir = repo_dir if repo_dir else config.get_path('pkgbuilds')
def setup_sources(package: Package, chroot: str, arch: str, pkgbuilds_dir: str = None):
pkgbuilds_dir = pkgbuilds_dir if pkgbuilds_dir else config.get_path('pkgbuilds')
makepkg_setup_args = [
'--nobuild',
'--holdver',
@ -343,7 +395,7 @@ def setup_sources(package: Package, chroot: str, arch: str, repo_dir: str = None
result = subprocess.run(
[os.path.join(chroot, 'usr/bin/makepkg')] + makepkg_cmd[1:] + makepkg_setup_args,
env=makepkg_cross_env | {'PACMAN_CHROOT': chroot},
cwd=os.path.join(repo_dir, package.path),
cwd=os.path.join(pkgbuilds_dir, package.path),
)
if result.returncode != 0:
raise Exception(f'Failed to check sources for {package.path}')
@ -366,16 +418,14 @@ def build_package(
target_chroot = setup_build_chroot(arch=arch, extra_packages=package.depends)
native_chroot = setup_build_chroot(arch=config.runtime['arch'], extra_packages=['base-devel']) if foreign_arch else target_chroot
cross = foreign_arch and package.mode == 'cross' and enable_crosscompile
umount_dirs = []
chroots = set([target_chroot, native_chroot])
# eliminate target_chroot == native_chroot with set()
for chroot in set([target_chroot, native_chroot]):
pkgs_path = config.get_path('packages')
chroot_pkgs = chroot + pkgs_path # NOTE: DO NOT USE PATH.JOIN, pkgs_path starts with /
os.makedirs(chroot_pkgs, exist_ok=True)
logging.debug(f'Mounting packages to {chroot_pkgs}')
result = mount(pkgs_path, chroot_pkgs)
if result.returncode != 0:
raise Exception(f'Unable to mount packages to {chroot_pkgs}')
for chroot, _arch in [(native_chroot, config.runtime['arch']), (target_chroot, arch)]:
logging.debug(f'Mounting packages to {chroot}')
dir = mount_packages(chroot, _arch)
umount_dirs += [dir]
if cross:
logging.info(f'Cross-compiling {package.path}')
@ -386,13 +436,18 @@ def build_package(
if enable_ccache:
env['PATH'] = f"/usr/lib/ccache:{env['PATH']}"
logging.info('Setting up dependencies for cross-compilation')
try_install_packages(package.depends + ['crossdirect', f"{GCC_HOSTSPECS[config.runtime['arch']][arch]}-gcc"], native_chroot)
# include crossdirect for ccache symlinks.
results = try_install_packages(package.depends + ['crossdirect', f"{GCC_HOSTSPECS[config.runtime['arch']][arch]}-gcc"], native_chroot)
if results['crossdirect'].returncode != 0:
raise Exception('Unable to install crossdirect')
# mount foreign arch chroot inside native chroot
chroot_relative = os.path.join('chroot', os.path.basename(target_chroot))
chroot_mount_path = os.path.join(native_chroot, chroot_relative)
write_cross_makepkg_conf(native_chroot=native_chroot, arch=arch, target_chroot_relative=chroot_relative)
makepkg_relative = write_cross_makepkg_conf(native_chroot=native_chroot, arch=arch, target_chroot_relative=chroot_relative)
makepkg_conf_path = os.path.join('/', makepkg_relative)
os.makedirs(chroot_mount_path)
mount(target_chroot, chroot_mount_path)
umount_dirs += [chroot_mount_path]
else:
logging.info(f'Host-compiling {package.path}')
build_root = target_chroot
@ -402,9 +457,9 @@ def build_package(
env['PATH'] = f"/usr/lib/ccache:{env['PATH']}"
if not foreign_arch:
logging.debug('Building for native arch, skipping crossdirect.')
elif enable_crossdirect and not package.name in CROSSDIRECT_PKGS:
elif enable_crossdirect and package.name not in CROSSDIRECT_PKGS:
env['PATH'] = f"/native/usr/lib/crossdirect/{arch}:{env['PATH']}"
mount_crossdirect(native_chroot=native_chroot, target_chroot=target_chroot, target_arch=arch)
umount_dirs += [mount_crossdirect(native_chroot=native_chroot, target_chroot=target_chroot, target_arch=arch)]
else:
logging.debug('Skipping crossdirect.')
@ -414,60 +469,22 @@ def build_package(
result = mount(config.get_path('pkgbuilds'), src_dir)
if result.returncode != 0:
raise Exception(f'Failed to bind mount pkgdirs to {build_root}/src')
raise Exception(f'Failed to bind mount pkgbuilds to {build_root}/src')
umount_dirs += [src_dir]
makepkg_conf_absolute = os.path.join('/', makepkg_conf_path)
build_cmd = f'cd /src/{package.path} && makepkg --config {makepkg_conf_absolute} --needed --noconfirm --ignorearch {" ".join(makepkg_compile_opts)}'
logging.debug(f'Building: Running {build_cmd}')
result = run_chroot_cmd(build_cmd, chroot_path=build_root, inner_env=env)
umount_result = umount(src_dir)
if umount_result != 0:
logging.warning(f'Failed to unmount {src_dir}')
if result.returncode != 0:
raise Exception(f'Failed to compile package {package.path}')
def add_package_to_repo(package: Package, arch: str):
logging.info(f'Adding {package.path} to repo')
binary_dir = os.path.join(config.get_path('packages'), package.repo)
pkgbuild_dir = os.path.join(config.get_path('pkgbuilds'), package.path)
pacman_cache_dir = os.path.join(config.get_path('pacman'), arch)
os.makedirs(binary_dir, exist_ok=True)
for file in os.listdir(pkgbuild_dir):
# Forced extension by makepkg.conf
if file.endswith('.pkg.tar.xz') or file.endswith('.pkg.tar.zst'):
shutil.move(
os.path.join(pkgbuild_dir, file),
os.path.join(binary_dir, file),
)
# clean up same name package from pacman cache
cache_file = os.path.join(pacman_cache_dir, file)
if os.path.exists(cache_file):
os.unlink(cache_file)
result = subprocess.run([
'repo-add',
'--remove',
'--new',
'--prevent-downgrade',
os.path.join(
binary_dir,
f'{package.repo}.db.tar.xz',
),
os.path.join(binary_dir, file),
])
if result.returncode != 0:
logging.fatal(f'Failed add package {package.path} to repo')
exit(1)
for repo in REPOSITORIES:
for ext in ['db', 'files']:
if os.path.exists(os.path.join(binary_dir, f'{repo}.{ext}.tar.xz')):
os.unlink(os.path.join(binary_dir, f'{repo}.{ext}'))
shutil.copyfile(
os.path.join(binary_dir, f'{repo}.{ext}.tar.xz'),
os.path.join(binary_dir, f'{repo}.{ext}'),
)
if os.path.exists(os.path.join(binary_dir, f'{repo}.{ext}.tar.xz.old')):
os.unlink(os.path.join(binary_dir, f'{repo}.{ext}.tar.xz.old'))
# cleanup
for dir in umount_dirs:
umount_result = umount(dir)
if umount_result != 0:
logging.warning(f'Failed to unmount {dir}')
@click.group(name='packages')
@ -485,7 +502,8 @@ def cmd_build(paths: list[str], force=False, arch=None):
# arch = config.get_profile()...
arch = 'aarch64'
check_prebuilts()
for _arch in set([arch, config.runtime['arch']]):
check_prebuilts(_arch)
paths = list(paths)
repo = discover_packages()
@ -522,7 +540,7 @@ def cmd_build(paths: list[str], force=False, arch=None):
@cmd_packages.command(name='clean')
def cmd_clean():
check_programs_wrap('git')
enforce_wrap()
result = subprocess.run([
'git',
'clean',

View file

@ -33,13 +33,13 @@ def mount(src: str, dest: str, options=['bind'], type=None) -> subprocess.Comple
if type:
type = ['-t', type]
result = subprocess.run(['mount'] + type + opts + [
src,
dest,
])
if result.returncode == 0:
atexit.register(umount, dest)
return result
result = subprocess.run(
['mount'] + type + opts + [
src,
dest,
],
capture_output=False,
)
if result.returncode == 0:
atexit.register(umount, dest)
return result