Split up repos by arches, always add already-built packages to repo file again, don't use repo-add --new, ...

Signed-off-by: InsanePrawn <insane.prawny@gmail.com>

chroot.py

@@ -139,23 +139,34 @@ def create_chroot_user(
         raise Exception('Failed to setup user')
 
 
-def try_install_packages(packages: list[str], chroot: str) -> dict[str, subprocess.CompletedProcess]:
+def try_install_packages(packages: list[str], chroot: str, refresh: bool = False, allow_fail: bool = True) -> dict[str, subprocess.CompletedProcess]:
-    """Try installing packages one by one"""
+    """Try installing packages, fall back to installing one by one"""
+    if refresh:
+        run_chroot_cmd('pacman -Syy --noconfirm', chroot)
+    cmd = 'pacman -S --noconfirm --needed'
+    result = run_chroot_cmd(f'{cmd} {" ".join(packages)}', chroot)
+    results = {package: result for package in packages}
+    if result.returncode != 0 and allow_fail:
         results = {}
+        logging.debug('Falling back to serial installation')
         for pkg in set(packages):
             # Don't check for errors here because there might be packages that are listed as dependencies but are not available on x86_64
-        results[pkg] = run_chroot_cmd(f'pacman -Syy --noconfirm --needed {pkg}', chroot)
+            results[pkg] = run_chroot_cmd(f'{cmd} {pkg}', chroot)
     return results
 
 
 def mount_crossdirect(native_chroot: str, target_chroot: str, target_arch: str, host_arch: str = None):
+    """
+    mount `native_chroot` at `target_chroot`/native
+    returns the absolute path that `native_chroot` has been mounted at.
+    """
     if host_arch is None:
         host_arch = config.runtime['arch']
     gcc = f'{GCC_HOSTSPECS[host_arch][target_arch]}-gcc'
 
     native_mount = os.path.join(target_chroot, 'native')
     logging.debug(f'Activating crossdirect in {native_mount}')
-    results = try_install_packages(CROSSDIRECT_PKGS + [gcc], native_chroot)
+    results = try_install_packages(CROSSDIRECT_PKGS + [gcc], native_chroot, refresh=True, allow_fail=False)
     if results[gcc].returncode != 0:
         logging.debug('Failed to install cross-compiler package {gcc}')
     if results['crossdirect'].returncode != 0:
@@ -170,18 +181,45 @@ def mount_crossdirect(native_chroot: str, target_chroot: str, target_arch: str,
     result = mount(native_chroot, native_mount)
     if result.returncode != 0:
         raise Exception(f'Failed to mount native chroot {native_chroot} to {native_mount}')
+    return native_mount
+
+
+def mount_relative(chroot_path: str, absolute_source: str, relative_target: str) -> tuple[subprocess.CompletedProcess, str]:
+    """returns the absolute path `relative_target` was mounted at"""
+    target = os.path.join(chroot_path, relative_target.lstrip('/'))
+    os.makedirs(target, exist_ok=True)
+    result = mount(absolute_source, target)
+    return result, target
+
+
+def mount_pacman_cache(chroot_path: str, arch: str) -> str:
+    global_cache = os.path.join(config.get_path('pacman'), arch)
+    relative = os.path.join('var', 'cache', 'pacman', arch)
+    result, absolute = mount_relative(chroot_path, global_cache, relative)
+    if result.returncode != 0:
+        raise Exception(f'Failed to mount {global_cache} to {absolute}')
+    return absolute
+
+
+def mount_packages(chroot_path, arch) -> str:
+    packages = config.get_package_dir(arch)
+    result, absolute = mount_relative(chroot_path, absolute_source=packages, relative_target=packages.lstrip('/'))
+    if result.returncode != 0:
+        raise Exception(f'Failed to mount {packages} to {absolute}: {result.returncode}')
+    return absolute
 
 
 def write_cross_makepkg_conf(native_chroot: str, arch: str, target_chroot_relative: str, cross: bool = True) -> str:
     """
     Generate a makepkg_cross_$arch.conf file in `native_chroot`/etc, building for `target_chroot_relative`
-    Returns the absolute (host) path to the makepkg config file.
+    Returns the relative (to `native_chroot`) path to written file, e.g. `etc/makepkg_cross_aarch64.conf`.
     """
     makepkg_cross_conf = generate_makepkg_conf(arch, cross=cross, chroot=target_chroot_relative)
-    makepkg_conf_path = os.path.join(native_chroot, 'etc', f'makepkg_cross_{arch}.conf')
+    makepkg_conf_path_relative = os.path.join('etc', f'makepkg_cross_{arch}.conf')
+    makepkg_conf_path = os.path.join(native_chroot, makepkg_conf_path_relative)
     with open(makepkg_conf_path, 'w') as f:
         f.write(makepkg_cross_conf)
-    return makepkg_conf_path
+    return makepkg_conf_path_relative
 
 
 @click.command('chroot')
@@ -226,5 +264,5 @@ def cmd_chroot(type: str = 'build', arch: str = None, enable_crossdirect=True):
             mount_crossdirect(native_chroot=native_chroot, target_chroot=chroot_path, target_arch=arch)
 
     cmd = ['arch-chroot', chroot_path, '/bin/bash']
-    logging.debug('Starting chroot: ' + repr(cmd))
+    logging.debug('Starting chroot shell: ' + repr(cmd))
     subprocess.call(cmd)

config.py

@@ -280,6 +280,9 @@ class ConfigStateHolder:
         paths = self.file['paths']
         return resolve_path_template(paths[path_name], paths)
 
+    def get_package_dir(self, arch: str):
+        return os.path.join(self.get_path('packages'), arch)
+
     def dump(self) -> str:
         dump_toml(self.file)
 

distro.py

@@ -201,4 +201,4 @@ def get_kupfer_https(arch: str) -> Distro:
 
 
 def get_kupfer_local(arch: str) -> Distro:
-    return get_kupfer(arch, f"file://{config.get_path('packages')}/$repo")
+    return get_kupfer(arch, f"file://{config.get_path('packages')}/$arch/$repo")

image.py

@@ -161,7 +161,7 @@ def cmd_build():
     rootfs_mount = get_chroot_path(chroot_name)
     mount_rootfs_image(image_name, rootfs_mount)
 
-    packages_dir = config.get_path('packages')
+    packages_dir = config.get_packages(arch)
     if os.path.exists(os.path.join(packages_dir, 'main')):
         extra_repos = get_kupfer_local(arch).repos
     else:

packages.py

@@ -9,7 +9,7 @@ from joblib import Parallel, delayed
 
 from constants import REPOSITORIES, CROSSDIRECT_PKGS, GCC_HOSTSPECS
 from config import config
-from chroot import create_chroot, run_chroot_cmd, try_install_packages, mount_crossdirect, write_cross_makepkg_conf
+from chroot import create_chroot, run_chroot_cmd, try_install_packages, mount_crossdirect, write_cross_makepkg_conf, mount_packages, mount_pacman_cache
 from distro import get_kupfer_local
 from wrapper import enforce_wrap, check_programs_wrap
 from utils import mount, umount
@@ -98,8 +98,8 @@ class Package:
         return f'package({self.name},{repr(self.names)})'
 
 
-def check_prebuilts(dir: str = None):
+def check_prebuilts(arch: str, dir: str = None):
-    prebuilts_dir = dir if dir else config.get_path('packages')
+    prebuilts_dir = dir if dir else config.get_package_dir(arch)
     os.makedirs(prebuilts_dir, exist_ok=True)
     for repo in REPOSITORIES:
         os.makedirs(os.path.join(prebuilts_dir, repo), exist_ok=True)
@@ -271,10 +271,60 @@ def generate_dependency_chain(package_repo: dict[str, Package], to_build: list[P
     return list([lvl for lvl in dep_levels[::-1] if lvl])
 
 
-def check_package_version_built(package: Package, arch) -> bool:
+def add_file_to_repo(file_path: str, repo_name: str, arch: str):
-    built = True
+    repo_dir = os.path.join(config.get_package_dir(arch), repo_name)
+    pacman_cache_dir = os.path.join(config.get_path('pacman'), arch)
+    file_name = os.path.basename(file_path)
+    target_file = os.path.join(repo_dir, file_name)
 
-    config_path = write_cross_makepkg_conf(native_chroot='/', arch=arch, target_chroot_relative=None, cross=False)
+    os.makedirs(repo_dir, exist_ok=True)
+    if file_path != target_file:
+        logging.debug(f'moving {file_path} to {target_file} ({repo_dir})')
+        shutil.copy(
+            file_path,
+            repo_dir,
+        )
+        os.unlink(file_path)
+
+    # clean up same name package from pacman cache
+    cache_file = os.path.join(pacman_cache_dir, file_name)
+    if os.path.exists(cache_file):
+        os.unlink(cache_file)
+    result = subprocess.run([
+        'repo-add',
+        '--remove',
+        '--prevent-downgrade',
+        os.path.join(
+            repo_dir,
+            f'{repo_name}.db.tar.xz',
+        ),
+        target_file,
+    ])
+    if result.returncode != 0:
+        raise Exception(f'Failed add package {target_file} to repo {repo_name}')
+    for ext in ['db', 'files']:
+        file = os.path.join(repo_dir, f'{repo_name}.{ext}')
+        if os.path.exists(file + '.tar.xz'):
+            os.unlink(file)
+            shutil.copyfile(file + '.tar.xz', file)
+        old = file + '.tar.xz.old'
+        if os.path.exists(old):
+            os.unlink(old)
+
+
+def add_package_to_repo(package: Package, arch: str):
+    logging.info(f'Adding {package.path} to repo {package.repo}')
+    pkgbuild_dir = os.path.join(config.get_path('pkgbuilds'), package.path)
+
+    for file in os.listdir(pkgbuild_dir):
+        # Forced extension by makepkg.conf
+        if file.endswith('.pkg.tar.xz') or file.endswith('.pkg.tar.zst'):
+            return add_file_to_repo(os.path.join(pkgbuild_dir, file), package.repo, arch)
+
+
+def check_package_version_built(package: Package, arch) -> bool:
+
+    config_path = '/' + write_cross_makepkg_conf(native_chroot='/', arch=arch, target_chroot_relative=None, cross=False)
 
     result = subprocess.run(
         makepkg_cmd + [
@@ -289,17 +339,17 @@ def check_package_version_built(package: Package, arch) -> bool:
         capture_output=True,
     )
     if result.returncode != 0:
-        logging.fatal(f'Failed to get package list for {package.path}:' + '\n' + result.stdout.decode() + '\n' + result.stderr.decode())
+        raise Exception(f'Failed to get package list for {package.path}:' + '\n' + result.stdout.decode() + '\n' + result.stderr.decode())
-        exit(1)
 
     for line in result.stdout.decode('utf-8').split('\n'):
         if line != "":
-            file = os.path.join(config.get_path('packages'), package.repo, os.path.basename(line))
+            file = os.path.join(config.get_package_dir(arch), package.repo, os.path.basename(line))
             logging.debug(f'Checking if {file} is built')
-            if not os.path.exists(file):
+            if os.path.exists(file):
-                built = False
+                add_file_to_repo(file, repo_name=package.repo, arch=arch)
+                return True
 
-    return built
+    return False
 
 
 def setup_build_chroot(arch: str, extra_packages=[]) -> str:
@@ -311,6 +361,7 @@ def setup_build_chroot(arch: str, extra_packages=[]) -> str:
         packages=list(set(['base-devel', 'git', 'ccache'] + extra_packages)),
         extra_repos=get_kupfer_local(arch).repos,
     )
+    pacman_cache = mount_pacman_cache(chroot_path, arch)
 
     logging.info(f'Updating chroot {chroot_name}')
     result = subprocess.run(
@@ -328,11 +379,12 @@ def setup_build_chroot(arch: str, extra_packages=[]) -> str:
         logging.fatal(result.stdout)
         logging.fatal(result.stderr)
         raise Exception(f'Failed to update chroot {chroot_name}')
+    umount(pacman_cache)
     return chroot_path
 
 
-def setup_sources(package: Package, chroot: str, arch: str, repo_dir: str = None):
+def setup_sources(package: Package, chroot: str, arch: str, pkgbuilds_dir: str = None):
-    repo_dir = repo_dir if repo_dir else config.get_path('pkgbuilds')
+    pkgbuilds_dir = pkgbuilds_dir if pkgbuilds_dir else config.get_path('pkgbuilds')
     makepkg_setup_args = [
         '--nobuild',
         '--holdver',
@@ -343,7 +395,7 @@ def setup_sources(package: Package, chroot: str, arch: str, repo_dir: str = None
     result = subprocess.run(
         [os.path.join(chroot, 'usr/bin/makepkg')] + makepkg_cmd[1:] + makepkg_setup_args,
         env=makepkg_cross_env | {'PACMAN_CHROOT': chroot},
-        cwd=os.path.join(repo_dir, package.path),
+        cwd=os.path.join(pkgbuilds_dir, package.path),
     )
     if result.returncode != 0:
         raise Exception(f'Failed to check sources for {package.path}')
@@ -366,16 +418,14 @@ def build_package(
     target_chroot = setup_build_chroot(arch=arch, extra_packages=package.depends)
     native_chroot = setup_build_chroot(arch=config.runtime['arch'], extra_packages=['base-devel']) if foreign_arch else target_chroot
     cross = foreign_arch and package.mode == 'cross' and enable_crosscompile
+    umount_dirs = []
+    chroots = set([target_chroot, native_chroot])
 
     # eliminate target_chroot == native_chroot with set()
-    for chroot in set([target_chroot, native_chroot]):
+    for chroot, _arch in [(native_chroot, config.runtime['arch']), (target_chroot, arch)]:
-        pkgs_path = config.get_path('packages')
+        logging.debug(f'Mounting packages to {chroot}')
-        chroot_pkgs = chroot + pkgs_path  # NOTE: DO NOT USE PATH.JOIN, pkgs_path starts with /
+        dir = mount_packages(chroot, _arch)
-        os.makedirs(chroot_pkgs, exist_ok=True)
+        umount_dirs += [dir]
-        logging.debug(f'Mounting packages to {chroot_pkgs}')
-        result = mount(pkgs_path, chroot_pkgs)
-        if result.returncode != 0:
-            raise Exception(f'Unable to mount packages to {chroot_pkgs}')
 
     if cross:
         logging.info(f'Cross-compiling {package.path}')
@@ -386,13 +436,18 @@ def build_package(
         if enable_ccache:
             env['PATH'] = f"/usr/lib/ccache:{env['PATH']}"
         logging.info('Setting up dependencies for cross-compilation')
-        try_install_packages(package.depends + ['crossdirect', f"{GCC_HOSTSPECS[config.runtime['arch']][arch]}-gcc"], native_chroot)
+        # include crossdirect for ccache symlinks.
+        results = try_install_packages(package.depends + ['crossdirect', f"{GCC_HOSTSPECS[config.runtime['arch']][arch]}-gcc"], native_chroot)
+        if results['crossdirect'].returncode != 0:
+            raise Exception('Unable to install crossdirect')
         # mount foreign arch chroot inside native chroot
         chroot_relative = os.path.join('chroot', os.path.basename(target_chroot))
         chroot_mount_path = os.path.join(native_chroot, chroot_relative)
-        write_cross_makepkg_conf(native_chroot=native_chroot, arch=arch, target_chroot_relative=chroot_relative)
+        makepkg_relative = write_cross_makepkg_conf(native_chroot=native_chroot, arch=arch, target_chroot_relative=chroot_relative)
+        makepkg_conf_path = os.path.join('/', makepkg_relative)
         os.makedirs(chroot_mount_path)
         mount(target_chroot, chroot_mount_path)
+        umount_dirs += [chroot_mount_path]
     else:
         logging.info(f'Host-compiling {package.path}')
         build_root = target_chroot
@@ -402,9 +457,9 @@ def build_package(
             env['PATH'] = f"/usr/lib/ccache:{env['PATH']}"
         if not foreign_arch:
             logging.debug('Building for native arch, skipping crossdirect.')
-        elif enable_crossdirect and not package.name in CROSSDIRECT_PKGS:
+        elif enable_crossdirect and package.name not in CROSSDIRECT_PKGS:
             env['PATH'] = f"/native/usr/lib/crossdirect/{arch}:{env['PATH']}"
-            mount_crossdirect(native_chroot=native_chroot, target_chroot=target_chroot, target_arch=arch)
+            umount_dirs += [mount_crossdirect(native_chroot=native_chroot, target_chroot=target_chroot, target_arch=arch)]
         else:
             logging.debug('Skipping crossdirect.')
 
@@ -414,60 +469,22 @@ def build_package(
 
     result = mount(config.get_path('pkgbuilds'), src_dir)
     if result.returncode != 0:
-        raise Exception(f'Failed to bind mount pkgdirs to {build_root}/src')
+        raise Exception(f'Failed to bind mount pkgbuilds to {build_root}/src')
+    umount_dirs += [src_dir]
 
     makepkg_conf_absolute = os.path.join('/', makepkg_conf_path)
     build_cmd = f'cd /src/{package.path} && makepkg --config {makepkg_conf_absolute} --needed --noconfirm --ignorearch {" ".join(makepkg_compile_opts)}'
+    logging.debug(f'Building: Running {build_cmd}')
     result = run_chroot_cmd(build_cmd, chroot_path=build_root, inner_env=env)
-    umount_result = umount(src_dir)
+
-    if umount_result != 0:
-        logging.warning(f'Failed to unmount {src_dir}')
     if result.returncode != 0:
         raise Exception(f'Failed to compile package {package.path}')
 
-
+    # cleanup
-def add_package_to_repo(package: Package, arch: str):
+    for dir in umount_dirs:
-    logging.info(f'Adding {package.path} to repo')
+        umount_result = umount(dir)
-    binary_dir = os.path.join(config.get_path('packages'), package.repo)
+        if umount_result != 0:
-    pkgbuild_dir = os.path.join(config.get_path('pkgbuilds'), package.path)
+            logging.warning(f'Failed to unmount {dir}')
-    pacman_cache_dir = os.path.join(config.get_path('pacman'), arch)
-    os.makedirs(binary_dir, exist_ok=True)
-
-    for file in os.listdir(pkgbuild_dir):
-        # Forced extension by makepkg.conf
-        if file.endswith('.pkg.tar.xz') or file.endswith('.pkg.tar.zst'):
-            shutil.move(
-                os.path.join(pkgbuild_dir, file),
-                os.path.join(binary_dir, file),
-            )
-            # clean up same name package from pacman cache
-            cache_file = os.path.join(pacman_cache_dir, file)
-            if os.path.exists(cache_file):
-                os.unlink(cache_file)
-            result = subprocess.run([
-                'repo-add',
-                '--remove',
-                '--new',
-                '--prevent-downgrade',
-                os.path.join(
-                    binary_dir,
-                    f'{package.repo}.db.tar.xz',
-                ),
-                os.path.join(binary_dir, file),
-            ])
-            if result.returncode != 0:
-                logging.fatal(f'Failed add package {package.path} to repo')
-                exit(1)
-    for repo in REPOSITORIES:
-        for ext in ['db', 'files']:
-            if os.path.exists(os.path.join(binary_dir, f'{repo}.{ext}.tar.xz')):
-                os.unlink(os.path.join(binary_dir, f'{repo}.{ext}'))
-                shutil.copyfile(
-                    os.path.join(binary_dir, f'{repo}.{ext}.tar.xz'),
-                    os.path.join(binary_dir, f'{repo}.{ext}'),
-                )
-            if os.path.exists(os.path.join(binary_dir, f'{repo}.{ext}.tar.xz.old')):
-                os.unlink(os.path.join(binary_dir, f'{repo}.{ext}.tar.xz.old'))
 
 
 @click.group(name='packages')
@@ -485,7 +502,8 @@ def cmd_build(paths: list[str], force=False, arch=None):
         # arch = config.get_profile()...
         arch = 'aarch64'
 
-    check_prebuilts()
+    for _arch in set([arch, config.runtime['arch']]):
+        check_prebuilts(_arch)
 
     paths = list(paths)
     repo = discover_packages()
@@ -522,7 +540,7 @@ def cmd_build(paths: list[str], force=False, arch=None):
 
 @cmd_packages.command(name='clean')
 def cmd_clean():
-    check_programs_wrap('git')
+    enforce_wrap()
     result = subprocess.run([
         'git',
         'clean',

utils.py

@@ -33,13 +33,13 @@ def mount(src: str, dest: str, options=['bind'], type=None) -> subprocess.Comple
     if type:
         type = ['-t', type]
 
-    result = subprocess.run(['mount'] + type + opts + [
+    result = subprocess.run(
+        ['mount'] + type + opts + [
             src,
             dest,
-    ])
+        ],
-    if result.returncode == 0:
+        capture_output=False,
-        atexit.register(umount, dest)
+    )
-    return result
     if result.returncode == 0:
         atexit.register(umount, dest)
     return result