Split up repos by arches, always add already-built packages to repo file again, don't use repo-add --new, ...
Signed-off-by: InsanePrawn <insane.prawny@gmail.com>
This commit is contained in:
parent
1ba4dcfaec
commit
0b2caa02af
6 changed files with 155 additions and 96 deletions
60
chroot.py
60
chroot.py
|
@ -139,23 +139,34 @@ def create_chroot_user(
|
|||
raise Exception('Failed to setup user')
|
||||
|
||||
|
||||
def try_install_packages(packages: list[str], chroot: str) -> dict[str, subprocess.CompletedProcess]:
|
||||
"""Try installing packages one by one"""
|
||||
results = {}
|
||||
for pkg in set(packages):
|
||||
# Don't check for errors here because there might be packages that are listed as dependencies but are not available on x86_64
|
||||
results[pkg] = run_chroot_cmd(f'pacman -Syy --noconfirm --needed {pkg}', chroot)
|
||||
def try_install_packages(packages: list[str], chroot: str, refresh: bool = False, allow_fail: bool = True) -> dict[str, subprocess.CompletedProcess]:
|
||||
"""Try installing packages, fall back to installing one by one"""
|
||||
if refresh:
|
||||
run_chroot_cmd('pacman -Syy --noconfirm', chroot)
|
||||
cmd = 'pacman -S --noconfirm --needed'
|
||||
result = run_chroot_cmd(f'{cmd} {" ".join(packages)}', chroot)
|
||||
results = {package: result for package in packages}
|
||||
if result.returncode != 0 and allow_fail:
|
||||
results = {}
|
||||
logging.debug('Falling back to serial installation')
|
||||
for pkg in set(packages):
|
||||
# Don't check for errors here because there might be packages that are listed as dependencies but are not available on x86_64
|
||||
results[pkg] = run_chroot_cmd(f'{cmd} {pkg}', chroot)
|
||||
return results
|
||||
|
||||
|
||||
def mount_crossdirect(native_chroot: str, target_chroot: str, target_arch: str, host_arch: str = None):
|
||||
"""
|
||||
mount `native_chroot` at `target_chroot`/native
|
||||
returns the absolute path that `native_chroot` has been mounted at.
|
||||
"""
|
||||
if host_arch is None:
|
||||
host_arch = config.runtime['arch']
|
||||
gcc = f'{GCC_HOSTSPECS[host_arch][target_arch]}-gcc'
|
||||
|
||||
native_mount = os.path.join(target_chroot, 'native')
|
||||
logging.debug(f'Activating crossdirect in {native_mount}')
|
||||
results = try_install_packages(CROSSDIRECT_PKGS + [gcc], native_chroot)
|
||||
results = try_install_packages(CROSSDIRECT_PKGS + [gcc], native_chroot, refresh=True, allow_fail=False)
|
||||
if results[gcc].returncode != 0:
|
||||
logging.debug('Failed to install cross-compiler package {gcc}')
|
||||
if results['crossdirect'].returncode != 0:
|
||||
|
@ -170,18 +181,45 @@ def mount_crossdirect(native_chroot: str, target_chroot: str, target_arch: str,
|
|||
result = mount(native_chroot, native_mount)
|
||||
if result.returncode != 0:
|
||||
raise Exception(f'Failed to mount native chroot {native_chroot} to {native_mount}')
|
||||
return native_mount
|
||||
|
||||
|
||||
def mount_relative(chroot_path: str, absolute_source: str, relative_target: str) -> tuple[subprocess.CompletedProcess, str]:
|
||||
"""returns the absolute path `relative_target` was mounted at"""
|
||||
target = os.path.join(chroot_path, relative_target.lstrip('/'))
|
||||
os.makedirs(target, exist_ok=True)
|
||||
result = mount(absolute_source, target)
|
||||
return result, target
|
||||
|
||||
|
||||
def mount_pacman_cache(chroot_path: str, arch: str) -> str:
|
||||
global_cache = os.path.join(config.get_path('pacman'), arch)
|
||||
relative = os.path.join('var', 'cache', 'pacman', arch)
|
||||
result, absolute = mount_relative(chroot_path, global_cache, relative)
|
||||
if result.returncode != 0:
|
||||
raise Exception(f'Failed to mount {global_cache} to {absolute}')
|
||||
return absolute
|
||||
|
||||
|
||||
def mount_packages(chroot_path, arch) -> str:
|
||||
packages = config.get_package_dir(arch)
|
||||
result, absolute = mount_relative(chroot_path, absolute_source=packages, relative_target=packages.lstrip('/'))
|
||||
if result.returncode != 0:
|
||||
raise Exception(f'Failed to mount {packages} to {absolute}: {result.returncode}')
|
||||
return absolute
|
||||
|
||||
|
||||
def write_cross_makepkg_conf(native_chroot: str, arch: str, target_chroot_relative: str, cross: bool = True) -> str:
|
||||
"""
|
||||
Generate a makepkg_cross_$arch.conf file in `native_chroot`/etc, building for `target_chroot_relative`
|
||||
Returns the absolute (host) path to the makepkg config file.
|
||||
Returns the relative (to `native_chroot`) path to written file, e.g. `etc/makepkg_cross_aarch64.conf`.
|
||||
"""
|
||||
makepkg_cross_conf = generate_makepkg_conf(arch, cross=cross, chroot=target_chroot_relative)
|
||||
makepkg_conf_path = os.path.join(native_chroot, 'etc', f'makepkg_cross_{arch}.conf')
|
||||
makepkg_conf_path_relative = os.path.join('etc', f'makepkg_cross_{arch}.conf')
|
||||
makepkg_conf_path = os.path.join(native_chroot, makepkg_conf_path_relative)
|
||||
with open(makepkg_conf_path, 'w') as f:
|
||||
f.write(makepkg_cross_conf)
|
||||
return makepkg_conf_path
|
||||
return makepkg_conf_path_relative
|
||||
|
||||
|
||||
@click.command('chroot')
|
||||
|
@ -226,5 +264,5 @@ def cmd_chroot(type: str = 'build', arch: str = None, enable_crossdirect=True):
|
|||
mount_crossdirect(native_chroot=native_chroot, target_chroot=chroot_path, target_arch=arch)
|
||||
|
||||
cmd = ['arch-chroot', chroot_path, '/bin/bash']
|
||||
logging.debug('Starting chroot: ' + repr(cmd))
|
||||
logging.debug('Starting chroot shell: ' + repr(cmd))
|
||||
subprocess.call(cmd)
|
||||
|
|
|
@ -280,6 +280,9 @@ class ConfigStateHolder:
|
|||
paths = self.file['paths']
|
||||
return resolve_path_template(paths[path_name], paths)
|
||||
|
||||
def get_package_dir(self, arch: str):
|
||||
return os.path.join(self.get_path('packages'), arch)
|
||||
|
||||
def dump(self) -> str:
|
||||
dump_toml(self.file)
|
||||
|
||||
|
|
|
@ -201,4 +201,4 @@ def get_kupfer_https(arch: str) -> Distro:
|
|||
|
||||
|
||||
def get_kupfer_local(arch: str) -> Distro:
|
||||
return get_kupfer(arch, f"file://{config.get_path('packages')}/$repo")
|
||||
return get_kupfer(arch, f"file://{config.get_path('packages')}/$arch/$repo")
|
||||
|
|
2
image.py
2
image.py
|
@ -161,7 +161,7 @@ def cmd_build():
|
|||
rootfs_mount = get_chroot_path(chroot_name)
|
||||
mount_rootfs_image(image_name, rootfs_mount)
|
||||
|
||||
packages_dir = config.get_path('packages')
|
||||
packages_dir = config.get_packages(arch)
|
||||
if os.path.exists(os.path.join(packages_dir, 'main')):
|
||||
extra_repos = get_kupfer_local(arch).repos
|
||||
else:
|
||||
|
|
170
packages.py
170
packages.py
|
@ -9,7 +9,7 @@ from joblib import Parallel, delayed
|
|||
|
||||
from constants import REPOSITORIES, CROSSDIRECT_PKGS, GCC_HOSTSPECS
|
||||
from config import config
|
||||
from chroot import create_chroot, run_chroot_cmd, try_install_packages, mount_crossdirect, write_cross_makepkg_conf
|
||||
from chroot import create_chroot, run_chroot_cmd, try_install_packages, mount_crossdirect, write_cross_makepkg_conf, mount_packages, mount_pacman_cache
|
||||
from distro import get_kupfer_local
|
||||
from wrapper import enforce_wrap, check_programs_wrap
|
||||
from utils import mount, umount
|
||||
|
@ -98,8 +98,8 @@ class Package:
|
|||
return f'package({self.name},{repr(self.names)})'
|
||||
|
||||
|
||||
def check_prebuilts(dir: str = None):
|
||||
prebuilts_dir = dir if dir else config.get_path('packages')
|
||||
def check_prebuilts(arch: str, dir: str = None):
|
||||
prebuilts_dir = dir if dir else config.get_package_dir(arch)
|
||||
os.makedirs(prebuilts_dir, exist_ok=True)
|
||||
for repo in REPOSITORIES:
|
||||
os.makedirs(os.path.join(prebuilts_dir, repo), exist_ok=True)
|
||||
|
@ -271,10 +271,60 @@ def generate_dependency_chain(package_repo: dict[str, Package], to_build: list[P
|
|||
return list([lvl for lvl in dep_levels[::-1] if lvl])
|
||||
|
||||
|
||||
def check_package_version_built(package: Package, arch) -> bool:
|
||||
built = True
|
||||
def add_file_to_repo(file_path: str, repo_name: str, arch: str):
|
||||
repo_dir = os.path.join(config.get_package_dir(arch), repo_name)
|
||||
pacman_cache_dir = os.path.join(config.get_path('pacman'), arch)
|
||||
file_name = os.path.basename(file_path)
|
||||
target_file = os.path.join(repo_dir, file_name)
|
||||
|
||||
config_path = write_cross_makepkg_conf(native_chroot='/', arch=arch, target_chroot_relative=None, cross=False)
|
||||
os.makedirs(repo_dir, exist_ok=True)
|
||||
if file_path != target_file:
|
||||
logging.debug(f'moving {file_path} to {target_file} ({repo_dir})')
|
||||
shutil.copy(
|
||||
file_path,
|
||||
repo_dir,
|
||||
)
|
||||
os.unlink(file_path)
|
||||
|
||||
# clean up same name package from pacman cache
|
||||
cache_file = os.path.join(pacman_cache_dir, file_name)
|
||||
if os.path.exists(cache_file):
|
||||
os.unlink(cache_file)
|
||||
result = subprocess.run([
|
||||
'repo-add',
|
||||
'--remove',
|
||||
'--prevent-downgrade',
|
||||
os.path.join(
|
||||
repo_dir,
|
||||
f'{repo_name}.db.tar.xz',
|
||||
),
|
||||
target_file,
|
||||
])
|
||||
if result.returncode != 0:
|
||||
raise Exception(f'Failed add package {target_file} to repo {repo_name}')
|
||||
for ext in ['db', 'files']:
|
||||
file = os.path.join(repo_dir, f'{repo_name}.{ext}')
|
||||
if os.path.exists(file + '.tar.xz'):
|
||||
os.unlink(file)
|
||||
shutil.copyfile(file + '.tar.xz', file)
|
||||
old = file + '.tar.xz.old'
|
||||
if os.path.exists(old):
|
||||
os.unlink(old)
|
||||
|
||||
|
||||
def add_package_to_repo(package: Package, arch: str):
|
||||
logging.info(f'Adding {package.path} to repo {package.repo}')
|
||||
pkgbuild_dir = os.path.join(config.get_path('pkgbuilds'), package.path)
|
||||
|
||||
for file in os.listdir(pkgbuild_dir):
|
||||
# Forced extension by makepkg.conf
|
||||
if file.endswith('.pkg.tar.xz') or file.endswith('.pkg.tar.zst'):
|
||||
return add_file_to_repo(os.path.join(pkgbuild_dir, file), package.repo, arch)
|
||||
|
||||
|
||||
def check_package_version_built(package: Package, arch) -> bool:
|
||||
|
||||
config_path = '/' + write_cross_makepkg_conf(native_chroot='/', arch=arch, target_chroot_relative=None, cross=False)
|
||||
|
||||
result = subprocess.run(
|
||||
makepkg_cmd + [
|
||||
|
@ -289,17 +339,17 @@ def check_package_version_built(package: Package, arch) -> bool:
|
|||
capture_output=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
logging.fatal(f'Failed to get package list for {package.path}:' + '\n' + result.stdout.decode() + '\n' + result.stderr.decode())
|
||||
exit(1)
|
||||
raise Exception(f'Failed to get package list for {package.path}:' + '\n' + result.stdout.decode() + '\n' + result.stderr.decode())
|
||||
|
||||
for line in result.stdout.decode('utf-8').split('\n'):
|
||||
if line != "":
|
||||
file = os.path.join(config.get_path('packages'), package.repo, os.path.basename(line))
|
||||
file = os.path.join(config.get_package_dir(arch), package.repo, os.path.basename(line))
|
||||
logging.debug(f'Checking if {file} is built')
|
||||
if not os.path.exists(file):
|
||||
built = False
|
||||
if os.path.exists(file):
|
||||
add_file_to_repo(file, repo_name=package.repo, arch=arch)
|
||||
return True
|
||||
|
||||
return built
|
||||
return False
|
||||
|
||||
|
||||
def setup_build_chroot(arch: str, extra_packages=[]) -> str:
|
||||
|
@ -311,6 +361,7 @@ def setup_build_chroot(arch: str, extra_packages=[]) -> str:
|
|||
packages=list(set(['base-devel', 'git', 'ccache'] + extra_packages)),
|
||||
extra_repos=get_kupfer_local(arch).repos,
|
||||
)
|
||||
pacman_cache = mount_pacman_cache(chroot_path, arch)
|
||||
|
||||
logging.info(f'Updating chroot {chroot_name}')
|
||||
result = subprocess.run(
|
||||
|
@ -328,11 +379,12 @@ def setup_build_chroot(arch: str, extra_packages=[]) -> str:
|
|||
logging.fatal(result.stdout)
|
||||
logging.fatal(result.stderr)
|
||||
raise Exception(f'Failed to update chroot {chroot_name}')
|
||||
umount(pacman_cache)
|
||||
return chroot_path
|
||||
|
||||
|
||||
def setup_sources(package: Package, chroot: str, arch: str, repo_dir: str = None):
|
||||
repo_dir = repo_dir if repo_dir else config.get_path('pkgbuilds')
|
||||
def setup_sources(package: Package, chroot: str, arch: str, pkgbuilds_dir: str = None):
|
||||
pkgbuilds_dir = pkgbuilds_dir if pkgbuilds_dir else config.get_path('pkgbuilds')
|
||||
makepkg_setup_args = [
|
||||
'--nobuild',
|
||||
'--holdver',
|
||||
|
@ -343,7 +395,7 @@ def setup_sources(package: Package, chroot: str, arch: str, repo_dir: str = None
|
|||
result = subprocess.run(
|
||||
[os.path.join(chroot, 'usr/bin/makepkg')] + makepkg_cmd[1:] + makepkg_setup_args,
|
||||
env=makepkg_cross_env | {'PACMAN_CHROOT': chroot},
|
||||
cwd=os.path.join(repo_dir, package.path),
|
||||
cwd=os.path.join(pkgbuilds_dir, package.path),
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise Exception(f'Failed to check sources for {package.path}')
|
||||
|
@ -366,16 +418,14 @@ def build_package(
|
|||
target_chroot = setup_build_chroot(arch=arch, extra_packages=package.depends)
|
||||
native_chroot = setup_build_chroot(arch=config.runtime['arch'], extra_packages=['base-devel']) if foreign_arch else target_chroot
|
||||
cross = foreign_arch and package.mode == 'cross' and enable_crosscompile
|
||||
umount_dirs = []
|
||||
chroots = set([target_chroot, native_chroot])
|
||||
|
||||
# eliminate target_chroot == native_chroot with set()
|
||||
for chroot in set([target_chroot, native_chroot]):
|
||||
pkgs_path = config.get_path('packages')
|
||||
chroot_pkgs = chroot + pkgs_path # NOTE: DO NOT USE PATH.JOIN, pkgs_path starts with /
|
||||
os.makedirs(chroot_pkgs, exist_ok=True)
|
||||
logging.debug(f'Mounting packages to {chroot_pkgs}')
|
||||
result = mount(pkgs_path, chroot_pkgs)
|
||||
if result.returncode != 0:
|
||||
raise Exception(f'Unable to mount packages to {chroot_pkgs}')
|
||||
for chroot, _arch in [(native_chroot, config.runtime['arch']), (target_chroot, arch)]:
|
||||
logging.debug(f'Mounting packages to {chroot}')
|
||||
dir = mount_packages(chroot, _arch)
|
||||
umount_dirs += [dir]
|
||||
|
||||
if cross:
|
||||
logging.info(f'Cross-compiling {package.path}')
|
||||
|
@ -386,13 +436,18 @@ def build_package(
|
|||
if enable_ccache:
|
||||
env['PATH'] = f"/usr/lib/ccache:{env['PATH']}"
|
||||
logging.info('Setting up dependencies for cross-compilation')
|
||||
try_install_packages(package.depends + ['crossdirect', f"{GCC_HOSTSPECS[config.runtime['arch']][arch]}-gcc"], native_chroot)
|
||||
# include crossdirect for ccache symlinks.
|
||||
results = try_install_packages(package.depends + ['crossdirect', f"{GCC_HOSTSPECS[config.runtime['arch']][arch]}-gcc"], native_chroot)
|
||||
if results['crossdirect'].returncode != 0:
|
||||
raise Exception('Unable to install crossdirect')
|
||||
# mount foreign arch chroot inside native chroot
|
||||
chroot_relative = os.path.join('chroot', os.path.basename(target_chroot))
|
||||
chroot_mount_path = os.path.join(native_chroot, chroot_relative)
|
||||
write_cross_makepkg_conf(native_chroot=native_chroot, arch=arch, target_chroot_relative=chroot_relative)
|
||||
makepkg_relative = write_cross_makepkg_conf(native_chroot=native_chroot, arch=arch, target_chroot_relative=chroot_relative)
|
||||
makepkg_conf_path = os.path.join('/', makepkg_relative)
|
||||
os.makedirs(chroot_mount_path)
|
||||
mount(target_chroot, chroot_mount_path)
|
||||
umount_dirs += [chroot_mount_path]
|
||||
else:
|
||||
logging.info(f'Host-compiling {package.path}')
|
||||
build_root = target_chroot
|
||||
|
@ -402,9 +457,9 @@ def build_package(
|
|||
env['PATH'] = f"/usr/lib/ccache:{env['PATH']}"
|
||||
if not foreign_arch:
|
||||
logging.debug('Building for native arch, skipping crossdirect.')
|
||||
elif enable_crossdirect and not package.name in CROSSDIRECT_PKGS:
|
||||
elif enable_crossdirect and package.name not in CROSSDIRECT_PKGS:
|
||||
env['PATH'] = f"/native/usr/lib/crossdirect/{arch}:{env['PATH']}"
|
||||
mount_crossdirect(native_chroot=native_chroot, target_chroot=target_chroot, target_arch=arch)
|
||||
umount_dirs += [mount_crossdirect(native_chroot=native_chroot, target_chroot=target_chroot, target_arch=arch)]
|
||||
else:
|
||||
logging.debug('Skipping crossdirect.')
|
||||
|
||||
|
@ -414,60 +469,22 @@ def build_package(
|
|||
|
||||
result = mount(config.get_path('pkgbuilds'), src_dir)
|
||||
if result.returncode != 0:
|
||||
raise Exception(f'Failed to bind mount pkgdirs to {build_root}/src')
|
||||
raise Exception(f'Failed to bind mount pkgbuilds to {build_root}/src')
|
||||
umount_dirs += [src_dir]
|
||||
|
||||
makepkg_conf_absolute = os.path.join('/', makepkg_conf_path)
|
||||
build_cmd = f'cd /src/{package.path} && makepkg --config {makepkg_conf_absolute} --needed --noconfirm --ignorearch {" ".join(makepkg_compile_opts)}'
|
||||
logging.debug(f'Building: Running {build_cmd}')
|
||||
result = run_chroot_cmd(build_cmd, chroot_path=build_root, inner_env=env)
|
||||
umount_result = umount(src_dir)
|
||||
if umount_result != 0:
|
||||
logging.warning(f'Failed to unmount {src_dir}')
|
||||
|
||||
if result.returncode != 0:
|
||||
raise Exception(f'Failed to compile package {package.path}')
|
||||
|
||||
|
||||
def add_package_to_repo(package: Package, arch: str):
|
||||
logging.info(f'Adding {package.path} to repo')
|
||||
binary_dir = os.path.join(config.get_path('packages'), package.repo)
|
||||
pkgbuild_dir = os.path.join(config.get_path('pkgbuilds'), package.path)
|
||||
pacman_cache_dir = os.path.join(config.get_path('pacman'), arch)
|
||||
os.makedirs(binary_dir, exist_ok=True)
|
||||
|
||||
for file in os.listdir(pkgbuild_dir):
|
||||
# Forced extension by makepkg.conf
|
||||
if file.endswith('.pkg.tar.xz') or file.endswith('.pkg.tar.zst'):
|
||||
shutil.move(
|
||||
os.path.join(pkgbuild_dir, file),
|
||||
os.path.join(binary_dir, file),
|
||||
)
|
||||
# clean up same name package from pacman cache
|
||||
cache_file = os.path.join(pacman_cache_dir, file)
|
||||
if os.path.exists(cache_file):
|
||||
os.unlink(cache_file)
|
||||
result = subprocess.run([
|
||||
'repo-add',
|
||||
'--remove',
|
||||
'--new',
|
||||
'--prevent-downgrade',
|
||||
os.path.join(
|
||||
binary_dir,
|
||||
f'{package.repo}.db.tar.xz',
|
||||
),
|
||||
os.path.join(binary_dir, file),
|
||||
])
|
||||
if result.returncode != 0:
|
||||
logging.fatal(f'Failed add package {package.path} to repo')
|
||||
exit(1)
|
||||
for repo in REPOSITORIES:
|
||||
for ext in ['db', 'files']:
|
||||
if os.path.exists(os.path.join(binary_dir, f'{repo}.{ext}.tar.xz')):
|
||||
os.unlink(os.path.join(binary_dir, f'{repo}.{ext}'))
|
||||
shutil.copyfile(
|
||||
os.path.join(binary_dir, f'{repo}.{ext}.tar.xz'),
|
||||
os.path.join(binary_dir, f'{repo}.{ext}'),
|
||||
)
|
||||
if os.path.exists(os.path.join(binary_dir, f'{repo}.{ext}.tar.xz.old')):
|
||||
os.unlink(os.path.join(binary_dir, f'{repo}.{ext}.tar.xz.old'))
|
||||
# cleanup
|
||||
for dir in umount_dirs:
|
||||
umount_result = umount(dir)
|
||||
if umount_result != 0:
|
||||
logging.warning(f'Failed to unmount {dir}')
|
||||
|
||||
|
||||
@click.group(name='packages')
|
||||
|
@ -485,7 +502,8 @@ def cmd_build(paths: list[str], force=False, arch=None):
|
|||
# arch = config.get_profile()...
|
||||
arch = 'aarch64'
|
||||
|
||||
check_prebuilts()
|
||||
for _arch in set([arch, config.runtime['arch']]):
|
||||
check_prebuilts(_arch)
|
||||
|
||||
paths = list(paths)
|
||||
repo = discover_packages()
|
||||
|
@ -522,7 +540,7 @@ def cmd_build(paths: list[str], force=False, arch=None):
|
|||
|
||||
@cmd_packages.command(name='clean')
|
||||
def cmd_clean():
|
||||
check_programs_wrap('git')
|
||||
enforce_wrap()
|
||||
result = subprocess.run([
|
||||
'git',
|
||||
'clean',
|
||||
|
|
14
utils.py
14
utils.py
|
@ -33,13 +33,13 @@ def mount(src: str, dest: str, options=['bind'], type=None) -> subprocess.Comple
|
|||
if type:
|
||||
type = ['-t', type]
|
||||
|
||||
result = subprocess.run(['mount'] + type + opts + [
|
||||
src,
|
||||
dest,
|
||||
])
|
||||
if result.returncode == 0:
|
||||
atexit.register(umount, dest)
|
||||
return result
|
||||
result = subprocess.run(
|
||||
['mount'] + type + opts + [
|
||||
src,
|
||||
dest,
|
||||
],
|
||||
capture_output=False,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
atexit.register(umount, dest)
|
||||
return result
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue