diff --git a/chroot.py b/chroot.py index 219688f..a761115 100644 --- a/chroot.py +++ b/chroot.py @@ -5,14 +5,14 @@ import os import atexit from glob import glob from shutil import rmtree +from shlex import quote as shell_quote from config import config -from distro import get_base_distro, get_kupfer_local, RepoInfo -from shlex import quote as shell_quote -from utils import mount, umount, check_findmnt, log_or_exception +from distro.distro import get_base_distro, get_kupfer_local, RepoInfo from wrapper import enforce_wrap from constants import Arch, GCC_HOSTSPECS, CROSSDIRECT_PKGS, BASE_PACKAGES, CHROOT_PATHS from generator import generate_makepkg_conf +from utils import mount, umount, check_findmnt, log_or_exception BIND_BUILD_DIRS = 'BINDBUILDDIRS' BASE_CHROOT_PREFIX = 'base_' diff --git a/constants.py b/constants.py index 5235663..0bb72fe 100644 --- a/constants.py +++ b/constants.py @@ -156,3 +156,10 @@ WRAPPER_TYPES = [ 'none', 'docker', ] + +MAKEPKG_CMD = [ + 'makepkg', + '--noconfirm', + '--ignorearch', + '--needed', +] diff --git a/distro/__init__.py b/distro/__init__.py index 2d5bf62..4621cbc 100644 --- a/distro/__init__.py +++ b/distro/__init__.py @@ -1,64 +1,2 @@ -from constants import ARCHES, BASE_DISTROS, REPOSITORIES, KUPFER_HTTPS, CHROOT_PATHS -from generator import generate_pacman_conf_body -from config import config - from .package import PackageInfo from .repo import RepoInfo, Repo - - -class Distro: - repos: dict[str, Repo] - arch: str - - def __init__(self, arch: str, repo_infos: dict[str, RepoInfo], scan=False): - assert (arch in ARCHES) - self.arch = arch - self.repos = dict[str, Repo]() - for repo_name, repo_info in repo_infos.items(): - self.repos[repo_name] = Repo( - name=repo_name, - arch=arch, - url_template=repo_info.url_template, - options=repo_info.options, - scan=scan, - ) - - def get_packages(self): - """ get packages from all repos, semantically overlaying them""" - results = dict[str, PackageInfo]() - for repo in self.repos.values().reverse(): - assert (repo.packages is not None) - for package in repo.packages: - results[package.name] = package - - def repos_config_snippet(self, extra_repos: dict[str, RepoInfo] = {}) -> str: - extras = [Repo(name, url_template=info.url_template, arch=self.arch, options=info.options, scan=False) for name, info in extra_repos.items()] - return '\n\n'.join(repo.config_snippet() for repo in (list(self.repos.values()) + extras)) - - def get_pacman_conf(self, extra_repos: dict[str, RepoInfo] = {}, check_space: bool = True): - body = generate_pacman_conf_body(self.arch, check_space=check_space) - return body + self.repos_config_snippet(extra_repos) - - -def get_base_distro(arch: str) -> Distro: - repos = {name: RepoInfo(url_template=url) for name, url in BASE_DISTROS[arch]['repos'].items()} - return Distro(arch=arch, repo_infos=repos, scan=False) - - -def get_kupfer(arch: str, url_template: str) -> Distro: - repos = {name: RepoInfo(url_template=url_template, options={'SigLevel': 'Never'}) for name in REPOSITORIES} - return Distro( - arch=arch, - repo_infos=repos, - ) - - -def get_kupfer_https(arch: str) -> Distro: - return get_kupfer(arch, KUPFER_HTTPS) - - -def get_kupfer_local(arch: str = None, in_chroot: bool = True) -> Distro: - if not arch: - arch = config.runtime['arch'] - dir = CHROOT_PATHS['packages'] if in_chroot else config.get_path('packages') - return get_kupfer(arch, f"file://{dir}/$arch/$repo") diff --git a/distro/distro.py b/distro/distro.py new file mode 100644 index 0000000..2d5bf62 --- /dev/null +++ b/distro/distro.py @@ -0,0 +1,64 @@ +from constants import ARCHES, BASE_DISTROS, REPOSITORIES, KUPFER_HTTPS, CHROOT_PATHS +from generator import generate_pacman_conf_body +from config import config + +from .package import PackageInfo +from .repo import RepoInfo, Repo + + +class Distro: + repos: dict[str, Repo] + arch: str + + def __init__(self, arch: str, repo_infos: dict[str, RepoInfo], scan=False): + assert (arch in ARCHES) + self.arch = arch + self.repos = dict[str, Repo]() + for repo_name, repo_info in repo_infos.items(): + self.repos[repo_name] = Repo( + name=repo_name, + arch=arch, + url_template=repo_info.url_template, + options=repo_info.options, + scan=scan, + ) + + def get_packages(self): + """ get packages from all repos, semantically overlaying them""" + results = dict[str, PackageInfo]() + for repo in self.repos.values().reverse(): + assert (repo.packages is not None) + for package in repo.packages: + results[package.name] = package + + def repos_config_snippet(self, extra_repos: dict[str, RepoInfo] = {}) -> str: + extras = [Repo(name, url_template=info.url_template, arch=self.arch, options=info.options, scan=False) for name, info in extra_repos.items()] + return '\n\n'.join(repo.config_snippet() for repo in (list(self.repos.values()) + extras)) + + def get_pacman_conf(self, extra_repos: dict[str, RepoInfo] = {}, check_space: bool = True): + body = generate_pacman_conf_body(self.arch, check_space=check_space) + return body + self.repos_config_snippet(extra_repos) + + +def get_base_distro(arch: str) -> Distro: + repos = {name: RepoInfo(url_template=url) for name, url in BASE_DISTROS[arch]['repos'].items()} + return Distro(arch=arch, repo_infos=repos, scan=False) + + +def get_kupfer(arch: str, url_template: str) -> Distro: + repos = {name: RepoInfo(url_template=url_template, options={'SigLevel': 'Never'}) for name in REPOSITORIES} + return Distro( + arch=arch, + repo_infos=repos, + ) + + +def get_kupfer_https(arch: str) -> Distro: + return get_kupfer(arch, KUPFER_HTTPS) + + +def get_kupfer_local(arch: str = None, in_chroot: bool = True) -> Distro: + if not arch: + arch = config.runtime['arch'] + dir = CHROOT_PATHS['packages'] if in_chroot else config.get_path('packages') + return get_kupfer(arch, f"file://{dir}/$arch/$repo") diff --git a/distro/pkgbuild.py b/distro/pkgbuild.py new file mode 100644 index 0000000..439b703 --- /dev/null +++ b/distro/pkgbuild.py @@ -0,0 +1,98 @@ +from copy import deepcopy +import logging +import os +import subprocess +from typing import Generator + +from chroot import Chroot +from constants import CHROOT_PATHS, MAKEPKG_CMD + +from .package import PackageInfo + + +class Pkgbuild(PackageInfo): + depends: list[str] = None + provides: list[str] = None + local_depends: list[PackageInfo] = None + subpackages: list[PackageInfo] = None + repo = '' + mode = '' + path = '' + pkgver = '' + pkgrel = '' + + def __init__( + self, + relative_path: str, + depends: list[str] = [], + provides: list[str] = [], + subpackages: list[PackageInfo] = [], + ) -> None: + self.version = None + self.path = relative_path + self.depends = deepcopy(depends) + self.provides = deepcopy(provides) + self.subpackages = deepcopy(subpackages) + + def __repr__(self): + return f'Package({self.name},{repr(self.path)},{self.version},{self.mode})' + + def names(self): + return [self.name] + self.provides + + +def parse_pkgbuild(relative_pkg_dir: str, native_chroot: Chroot) -> list[Pkgbuild]: + mode = None + with open(os.path.join(native_chroot.get_path(CHROOT_PATHS['pkgbuilds']), relative_pkg_dir, 'PKGBUILD'), 'r') as file: + for line in file.read().split('\n'): + if line.startswith('_mode='): + mode = line.split('=')[1] + break + if mode not in ['host', 'cross']: + raise Exception((f'{relative_pkg_dir}/PKGBUILD has {"no" if mode is None else "an invalid"} mode configured') + + (f': "{mode}"' if mode is not None else '')) + + base_package = Pkgbuild(relative_pkg_dir) + base_package.mode = mode + base_package.repo = relative_pkg_dir.split('/')[0] + srcinfo = native_chroot.run_cmd( + MAKEPKG_CMD + ['--printsrcinfo'], + cwd=os.path.join(CHROOT_PATHS['pkgbuilds'], base_package.path), + stdout=subprocess.PIPE, + ) + lines = srcinfo.stdout.decode('utf-8').split('\n') + + current = base_package + multi_pkgs = False + for line_raw in lines: + line = line_raw.strip() + if not line: + continue + splits = line.split(' = ') + if line.startswith('pkgbase'): + base_package.name = splits[1] + multi_pkgs = True + elif line.startswith('pkgname'): + if multi_pkgs: + if current is not base_package: + base_package.subpackages.append(current) + current = deepcopy(base_package) + current.name = splits[1] + elif line.startswith('pkgver'): + current.pkgver = splits[1] + elif line.startswith('pkgrel'): + current.pkgrel = splits[1] + elif line.startswith('provides'): + current.provides.append(splits[1]) + elif line.startswith('depends') or line.startswith('makedepends') or line.startswith('checkdepends') or line.startswith('optdepends'): + current.depends.append(splits[1].split('=')[0].split(': ')[0]) + current.depends = list(set(current.depends)) + + results = base_package.subpackages or [base_package] + for pkg in results: + pkg.mode = mode + pkg.version = f'{pkg.pkgver}-{pkg.pkgrel}' + if not (pkg.pkgver == base_package.pkgver and pkg.pkgrel == base_package.pkgrel): + raise Exception('subpackage malformed! pkgver differs!') + + return results diff --git a/image.py b/image.py index 50766e8..70b6af3 100644 --- a/image.py +++ b/image.py @@ -12,7 +12,7 @@ from subprocess import run, CompletedProcess from chroot import Chroot, get_device_chroot from constants import BASE_PACKAGES, DEVICES, FLAVOURS from config import config -from distro import get_base_distro, get_kupfer_https, get_kupfer_local +from distro.distro import get_base_distro, get_kupfer_https, get_kupfer_local from packages import build_enable_qemu_binfmt, discover_packages, build_packages from ssh import copy_ssh_keys from wrapper import enforce_wrap diff --git a/packages.py b/packages.py index 51b273d..f32ae11 100644 --- a/packages.py +++ b/packages.py @@ -9,20 +9,14 @@ from joblib import Parallel, delayed from glob import glob from shutil import rmtree -from constants import REPOSITORIES, CROSSDIRECT_PKGS, QEMU_BINFMT_PKGS, GCC_HOSTSPECS, ARCHES, Arch, CHROOT_PATHS +from constants import REPOSITORIES, CROSSDIRECT_PKGS, QEMU_BINFMT_PKGS, GCC_HOSTSPECS, ARCHES, Arch, CHROOT_PATHS, MAKEPKG_CMD from config import config from chroot import get_build_chroot, Chroot from ssh import run_ssh_command, scp_put_files from wrapper import enforce_wrap from utils import git from binfmt import register as binfmt_register - -makepkg_cmd = [ - 'makepkg', - '--noconfirm', - '--ignorearch', - '--needed', -] +from distro.pkgbuild import Pkgbuild as Package, parse_pkgbuild pacman_cmd = [ 'pacman', @@ -44,68 +38,6 @@ def get_makepkg_env(): } -class Package: - name = '' - names: list[str] = [] - depends: list[str] = [] - local_depends = None - repo = '' - mode = '' - - def __init__( - self, - path: str, - native_chroot: Chroot, - ) -> None: - self.path = path - self._loadinfo(native_chroot) - - def _loadinfo(self, native_chroot: Chroot): - result = native_chroot.run_cmd( - makepkg_cmd + ['--printsrcinfo'], - cwd=os.path.join(CHROOT_PATHS['pkgbuilds'], self.path), - stdout=subprocess.PIPE, - ) - lines = result.stdout.decode('utf-8').split('\n') - names = [] - depends = [] - multi_pkgs = False - - for line_raw in lines: - line = line_raw.lstrip() - if line.startswith('pkgbase'): - self.name = line.split(' = ')[1] - names.append(self.name) - multi_pkgs = True - if line.startswith('pkgname'): - names.append(line.split(' = ')[1]) - if not multi_pkgs: - self.name = line.split(' = ')[1] - if line.startswith('pkgbase') or line.startswith('provides'): - names.append(line.split(' = ')[1]) - if line.startswith('depends') or line.startswith('makedepends') or line.startswith('checkdepends') or line.startswith('optdepends'): - depends.append(line.split(' = ')[1].split('=')[0].split(': ')[0]) - self.names = list(set(names)) - self.depends = list(set(depends)) - - self.repo = self.path.split('/')[0] - - mode = '' - logging.debug(config) - with open(os.path.join(native_chroot.get_path(CHROOT_PATHS['pkgbuilds']), self.path, 'PKGBUILD'), 'r') as file: - for line in file.read().split('\n'): - if line.startswith('_mode='): - mode = line.split('=')[1] - break - if mode not in ['host', 'cross']: - logging.fatal(f'Package {self.path} has an invalid mode configured: \'{mode}\'') - exit(1) - self.mode = mode - - def __repr__(self): - return f'Package({self.name},{repr(self.names)},{repr(self.path)})' - - def clone_pkbuilds(pkgbuilds_dir: str, repo_url: str, branch: str, interactive=False, update=True): git_dir = os.path.join(pkgbuilds_dir, '.git') if not os.path.exists(git_dir): @@ -162,7 +94,7 @@ def init_prebuilts(arch: Arch, dir: str = None): exit(1) -def discover_packages() -> dict[str, Package]: +def discover_packages(parallel: bool = True) -> dict[str, Package]: pkgbuilds_dir = config.get_path('pkgbuilds') packages = {} paths = [] @@ -172,8 +104,20 @@ def discover_packages() -> dict[str, Package]: paths.append(os.path.join(repo, dir)) native_chroot = setup_build_chroot(config.runtime['arch'], add_kupfer_repos=False) - results = Parallel(n_jobs=multiprocessing.cpu_count() * 4)(delayed(Package)(path, native_chroot) for path in paths) + results = [] + + if parallel: + chunks = (Parallel(n_jobs=multiprocessing.cpu_count() * 4)(delayed(parse_pkgbuild)(path, native_chroot) for path in paths)) + else: + chunks = (parse_pkgbuild(path, native_chroot) for path in paths) + + for pkglist in chunks: + results += pkglist + + logging.debug('Building package dictionary!') for package in results: + if package.name in packages: + logging.warn(f'Overriding {packages[package.name]} with {package}') packages[package.name] = package # This filters the deps to only include the ones that are provided in this repo @@ -184,7 +128,7 @@ def discover_packages() -> dict[str, Package]: for p in packages.values(): if found: break - for name in p.names: + for name in p.names(): if dep == name: logging.debug(f'Found {p.name} that provides {dep}') found = True @@ -221,7 +165,7 @@ def generate_dependency_chain(package_repo: dict[str, Package], to_build: list[P def visit(package: Package, visited=visited, visited_names=visited_names): visited.add(package) - visited_names.update(package.names) + visited_names.update(package.names()) def join_levels(levels: list[set[Package]]) -> dict[Package, int]: result = dict[Package, int]() @@ -285,7 +229,7 @@ def generate_dependency_chain(package_repo: dict[str, Package], to_build: list[P if type(other_pkg) != Package: raise Exception('Not a Package object:' + repr(other_pkg)) for dep_name in other_pkg.depends: - if dep_name in pkg.names: + if dep_name in pkg.names(): dep_levels[level].remove(pkg) dep_levels[level + 1].add(pkg) logging.debug(f'Moving {pkg.name} to level {level+1} because {other_pkg.name} depends on it as {dep_name}') @@ -380,7 +324,7 @@ def check_package_version_built(package: Package, arch: Arch) -> bool: cross=True, ) - cmd = ['cd', os.path.join(CHROOT_PATHS['pkgbuilds'], package.path), '&&'] + makepkg_cmd + [ + cmd = ['cd', os.path.join(CHROOT_PATHS['pkgbuilds'], package.path), '&&'] + MAKEPKG_CMD + [ '--config', config_path, '--nobuild', @@ -440,7 +384,7 @@ def setup_sources(package: Package, chroot: Chroot, makepkg_conf_path='/etc/make ] logging.info(f'Setting up sources for {package.path} in {chroot.name}') - result = chroot.run_cmd(makepkg_cmd + makepkg_setup_args, cwd=os.path.join(CHROOT_PATHS['pkgbuilds'], package.path)) + result = chroot.run_cmd(MAKEPKG_CMD + makepkg_setup_args, cwd=os.path.join(CHROOT_PATHS['pkgbuilds'], package.path)) if result.returncode != 0: raise Exception(f'Failed to check sources for {package.path}') @@ -458,7 +402,7 @@ def build_package( makepkg_conf_path = 'etc/makepkg.conf' repo_dir = repo_dir if repo_dir else config.get_path('pkgbuilds') foreign_arch = config.runtime['arch'] != arch - deps = (list(set(package.depends) - set(package.names))) + deps = (list(set(package.depends) - set(package.names()))) target_chroot = setup_build_chroot( arch=arch, extra_packages=deps, @@ -531,7 +475,7 @@ def get_unbuilt_package_levels(repo: dict[str, Package], packages: list[Package] if ((not check_package_version_built(package, arch)) or set.intersection(set(package.depends), set(build_names)) or (force and package in packages)): level.add(package) - build_names.update(package.names) + build_names.update(package.names()) if level: build_levels.append(level) logging.debug(f'Adding to level {i}:' + '\n' + ('\n'.join([p.name for p in level])))