From 7666b91efc7a5d7310a4bab632ebee6e88bcc03e Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Mon, 17 Apr 2023 06:21:34 +0200 Subject: [PATCH 01/20] distro/package: acquire(): allow overriding filename and use utils.download_file() --- distro/package.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/distro/package.py b/distro/package.py index 4a5b5a6..721dfe3 100644 --- a/distro/package.py +++ b/distro/package.py @@ -1,11 +1,10 @@ import logging import os -from shutil import copyfileobj from typing import Optional, Union -from urllib.request import urlopen from exec.file import get_temp_dir, makedir +from utils import download_file class PackageInfo: @@ -77,17 +76,16 @@ class LocalPackage(BinaryPackage): class RemotePackage(BinaryPackage): - def acquire(self, dest_dir: Optional[str] = None) -> str: + def acquire(self, dest_dir: Optional[str] = None, filename: Optional[str] = None) -> str: assert self.resolved_url and '.pkg.tar.' in self.resolved_url url = f"{self.resolved_url}" assert url dest_dir = dest_dir or get_temp_dir() makedir(dest_dir) - dest_file_path = os.path.join(dest_dir, self.filename) + dest_file_path = os.path.join(dest_dir, filename or self.filename) logging.info(f"Trying to download package {url}") - with urlopen(url) as fsrc, open(dest_file_path, 'wb') as fdst: - copyfileobj(fsrc, fdst) + download_file(dest_file_path, url) logging.info(f"{self.filename} downloaded from repos") return dest_file_path From ba5aa209ddbd00666b2e4aa367e6900457b12282 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Mon, 17 Apr 2023 07:00:35 +0200 Subject: [PATCH 02/20] exec/file: add copy_file() --- exec/file.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/exec/file.py b/exec/file.py index 00653aa..c677e09 100644 --- a/exec/file.py +++ b/exec/file.py @@ -4,7 +4,7 @@ import os import stat import subprocess -from shutil import rmtree +from shutil import copyfile, rmtree from tempfile import mkdtemp from typing import Optional, Union @@ -41,7 +41,7 @@ def chown(path: str, user: Optional[Union[str, int]] = None, group: Optional[Uni raise Exception(f"Failed to change owner of '{path}' to '{owner}'") -def chmod(path, mode: Union[int, str] = 0o0755, force_sticky=True, privileged: bool = True): +def chmod(path: str, mode: Union[int, str] = 0o0755, force_sticky=True, privileged: bool = True): if not isinstance(mode, str): octal = oct(mode)[2:] else: @@ -60,11 +60,14 @@ def chmod(path, mode: Union[int, str] = 0o0755, force_sticky=True, privileged: b raise Exception(f"Failed to set mode of '{path}' to '{chmod}'") -def root_check_exists(path): +copy_file = copyfile + + +def root_check_exists(path: str): return os.path.exists(path) or run_root_cmd(['[', '-e', path, ']']).returncode == 0 -def root_check_is_dir(path): +def root_check_is_dir(path: str): return os.path.isdir(path) or run_root_cmd(['[', '-d', path, ']']) From d5277694737911ffe49a9791bd9d152a6883648e Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Mon, 17 Apr 2023 07:01:47 +0200 Subject: [PATCH 03/20] distro/package: add acquire() parameters to Distro interface: `dest_dir: Optional[str], filename: Optional[str]` --- distro/package.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/distro/package.py b/distro/package.py index 721dfe3..3298247 100644 --- a/distro/package.py +++ b/distro/package.py @@ -3,8 +3,8 @@ import os from typing import Optional, Union -from exec.file import get_temp_dir, makedir -from utils import download_file +from exec.file import copy_file, get_temp_dir, makedir +from utils import download_file, sha256sum class PackageInfo: @@ -61,16 +61,20 @@ class BinaryPackage(PackageInfo): p._desc = desc return p - def acquire(self) -> str: + def acquire(self, dest_dir: Optional[str] = None, filename: Optional[str] = None) -> str: raise NotImplementedError() class LocalPackage(BinaryPackage): - def acquire(self) -> str: + def acquire(self, dest_dir: Optional[str] = None, filename: Optional[str] = None) -> str: assert self.resolved_url and self.filename and self.filename in self.resolved_url path = f'{self.resolved_url.split("file://")[1]}' - assert os.path.exists(path) or print(path) + if dest_dir: + target = os.path.join(dest_dir, filename or self.filename) + if os.path.getsize(path) != os.path.getsize(target) or sha256sum(path) != sha256sum(target): + copy_file(path, target, follow_symlinks=True) + return target return path From 0c56038ed666941471cba3933f7587a94b3d1b1e Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Mon, 17 Apr 2023 07:17:12 +0200 Subject: [PATCH 04/20] distro/package: BinaryPackage.acquire(): return (path: str, changed: bool) --- devices/device.py | 2 +- distro/package.py | 16 +++++++++------- packages/build.py | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/devices/device.py b/devices/device.py index c386053..c4fd067 100644 --- a/devices/device.py +++ b/devices/device.py @@ -80,7 +80,7 @@ class Device(DictScheme): if self.package.name not in pkgs: raise Exception(f"device package {self.package.name} somehow not in repos, this is a kupferbootstrap bug") pkg = pkgs[self.package.name] - file_path = pkg.acquire() + file_path, _ = pkg.acquire() assert file_path assert os.path.exists(file_path) deviceinfo_path = 'etc/kupfer/deviceinfo' diff --git a/distro/package.py b/distro/package.py index 3298247..a6c06a9 100644 --- a/distro/package.py +++ b/distro/package.py @@ -61,26 +61,28 @@ class BinaryPackage(PackageInfo): p._desc = desc return p - def acquire(self, dest_dir: Optional[str] = None, filename: Optional[str] = None) -> str: + def acquire(self, dest_dir: Optional[str] = None, filename: Optional[str] = None) -> tuple[str, bool]: raise NotImplementedError() class LocalPackage(BinaryPackage): - def acquire(self, dest_dir: Optional[str] = None, filename: Optional[str] = None) -> str: + def acquire(self, dest_dir: Optional[str] = None, filename: Optional[str] = None) -> tuple[str, bool]: + changed = False assert self.resolved_url and self.filename and self.filename in self.resolved_url path = f'{self.resolved_url.split("file://")[1]}' if dest_dir: target = os.path.join(dest_dir, filename or self.filename) if os.path.getsize(path) != os.path.getsize(target) or sha256sum(path) != sha256sum(target): copy_file(path, target, follow_symlinks=True) - return target - return path + changed = True + path = target + return path, changed class RemotePackage(BinaryPackage): - def acquire(self, dest_dir: Optional[str] = None, filename: Optional[str] = None) -> str: + def acquire(self, dest_dir: Optional[str] = None, filename: Optional[str] = None) -> tuple[str, bool]: assert self.resolved_url and '.pkg.tar.' in self.resolved_url url = f"{self.resolved_url}" assert url @@ -90,6 +92,6 @@ class RemotePackage(BinaryPackage): dest_file_path = os.path.join(dest_dir, filename or self.filename) logging.info(f"Trying to download package {url}") - download_file(dest_file_path, url) + changed = download_file(dest_file_path, url) logging.info(f"{self.filename} downloaded from repos") - return dest_file_path + return dest_file_path, changed diff --git a/packages/build.py b/packages/build.py index b388221..d7f56d4 100644 --- a/packages/build.py +++ b/packages/build.py @@ -316,7 +316,7 @@ def try_download_package(dest_file_path: str, package: Pkgbuild, arch: Arch) -> url = repo_pkg.resolved_url assert url try: - path = repo_pkg.acquire() + path, _ = repo_pkg.acquire() assert os.path.exists(path) return path except HTTPError as e: From e068b3587e9aa67ffc23c8ff6745c5e63ff0a8b5 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Mon, 17 Apr 2023 16:27:36 +0200 Subject: [PATCH 05/20] WIP: keyring init --- constants.py | 20 ++++++++++++-------- distro/package.py | 1 + distro/repo_config.py | 14 ++++++++------ utils.py | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 57 insertions(+), 14 deletions(-) diff --git a/constants.py b/constants.py index 2ddd686..0f32cde 100644 --- a/constants.py +++ b/constants.py @@ -1,4 +1,4 @@ -from typehelpers import TypeAlias +from typehelpers import TypeAlias, Union FASTBOOT = 'fastboot' FLASH_PARTS = { @@ -56,6 +56,8 @@ ARCHES = [ DistroArch: TypeAlias = Arch TargetArch: TypeAlias = Arch +KEYRINGS_KEY = 'keyrings' + ALARM_REPOS = { 'core': 'http://mirror.archlinuxarm.org/$arch/$repo', 'extra': 'http://mirror.archlinuxarm.org/$arch/$repo', @@ -64,20 +66,22 @@ ALARM_REPOS = { 'aur': 'http://mirror.archlinuxarm.org/$arch/$repo', } -BASE_DISTROS: dict[DistroArch, dict[str, dict[str, str]]] = { +ALARM_DISTRO: dict[str, Union[dict[str, str], list[str]]] = { + 'repos': ALARM_REPOS, + KEYRINGS_KEY: ['archlinuxarm-keyring'], +} + +BASE_DISTROS: dict[DistroArch, dict[str, Union[dict[str, str], list[str]]]] = { 'x86_64': { 'repos': { 'core': 'https://geo.mirror.pkgbuild.com/$repo/os/$arch', 'extra': 'https://geo.mirror.pkgbuild.com/$repo/os/$arch', 'community': 'https://geo.mirror.pkgbuild.com/$repo/os/$arch', }, + KEYRINGS_KEY: ['archlinux-keyring'], }, - 'aarch64': { - 'repos': ALARM_REPOS, - }, - 'armv7h': { - 'repos': ALARM_REPOS, - }, + 'aarch64': ALARM_DISTRO, + 'armv7h': ALARM_DISTRO, } COMPILE_ARCHES: dict[Arch, str] = { diff --git a/distro/package.py b/distro/package.py index a6c06a9..200a7ef 100644 --- a/distro/package.py +++ b/distro/package.py @@ -72,6 +72,7 @@ class LocalPackage(BinaryPackage): assert self.resolved_url and self.filename and self.filename in self.resolved_url path = f'{self.resolved_url.split("file://")[1]}' if dest_dir: + makedir(dest_dir) target = os.path.join(dest_dir, filename or self.filename) if os.path.getsize(path) != os.path.getsize(target) or sha256sum(path) != sha256sum(target): copy_file(path, target, follow_symlinks=True) diff --git a/distro/repo_config.py b/distro/repo_config.py index 5f32d2b..341a573 100644 --- a/distro/repo_config.py +++ b/distro/repo_config.py @@ -9,7 +9,7 @@ from copy import deepcopy from typing import ClassVar, Optional, Mapping, Union from config.state import config -from constants import Arch, BASE_DISTROS, KUPFER_HTTPS, REPOS_CONFIG_FILE, REPOSITORIES +from constants import Arch, BASE_DISTROS, KUPFER_HTTPS, KEYRINGS_KEY, REPOS_CONFIG_FILE, REPOSITORIES from dictscheme import DictScheme, toml_inline_dicts, TomlPreserveInlineDictEncoder from utils import sha256sum @@ -39,11 +39,13 @@ class RepoConfig(AbstrRepoConfig): class BaseDistro(DictScheme): remote_url: Optional[str] + keyrings: Optional[list[str]] repos: dict[str, BaseDistroRepo] class ReposConfigFile(DictScheme): remote_url: Optional[str] + keyrings: Optional[list[str]] repos: dict[str, RepoConfig] base_distros: dict[Arch, BaseDistro] _path: Optional[str] @@ -106,6 +108,7 @@ REPOS_CONFIG_DEFAULT = ReposConfigFile({ '_path': '__DEFAULTS__', '_checksum': None, REMOTEURL_KEY: KUPFER_HTTPS, + KEYRINGS_KEY: [], REPOS_KEY: { 'kupfer_local': REPO_DEFAULTS | { LOCALONLY_KEY: True @@ -117,11 +120,10 @@ REPOS_CONFIG_DEFAULT = ReposConfigFile({ BASEDISTROS_KEY: { arch: { REMOTEURL_KEY: None, - 'repos': { - k: { - 'remote_url': v - } for k, v in arch_def['repos'].items() - }, + KEYRINGS_KEY: arch_def.get(KEYRINGS_KEY, None), + 'repos': {k: { + 'remote_url': v + } for k, v in arch_def['repos'].items()}, # type: ignore[union-attr] } for arch, arch_def in BASE_DISTROS.items() }, }) diff --git a/utils.py b/utils.py index df40a97..02048ac 100644 --- a/utils.py +++ b/utils.py @@ -138,6 +138,42 @@ def read_files_from_tar(tar_file: str, files: Sequence[str]) -> Generator[tuple[ yield path, fd +def read_files_from_tar_recursive(tar_file: str, paths: Sequence[str], append_slash: bool = True) -> Generator[tuple[str, IO], None, None]: + """ + Returns tar FDs to files that lie under the directories specified in paths. + HINT: deactivate append_slash to get glob-like behaviour, as if all paths ended with * + """ + assert os.path.exists(tar_file) + paths = [f"{p.strip('/')}/" for p in paths] + with tarfile.open(tar_file) as index: + for member in index.getmembers(): + for path in paths: + if member.isfile() and member.path.startswith(path): + fd = index.extractfile(member) + assert fd + yield member.path, fd + break + continue + + +def extract_files_from_tar_generator( + tar_generator: Generator[tuple[str, IO], None, None], + output_dir: str, + remove_prefix: str = '', + append_slash: bool = True, +): + assert os.path.exists(output_dir) + remove_prefix = remove_prefix.strip('/') + if append_slash and remove_prefix: + remove_prefix += '/' + for file_path, fd in tar_generator: + assert file_path.startswith(remove_prefix) + output_path = os.path.join(output_dir, file_path[len(remove_prefix):].lstrip('/')) + os.makedirs(os.path.dirname(output_path), exist_ok=True) + with open(output_path, 'wb') as f: + f.write(fd.read()) + + def download_file(path: str, url: str, update: bool = True): """Download a file over http[s]. With `update`, tries to use mtime timestamps to download only changed files.""" url_time = None From 38edce080fed66fe7659b380078ca8dea0338473 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Mon, 17 Apr 2023 17:29:20 +0200 Subject: [PATCH 06/20] WIP: keyring init done(?) --- constants.py | 5 ++ distro/keyring.py | 188 ++++++++++++++++++++++++++++++++++++++++++ distro/repo_config.py | 6 +- 3 files changed, 197 insertions(+), 2 deletions(-) create mode 100644 distro/keyring.py diff --git a/constants.py b/constants.py index 0f32cde..ec6a2f8 100644 --- a/constants.py +++ b/constants.py @@ -57,6 +57,11 @@ DistroArch: TypeAlias = Arch TargetArch: TypeAlias = Arch KEYRINGS_KEY = 'keyrings' +KEYRINGS_LOCAL_KEY = 'local_keyring' + +KEYRING_REMOTE_NAME = "kupfer-keyring" +KEYRINGS_LOCAL_NAME = KEYRING_REMOTE_NAME + '-local' + ALARM_REPOS = { 'core': 'http://mirror.archlinuxarm.org/$arch/$repo', diff --git a/distro/keyring.py b/distro/keyring.py new file mode 100644 index 0000000..b0c6ae3 --- /dev/null +++ b/distro/keyring.py @@ -0,0 +1,188 @@ +import logging +import os + +from enum import auto, Enum +from typing import Optional + +from config.state import config +from constants import Arch, KEYRINGS_KEY, KEYRINGS_LOCAL_KEY +from exec.cmd import CompletedProcess, run_cmd +from exec.file import makedir, remove_file +from repo_config import get_repo_config +from utils import extract_files_from_tar_generator, read_files_from_tar_recursive + +from .distro import Distro, get_base_distro, get_kupfer_local, get_kupfer_https +from .package import BinaryPackage + +KEYRING_DIR = 'keyrings' +KEYRING_DIST_DIR = 'dist' +KEYRING_GPG_DIR = 'keyring' + +PKGNAME_MARKER = '.pkg.tar' + +PKG_KEYRING_FOLDER = 'usr/share/pacman/keyrings/' + + +class DistroType(Enum): + BASE = auto + LOCAL = auto + REMOTE = auto + + +KEYRING_LOCATIONS: dict[DistroType, str] = { + DistroType.BASE: 'base', + DistroType.LOCAL: 'local', + DistroType.REMOTE: 'kupfer', +} + +keyring_created: dict[tuple[Arch, DistroType], bool] = {} + + +def keyring_is_created(arch: Arch, distro_type: DistroType) -> bool: + return keyring_created.get((arch, distro_type), False) + + +def init_keyring_dir( + arch: Arch, + distro_type: DistroType, + target_path: Optional[str] = None, + lazy: bool = True, +) -> dict[str, bool]: + base_dir = target_path or get_keyring_path(arch, distro_type) + keyring_dists = init_keyring_dist_dir(arch, distro_type, base_dir, lazy) + gpg_changed = init_keyring_gpg_dir(arch, distro_type, keyring_dists, base_dir, lazy) + keyring_created[(arch, distro_type)] = True + return gpg_changed + + +def init_keyring_gpg_dir( + arch: Arch, + distro_type: DistroType, + keyring_dists: dict[str, tuple[str, bool]], + base_dir: Optional[str] = None, + lazy: bool = True, +) -> dict[str, bool]: + base_dir = base_dir or get_keyring_path(arch, distro_type) + gpg_dir = get_keyring_gpg_path(base_dir) + exists = os.path.exists(gpg_dir) + if exists and not lazy: + remove_file(gpg_dir) + exists = False + lazy = lazy and exists + makedir(gpg_dir) + results = {} + for name, val in keyring_dists.items(): + dist_dir, dist_changed = val + if lazy and not dist_changed: + results[name] = False + continue + import_dist_keyring(gpg_dir, dist_dir) + results[name] = True + return results + + +def import_dist_keyring( + gpg_dir: str, + dist_dir: str, +) -> CompletedProcess: + assert gpg_dir and dist_dir and config.runtime.script_source_dir + pacman_key = os.path.join(config.runtime.script_source_dir, 'bin', 'pacman-key-user') + r = run_cmd([pacman_key, '--populate-from', dist_dir, '--populate', '--gpgdir', gpg_dir]) + assert isinstance(r, CompletedProcess) + return r + + +def init_keyring_dist_dir( + arch: Arch, + distro_type: DistroType, + base_dir: Optional[str] = None, + lazy: bool = True, +) -> dict[str, tuple[str, bool]]: + """ + create keyrings/{arch}/dist. Returns a boolean indicating whether changes were made + """ + repo_config = get_repo_config()[0] + base_dir = base_dir or get_keyring_path(arch, distro_type) + dist_dir = get_keyring_dist_path(base_dir) + + pkg_names: list[str] = [] + distro: Distro + if distro_type == DistroType.BASE: + pkg_names = repo_config.base_distros.get(arch, {}).get(KEYRINGS_KEY, None) or [] + distro = get_base_distro(arch, scan=False) + elif distro_type == DistroType.LOCAL: + pkg_name = repo_config.get(KEYRINGS_LOCAL_KEY, None) + pkg_names = [pkg_name] if pkg_name else [] + distro = get_kupfer_local(arch, scan=False, in_chroot=False) + elif distro_type == DistroType.REMOTE: + pkg_names = repo_config.get(KEYRINGS_KEY, None) or [] + distro = get_kupfer_https(arch, scan=False) + dist_pkgs, changed = acquire_dist_pkgs(pkg_names, distro, base_dir) + if lazy and dist_pkgs and not changed and os.path.exists(dist_dir): # and keyring_is_created(arch, distro_type): + return {name: (val[0], False) for name, val in dist_pkgs.items()} + + makedir(dist_dir) + dist_dirs = [] + results = {} + for name, _val in dist_pkgs.items(): + dist_pkg, changed = _val + _dir = os.path.join(dist_dir, name) + results[name] = _dir, False + if lazy and not changed and os.path.exists(_dir): + continue + extract_keyring_pkg(dist_pkg, _dir) + dist_dirs.append(_dir) + results[name] = dist_pkg, True + return results + + +def acquire_dist_pkgs(keyring_packages: list[str], distro: Distro, dist_dir: str) -> tuple[dict[str, tuple[str, bool]], bool]: + if not keyring_packages: + return {}, False + pkgs = {} + not_found = [] + distro.scan(lazy=True) + repos: dict[str, BinaryPackage] = distro.get_packages() + pkg: BinaryPackage + for name in keyring_packages: + if name not in repos: + not_found.append(name) + continue + pkg = repos[name] + pkgs[name] = pkg + if not_found: + raise Exception(f"Keyring packages for {distro.arch} not found: {not_found}") + + changed = False + results = {} + for name in pkgs: + assert isinstance(pkgs[name], BinaryPackage) + pkg = pkgs[name] + assert PKGNAME_MARKER in pkg.filename + comp_ext = pkg.filename.rsplit(PKGNAME_MARKER, 1)[1] + target_path, _changed = pkg.acquire(dist_dir, f'{name}.tar{comp_ext}') + results[name] = target_path, _changed + if _changed: + logging.debug(f"{target_path} changed") + changed = True + return results, changed + + +def extract_keyring_pkg(pkg_path: str, dest_path: str): + extract_files_from_tar_generator( + read_files_from_tar_recursive(pkg_path, PKG_KEYRING_FOLDER), + dest_path, + remove_prefix=PKG_KEYRING_FOLDER, + ) + + +def get_keyring_path(arch: Arch, distro_type: DistroType, *extra_paths) -> str: + return os.path.join(config.get_path('pacman'), KEYRING_DIR, arch, KEYRING_LOCATIONS[distro_type], *extra_paths) + + +def get_keyring_dist_path(base_dir: str) -> str: + return os.path.join(base_dir, KEYRING_DIST_DIR) + + +def get_keyring_gpg_path(base_dir: str) -> str: + return os.path.join(base_dir, KEYRING_GPG_DIR) diff --git a/distro/repo_config.py b/distro/repo_config.py index 341a573..c8c067b 100644 --- a/distro/repo_config.py +++ b/distro/repo_config.py @@ -9,7 +9,7 @@ from copy import deepcopy from typing import ClassVar, Optional, Mapping, Union from config.state import config -from constants import Arch, BASE_DISTROS, KUPFER_HTTPS, KEYRINGS_KEY, REPOS_CONFIG_FILE, REPOSITORIES +from constants import Arch, BASE_DISTROS, KUPFER_HTTPS, KEYRINGS_KEY, KEYRINGS_LOCAL_KEY, KEYRINGS_LOCAL_NAME, KEYRING_REMOTE_NAME, REPOS_CONFIG_FILE, REPOSITORIES from dictscheme import DictScheme, toml_inline_dicts, TomlPreserveInlineDictEncoder from utils import sha256sum @@ -46,6 +46,7 @@ class BaseDistro(DictScheme): class ReposConfigFile(DictScheme): remote_url: Optional[str] keyrings: Optional[list[str]] + local_keyring: Optional[str] repos: dict[str, RepoConfig] base_distros: dict[Arch, BaseDistro] _path: Optional[str] @@ -108,7 +109,8 @@ REPOS_CONFIG_DEFAULT = ReposConfigFile({ '_path': '__DEFAULTS__', '_checksum': None, REMOTEURL_KEY: KUPFER_HTTPS, - KEYRINGS_KEY: [], + KEYRINGS_KEY: [KEYRING_REMOTE_NAME], + KEYRINGS_LOCAL_KEY: KEYRINGS_LOCAL_NAME, REPOS_KEY: { 'kupfer_local': REPO_DEFAULTS | { LOCALONLY_KEY: True From a982f8c9668cc009417e5ea17c6e346d64e615d9 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Mon, 17 Apr 2023 18:56:11 +0200 Subject: [PATCH 07/20] utils: add decompress_if_zstd --- utils.py | 47 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/utils.py b/utils.py index 02048ac..0cfdae8 100644 --- a/utils.py +++ b/utils.py @@ -129,9 +129,52 @@ def get_gid(group: Union[int, str]) -> int: return grp.getgrnam(group).gr_gid +def is_zstd(data): + """ + Returns True if the given byte stream is compressed with the zstd algorithm, + False otherwise. This function performs a simplified version of the actual zstd + header validation, using hardcoded values. + """ + # Check for the magic number at the beginning of the stream + if len(data) < 4 or data[:4] != b"\x28\xb5\x2f\xfd": + logging.debug("zstd header not found") + return False + # Check the frame descriptor block size + if len(data) < 8: + return False + frame_size = data[4] & 0x7F | (data[5] & 0x7F) << 7 | (data[6] & 0x7F) << 14 | (data[7] & 0x07) << 21 + if frame_size < 1 or frame_size > 1 << 31: + return False + # Check the frame descriptor block for the checksum + if len(data) < 18: + return False + return True + + +def decompress_if_zstd(stream): + """ + Given a byte stream, returns either the original stream or the decompressed stream + if it is compressed with the zstd algorithm. + """ + if isinstance(stream, str): + stream = open(stream, 'rb') + data = stream.peek(18)[:18] + if not is_zstd(data): + logging.debug(f"{data=} Not zstd, skipping") + return tarfile.open(fileobj=stream) + logging.debug(f"Decompressing {stream=}") + import zstandard as zstd + dctx = zstd.ZstdDecompressor() + return tarfile.open(fileobj=dctx.stream_reader(stream, read_size=4096), mode='r|') + + +def open_tar(tar_file: str) -> tarfile.TarFile: + return decompress_if_zstd(tar_file) + + def read_files_from_tar(tar_file: str, files: Sequence[str]) -> Generator[tuple[str, IO], None, None]: assert os.path.exists(tar_file) - with tarfile.open(tar_file) as index: + with open_tar(tar_file) as index: for path in files: fd = index.extractfile(index.getmember(path)) assert fd @@ -145,7 +188,7 @@ def read_files_from_tar_recursive(tar_file: str, paths: Sequence[str], append_sl """ assert os.path.exists(tar_file) paths = [f"{p.strip('/')}/" for p in paths] - with tarfile.open(tar_file) as index: + with open_tar(tar_file) as index: for member in index.getmembers(): for path in paths: if member.isfile() and member.path.startswith(path): From 30c3fa77fdf2c700da040668b0da5c4da1e078d8 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Mon, 17 Apr 2023 18:56:24 +0200 Subject: [PATCH 08/20] WIP: keyrings 2 --- constants.py | 5 ++--- distro/distro.py | 8 ++++---- distro/keyring.py | 21 ++++++++++++++------- distro/repo_config.py | 21 +++++++++++++-------- 4 files changed, 33 insertions(+), 22 deletions(-) diff --git a/constants.py b/constants.py index ec6a2f8..8fde0d2 100644 --- a/constants.py +++ b/constants.py @@ -62,7 +62,6 @@ KEYRINGS_LOCAL_KEY = 'local_keyring' KEYRING_REMOTE_NAME = "kupfer-keyring" KEYRINGS_LOCAL_NAME = KEYRING_REMOTE_NAME + '-local' - ALARM_REPOS = { 'core': 'http://mirror.archlinuxarm.org/$arch/$repo', 'extra': 'http://mirror.archlinuxarm.org/$arch/$repo', @@ -85,8 +84,8 @@ BASE_DISTROS: dict[DistroArch, dict[str, Union[dict[str, str], list[str]]]] = { }, KEYRINGS_KEY: ['archlinux-keyring'], }, - 'aarch64': ALARM_DISTRO, - 'armv7h': ALARM_DISTRO, + 'aarch64': ALARM_DISTRO.copy(), + 'armv7h': ALARM_DISTRO.copy(), } COMPILE_ARCHES: dict[Arch, str] = { diff --git a/distro/distro.py b/distro/distro.py index bbc6329..215a665 100644 --- a/distro/distro.py +++ b/distro/distro.py @@ -8,7 +8,7 @@ from generator import generate_pacman_conf_body from config.state import config from .repo import BinaryPackageType, RepoInfo, Repo, LocalRepo, RemoteRepo -from .repo_config import AbstrRepoConfig, BaseDistro, ReposConfigFile, REPOS_CONFIG_DEFAULT, get_repo_config as _get_repo_config +from .repo_config import AbstrRepoConfig, BaseDistro, REMOTEURL_KEY, ReposConfigFile, REPOS_CONFIG_DEFAULT, get_repo_config as _get_repo_config class DistroLocation(IntFlag): @@ -138,7 +138,7 @@ def get_kupfer_repo_names(local) -> list[str]: def get_RepoInfo(arch: Arch, repo_config: AbstrRepoConfig, default_url: Optional[str]) -> RepoInfo: - url = repo_config.remote_url or default_url + url = repo_config.get(REMOTEURL_KEY, None) or default_url if isinstance(url, dict): if arch not in url and not default_url: raise Exception(f"Invalid repo config: Architecture {arch} not in remote_url mapping: {url}") @@ -161,7 +161,7 @@ def get_base_distro(arch: Arch, scan: bool = False, unsigned: bool = True, cache for repo, repo_config in distro_config.repos.items(): if unsigned: repo_config['options'] = (repo_config.get('options', None) or {}) | {'SigLevel': 'Never'} - repos[repo] = get_RepoInfo(arch, repo_config, default_url=distro_config.remote_url) + repos[repo] = get_RepoInfo(arch, repo_config, default_url=distro_config.get(REMOTEURL_KEY, None)) distro = RemoteDistro(arch=arch, repo_infos=repos, scan=False) if cache_db: @@ -187,7 +187,7 @@ def get_kupfer_distro( if location == DistroLocation.REMOTE: remote = True cache = _kupfer_https - default_url = repo_config.remote_url or KUPFER_HTTPS + default_url = repo_config.get(REMOTEURL_KEY, None) or KUPFER_HTTPS repos = {repo: get_RepoInfo(arch, conf, default_url) for repo, conf in repo_config.repos.items() if not conf.local_only} cls = RemoteDistro elif location in [DistroLocation.CHROOT, DistroLocation.LOCAL]: diff --git a/distro/keyring.py b/distro/keyring.py index b0c6ae3..59d8044 100644 --- a/distro/keyring.py +++ b/distro/keyring.py @@ -6,9 +6,9 @@ from typing import Optional from config.state import config from constants import Arch, KEYRINGS_KEY, KEYRINGS_LOCAL_KEY +from distro.repo_config import get_repo_config from exec.cmd import CompletedProcess, run_cmd from exec.file import makedir, remove_file -from repo_config import get_repo_config from utils import extract_files_from_tar_generator, read_files_from_tar_recursive from .distro import Distro, get_base_distro, get_kupfer_local, get_kupfer_https @@ -24,9 +24,9 @@ PKG_KEYRING_FOLDER = 'usr/share/pacman/keyrings/' class DistroType(Enum): - BASE = auto - LOCAL = auto - REMOTE = auto + BASE = auto() + LOCAL = auto() + REMOTE = auto() KEYRING_LOCATIONS: dict[DistroType, str] = { @@ -69,13 +69,15 @@ def init_keyring_gpg_dir( remove_file(gpg_dir) exists = False lazy = lazy and exists - makedir(gpg_dir) + if not lazy: + run_cmd([get_pacman_key_binary(), '--init', '--gpgdir', gpg_dir]) results = {} for name, val in keyring_dists.items(): dist_dir, dist_changed = val if lazy and not dist_changed: results[name] = False continue + logging.info(f"Importing dir {dist_dir} into {gpg_dir}") import_dist_keyring(gpg_dir, dist_dir) results[name] = True return results @@ -86,8 +88,7 @@ def import_dist_keyring( dist_dir: str, ) -> CompletedProcess: assert gpg_dir and dist_dir and config.runtime.script_source_dir - pacman_key = os.path.join(config.runtime.script_source_dir, 'bin', 'pacman-key-user') - r = run_cmd([pacman_key, '--populate-from', dist_dir, '--populate', '--gpgdir', gpg_dir]) + r = run_cmd([get_pacman_key_binary(), '--populate-from', dist_dir, '--populate', '--gpgdir', gpg_dir]) assert isinstance(r, CompletedProcess) return r @@ -117,6 +118,7 @@ def init_keyring_dist_dir( elif distro_type == DistroType.REMOTE: pkg_names = repo_config.get(KEYRINGS_KEY, None) or [] distro = get_kupfer_https(arch, scan=False) + logging.debug(f"Acquiring keyrings from {distro}: {pkg_names}") dist_pkgs, changed = acquire_dist_pkgs(pkg_names, distro, base_dir) if lazy and dist_pkgs and not changed and os.path.exists(dist_dir): # and keyring_is_created(arch, distro_type): return {name: (val[0], False) for name, val in dist_pkgs.items()} @@ -169,6 +171,7 @@ def acquire_dist_pkgs(keyring_packages: list[str], distro: Distro, dist_dir: str def extract_keyring_pkg(pkg_path: str, dest_path: str): + makedir(dest_path) extract_files_from_tar_generator( read_files_from_tar_recursive(pkg_path, PKG_KEYRING_FOLDER), dest_path, @@ -186,3 +189,7 @@ def get_keyring_dist_path(base_dir: str) -> str: def get_keyring_gpg_path(base_dir: str) -> str: return os.path.join(base_dir, KEYRING_GPG_DIR) + + +def get_pacman_key_binary() -> str: + return os.path.join(config.runtime.script_source_dir, 'bin', 'pacman-key-user') diff --git a/distro/repo_config.py b/distro/repo_config.py index c8c067b..32688b1 100644 --- a/distro/repo_config.py +++ b/distro/repo_config.py @@ -70,10 +70,11 @@ class ReposConfigFile(DictScheme): repos[name] = repo_cls(_repo, **kwargs) @staticmethod - def parse_config(path: str) -> ReposConfigFile: + def parse_config(path: str, insert_defaults: bool = True) -> ReposConfigFile: + defaults = REPOS_CONFIG_DEFAULTS_DICT if insert_defaults else {} try: with open(path, 'r') as fd: - data = yaml.safe_load(fd) + data = defaults | yaml.safe_load(fd) data['_path'] = path data['_checksum'] = sha256sum(path) return ReposConfigFile(data, validate=True) @@ -105,7 +106,7 @@ BASE_DISTRO_DEFAULTS = { OPTIONS_KEY: None, } -REPOS_CONFIG_DEFAULT = ReposConfigFile({ +REPOS_CONFIG_DEFAULTS_DICT = { '_path': '__DEFAULTS__', '_checksum': None, REMOTEURL_KEY: KUPFER_HTTPS, @@ -122,13 +123,17 @@ REPOS_CONFIG_DEFAULT = ReposConfigFile({ BASEDISTROS_KEY: { arch: { REMOTEURL_KEY: None, - KEYRINGS_KEY: arch_def.get(KEYRINGS_KEY, None), - 'repos': {k: { - 'remote_url': v - } for k, v in arch_def['repos'].items()}, # type: ignore[union-attr] + KEYRINGS_KEY: arch_def.get(KEYRINGS_KEY, None).copy(), + 'repos': { + k: { + 'remote_url': v + } for k, v in arch_def['repos'].items() # type: ignore[union-attr] + }, } for arch, arch_def in BASE_DISTROS.items() }, -}) +} + +REPOS_CONFIG_DEFAULT = ReposConfigFile(REPOS_CONFIG_DEFAULTS_DICT) _current_config = None From c576dc8a517908c694c0ed19b430f927ae70c889 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Mon, 17 Apr 2023 21:57:30 +0200 Subject: [PATCH 09/20] utils: handle zstd compressed tarfiles --- utils.py | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 75 insertions(+), 9 deletions(-) diff --git a/utils.py b/utils.py index 0cfdae8..26a65ed 100644 --- a/utils.py +++ b/utils.py @@ -11,6 +11,7 @@ import subprocess import tarfile from dateutil.parser import parse as parsedate +from io import BytesIO from shutil import which from typing import Any, Generator, IO, Optional, Union, Sequence @@ -151,6 +152,63 @@ def is_zstd(data): return True +class BackwardsReadableStream: + def __init__(self, stream): + self.stream = stream + self.buffer = bytearray() + self.position = 0 + + def read(self, size=-1): + data = b'' + if size == -1: + # read all remaining data in stream + data = self.stream.read() + else: + not_read = (self.position + size) - len(self.buffer) + if not_read > 0: + # read up to size bytes from stream + data = self.stream.read(not_read) + else: + data = self.buffer[self.position:self.position+size+1] + + old_position = self.position + new_position = self.position + len(data) + self.buffer.extend(data) + self.position = new_position + return self.buffer[old_position:new_position+1] + + def seek(self, offset, whence=0): + if whence == 0: + # seek from beginning of buffer + self.position = offset + elif whence == 1: + # seek from current position + self.position += offset + elif whence == 2: + # seek from end of buffer + self.position = len(self.buffer) + offset + else: + raise ValueError("Invalid whence value") + + # adjust position to be within buffer bounds + self.position = max(0, min(self.position, len(self.buffer))) + + def tell(self): + return self.position + + def readable(self): + return True + + def seekable(self): + return True + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.stream.__exit__(exc_type, exc_value, traceback) + + def decompress_if_zstd(stream): """ Given a byte stream, returns either the original stream or the decompressed stream @@ -165,7 +223,7 @@ def decompress_if_zstd(stream): logging.debug(f"Decompressing {stream=}") import zstandard as zstd dctx = zstd.ZstdDecompressor() - return tarfile.open(fileobj=dctx.stream_reader(stream, read_size=4096), mode='r|') + return tarfile.open(fileobj=BytesIO(dctx.stream_reader(stream).read()), mode='r:tar') def open_tar(tar_file: str) -> tarfile.TarFile: @@ -190,13 +248,21 @@ def read_files_from_tar_recursive(tar_file: str, paths: Sequence[str], append_sl paths = [f"{p.strip('/')}/" for p in paths] with open_tar(tar_file) as index: for member in index.getmembers(): - for path in paths: - if member.isfile() and member.path.startswith(path): - fd = index.extractfile(member) - assert fd - yield member.path, fd - break - continue + file_path = member.path + if member.isfile() and check_file_matches(file_path, paths): + logging.debug(f"tar: Returning {file_path}") + fd = index.extractfile(member) + assert fd + yield file_path, fd + else: + logging.debug(f'tar: unmatched {file_path} for query {paths}') + + +def check_file_matches(file_path: str, queries: list[str]) -> bool: + for query in queries: + if file_path.startswith(query): + return True + return False def extract_files_from_tar_generator( @@ -205,7 +271,6 @@ def extract_files_from_tar_generator( remove_prefix: str = '', append_slash: bool = True, ): - assert os.path.exists(output_dir) remove_prefix = remove_prefix.strip('/') if append_slash and remove_prefix: remove_prefix += '/' @@ -214,6 +279,7 @@ def extract_files_from_tar_generator( output_path = os.path.join(output_dir, file_path[len(remove_prefix):].lstrip('/')) os.makedirs(os.path.dirname(output_path), exist_ok=True) with open(output_path, 'wb') as f: + logging.debug(f"Extracting {file_path}") f.write(fd.read()) From 38b23de9ad94c60e028d2a663fc0fb8e3da63207 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Mon, 17 Apr 2023 21:57:42 +0200 Subject: [PATCH 10/20] distro/keyrings: extraction works --- distro/distro.py | 9 +++++++++ distro/keyring.py | 42 ++++++++++++++++++++++-------------------- distro/package.py | 2 +- distro/repo_config.py | 2 +- requirements.txt | 1 + 5 files changed, 34 insertions(+), 22 deletions(-) diff --git a/distro/distro.py b/distro/distro.py index 215a665..7549e95 100644 --- a/distro/distro.py +++ b/distro/distro.py @@ -49,6 +49,15 @@ class Distro(Generic[RepoType]): results.update(repo.packages) return results + def find_package(self, name: str) -> Optional[BinaryPackageType]: + for repo in self.repos.values(): + if not repo.scanned: + repo.scan() + p = repo.packages.get(name, None) + if p: + return p + return None + def repos_config_snippet(self, extra_repos: Mapping[str, RepoInfo] = {}) -> str: extras: list[Repo] = [ Repo(name, url_template=info.url_template, arch=self.arch, options=info.options, scan=False) for name, info in extra_repos.items() diff --git a/distro/keyring.py b/distro/keyring.py index 59d8044..a7d609e 100644 --- a/distro/keyring.py +++ b/distro/keyring.py @@ -9,7 +9,7 @@ from constants import Arch, KEYRINGS_KEY, KEYRINGS_LOCAL_KEY from distro.repo_config import get_repo_config from exec.cmd import CompletedProcess, run_cmd from exec.file import makedir, remove_file -from utils import extract_files_from_tar_generator, read_files_from_tar_recursive +from utils import extract_files_from_tar_generator, read_files_from_tar_recursive, sha256sum from .distro import Distro, get_base_distro, get_kupfer_local, get_kupfer_https from .package import BinaryPackage @@ -64,7 +64,7 @@ def init_keyring_gpg_dir( ) -> dict[str, bool]: base_dir = base_dir or get_keyring_path(arch, distro_type) gpg_dir = get_keyring_gpg_path(base_dir) - exists = os.path.exists(gpg_dir) + exists = os.path.exists(os.path.join(gpg_dir, 'trustdb.gpg')) if exists and not lazy: remove_file(gpg_dir) exists = False @@ -109,7 +109,7 @@ def init_keyring_dist_dir( pkg_names: list[str] = [] distro: Distro if distro_type == DistroType.BASE: - pkg_names = repo_config.base_distros.get(arch, {}).get(KEYRINGS_KEY, None) or [] + pkg_names = repo_config.base_distros.get(arch, {}).get(KEYRINGS_KEY, None) or [] # type: ignore[call-overload] distro = get_base_distro(arch, scan=False) elif distro_type == DistroType.LOCAL: pkg_name = repo_config.get(KEYRINGS_LOCAL_KEY, None) @@ -119,9 +119,9 @@ def init_keyring_dist_dir( pkg_names = repo_config.get(KEYRINGS_KEY, None) or [] distro = get_kupfer_https(arch, scan=False) logging.debug(f"Acquiring keyrings from {distro}: {pkg_names}") - dist_pkgs, changed = acquire_dist_pkgs(pkg_names, distro, base_dir) - if lazy and dist_pkgs and not changed and os.path.exists(dist_dir): # and keyring_is_created(arch, distro_type): - return {name: (val[0], False) for name, val in dist_pkgs.items()} + dist_pkgs, changed = acquire_dist_pkgs(pkg_names, distro, dist_dir) + #if lazy and dist_pkgs and not changed and os.path.exists(dist_dir): # and keyring_is_created(arch, distro_type): + # return {name: (get_keyring_dist_path(base_dir, name), False) for name, val in dist_pkgs.items()} makedir(dist_dir) dist_dirs = [] @@ -131,26 +131,25 @@ def init_keyring_dist_dir( _dir = os.path.join(dist_dir, name) results[name] = _dir, False if lazy and not changed and os.path.exists(_dir): + logging.debug(f"Skipping extracting keyring pkg for {name}: dir exists and file unchanged") continue extract_keyring_pkg(dist_pkg, _dir) dist_dirs.append(_dir) - results[name] = dist_pkg, True + results[name] = _dir, True return results def acquire_dist_pkgs(keyring_packages: list[str], distro: Distro, dist_dir: str) -> tuple[dict[str, tuple[str, bool]], bool]: if not keyring_packages: return {}, False - pkgs = {} + pkgs: dict[str, BinaryPackage] = {} not_found = [] - distro.scan(lazy=True) - repos: dict[str, BinaryPackage] = distro.get_packages() - pkg: BinaryPackage + pkg: Optional[BinaryPackage] for name in keyring_packages: - if name not in repos: + pkg = distro.find_package(name) + if not pkg: not_found.append(name) continue - pkg = repos[name] pkgs[name] = pkg if not_found: raise Exception(f"Keyring packages for {distro.arch} not found: {not_found}") @@ -158,22 +157,25 @@ def acquire_dist_pkgs(keyring_packages: list[str], distro: Distro, dist_dir: str changed = False results = {} for name in pkgs: - assert isinstance(pkgs[name], BinaryPackage) pkg = pkgs[name] assert PKGNAME_MARKER in pkg.filename comp_ext = pkg.filename.rsplit(PKGNAME_MARKER, 1)[1] - target_path, _changed = pkg.acquire(dist_dir, f'{name}.tar{comp_ext}') + filename = f'{name}.tar{comp_ext}' + filepath = os.path.join(dist_dir, filename) + checksum = None if not os.path.exists(filepath) else sha256sum(filepath) + target_path, _changed = pkg.acquire(dist_dir, filename) + _changed = _changed and checksum != sha256sum(filepath) results[name] = target_path, _changed if _changed: - logging.debug(f"{target_path} changed") changed = True + logging.debug(f"{target_path} changed") return results, changed def extract_keyring_pkg(pkg_path: str, dest_path: str): - makedir(dest_path) + logging.debug(f"Extracting {pkg_path} to {dest_path}") extract_files_from_tar_generator( - read_files_from_tar_recursive(pkg_path, PKG_KEYRING_FOLDER), + read_files_from_tar_recursive(pkg_path, [PKG_KEYRING_FOLDER]), dest_path, remove_prefix=PKG_KEYRING_FOLDER, ) @@ -183,8 +185,8 @@ def get_keyring_path(arch: Arch, distro_type: DistroType, *extra_paths) -> str: return os.path.join(config.get_path('pacman'), KEYRING_DIR, arch, KEYRING_LOCATIONS[distro_type], *extra_paths) -def get_keyring_dist_path(base_dir: str) -> str: - return os.path.join(base_dir, KEYRING_DIST_DIR) +def get_keyring_dist_path(base_dir: str, *name) -> str: + return os.path.join(base_dir, KEYRING_DIST_DIR, *name) def get_keyring_gpg_path(base_dir: str) -> str: diff --git a/distro/package.py b/distro/package.py index 200a7ef..672eb98 100644 --- a/distro/package.py +++ b/distro/package.py @@ -94,5 +94,5 @@ class RemotePackage(BinaryPackage): logging.info(f"Trying to download package {url}") changed = download_file(dest_file_path, url) - logging.info(f"{self.filename} downloaded from repos") + logging.info(f"{self.filename} {'already ' if not changed else ''}downloaded from repos") return dest_file_path, changed diff --git a/distro/repo_config.py b/distro/repo_config.py index 32688b1..e06a40c 100644 --- a/distro/repo_config.py +++ b/distro/repo_config.py @@ -123,7 +123,7 @@ REPOS_CONFIG_DEFAULTS_DICT = { BASEDISTROS_KEY: { arch: { REMOTEURL_KEY: None, - KEYRINGS_KEY: arch_def.get(KEYRINGS_KEY, None).copy(), + KEYRINGS_KEY: arch_def[KEYRINGS_KEY].copy() if KEYRINGS_KEY in arch_def else None, 'repos': { k: { 'remote_url': v diff --git a/requirements.txt b/requirements.txt index df53ed4..3e26b57 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ requests python-dateutil enlighten PyYAML +zstandard From d2e0fad436b723b52e450e3f052849d91f75576f Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Tue, 18 Apr 2023 00:38:45 +0200 Subject: [PATCH 11/20] utils: remove BackwardsReadablestream --- utils.py | 57 -------------------------------------------------------- 1 file changed, 57 deletions(-) diff --git a/utils.py b/utils.py index 26a65ed..14c5689 100644 --- a/utils.py +++ b/utils.py @@ -152,63 +152,6 @@ def is_zstd(data): return True -class BackwardsReadableStream: - def __init__(self, stream): - self.stream = stream - self.buffer = bytearray() - self.position = 0 - - def read(self, size=-1): - data = b'' - if size == -1: - # read all remaining data in stream - data = self.stream.read() - else: - not_read = (self.position + size) - len(self.buffer) - if not_read > 0: - # read up to size bytes from stream - data = self.stream.read(not_read) - else: - data = self.buffer[self.position:self.position+size+1] - - old_position = self.position - new_position = self.position + len(data) - self.buffer.extend(data) - self.position = new_position - return self.buffer[old_position:new_position+1] - - def seek(self, offset, whence=0): - if whence == 0: - # seek from beginning of buffer - self.position = offset - elif whence == 1: - # seek from current position - self.position += offset - elif whence == 2: - # seek from end of buffer - self.position = len(self.buffer) + offset - else: - raise ValueError("Invalid whence value") - - # adjust position to be within buffer bounds - self.position = max(0, min(self.position, len(self.buffer))) - - def tell(self): - return self.position - - def readable(self): - return True - - def seekable(self): - return True - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.stream.__exit__(exc_type, exc_value, traceback) - - def decompress_if_zstd(stream): """ Given a byte stream, returns either the original stream or the decompressed stream From 3e957254f5047f56608564e7abdafb85e64309e6 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Mon, 1 Apr 2024 17:57:12 +0200 Subject: [PATCH 12/20] distro/keyring: use vanilla pacman-key now that --gpgdir is merged --- distro/keyring.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/distro/keyring.py b/distro/keyring.py index a7d609e..25544c6 100644 --- a/distro/keyring.py +++ b/distro/keyring.py @@ -70,7 +70,7 @@ def init_keyring_gpg_dir( exists = False lazy = lazy and exists if not lazy: - run_cmd([get_pacman_key_binary(), '--init', '--gpgdir', gpg_dir]) + run_cmd(['pacman-key', '--init', '--gpgdir', gpg_dir]) results = {} for name, val in keyring_dists.items(): dist_dir, dist_changed = val @@ -88,7 +88,7 @@ def import_dist_keyring( dist_dir: str, ) -> CompletedProcess: assert gpg_dir and dist_dir and config.runtime.script_source_dir - r = run_cmd([get_pacman_key_binary(), '--populate-from', dist_dir, '--populate', '--gpgdir', gpg_dir]) + r = run_cmd(['pacman-key', '--populate-from', dist_dir, '--populate', '--gpgdir', gpg_dir]) assert isinstance(r, CompletedProcess) return r @@ -191,7 +191,3 @@ def get_keyring_dist_path(base_dir: str, *name) -> str: def get_keyring_gpg_path(base_dir: str) -> str: return os.path.join(base_dir, KEYRING_GPG_DIR) - - -def get_pacman_key_binary() -> str: - return os.path.join(config.runtime.script_source_dir, 'bin', 'pacman-key-user') From e79859b0a091c9d9254c4f1596bde0e47a3805c0 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Mon, 1 Apr 2024 21:46:22 +0200 Subject: [PATCH 13/20] distro/gpg: add helpers for generating secret keys --- distro/gpg.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 distro/gpg.py diff --git a/distro/gpg.py b/distro/gpg.py new file mode 100644 index 0000000..4fe15df --- /dev/null +++ b/distro/gpg.py @@ -0,0 +1,50 @@ +import logging +import os + +from exec.cmd import run_cmd +from exec.file import get_temp_dir, makedir, write_file + +GPG_ARGS = ["--batch", "--no-tty"] + + +def get_gpg_creation_script( + key_name: str = "Kupfer Local Signing", + email: str = "local@kupfer.mobi", + comment: str = "Generated by kupferbootstrap", +): + return f""" +%echo Generating a new ed25519 GPG key for "{key_name} <{email}> # {comment}" + +%no-protection + +Key-Type: eddsa +Key-Curve: Ed25519 +Key-Usage: cert,sign +Subkey-Type: ecdh +Subkey-Curve: Curve25519 +Subkey-Usage: encrypt + +Name-Real: {key_name} +Name-Comment: {comment} +Name-Email: {email} +Expire-Date: 0 +# Do a commit here, so that we can later print "done" +%commit +%echo done +""" + + +def create_secret_key(location: str, *, gpg_binary: str = "gpg", **creation_args): + makedir(os.path.dirname(location)) + temp_dir = get_temp_dir() + script_file = os.path.join(temp_dir, "__gpg_creation_script") + write_file(script_file, content=get_gpg_creation_script(**creation_args)) + logging.info(f"Creating new GPG key for {location!r}") + run_cmd([gpg_binary, *GPG_ARGS, "--homedir", temp_dir, "--generate-key", script_file]).check_returncode() + res = run_cmd( + [gpg_binary, *GPG_ARGS, "--homedir", temp_dir, "--armor", "--export-secret-keys"], capture_output=True + ) + if not (res.stdout and res.stdout.strip()): + raise Exception(f"Failed to get secret GPG key from stdout: {res.stdout=}\n{res.stderr=}") + logging.debug(f"Writing GPG private key to {location}") + write_file(location, content=res.stdout, mode="600") From 3034afe5a849cbf77681eaf3440b81653480852f Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Mon, 1 Apr 2024 23:50:44 +0200 Subject: [PATCH 14/20] distro/gpg: add initialisation script for key files and gpghome --- distro/gpg.py | 112 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 103 insertions(+), 9 deletions(-) diff --git a/distro/gpg.py b/distro/gpg.py index 4fe15df..c354f59 100644 --- a/distro/gpg.py +++ b/distro/gpg.py @@ -1,16 +1,34 @@ import logging import os -from exec.cmd import run_cmd +from typing import Optional, TypedDict + +from config.state import config +from exec.cmd import run_cmd, CompletedProcess from exec.file import get_temp_dir, makedir, write_file +PKG_KEY_FILE = "package_signing_key.pgp" +REPO_KEY_FILE = "repo_signing_key.pgp" + +GPG_HOME_DIR = "gpghome" + +KUPFER_DEFAULT_NAME = "Kupfer Local Signing" +KUFER_DEFAULT_EMAIL = "local@kupfer.mobi" +KUPFER_DEFAULT_COMMENT = "Generated by kupferbootstrap" + GPG_ARGS = ["--batch", "--no-tty"] +class Fingerprints(TypedDict): + pkg: str + repo: str + + + def get_gpg_creation_script( - key_name: str = "Kupfer Local Signing", - email: str = "local@kupfer.mobi", - comment: str = "Generated by kupferbootstrap", + key_name: str = KUPFER_DEFAULT_NAME, + email: str = KUFER_DEFAULT_EMAIL, + comment: str = KUPFER_DEFAULT_COMMENT, ): return f""" %echo Generating a new ed25519 GPG key for "{key_name} <{email}> # {comment}" @@ -39,12 +57,88 @@ def create_secret_key(location: str, *, gpg_binary: str = "gpg", **creation_args temp_dir = get_temp_dir() script_file = os.path.join(temp_dir, "__gpg_creation_script") write_file(script_file, content=get_gpg_creation_script(**creation_args)) - logging.info(f"Creating new GPG key for {location!r}") - run_cmd([gpg_binary, *GPG_ARGS, "--homedir", temp_dir, "--generate-key", script_file]).check_returncode() - res = run_cmd( - [gpg_binary, *GPG_ARGS, "--homedir", temp_dir, "--armor", "--export-secret-keys"], capture_output=True - ) + run_cmd([gpg_binary, *GPG_ARGS, "--homedir", temp_dir, "--generate-key", script_file], capture_output=True).check_returncode() # type: ignore[union-attr] + res = run_cmd([gpg_binary, *GPG_ARGS, "--homedir", temp_dir, "--armor", "--export-secret-keys"], capture_output=True) + assert isinstance(res, CompletedProcess) if not (res.stdout and res.stdout.strip()): raise Exception(f"Failed to get secret GPG key from stdout: {res.stdout=}\n{res.stderr=}") logging.debug(f"Writing GPG private key to {location}") write_file(location, content=res.stdout, mode="600") + + +def import_gpg_key( + key_file: str, + gpgdir: str, + *, + gpg_binary: str = "gpg", +): + res = run_cmd([gpg_binary, "--homedir", gpgdir, *GPG_ARGS, "--import", key_file], capture_output=True) + assert isinstance(res, CompletedProcess) + res.check_returncode() + + +def detect_key_id(location: str, gpg_binary: str = "gpg"): + res = run_cmd([gpg_binary, *GPG_ARGS, "--with-colons", "--show-keys", location], capture_output=True) + assert isinstance(res, CompletedProcess) + if res.returncode: + raise Exception(f"Failed to scan {location} for a gpg key id:\n{res.stdout=}\n\n{res.stderr=}") + text = res.stdout.decode().strip() + for line in text.split("\n"): + if line.startswith("fpr:"): + fp: str = line.rstrip(":").rsplit(":")[-1] + if not fp or not fp.isalnum(): + raise Exception(f"Failed to detect GPG fingerprint fron line {line}") + return fp.strip() + raise Exception(f"GPG Fingerprint line (fpr:) not found in GPG stdout: {text!r}") + + +def ensure_gpg_initialised( + gpg_base_dir: str, + gpg_binary: str = "gpg", + email: str = KUFER_DEFAULT_EMAIL, + gpgdir: Optional[str] = None, +) -> Fingerprints: + repo_key = os.path.join(gpg_base_dir, REPO_KEY_FILE) + pkg_key = os.path.join(gpg_base_dir, PKG_KEY_FILE) + gpgdir = gpgdir or os.path.join(gpg_base_dir, GPG_HOME_DIR) + makedir(gpgdir) + names = {"repo": "Repo Signing", "pkg": "Package Signing"} + fingerprints: Fingerprints = {} # type: ignore[typeddict-item] + for key_type, key_file in {"repo": repo_key, "pkg": pkg_key}.items(): + if not os.path.exists(key_file): + key_name = f"Kupfer Local {names[key_type]}" + logging.info(f"Creating new GPG key for {key_name!r} <{email}> at {key_file!r}") + create_secret_key(key_file, key_name=key_name) + import_gpg_key(key_file, gpg_binary=gpg_binary, gpgdir=gpgdir) + fingerprints[key_type] = detect_key_id(key_file) # type: ignore[literal-required] + pkg_fp = fingerprints["pkg"] + repo_fp = fingerprints["repo"] + logging.debug(f"Ensuring package build GPG key {pkg_fp!r} is signed by repo key {repo_fp}") + res = run_cmd( + [ + gpg_binary, + *GPG_ARGS, + "--yes", + "--homedir", + gpgdir, + "--default-key", + repo_fp, + "--trusted-key", + pkg_fp, + "--sign-key", + pkg_fp, + ], + capture_output=True, + ) + assert isinstance(res, CompletedProcess) + if res.returncode: + raise Exception(f"Failed to sign package GPG key {pkg_fp!r} with repo key {repo_fp!r}:\n{res.stdout=}\n{res.stderr=}") + logging.debug("GPG setup done") + return fingerprints + +def init_keys(*kargs, lazy: bool = True, **kwargs) -> None: + if lazy and config.runtime.gpg_initialized: + return + fps = ensure_gpg_initialised(*kargs, **kwargs) + config.runtime.gpg_pkg_key = fps["pkg"] + config.runtime.gpg_repo_key = fps["repo"] From 4a48e78ec0e0c971fb314b3650b7871420337259 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Tue, 2 Apr 2024 01:39:01 +0200 Subject: [PATCH 15/20] config: add gpg dir to chroot paths --- config/scheme.py | 1 + config/state.py | 1 + constants.py | 1 + 3 files changed, 3 insertions(+) diff --git a/config/scheme.py b/config/scheme.py index a5846ba..c27ee16 100644 --- a/config/scheme.py +++ b/config/scheme.py @@ -67,6 +67,7 @@ class PathsSection(DictScheme): images: str ccache: str rust: str + gpg: str class ProfilesSection(DictScheme): diff --git a/config/state.py b/config/state.py index 2d1ba42..ab144e1 100644 --- a/config/state.py +++ b/config/state.py @@ -44,6 +44,7 @@ CONFIG_DEFAULTS_DICT = { 'images': os.path.join('%cache_dir%', 'images'), 'ccache': os.path.join('%cache_dir%', 'ccache'), 'rust': os.path.join('%cache_dir%', 'rust'), + 'gpg': os.path.join('%cache_dir%', 'gpg'), }, 'profiles': { 'current': 'default', diff --git a/constants.py b/constants.py index 8fde0d2..abef1ad 100644 --- a/constants.py +++ b/constants.py @@ -156,6 +156,7 @@ CHROOT_PATHS = { 'packages': '/packages', 'pkgbuilds': '/pkgbuilds', 'images': '/images', + 'gpg': '/gpg', } WRAPPER_TYPES = [ From 6f09fe4403388f5bddb11fb68c283ca024900609 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Tue, 2 Apr 2024 02:49:18 +0200 Subject: [PATCH 16/20] packages/build: pass `try_download` to build_enable_qemu_binfmt() --- image/image.py | 2 +- packages/build.py | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/image/image.py b/image/image.py index 6532af7..a79978d 100644 --- a/image/image.py +++ b/image/image.py @@ -446,7 +446,7 @@ def cmd_build( packages_extra = BASE_PACKAGES + profile.pkgs_include if arch != config.runtime.arch: - build_enable_qemu_binfmt(arch) + build_enable_qemu_binfmt(arch, try_download=not no_download_pkgs) if local_repos and build_pkgs: logging.info("Making sure all packages are built") diff --git a/packages/build.py b/packages/build.py index d7f56d4..e297d05 100644 --- a/packages/build.py +++ b/packages/build.py @@ -440,10 +440,11 @@ def setup_build_chroot( add_kupfer_repos: bool = True, clean_chroot: bool = False, repo: Optional[dict[str, Pkgbuild]] = None, + try_download: bool = True, ) -> BuildChroot: assert config.runtime.arch if arch != config.runtime.arch: - build_enable_qemu_binfmt(arch, repo=repo or discover_pkgbuilds(), lazy=False) + build_enable_qemu_binfmt(arch, repo=repo or discover_pkgbuilds(), try_download=try_download, lazy=False) init_prebuilts(arch) chroot = get_build_chroot(arch, add_kupfer_repos=add_kupfer_repos) chroot.mount_packages() @@ -513,6 +514,7 @@ def build_package( clean_chroot: bool = False, build_user: str = 'kupfer', repo: Optional[dict[str, Pkgbuild]] = None, + try_download: bool = False, ): makepkg_compile_opts = ['--holdver'] makepkg_conf_path = 'etc/makepkg.conf' @@ -533,6 +535,7 @@ def build_package( extra_packages=deps, clean_chroot=clean_chroot, repo=repo, + try_download=try_download, ) assert config.runtime.arch native_chroot = target_chroot @@ -543,6 +546,7 @@ def build_package( extra_packages=['base-devel'] + CROSSDIRECT_PKGS, clean_chroot=clean_chroot, repo=repo, + try_download=try_download, ) if not package.mode: logging.warning(f'Package {package.path} has no _mode set, assuming "host"') @@ -762,6 +766,7 @@ def build_packages( enable_ccache=enable_ccache, clean_chroot=clean_chroot, repo=repo, + try_download=try_download, ) files += add_package_to_repo(package, arch) updated_repos.add(package.repo) @@ -816,7 +821,12 @@ def build_packages_by_paths( _qemu_enabled: dict[Arch, bool] = {arch: False for arch in ARCHES} -def build_enable_qemu_binfmt(arch: Arch, repo: Optional[dict[str, Pkgbuild]] = None, lazy: bool = True, native_chroot: Optional[BuildChroot] = None): +def build_enable_qemu_binfmt( + arch: Arch, repo: Optional[dict[str, Pkgbuild]] = None, + lazy: bool = True, + native_chroot: Optional[BuildChroot] = None, + try_download: bool = True, +) -> None: """ Build and enable qemu-user-static, binfmt and crossdirect Specify lazy=False to force building the packages. @@ -852,7 +862,7 @@ def build_enable_qemu_binfmt(arch: Arch, repo: Optional[dict[str, Pkgbuild]] = N packages, native, repo=repo, - try_download=True, + try_download=try_download, enable_crosscompile=False, enable_crossdirect=False, enable_ccache=False, From 166a8620a77143a4cb39fb731ebf03185a05d06b Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Tue, 2 Apr 2024 12:22:22 +0200 Subject: [PATCH 17/20] config: add gpg folder and signing options --- config/scheme.py | 5 +++++ config/state.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/config/scheme.py b/config/scheme.py index c27ee16..7ee2f47 100644 --- a/config/scheme.py +++ b/config/scheme.py @@ -44,6 +44,8 @@ class BuildSection(DictScheme): crosscompile: bool crossdirect: bool threads: int + sign_pkgs: bool + sign_repos: bool class PkgbuildsSection(DictScheme): @@ -141,6 +143,9 @@ class RuntimeConfiguration(DictScheme): uid: Optional[int] progress_bars: Optional[bool] colors: Optional[bool] + gpg_initialized: bool + gpg_pkg_key: Optional[str] + gpg_repo_key: Optional[str] class ConfigLoadState(DictScheme): diff --git a/config/state.py b/config/state.py index ab144e1..8038285 100644 --- a/config/state.py +++ b/config/state.py @@ -24,6 +24,8 @@ CONFIG_DEFAULTS_DICT = { 'crosscompile': True, 'crossdirect': True, 'threads': 0, + 'sign_pkgs': True, + 'sign_repos': False, }, 'pkgbuilds': { 'git_repo': 'https://gitlab.com/kupfer/packages/pkgbuilds.git', @@ -64,6 +66,9 @@ CONFIG_RUNTIME_DEFAULTS: RuntimeConfiguration = RuntimeConfiguration.fromDict({ 'uid': None, 'progress_bars': None, 'colors': None, + 'gpg_initialized': False, + 'gpg_pkg_key': None, + 'gpg_repo_key': None, }) From 871c4f27c7b51587859bb2469764a17bf934d0b2 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Tue, 2 Apr 2024 12:22:50 +0200 Subject: [PATCH 18/20] chroot/abstract: run_cmd(): use unshare --pid to get rid of leftover processes --- chroot/abstract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chroot/abstract.py b/chroot/abstract.py index ffef4a6..acf28a8 100644 --- a/chroot/abstract.py +++ b/chroot/abstract.py @@ -248,7 +248,7 @@ class Chroot(AbstractChroot): inner_cmd = generate_cmd_su(script, switch_user=switch_user, elevation_method='none', force_su=True) else: inner_cmd = wrap_in_bash(script, flatten_result=False) - cmd = flatten_shell_script(['chroot', self.path] + env_cmd + inner_cmd, shell_quote_items=True) + cmd = flatten_shell_script(["unshare", "--fork", "--pid", 'chroot', self.path] + env_cmd + inner_cmd, shell_quote_items=True) return run_root_cmd(cmd, env=outer_env, attach_tty=attach_tty, capture_output=capture_output, stdout=stdout, stderr=stderr) From 07436a0ad2b0988e8455ffd12fc18dfb9c6df1c7 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Tue, 2 Apr 2024 12:23:52 +0200 Subject: [PATCH 19/20] chroot/build: add mount_gpg() --- chroot/build.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/chroot/build.py b/chroot/build.py index 40b123d..b33000e 100644 --- a/chroot/build.py +++ b/chroot/build.py @@ -1,3 +1,4 @@ +import atexit import logging import os import subprocess @@ -7,6 +8,7 @@ from typing import ClassVar, Optional from config.state import config from constants import Arch, GCC_HOSTSPECS, CROSSDIRECT_PKGS, CHROOT_PATHS from distro.distro import get_kupfer_local +from distro.gpg import GPG_HOME_DIR from exec.cmd import run_root_cmd from exec.file import makedir, remove_file, root_makedir, root_write_file, symlink @@ -159,6 +161,25 @@ class BuildChroot(Chroot): )) return results + def mount_gpg(self, fail_if_mounted: bool = False, schedule_gpg_kill: bool = True) -> str: + res = self.mount( + absolute_source=config.get_path('gpg'), + relative_destination=CHROOT_PATHS['gpg'].lstrip('/'), + fail_if_mounted=fail_if_mounted, + ) + if schedule_gpg_kill: + atexit.register(self.kill_gpg_agent) + return res + + def get_gpg_home(self, host_path: bool = False) -> str: + gpg_home = os.path.join(CHROOT_PATHS['gpg']. GPG_HOME_DIR) + if host_path: + gpg_home = self.get_path(gpg_home) + return gpg_home + + def kill_gpg_agent(self) -> subprocess.CompletedProcess: + res = self.run_cmd(["timeout", "2s", "gpgconf", "--kill", "gpg-agent"], inner_env={"GNUPGHOME": self.get_gpg_home()}) + logging.debug(f"GPG agent killed: {res.returncode=}, {res.stdout=}, {res.stderr}") def get_build_chroot(arch: Arch, add_kupfer_repos: bool = True, **kwargs) -> BuildChroot: name = build_chroot_name(arch) From aaf94de0acb1d298358fd02604f2209edc773115 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Tue, 2 Apr 2024 12:24:39 +0200 Subject: [PATCH 20/20] packages/build: add pkg signing --- packages/build.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/packages/build.py b/packages/build.py index e297d05..0ca7717 100644 --- a/packages/build.py +++ b/packages/build.py @@ -16,6 +16,7 @@ from exec.cmd import run_cmd, run_root_cmd from exec.file import makedir, remove_file, symlink from chroot.build import get_build_chroot, BuildChroot from distro.distro import get_kupfer_https, get_kupfer_local, get_kupfer_repo_names +from distro.gpg import init_keys, GPG_HOME_DIR from distro.package import RemotePackage, LocalPackage from distro.repo import LocalRepo from progressbar import BAR_PADDING, get_levels_bar @@ -213,6 +214,14 @@ def add_file_to_repo(file_path: str, repo_name: str, arch: Arch, remove_original ) if remove_original: remove_file(file_path) + sig_file = "{file_path}.sig" + if os.path.exists(sig_file): + shutil.copy( + sig_file, + repo_dir, + ) + if remove_original: + remove_file(sig_file) # clean up same name package from pacman cache cache_file = os.path.join(pacman_cache_dir, file_name) @@ -602,6 +611,13 @@ def build_package( makepkg_conf_absolute = os.path.join('/', makepkg_conf_path) build_cmd = ['source', '/etc/profile', '&&', *MAKEPKG_CMD, '--config', makepkg_conf_absolute, '--skippgpcheck', *makepkg_compile_opts] + if config.file.build.sign_pkgs: + logging.debug("Package signing requested; adding makepkg args and GNUPGHOME env var") + init_keys(config.get_path("gpg"), lazy=True) + assert config.runtime.gpg_pkg_key + build_cmd.extend(["--sign", "--key", config.runtime.gpg_pkg_key]) + env["GNUPGHOME"] = os.path.join(CHROOT_PATHS["gpg"], GPG_HOME_DIR) + target_chroot.mount_gpg() logging.debug(f'Building: Running {build_cmd}') result = build_root.run_cmd( build_cmd,