From 8437613e6ed3224284f14262be70f21464788755 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Sat, 8 Jul 2023 14:13:19 +0200 Subject: [PATCH 01/30] image/image: move CLI methods to image/cli.py --- chroot/cli.py | 2 +- image/cli.py | 210 ++++++++++++++++++++++++++++++++++++++++++++++++- image/image.py | 188 ++----------------------------------------- 3 files changed, 215 insertions(+), 185 deletions(-) diff --git a/chroot/cli.py b/chroot/cli.py index 0a99f48..c214a90 100644 --- a/chroot/cli.py +++ b/chroot/cli.py @@ -7,6 +7,7 @@ from typing import Optional from config.state import config from wrapper import enforce_wrap from devices.device import get_profile_device +from image.cli import cmd_inspect from .abstract import Chroot from .base import get_base_chroot @@ -30,7 +31,6 @@ def cmd_chroot(ctx: click.Context, type: str = 'build', name: Optional[str] = No raise Exception(f'Unknown chroot type: "{type}"') if type == 'rootfs': - from image.image import cmd_inspect assert isinstance(cmd_inspect, click.Command) ctx.invoke(cmd_inspect, profile=name, shell=True) return diff --git a/image/cli.py b/image/cli.py index 866590d..c4b4143 100644 --- a/image/cli.py +++ b/image/cli.py @@ -1,6 +1,214 @@ +import click +import logging +import os + +from signal import pause +from typing import Optional + +from config.state import config, Profile +from constants import BASE_LOCAL_PACKAGES, BASE_PACKAGES +from devices.device import get_profile_device +from exec.file import makedir +from flavours.flavour import get_profile_flavour +from packages.build import build_enable_qemu_binfmt, build_packages, filter_pkgbuilds +from wrapper import enforce_wrap + from .boot import cmd_boot from .flash import cmd_flash -from .image import cmd_image +from .image import ( + IMG_FILE_BOOT_DEFAULT_SIZE, + create_boot_fs, + create_img_file, + create_root_fs, + dd_image, + get_device_chroot, + get_image_path, + install_rootfs, + losetup_rootfs_image, + mount_chroot, + partprobe, + partition_device, +) + + +@click.group(name='image') +def cmd_image(): + """Build, flash and boot device images""" + for cmd in [cmd_boot, cmd_flash]: cmd_image.add_command(cmd) + +sectorsize_option = click.option( + '-b', + '--sector-size', + help="Override the device's sector size", + type=int, + default=None, +) + + +@cmd_image.command(name='build') +@click.argument('profile_name', required=False) +@click.option( + '--local-repos/--no-local-repos', + '-l/-L', + help='Whether to use local package repos at all or only use HTTPS repos.', + default=True, + show_default=True, + is_flag=True, +) +@click.option( + '--build-pkgs/--no-build-pkgs', + '-p/-P', + help='Whether to build missing/outdated local packages if local repos are enabled.', + default=True, + show_default=True, + is_flag=True, +) +@click.option( + '--no-download-pkgs', + help='Disable trying to download packages instead of building if building is enabled.', + default=False, + is_flag=True, +) +@click.option( + '--block-target', + help='Override the block device file to write the final image to', + type=click.Path(), + default=None, +) +@click.option( + '--skip-part-images', + help='Skip creating image files for the partitions and directly work on the target block device.', + default=False, + is_flag=True, +) +@sectorsize_option +def cmd_build( + profile_name: Optional[str] = None, + local_repos: bool = True, + build_pkgs: bool = True, + no_download_pkgs=False, + block_target: Optional[str] = None, + sector_size: Optional[int] = None, + skip_part_images: bool = False, +): + """ + Build a device image. + + Unless overriden, required packages will be built or preferably downloaded from HTTPS repos. + """ + + config.enforce_profile_device_set() + config.enforce_profile_flavour_set() + enforce_wrap() + device = get_profile_device(profile_name) + arch = device.arch + # check_programs_wrap(['makepkg', 'pacman', 'pacstrap']) + profile: Profile = config.get_profile(profile_name) + flavour = get_profile_flavour(profile_name) + rootfs_size_mb = flavour.parse_flavourinfo().rootfs_size * 1000 + int(profile.size_extra_mb) + + packages = BASE_LOCAL_PACKAGES + [device.package.name, flavour.pkgbuild.name] + packages_extra = BASE_PACKAGES + profile.pkgs_include + + if arch != config.runtime.arch: + build_enable_qemu_binfmt(arch) + + if local_repos and build_pkgs: + logging.info("Making sure all packages are built") + # enforce that local base packages are built + pkgbuilds = set(filter_pkgbuilds(packages, arch=arch, allow_empty_results=False, use_paths=False)) + # extra packages might be a mix of package names that are in our PKGBUILDs and packages from the base distro + pkgbuilds |= set(filter_pkgbuilds(packages_extra, arch=arch, allow_empty_results=True, use_paths=False)) + build_packages(pkgbuilds, arch, try_download=not no_download_pkgs) + + sector_size = sector_size or device.get_image_sectorsize() + + image_path = block_target or get_image_path(device, flavour.name) + + makedir(os.path.dirname(image_path)) + + logging.info(f'Creating new file at {image_path}') + create_img_file(image_path, f"{rootfs_size_mb}M") + + loop_device = losetup_rootfs_image(image_path, sector_size or device.get_image_sectorsize_default()) + + partition_device(loop_device) + partprobe(loop_device) + + boot_dev: str + root_dev: str + loop_boot = loop_device + 'p1' + loop_root = loop_device + 'p2' + if skip_part_images: + boot_dev = loop_boot + root_dev = loop_root + else: + logging.info('Creating per-partition image files') + boot_dev = create_img_file(get_image_path(device, flavour, 'boot'), IMG_FILE_BOOT_DEFAULT_SIZE) + root_dev = create_img_file(get_image_path(device, flavour, 'root'), f'{rootfs_size_mb - 200}M') + + create_boot_fs(boot_dev, sector_size) + create_root_fs(root_dev, sector_size) + + install_rootfs( + root_dev, + boot_dev, + device, + flavour, + arch, + list(set(packages) | set(packages_extra)), + local_repos, + profile, + ) + + if not skip_part_images: + logging.info('Copying partition image files into full image:') + logging.info(f'Block-copying /boot to {image_path}') + dd_image(input=boot_dev, output=loop_boot) + logging.info(f'Block-copying rootfs to {image_path}') + dd_image(input=root_dev, output=loop_root) + + logging.info(f'Done! Image saved to {image_path}') + + +@cmd_image.command(name='inspect') +@click.option('--shell', '-s', is_flag=True) +@click.option('--use-local-repos', '-l', is_flag=True) +@sectorsize_option +@click.argument('profile', required=False) +def cmd_inspect( + profile: Optional[str] = None, + shell: bool = False, + sector_size: Optional[int] = None, + use_local_repos: bool = False, +): + """Loop-mount the device image for inspection.""" + config.enforce_profile_device_set() + config.enforce_profile_flavour_set() + enforce_wrap() + device = get_profile_device(profile) + arch = device.arch + flavour = get_profile_flavour(profile).name + sector_size = sector_size or device.get_image_sectorsize_default() + + chroot = get_device_chroot(device.name, flavour, arch, packages=[], use_local_repos=use_local_repos) + image_path = get_image_path(device, flavour) + loop_device = losetup_rootfs_image(image_path, sector_size) + partprobe(loop_device) + mount_chroot(loop_device + 'p2', loop_device + 'p1', chroot) + + logging.info(f'Inspect the rootfs image at {chroot.path}') + + if shell: + chroot.initialized = True + chroot.activate() + if arch != config.runtime.arch: + logging.info('Installing requisites for foreign-arch shell') + build_enable_qemu_binfmt(arch) + logging.info('Starting inspection shell') + chroot.run_cmd('/bin/bash') + else: + pause() diff --git a/image/image.py b/image/image.py index 0cb0bcc..8efe33f 100644 --- a/image/image.py +++ b/image/image.py @@ -1,25 +1,21 @@ import atexit import json +import logging import os import re import subprocess -import click -import logging -from signal import pause from subprocess import CompletedProcess from typing import Optional, Union from config.state import config, Profile from chroot.device import DeviceChroot, get_device_chroot -from constants import Arch, BASE_LOCAL_PACKAGES, BASE_PACKAGES, POST_INSTALL_CMDS +from constants import Arch, POST_INSTALL_CMDS from distro.distro import get_base_distro, get_kupfer_https -from devices.device import Device, get_profile_device +from devices.device import Device from exec.cmd import run_root_cmd, generate_cmd_su -from exec.file import get_temp_dir, root_write_file, root_makedir, makedir -from flavours.flavour import Flavour, get_profile_flavour +from exec.file import get_temp_dir, root_write_file, root_makedir +from flavours.flavour import Flavour from net.ssh import copy_ssh_keys -from packages.build import build_enable_qemu_binfmt, build_packages, filter_pkgbuilds -from wrapper import enforce_wrap # image files need to be slightly smaller than partitions to fit IMG_FILE_ROOT_DEFAULT_SIZE = "1800M" @@ -363,177 +359,3 @@ def install_rootfs( res = run_root_cmd(['umount', chroot.path]) assert isinstance(res, CompletedProcess) logging.debug(f'rc: {res.returncode}') - - -@click.group(name='image') -def cmd_image(): - """Build, flash and boot device images""" - - -sectorsize_option = click.option( - '-b', - '--sector-size', - help="Override the device's sector size", - type=int, - default=None, -) - - -@cmd_image.command(name='build') -@click.argument('profile_name', required=False) -@click.option( - '--local-repos/--no-local-repos', - '-l/-L', - help='Whether to use local package repos at all or only use HTTPS repos.', - default=True, - show_default=True, - is_flag=True, -) -@click.option( - '--build-pkgs/--no-build-pkgs', - '-p/-P', - help='Whether to build missing/outdated local packages if local repos are enabled.', - default=True, - show_default=True, - is_flag=True, -) -@click.option( - '--no-download-pkgs', - help='Disable trying to download packages instead of building if building is enabled.', - default=False, - is_flag=True, -) -@click.option( - '--block-target', - help='Override the block device file to write the final image to', - type=click.Path(), - default=None, -) -@click.option( - '--skip-part-images', - help='Skip creating image files for the partitions and directly work on the target block device.', - default=False, - is_flag=True, -) -@sectorsize_option -def cmd_build( - profile_name: Optional[str] = None, - local_repos: bool = True, - build_pkgs: bool = True, - no_download_pkgs=False, - block_target: Optional[str] = None, - sector_size: Optional[int] = None, - skip_part_images: bool = False, -): - """ - Build a device image. - - Unless overriden, required packages will be built or preferably downloaded from HTTPS repos. - """ - - config.enforce_profile_device_set() - config.enforce_profile_flavour_set() - enforce_wrap() - device = get_profile_device(profile_name) - arch = device.arch - # check_programs_wrap(['makepkg', 'pacman', 'pacstrap']) - profile: Profile = config.get_profile(profile_name) - flavour = get_profile_flavour(profile_name) - rootfs_size_mb = flavour.parse_flavourinfo().rootfs_size * 1000 + int(profile.size_extra_mb) - - packages = BASE_LOCAL_PACKAGES + [device.package.name, flavour.pkgbuild.name] - packages_extra = BASE_PACKAGES + profile.pkgs_include - - if arch != config.runtime.arch: - build_enable_qemu_binfmt(arch) - - if local_repos and build_pkgs: - logging.info("Making sure all packages are built") - # enforce that local base packages are built - pkgbuilds = set(filter_pkgbuilds(packages, arch=arch, allow_empty_results=False, use_paths=False)) - # extra packages might be a mix of package names that are in our PKGBUILDs and packages from the base distro - pkgbuilds |= set(filter_pkgbuilds(packages_extra, arch=arch, allow_empty_results=True, use_paths=False)) - build_packages(pkgbuilds, arch, try_download=not no_download_pkgs) - - sector_size = sector_size or device.get_image_sectorsize() - - image_path = block_target or get_image_path(device, flavour.name) - - makedir(os.path.dirname(image_path)) - - logging.info(f'Creating new file at {image_path}') - create_img_file(image_path, f"{rootfs_size_mb}M") - - loop_device = losetup_rootfs_image(image_path, sector_size or device.get_image_sectorsize_default()) - - partition_device(loop_device) - partprobe(loop_device) - - boot_dev: str - root_dev: str - loop_boot = loop_device + 'p1' - loop_root = loop_device + 'p2' - if skip_part_images: - boot_dev = loop_boot - root_dev = loop_root - else: - logging.info('Creating per-partition image files') - boot_dev = create_img_file(get_image_path(device, flavour, 'boot'), IMG_FILE_BOOT_DEFAULT_SIZE) - root_dev = create_img_file(get_image_path(device, flavour, 'root'), f'{rootfs_size_mb - 200}M') - - create_boot_fs(boot_dev, sector_size) - create_root_fs(root_dev, sector_size) - - install_rootfs( - root_dev, - boot_dev, - device, - flavour, - arch, - list(set(packages) | set(packages_extra)), - local_repos, - profile, - ) - - if not skip_part_images: - logging.info('Copying partition image files into full image:') - logging.info(f'Block-copying /boot to {image_path}') - dd_image(input=boot_dev, output=loop_boot) - logging.info(f'Block-copying rootfs to {image_path}') - dd_image(input=root_dev, output=loop_root) - - logging.info(f'Done! Image saved to {image_path}') - - -@cmd_image.command(name='inspect') -@click.option('--shell', '-s', is_flag=True) -@sectorsize_option -@click.argument('profile', required=False) -def cmd_inspect(profile: Optional[str] = None, shell: bool = False, sector_size: Optional[int] = None): - """Loop-mount the device image for inspection.""" - config.enforce_profile_device_set() - config.enforce_profile_flavour_set() - enforce_wrap() - device = get_profile_device(profile) - arch = device.arch - flavour = get_profile_flavour(profile).name - sector_size = sector_size or device.get_image_sectorsize_default() - - chroot = get_device_chroot(device.name, flavour, arch) - image_path = get_image_path(device, flavour) - loop_device = losetup_rootfs_image(image_path, sector_size) - partprobe(loop_device) - mount_chroot(loop_device + 'p2', loop_device + 'p1', chroot) - - logging.info(f'Inspect the rootfs image at {chroot.path}') - - if shell: - chroot.initialized = True - chroot.activate() - if arch != config.runtime.arch: - logging.info('Installing requisites for foreign-arch shell') - build_enable_qemu_binfmt(arch) - logging.info('Starting inspection shell') - chroot.run_cmd('/bin/bash') - else: - pause() From 829d80ede02282f343dcc20741c6392d48381691 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Sat, 8 Jul 2023 15:16:46 +0200 Subject: [PATCH 02/30] chroot/abstract: add switch_user param to chroot.run_cmd()'s signature --- chroot/abstract.py | 1 + 1 file changed, 1 insertion(+) diff --git a/chroot/abstract.py b/chroot/abstract.py index ffef4a6..ef78371 100644 --- a/chroot/abstract.py +++ b/chroot/abstract.py @@ -62,6 +62,7 @@ class AbstractChroot(Protocol): fail_inactive: bool, stdout: Optional[FileDescriptor], stderr: Optional[FileDescriptor], + switch_user: Optional[str], ): pass From 1e446e6f80d380c33c177762f2a8c625dcce25b2 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Sat, 8 Jul 2023 18:38:45 +0200 Subject: [PATCH 03/30] config/state: remove superflous attrs from Profile, as they already exist in SparseProfile --- config/scheme.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/config/scheme.py b/config/scheme.py index a5846ba..605dca6 100644 --- a/config/scheme.py +++ b/config/scheme.py @@ -23,14 +23,12 @@ class SparseProfile(DictScheme): class Profile(SparseProfile): - parent: Optional[str] device: str flavour: str pkgs_include: list[str] pkgs_exclude: list[str] hostname: str username: str - password: Optional[str] size_extra_mb: Union[str, int] From d6900172fed9f7270d19c7eb72aba412fcd1c3e4 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Sat, 8 Jul 2023 18:45:09 +0200 Subject: [PATCH 04/30] exec/cmd: run_cmd(): add new params: stdin, stdin_input, check asddsadsa --- chroot/abstract.py | 24 ++++++++++++++++++++---- exec/cmd.py | 7 +++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/chroot/abstract.py b/chroot/abstract.py index ef78371..b158fc9 100644 --- a/chroot/abstract.py +++ b/chroot/abstract.py @@ -60,9 +60,12 @@ class AbstractChroot(Protocol): capture_output: bool, cwd: str, fail_inactive: bool, + switch_user: Optional[str], stdout: Optional[FileDescriptor], stderr: Optional[FileDescriptor], - switch_user: Optional[str], + stdin: Optional[FileDescriptor], + stdin_input: Optional[str], + check: Optional[bool], ): pass @@ -226,9 +229,12 @@ class Chroot(AbstractChroot): capture_output: bool = False, cwd: Optional[str] = None, fail_inactive: bool = True, + switch_user: Optional[str] = None, stdout: Optional[FileDescriptor] = None, stderr: Optional[FileDescriptor] = None, - switch_user: Optional[str] = None, + stdin: Optional[FileDescriptor] = None, + stdin_input: Optional[str] = None, + check: Optional[bool] = None, ) -> Union[int, subprocess.CompletedProcess]: if not self.active and fail_inactive: raise Exception(f'Chroot {self.name} is inactive, not running command! Hint: pass `fail_inactive=False`') @@ -245,13 +251,23 @@ class Chroot(AbstractChroot): script = flatten_shell_script(script, shell_quote_items=False, wrap_in_shell_quote=False) if cwd: script = f"cd {shell_quote(cwd)} && ( {script} )" - if switch_user: + if switch_user and switch_user != 'root': inner_cmd = generate_cmd_su(script, switch_user=switch_user, elevation_method='none', force_su=True) else: inner_cmd = wrap_in_bash(script, flatten_result=False) cmd = flatten_shell_script(['chroot', self.path] + env_cmd + inner_cmd, shell_quote_items=True) - return run_root_cmd(cmd, env=outer_env, attach_tty=attach_tty, capture_output=capture_output, stdout=stdout, stderr=stderr) + return run_root_cmd( + cmd, + env=outer_env, + attach_tty=attach_tty, + capture_output=capture_output, + stdout=stdout, + stderr=stderr, + stdin=stdin, + stdin_input=stdin_input, + check=check, + ) def mount_pkgbuilds(self, fail_if_mounted: bool = False) -> str: return self.mount( diff --git a/exec/cmd.py b/exec/cmd.py index d653d05..916f587 100644 --- a/exec/cmd.py +++ b/exec/cmd.py @@ -97,6 +97,9 @@ def run_cmd( elevation_method: Optional[ElevationMethod] = None, stdout: Optional[FileDescriptor] = None, stderr: Optional[FileDescriptor] = None, + stdin: Optional[FileDescriptor] = None, + stdin_input: Optional[str] = None, + check: Optional[bool] = None, ) -> Union[CompletedProcess, int]: "execute `script` as `switch_user`, elevating and su'ing as necessary" kwargs: dict = {} @@ -111,6 +114,10 @@ def run_cmd( for name, fd in {'stdout': stdout, 'stderr': stderr}.items(): if fd is not None: kwargs[name] = fd + for name, value in {'stdin': stdin, 'input': stdin_input, 'check': check}.items(): + if value is not None: + kwargs[name] = value + script = flatten_shell_script(script) if cwd: kwargs['cwd'] = cwd From 370510e09f1e4e076308b1066356d010939d268c Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Sat, 8 Jul 2023 18:46:46 +0200 Subject: [PATCH 05/30] config/scheme: add Profile.encryption and Profile.encryption_password --- config/profile.py | 2 ++ config/scheme.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/config/profile.py b/config/profile.py index ff0ba95..b064d64 100644 --- a/config/profile.py +++ b/config/profile.py @@ -15,6 +15,8 @@ PROFILE_DEFAULTS_DICT = { 'username': 'kupfer', 'password': None, 'size_extra_mb': "0", + 'encryption': None, + 'encryption_password': None, } PROFILE_DEFAULTS = Profile.fromDict(PROFILE_DEFAULTS_DICT) diff --git a/config/scheme.py b/config/scheme.py index 605dca6..db36d3a 100644 --- a/config/scheme.py +++ b/config/scheme.py @@ -17,6 +17,8 @@ class SparseProfile(DictScheme): username: Optional[str] password: Optional[str] size_extra_mb: Optional[Union[str, int]] + encryption: Optional[bool] + encryption_password: Optional[str] def __repr__(self): return f'{type(self)}{dict.__repr__(self.toDict())}' From 6de8137c90a17dc015172cd15229503a4b76033c Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Sat, 8 Jul 2023 18:43:28 +0200 Subject: [PATCH 06/30] image: add new module: cryptsetup --- constants.py | 3 + image/cryptsetup.py | 138 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+) create mode 100644 image/cryptsetup.py diff --git a/constants.py b/constants.py index 723dcc0..29b7bad 100644 --- a/constants.py +++ b/constants.py @@ -172,3 +172,6 @@ SRCINFO_TARBALL_URL = f'{KUPFER_HTTPS_BASE}/{SRCINFO_TARBALL_FILE}' FLAVOUR_INFO_FILE = 'flavourinfo.json' FLAVOUR_DESCRIPTION_PREFIX = 'kupfer flavour:' + +LUKS_LABEL_DEFAULT = 'kupfer_crypt' +LUKS_MAPPER_DEFAULT = 'kupfer-crypt' diff --git a/image/cryptsetup.py b/image/cryptsetup.py new file mode 100644 index 0000000..b7edce7 --- /dev/null +++ b/image/cryptsetup.py @@ -0,0 +1,138 @@ +import atexit +import logging +import os + +from typing import Optional + +from constants import LUKS_LABEL_DEFAULT +from chroot.build import BuildChroot +from exec.cmd import run_cmd, CompletedProcess + + +def get_accessible_user(path): + return None if os.access(path, os.R_OK) else 'root' + + +def check_dev_exists(device_path: str, verb: str = 'find'): + if not os.path.exists(device_path): + raise Exception(f"Can't {verb} LUKS on {device_path!r}: file does not exist") + + +def get_cryptmapper_path(mapper_name: str) -> str: + return f'/dev/mapper/{mapper_name}' + + +def mapper_exists(mapper_name: str, chroot: Optional[BuildChroot]) -> bool: + path = get_cryptmapper_path(mapper_name) + paths = [path] + if chroot: + paths.append(chroot.get_path(path)) + for p in paths: + if os.path.exists(p): + return True + return False + + +def get_password_io(password: Optional[str]) -> Optional[bytes]: + return password.encode() if password else None + + +def is_luks(device_path: str, native_chroot: Optional[BuildChroot] = None) -> bool: + check_dev_exists(device_path, 'check') + run_func = native_chroot.run_cmd if native_chroot else run_cmd + user = get_accessible_user(device_path) + cmd = ["blkid", '--match-token', 'TYPE=crypto_LUKS', device_path] + result = run_func(cmd, capture_output=True, switch_user=user) # type: ignore[operator] + assert isinstance(result, CompletedProcess) + return bool(result.stdout and result.stdout.strip()) + + +def luks_create( + backing_device: str, + label: str = LUKS_LABEL_DEFAULT, + native_chroot: Optional[BuildChroot] = None, + password: Optional[str] = None, + extra_opts: list[str] = [], + use_random: bool = True, + cipher: Optional[str] = None, + pbkdf: Optional[str] = None, + iter_time: Optional[int] = None, +): + check_dev_exists(backing_device, 'create') + run_func = native_chroot.run_cmd if native_chroot else run_cmd + extra_opts = list(extra_opts) # copy list before modification + if use_random: + extra_opts += ['--use-random'] + if cipher: + extra_opts += ['--cipher', cipher] + if pbkdf: + extra_opts += ['--pbkdf', pbkdf] + if iter_time is not None: + extra_opts += ['--iter-time', str(iter_time)] + if label: + extra_opts += ['--label', label] + logging.info(f"Creating LUKS volume at {backing_device!r}{' (unattended)' if password else ''}") + result = run_func( # type: ignore[operator] + ['cryptsetup', '-q', 'luksFormat', *extra_opts, backing_device], + switch_user=get_accessible_user(backing_device), + attach_tty=not password, + stdin_input=get_password_io(password), + ) + rc = result if isinstance(result, int) else result.returncode + if rc: + raise Exception("Failed to format LUKS device: cryptsetup error^^^^") + + +def luks_open( + backing_device: str, + mapper_name: str, + extra_opts: list[str] = [], + password: Optional[str] = None, + native_chroot: Optional[BuildChroot] = None, + schedule_close: bool = True, + idempotent: bool = False, +): + check_dev_exists(backing_device, 'open') + run_func = native_chroot.run_cmd if native_chroot else run_cmd + if mapper_exists(mapper_name, native_chroot): + if idempotent: + logging.debug(f"LUKS mapper {mapper_name!r} already open") + return + raise Exception(f"Can't open LUKS for {backing_device!r} with mapper name {mapper_name!r}: " + "mapper file already exists") + logging.info(f"Opening LUKS mapper {mapper_name!r} for {backing_device!r}") + result = run_func( # type: ignore[operator] + ['cryptsetup', 'luksOpen', *extra_opts, backing_device, mapper_name], + switch_user='root', + attach_tty=not password, + stdin_input=get_password_io(password), + ) + rc = result if isinstance(result, int) else result.returncode + if rc: + raise Exception("Failed to open LUKS device: cryptsetup error^^^^") + if schedule_close: + atexit.register(luks_close, mapper_name, native_chroot=native_chroot, idempotent=True) + logging.info(f"LUKS mapper {mapper_name!r} opened!") + + +def luks_close( + mapper_name: str, + native_chroot: Optional[BuildChroot] = None, + extra_opts: list[str] = [], + idempotent: bool = False, +): + run_func = native_chroot.run_cmd if native_chroot else run_cmd + if not mapper_exists(mapper_name, native_chroot): + if idempotent: + logging.debug(f"LUKS mapper {mapper_name!r} already closed") + return 0 + raise Exception(f"Can't close LUKS mapper {mapper_name!r}: mapper doesn't exist") + logging.info(f"Closing LUKS mapper {mapper_name!r}") + result = run_func( # type: ignore[operator] + ['cryptsetup', 'close', *extra_opts, mapper_name], + switch_user='root', + ) + rc = result if isinstance(result, int) else result.returncode + if rc: + raise Exception("Failed to close LUKS device: cryptsetup error^^^^") + logging.info(f"LUKS mapper {mapper_name!r} closed.") From a9cd8178c8001fa4a8daac5c772c632d880426b1 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Sun, 9 Jul 2023 03:20:33 +0200 Subject: [PATCH 07/30] image: add LUKS support and --[no-]encryption CLI flag to build & inspect subcommands --- exec/file.py | 4 +- image/cli.py | 41 +++++++++- image/cryptsetup.py | 41 ++++++++++ image/flash.py | 15 +++- image/image.py | 184 ++++++++++++++++++++++++++++++++++---------- 5 files changed, 236 insertions(+), 49 deletions(-) diff --git a/exec/file.py b/exec/file.py index 852ad48..167d3f8 100644 --- a/exec/file.py +++ b/exec/file.py @@ -171,9 +171,9 @@ def symlink(source, target): raise Exception(f'Symlink creation of {target} pointing at {source} failed') -def get_temp_dir(register_cleanup=True, mode: int = 0o0755): +def get_temp_dir(register_cleanup=True, mode: int = 0o0755, prefix='kupfertmp_'): "create a new tempdir and sanitize ownership so root can access user files as god intended" - t = mkdtemp() + t = mkdtemp(prefix=prefix) chmod(t, mode, privileged=False) if register_cleanup: atexit.register(remove_file, t, recursive=True) diff --git a/image/cli.py b/image/cli.py index c4b4143..3833339 100644 --- a/image/cli.py +++ b/image/cli.py @@ -6,7 +6,7 @@ from signal import pause from typing import Optional from config.state import config, Profile -from constants import BASE_LOCAL_PACKAGES, BASE_PACKAGES +from constants import BASE_LOCAL_PACKAGES, BASE_PACKAGES, LUKS_MAPPER_DEFAULT from devices.device import get_profile_device from exec.file import makedir from flavours.flavour import get_profile_flavour @@ -14,6 +14,7 @@ from packages.build import build_enable_qemu_binfmt, build_packages, filter_pkgb from wrapper import enforce_wrap from .boot import cmd_boot +from .cryptsetup import encryption_option, get_cryptmapper_path, luks_close, luks_create, luks_open from .flash import cmd_flash from .image import ( IMG_FILE_BOOT_DEFAULT_SIZE, @@ -84,6 +85,7 @@ sectorsize_option = click.option( default=False, is_flag=True, ) +@encryption_option @sectorsize_option def cmd_build( profile_name: Optional[str] = None, @@ -93,6 +95,9 @@ def cmd_build( block_target: Optional[str] = None, sector_size: Optional[int] = None, skip_part_images: bool = False, + encryption: Optional[bool] = None, + encryption_password: Optional[str] = None, + encryption_mapper: str = LUKS_MAPPER_DEFAULT, ): """ Build a device image. @@ -110,9 +115,15 @@ def cmd_build( flavour = get_profile_flavour(profile_name) rootfs_size_mb = flavour.parse_flavourinfo().rootfs_size * 1000 + int(profile.size_extra_mb) + if encryption is None: + encryption = profile.encryption + packages = BASE_LOCAL_PACKAGES + [device.package.name, flavour.pkgbuild.name] packages_extra = BASE_PACKAGES + profile.pkgs_include + if encryption: + packages_extra += ['cryptsetup', 'util-linux'] # TODO: select osk-sdl here somehow + if arch != config.runtime.arch: build_enable_qemu_binfmt(arch) @@ -140,6 +151,7 @@ def cmd_build( boot_dev: str root_dev: str + root_dev_raw: str loop_boot = loop_device + 'p1' loop_root = loop_device + 'p2' if skip_part_images: @@ -150,8 +162,23 @@ def cmd_build( boot_dev = create_img_file(get_image_path(device, flavour, 'boot'), IMG_FILE_BOOT_DEFAULT_SIZE) root_dev = create_img_file(get_image_path(device, flavour, 'root'), f'{rootfs_size_mb - 200}M') - create_boot_fs(boot_dev, sector_size) + root_dev_raw = root_dev + + if encryption: + encryption_password = encryption_password or profile.encryption_password + if not encryption_password: + encryption_password = click.prompt( + "Please enter your encryption password (input hidden)", + hide_input=True, + confirmation_prompt=True, + ) + luks_create(root_dev, password=encryption_password) + luks_open(root_dev, mapper_name=encryption_mapper, password=encryption_password) + root_dev = get_cryptmapper_path(encryption_mapper) + + assert os.path.exists(root_dev) create_root_fs(root_dev, sector_size) + create_boot_fs(boot_dev, sector_size) install_rootfs( root_dev, @@ -162,14 +189,17 @@ def cmd_build( list(set(packages) | set(packages_extra)), local_repos, profile, + encrypted=bool(encryption), ) + if encryption: + luks_close(mapper_name=encryption_mapper) if not skip_part_images: logging.info('Copying partition image files into full image:') logging.info(f'Block-copying /boot to {image_path}') dd_image(input=boot_dev, output=loop_boot) logging.info(f'Block-copying rootfs to {image_path}') - dd_image(input=root_dev, output=loop_root) + dd_image(input=root_dev_raw, output=loop_root) logging.info(f'Done! Image saved to {image_path}') @@ -178,17 +208,20 @@ def cmd_build( @click.option('--shell', '-s', is_flag=True) @click.option('--use-local-repos', '-l', is_flag=True) @sectorsize_option +@encryption_option @click.argument('profile', required=False) def cmd_inspect( profile: Optional[str] = None, shell: bool = False, sector_size: Optional[int] = None, use_local_repos: bool = False, + encryption: Optional[bool] = None, ): """Loop-mount the device image for inspection.""" config.enforce_profile_device_set() config.enforce_profile_flavour_set() enforce_wrap() + profile_conf = config.get_profile(profile) device = get_profile_device(profile) arch = device.arch flavour = get_profile_flavour(profile).name @@ -198,7 +231,7 @@ def cmd_inspect( image_path = get_image_path(device, flavour) loop_device = losetup_rootfs_image(image_path, sector_size) partprobe(loop_device) - mount_chroot(loop_device + 'p2', loop_device + 'p1', chroot) + mount_chroot(loop_device + 'p2', loop_device + 'p1', chroot, password=profile_conf.encryption_password) logging.info(f'Inspect the rootfs image at {chroot.path}') diff --git a/image/cryptsetup.py b/image/cryptsetup.py index b7edce7..d4dfaa7 100644 --- a/image/cryptsetup.py +++ b/image/cryptsetup.py @@ -1,4 +1,5 @@ import atexit +import click import logging import os @@ -8,6 +9,14 @@ from constants import LUKS_LABEL_DEFAULT from chroot.build import BuildChroot from exec.cmd import run_cmd, CompletedProcess +encryption_option = click.option( + '--encryption/--no-encryption', + help="Force applying/ignoring LUKS encryption when handling the device image." + "Defaults to using the Profile's setting.", + default=None, + is_flag=True, +) + def get_accessible_user(path): return None if os.access(path, os.R_OK) else 'root' @@ -47,6 +56,38 @@ def is_luks(device_path: str, native_chroot: Optional[BuildChroot] = None) -> bo return bool(result.stdout and result.stdout.strip()) +def get_luks_offset( + mapper_name: str, + native_chroot: Optional[BuildChroot] = None, +) -> tuple[int, int]: + device_path = get_cryptmapper_path(mapper_name) + check_dev_exists(device_path, 'get offset of') + run_func = native_chroot.run_cmd if native_chroot else run_cmd + user = get_accessible_user(device_path) + stdout: str = '' + cmd = ['cryptsetup', 'status', mapper_name] + result = run_func(cmd, capture_output=True, switch_user=user) # type: ignore[operator] + assert isinstance(result, CompletedProcess) + if not (result.stdout and (stdout := result.stdout.strip())): + raise Exception(f"Couldn't get LUKS offset for {mapper_name!r} from 'cryptsetup status': empty stdout: {stdout!r}") + markers = {'offset': -1, 'sector size': -1} + for line in stdout.decode().split('\n'): + line = line.strip() + for item in markers: + offset_marker = f'{item}:' + if line.startswith(offset_marker): + try: + markers[item] = int(line.split(offset_marker)[-1].strip().split(' ')[0]) + except Exception as ex: + raise Exception(f"Couldn't get LUKS {item=} for {mapper_name!r} due to an exception parsing cryptsetup output: {ex}") + for i in markers.values(): + if i != -1: + continue + logging.debug(f"Failed to find ':' in stdout: {stdout}") + raise Exception(f"Failed to find LUKS offset for {mapper_name!r}: Offset line not found") + return markers['offset'], markers['sector size'] + + def luks_create( backing_device: str, label: str = LUKS_LABEL_DEFAULT, diff --git a/image/flash.py b/image/flash.py index 5def17c..9f11ed3 100644 --- a/image/flash.py +++ b/image/flash.py @@ -5,6 +5,7 @@ import logging from typing import Optional +from config.state import config from constants import FLASH_PARTS, LOCATIONS, FASTBOOT, JUMPDRIVE from exec.cmd import run_root_cmd from exec.file import get_temp_dir @@ -15,6 +16,7 @@ from wrapper import enforce_wrap from .fastboot import fastboot_flash from .image import dd_image, dump_aboot, dump_lk2nd, dump_qhypstub, get_image_path, losetup_destroy, losetup_rootfs_image, partprobe, shrink_fs +from .cryptsetup import encryption_option ABOOT = FLASH_PARTS['ABOOT'] LK2ND = FLASH_PARTS['LK2ND'] @@ -47,7 +49,7 @@ def test_blockdev(path: str): 'microSD inserted or no microSD card slot installed in the device) or corrupt or defect') -def prepare_minimal_image(source_path: str, sector_size: int) -> str: +def prepare_minimal_image(source_path: str, sector_size: int, encrypted: Optional[bool], encryption_password: Optional[str]) -> str: minimal_image_dir = get_temp_dir(register_cleanup=True) minimal_image_path = os.path.join(minimal_image_dir, f'minimal-{os.path.basename(source_path)}') logging.info(f"Copying image {os.path.basename(source_path)} to {minimal_image_dir} for shrinking") @@ -55,7 +57,7 @@ def prepare_minimal_image(source_path: str, sector_size: int) -> str: loop_device = losetup_rootfs_image(minimal_image_path, sector_size) partprobe(loop_device) - shrink_fs(loop_device, minimal_image_path, sector_size) + shrink_fs(loop_device, minimal_image_path, sector_size, encrypted, encryption_password) losetup_destroy(loop_device) return minimal_image_path @@ -67,6 +69,7 @@ def prepare_minimal_image(source_path: str, sector_size: int) -> str: @click.option('--shrink/--no-shrink', is_flag=True, default=True, help="Copy and shrink the image file to minimal size") @click.option('-b', '--sector-size', type=int, help="Override the device's sector size", default=None) @click.option('--confirm', is_flag=True, help="Ask for confirmation before executing fastboot commands") +@encryption_option @click.argument('what', type=click.Choice(list(FLASH_PARTS.values()))) @click.argument('location', type=str, required=False) def cmd_flash( @@ -78,6 +81,7 @@ def cmd_flash( shrink: bool = True, sector_size: Optional[int] = None, confirm: bool = False, + encryption: Optional[bool] = None, ): """ Flash a partition onto a device. @@ -115,7 +119,12 @@ def cmd_flash( if not location: raise Exception(f'You need to specify a location to flash {what} to') path = '' - image_path = prepare_minimal_image(device_image_path, sector_size) if shrink else device_image_path + image_path = prepare_minimal_image( + device_image_path, + sector_size, + encrypted=encryption, + encryption_password=config.get_profile(profile).encryption_password, + ) if shrink else device_image_path if method == FASTBOOT: fastboot_flash( partition=location, diff --git a/image/image.py b/image/image.py index 8efe33f..bebe82d 100644 --- a/image/image.py +++ b/image/image.py @@ -7,15 +7,21 @@ import subprocess from subprocess import CompletedProcess from typing import Optional, Union -from config.state import config, Profile +from chroot.build import BuildChroot, get_build_chroot from chroot.device import DeviceChroot, get_device_chroot -from constants import Arch, POST_INSTALL_CMDS +from config.state import config, Profile +from constants import Arch, LUKS_MAPPER_DEFAULT, POST_INSTALL_CMDS from distro.distro import get_base_distro, get_kupfer_https from devices.device import Device from exec.cmd import run_root_cmd, generate_cmd_su from exec.file import get_temp_dir, root_write_file, root_makedir from flavours.flavour import Flavour from net.ssh import copy_ssh_keys +from utils import programs_available + +from .cryptsetup import is_luks, get_luks_offset, luks_close, luks_open + +MAPPER_DIR = '/dev/mapper/' # image files need to be slightly smaller than partitions to fit IMG_FILE_ROOT_DEFAULT_SIZE = "1800M" @@ -72,24 +78,58 @@ def align_bytes(size_bytes: int, alignment: int = 4096) -> int: return size_bytes -def shrink_fs(loop_device: str, file: str, sector_size: int): +def shrink_fs( + loop_device: str, + file: str, + sector_size: int, + encrypted: Optional[bool] = None, + encryption_password: Optional[str] = None, + crypt_mapper=LUKS_MAPPER_DEFAULT, +): partprobe(loop_device) - logging.debug(f"Checking filesystem at {loop_device}p2") - result = run_root_cmd(['e2fsck', '-fy', f'{loop_device}p2']) + root_partition = f'{loop_device}p2' + root_partition_fs = root_partition + if not (encrypted is False): + root_partition_fs, native_chroot, encrypted = resolve_rootfs_crypt( + root_partition, + fail_on_unencrypted=bool(encrypted), + crypt_mapper=crypt_mapper, + password=encryption_password, + ) + logging.debug(f"Checking filesystem at {root_partition_fs}") + result = run_root_cmd(['e2fsck', '-fy', root_partition_fs]) if result.returncode > 2: # https://man7.org/linux/man-pages/man8/e2fsck.8.html#EXIT_CODE - raise Exception(f'Failed to e2fsck {loop_device}p2 with exit code {result.returncode}') + raise Exception(f'Failed to e2fsck {root_partition_fs} with exit code {result.returncode}') - logging.info(f'Shrinking filesystem at {loop_device}p2') - result = run_root_cmd(['resize2fs', '-M', f'{loop_device}p2']) + logging.info(f'Shrinking filesystem at {root_partition_fs}') + result = run_root_cmd(['resize2fs', '-M', root_partition_fs]) if result.returncode != 0: - raise Exception(f'Failed to resize2fs {loop_device}p2') + raise Exception(f'Failed to resize2fs {root_partition_fs}') - logging.debug(f'Reading size of shrunken filesystem on {loop_device}p2') - fs_blocks, fs_block_size = get_fs_size(f'{loop_device}p2') + logging.debug(f'Reading size of shrunken filesystem on {root_partition_fs}') + fs_blocks, fs_block_size = get_fs_size(root_partition_fs) sectors = bytes_to_sectors(fs_blocks * fs_block_size, sector_size) + logging.debug(f"shrunken FS length is {fs_blocks} blocks * {fs_block_size} bytes = {sectors} bytes") - logging.info(f'Shrinking partition at {loop_device}p2 to {sectors} sectors ({sectors * sector_size} bytes)') + _, image_size = find_end_sector(loop_device, root_partition, sector_size) + if image_size == -1: + raise Exception(f'Failed to find pre-repartition size of {loop_device}') + + if encrypted: + if sectors > image_size: + raise Exception("Shrunk FS size allegedly larger than the image itself; this is probably " + f"a kupferbootstrap parsing bug. shrunk partition end={sectors}, image size={image_size}, {sector_size=}") + old_sectors = sectors + luks_offset, luks_sector_size = get_luks_offset(crypt_mapper, native_chroot) + #luks_offset_bytes = align_bytes((luks_offset + 1) * luks_sector_size, sector_size) + luks_offset_normalized = bytes_to_sectors(luks_offset * luks_sector_size, sector_size) + logging.debug(f"Discovered LUKS attrs: {luks_offset=}, {luks_sector_size=}, {luks_offset_normalized=}") + luks_close(crypt_mapper, native_chroot) + sectors += luks_offset_normalized + 1024 + logging.debug(f"Increasing sectors from {old_sectors} to {sectors} ({sectors - old_sectors}) to leave space for the LUKS header") + + logging.info(f'Shrinking partition at {root_partition} to {sectors} {sector_size}b sectors ({sectors * sector_size} bytes)') child_proccess = subprocess.Popen( generate_cmd_su(['fdisk', '-b', str(sector_size), loop_device], switch_user='root'), # type: ignore stdin=subprocess.PIPE, @@ -113,27 +153,18 @@ def shrink_fs(loop_device: str, file: str, sector_size: int): # For some reason re-reading the partition table fails, but that is not a problem partprobe(loop_device) if returncode > 1: - raise Exception(f'Failed to shrink partition size of {loop_device}p2 with fdisk') + raise Exception(f'Failed to shrink partition size of {root_partition} with fdisk') partprobe(loop_device).check_returncode() - logging.debug(f'Finding end sector of partition at {loop_device}p2') - result = run_root_cmd(['fdisk', '-b', str(sector_size), '-l', loop_device], capture_output=True) - if result.returncode != 0: - print(result.stdout) - print(result.stderr) - raise Exception(f'Failed to fdisk -l {loop_device}') + end_sector, _ = find_end_sector(loop_device, root_partition, sector_size) + if end_sector == -1: + raise Exception(f'Failed to find end sector of {root_partition}') - end_sector = 0 - for line in result.stdout.decode('utf-8').split('\n'): - if line.startswith(f'{loop_device}p2'): - parts = list(filter(lambda part: part != '', line.split(' '))) - end_sector = int(parts[2]) - - if end_sector == 0: - raise Exception(f'Failed to find end sector of {loop_device}p2') - - end_size = align_bytes((end_sector + 1) * sector_size, 4096) + if end_sector > image_size: + logging.warning(f"Clipping sectors ({end_sector}) to {image_size=}") + end_sector = image_size + end_size = align_bytes((end_sector + 1024) * sector_size, 4096) logging.debug(f'({end_sector} + 1) sectors * {sector_size} bytes/sector = {end_size} bytes') logging.info(f'Truncating {file} to {end_size} bytes') @@ -143,6 +174,26 @@ def shrink_fs(loop_device: str, file: str, sector_size: int): partprobe(loop_device) +def find_end_sector(device: str, partition: str, sector_size: int) -> tuple[int, int]: + """Return (last_sector_index, sector_count) of a partition on a device, returns (-1, -1) if not found""" + logging.debug(f'Finding end sector of partition at {partition}') + result = run_root_cmd(['fdisk', '-b', str(sector_size), '-l', device], capture_output=True) + if result.returncode != 0: + print(result.stdout) + print(result.stderr) + raise Exception(f'Failed to fdisk -l {device}') + + end_sector = -1 + num_sectors = -1 + for line in result.stdout.decode('utf-8').split('\n'): + if line.startswith(partition): + parts = list(filter(lambda part: part != '', line.split(' '))) + end_sector = int(parts[2]) + num_sectors = int(parts[3]) + + return end_sector, num_sectors + + def losetup_destroy(loop_device): logging.debug(f'Destroying loop device {loop_device}') run_root_cmd( @@ -210,16 +261,59 @@ def losetup_rootfs_image(image_path: str, sector_size: int) -> str: return loop_device -def mount_chroot(rootfs_source: str, boot_src: str, chroot: DeviceChroot): - logging.debug(f'Mounting {rootfs_source} at {chroot.path}') +def resolve_rootfs_crypt( + rootfs_source: str, + password: Optional[str] = None, + crypt_mapper: str = LUKS_MAPPER_DEFAULT, + native_chroot: Optional[BuildChroot] = None, + fail_on_unencrypted: bool = True, +) -> tuple[str, Optional[BuildChroot], bool]: + assert config.runtime.arch + is_encrypted = False + if not (native_chroot or programs_available(['blkid'])): + native_chroot = get_build_chroot(config.runtime.arch, packages=['base', 'util-linux']) + if is_luks(rootfs_source, native_chroot=native_chroot): + if not (native_chroot or programs_available(['cryptsetup'])): + native_chroot = get_build_chroot(config.runtime.arch, packages=['base', 'cryptsetup', 'util-linux']) + luks_open(rootfs_source, crypt_mapper, password=password, native_chroot=native_chroot) + rootfs_source = f'{MAPPER_DIR}{crypt_mapper}' + is_encrypted = True + elif fail_on_unencrypted: + hint = '' + if rootfs_source.startswith(MAPPER_DIR): + hint = (f' HINT: path starts with {MAPPER_DIR!r}, probably already a decrypted volume.' + ' This is likely a kupferbootstrap bug.') + raise Exception(f"Error: {rootfs_source!r} is not an encrypted LUKS volume.{hint}") + return rootfs_source, native_chroot, is_encrypted - chroot.mount_rootfs(rootfs_source) - assert (os.path.ismount(chroot.path)) - root_makedir(chroot.get_path('boot')) +def mount_chroot( + rootfs_source: str, + boot_src: str, + device_chroot: DeviceChroot, + encrypted: Optional[bool] = None, + password: Optional[str] = None, + native_chroot: Optional[BuildChroot] = None, + crypt_mapper: str = LUKS_MAPPER_DEFAULT, +): + if encrypted is not False: + rootfs_source, native_chroot, encrypted = resolve_rootfs_crypt( + rootfs_source, + native_chroot=native_chroot, + crypt_mapper=crypt_mapper, + fail_on_unencrypted=bool(encrypted), + password=password, + ) - logging.debug(f'Mounting {boot_src} at {chroot.path}/boot') - chroot.mount(boot_src, '/boot', options=['defaults']) + logging.debug(f'Mounting {rootfs_source} at {device_chroot.path}') + + device_chroot.mount_rootfs(rootfs_source) + assert (os.path.ismount(device_chroot.path)) + + root_makedir(device_chroot.get_path('boot')) + + logging.debug(f'Mounting {boot_src} at {device_chroot.path}/boot') + device_chroot.mount(boot_src, '/boot', options=['defaults']) def dump_file_from_image(image_path: str, file_path: str, target_path: Optional[str] = None): @@ -314,18 +408,28 @@ def install_rootfs( packages: list[str], use_local_repos: bool, profile: Profile, + encrypted: bool, ): - user = profile['username'] or 'kupfer' + user = profile.username or 'kupfer' chroot = get_device_chroot(device=get_device_name(device), flavour=flavour.name, arch=arch, packages=packages, use_local_repos=use_local_repos) - mount_chroot(rootfs_device, bootfs_device, chroot) + # rootfs_device must be passed the crypt_mapper if encrypted is True + if encrypted: + assert rootfs_device.startswith(MAPPER_DIR) + + mount_chroot( + rootfs_device, + bootfs_device, + chroot, + encrypted=False, # rootfs_device is already the crypt_mapper + ) chroot.mount_pacman_cache() chroot.initialize() chroot.activate() chroot.create_user( user=user, - password=profile['password'], + password=profile.password, ) chroot.add_sudo_config(config_name='wheel', privilegee='%wheel', password_required=True) copy_ssh_keys( @@ -338,7 +442,7 @@ def install_rootfs( extra_repos=get_kupfer_https(arch).repos, in_chroot=True, ), - 'etc/hostname': profile['hostname'] or 'kupfer', + 'etc/hostname': profile.hostname or 'kupfer', } for target, content in files.items(): root_write_file(os.path.join(chroot.path, target.lstrip('/')), content) From dd4a4212a3754fd008983bdaf25309a22f5ea34d Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Sun, 9 Jul 2023 07:16:06 +0200 Subject: [PATCH 08/30] config/cli: init PKGBUILDs after main config is complete --- config/cli.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/config/cli.py b/config/cli.py index 740ab17..4b84830 100644 --- a/config/cli.py +++ b/config/cli.py @@ -1,5 +1,6 @@ import click import logging +import os from copy import deepcopy from typing import Any, Callable, Iterable, Mapping, Optional, Union @@ -260,6 +261,7 @@ def cmd_config_init( ): """Initialize the config file""" if not non_interactive: + from packages.cli import cmd_init as cmd_init_pkgbuilds logging.info(CONFIG_MSG) results: dict[str, dict] = {} for section in sections: @@ -282,6 +284,13 @@ def cmd_config_init( config.write() else: return + if not non_interactive and not os.path.exists(os.path.join(config.get_path('pkgbuilds'), '.git')): + extra_msg = " This way, we can give you a list of devices and flavours later" if 'profiles' in sections else '' + if click.confirm( + f"It seems you don't have our PKGBUILDs checked out yet.\nWould you like KBS to fetch them?{extra_msg}", + default=True, + ): + execute_without_exit(click.Context(cmd_config).invoke, ['packages', 'init'], cmd_init_pkgbuilds) if 'profiles' in sections: print("Configuring profiles") current_profile = 'default' if 'current' not in config.file.profiles else config.file.profiles.current From 095ecb672f25cba7d6fd11671c3d5dbd46b22573 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Sun, 9 Jul 2023 19:57:24 +0200 Subject: [PATCH 09/30] image: use IMG_FILE_BOOT_DEFAULT_SIZE to calculate shrunk boot partition size --- image/cli.py | 16 +++++++++++++--- image/image.py | 7 +++++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/image/cli.py b/image/cli.py index 3833339..9cc106c 100644 --- a/image/cli.py +++ b/image/cli.py @@ -114,6 +114,16 @@ def cmd_build( profile: Profile = config.get_profile(profile_name) flavour = get_profile_flavour(profile_name) rootfs_size_mb = flavour.parse_flavourinfo().rootfs_size * 1000 + int(profile.size_extra_mb) + bootfs_size_str = IMG_FILE_BOOT_DEFAULT_SIZE + bootfs_size_mb = -1 + if bootfs_size_str.endswith('M'): + bootfs_size_mb = int(bootfs_size_str.rstrip('M')) + elif bootfs_size_str.endswith('G'): + bootfs_size_mb = int(bootfs_size_str.rstrip('G')) * 1024 + elif not bootfs_size_str.isdecimal(): + raise Exception(f"Couldn't part bootfs target size as megabytes: {bootfs_size_str}") + else: + bootfs_size_mb = int(bootfs_size_str) if encryption is None: encryption = profile.encryption @@ -142,11 +152,11 @@ def cmd_build( makedir(os.path.dirname(image_path)) logging.info(f'Creating new file at {image_path}') - create_img_file(image_path, f"{rootfs_size_mb}M") + create_img_file(image_path, f"{rootfs_size_mb + bootfs_size_mb}M") loop_device = losetup_rootfs_image(image_path, sector_size or device.get_image_sectorsize_default()) - partition_device(loop_device) + partition_device(loop_device, boot_partition_size=bootfs_size_str) partprobe(loop_device) boot_dev: str @@ -159,7 +169,7 @@ def cmd_build( root_dev = loop_root else: logging.info('Creating per-partition image files') - boot_dev = create_img_file(get_image_path(device, flavour, 'boot'), IMG_FILE_BOOT_DEFAULT_SIZE) + boot_dev = create_img_file(get_image_path(device, flavour, 'boot'), f'{bootfs_size_mb}M') root_dev = create_img_file(get_image_path(device, flavour, 'root'), f'{rootfs_size_mb - 200}M') root_dev_raw = root_dev diff --git a/image/image.py b/image/image.py index bebe82d..557b7fb 100644 --- a/image/image.py +++ b/image/image.py @@ -356,8 +356,11 @@ def create_img_file(image_path: str, size_str: str): return image_path -def partition_device(device: str): - boot_partition_size = '100MiB' +def partition_device(device: str, boot_partition_size: Optional[str] = None): + if boot_partition_size is None: + boot_partition_size = IMG_FILE_BOOT_DEFAULT_SIZE + if boot_partition_size and boot_partition_size[-1] in ['M', 'G', 'K']: + boot_partition_size = f'{boot_partition_size}iB' create_partition_table = ['mklabel', 'msdos'] create_boot_partition = ['mkpart', 'primary', 'ext2', '0%', boot_partition_size] create_root_partition = ['mkpart', 'primary', boot_partition_size, '100%'] From 03c95dcb6a3c874b8ac2c3c08ff45b722adf39e2 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Sun, 9 Jul 2023 21:49:12 +0200 Subject: [PATCH 10/30] image/image: rename losetup_rootfs_image() to losetup_setup_image() --- image/boot.py | 4 ++-- image/cli.py | 6 +++--- image/flash.py | 6 +++--- image/image.py | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/image/boot.py b/image/boot.py index 33e1f8b..d60efb5 100644 --- a/image/boot.py +++ b/image/boot.py @@ -13,7 +13,7 @@ from flavours.cli import profile_option from wrapper import enforce_wrap from .fastboot import fastboot_boot, fastboot_erase -from .image import get_device_name, losetup_rootfs_image, get_image_path, dump_aboot, dump_lk2nd +from .image import get_device_name, losetup_setup_image, get_image_path, dump_aboot, dump_lk2nd LK2ND = FLASH_PARTS['LK2ND'] ABOOT = FLASH_PARTS['ABOOT'] @@ -61,7 +61,7 @@ def cmd_boot( if not os.path.exists(path): urllib.request.urlretrieve(f'https://github.com/dreemurrs-embedded/Jumpdrive/releases/download/{JUMPDRIVE_VERSION}/{file}', path) else: - loop_device = losetup_rootfs_image(image_path, sector_size) + loop_device = losetup_setup_image(image_path, sector_size) if type == LK2ND: path = dump_lk2nd(loop_device + 'p1') elif type == ABOOT: diff --git a/image/cli.py b/image/cli.py index 9cc106c..241f66b 100644 --- a/image/cli.py +++ b/image/cli.py @@ -25,7 +25,7 @@ from .image import ( get_device_chroot, get_image_path, install_rootfs, - losetup_rootfs_image, + losetup_setup_image, mount_chroot, partprobe, partition_device, @@ -154,7 +154,7 @@ def cmd_build( logging.info(f'Creating new file at {image_path}') create_img_file(image_path, f"{rootfs_size_mb + bootfs_size_mb}M") - loop_device = losetup_rootfs_image(image_path, sector_size or device.get_image_sectorsize_default()) + loop_device = losetup_setup_image(image_path, sector_size or device.get_image_sectorsize_default()) partition_device(loop_device, boot_partition_size=bootfs_size_str) partprobe(loop_device) @@ -239,7 +239,7 @@ def cmd_inspect( chroot = get_device_chroot(device.name, flavour, arch, packages=[], use_local_repos=use_local_repos) image_path = get_image_path(device, flavour) - loop_device = losetup_rootfs_image(image_path, sector_size) + loop_device = losetup_setup_image(image_path, sector_size) partprobe(loop_device) mount_chroot(loop_device + 'p2', loop_device + 'p1', chroot, password=profile_conf.encryption_password) diff --git a/image/flash.py b/image/flash.py index 9f11ed3..8fa2c90 100644 --- a/image/flash.py +++ b/image/flash.py @@ -15,7 +15,7 @@ from flavours.cli import profile_option from wrapper import enforce_wrap from .fastboot import fastboot_flash -from .image import dd_image, dump_aboot, dump_lk2nd, dump_qhypstub, get_image_path, losetup_destroy, losetup_rootfs_image, partprobe, shrink_fs +from .image import dd_image, dump_aboot, dump_lk2nd, dump_qhypstub, get_image_path, losetup_destroy, losetup_setup_image, partprobe, shrink_fs from .cryptsetup import encryption_option ABOOT = FLASH_PARTS['ABOOT'] @@ -55,7 +55,7 @@ def prepare_minimal_image(source_path: str, sector_size: int, encrypted: Optiona logging.info(f"Copying image {os.path.basename(source_path)} to {minimal_image_dir} for shrinking") shutil.copyfile(source_path, minimal_image_path) - loop_device = losetup_rootfs_image(minimal_image_path, sector_size) + loop_device = losetup_setup_image(minimal_image_path, sector_size) partprobe(loop_device) shrink_fs(loop_device, minimal_image_path, sector_size, encrypted, encryption_password) losetup_destroy(loop_device) @@ -145,7 +145,7 @@ def cmd_flash( else: if method and method != FASTBOOT: raise Exception(f'Flashing "{what}" with method "{method}" not supported, try no parameter or "{FASTBOOT}"') - loop_device = losetup_rootfs_image(device_image_path, sector_size) + loop_device = losetup_setup_image(device_image_path, sector_size) if what == ABOOT: path = dump_aboot(f'{loop_device}p1') fastboot_flash(location or 'boot', path, confirm=confirm) diff --git a/image/image.py b/image/image.py index 557b7fb..e43bb41 100644 --- a/image/image.py +++ b/image/image.py @@ -224,7 +224,7 @@ def get_image_path(device: Union[str, Device], flavour: Union[str, Flavour], img return os.path.join(config.get_path('images'), get_image_name(device, flavour, img_type)) -def losetup_rootfs_image(image_path: str, sector_size: int) -> str: +def losetup_setup_image(image_path: str, sector_size: int) -> str: logging.debug(f'Creating loop device for {image_path} with sector size {sector_size}') result = run_root_cmd([ 'losetup', From fbe43456f8ee81e44e20eb6d65e3b9ca627f718d Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Sun, 9 Jul 2023 21:51:10 +0200 Subject: [PATCH 11/30] image: don't pass block size to mkfs.ext* --- image/cli.py | 4 ++-- image/image.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/image/cli.py b/image/cli.py index 241f66b..95e65de 100644 --- a/image/cli.py +++ b/image/cli.py @@ -187,8 +187,8 @@ def cmd_build( root_dev = get_cryptmapper_path(encryption_mapper) assert os.path.exists(root_dev) - create_root_fs(root_dev, sector_size) - create_boot_fs(boot_dev, sector_size) + create_root_fs(root_dev) + create_boot_fs(boot_dev) install_rootfs( root_dev, diff --git a/image/image.py b/image/image.py index e43bb41..5b5412a 100644 --- a/image/image.py +++ b/image/image.py @@ -374,7 +374,7 @@ def partition_device(device: str, boot_partition_size: Optional[str] = None): raise Exception(f'Failed to create partitions on {device}') -def create_filesystem(device: str, blocksize: Optional[int], label=None, options=[], fstype='ext4'): +def create_filesystem(device: str, blocksize: Optional[int] = None, label=None, options=[], fstype='ext4'): """Creates a new filesystem. Blocksize defaults""" labels = ['-L', label] if label else [] cmd = [f'mkfs.{fstype}', '-F', *labels] @@ -394,12 +394,12 @@ def create_filesystem(device: str, blocksize: Optional[int], label=None, options raise Exception(f'Failed to create {fstype} filesystem on {device} with CMD: {cmd}') -def create_root_fs(device: str, blocksize: Optional[int]): - create_filesystem(device, blocksize=blocksize, label='kupfer_root', options=['-O', '^metadata_csum', '-N', '100000']) +def create_root_fs(device: str): + create_filesystem(device, label='kupfer_root', options=['-O', '^metadata_csum', '-N', '100000']) -def create_boot_fs(device: str, blocksize: Optional[int]): - create_filesystem(device, blocksize=blocksize, label='kupfer_boot', fstype='ext2') +def create_boot_fs(device: str): + create_filesystem(device, label='kupfer_boot', fstype='ext2') def install_rootfs( From d2942945e4ce2c6e3b4921a6b582fa27c8c3ca6f Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Sun, 9 Jul 2023 21:52:03 +0200 Subject: [PATCH 12/30] image: pass sector_size to partition_device() --- image/cli.py | 17 ++++------------- image/image.py | 16 ++++++---------- 2 files changed, 10 insertions(+), 23 deletions(-) diff --git a/image/cli.py b/image/cli.py index 95e65de..06091f0 100644 --- a/image/cli.py +++ b/image/cli.py @@ -17,7 +17,7 @@ from .boot import cmd_boot from .cryptsetup import encryption_option, get_cryptmapper_path, luks_close, luks_create, luks_open from .flash import cmd_flash from .image import ( - IMG_FILE_BOOT_DEFAULT_SIZE, + IMG_DEFAULT_SIZE_BOOT_MB, create_boot_fs, create_img_file, create_root_fs, @@ -114,16 +114,7 @@ def cmd_build( profile: Profile = config.get_profile(profile_name) flavour = get_profile_flavour(profile_name) rootfs_size_mb = flavour.parse_flavourinfo().rootfs_size * 1000 + int(profile.size_extra_mb) - bootfs_size_str = IMG_FILE_BOOT_DEFAULT_SIZE - bootfs_size_mb = -1 - if bootfs_size_str.endswith('M'): - bootfs_size_mb = int(bootfs_size_str.rstrip('M')) - elif bootfs_size_str.endswith('G'): - bootfs_size_mb = int(bootfs_size_str.rstrip('G')) * 1024 - elif not bootfs_size_str.isdecimal(): - raise Exception(f"Couldn't part bootfs target size as megabytes: {bootfs_size_str}") - else: - bootfs_size_mb = int(bootfs_size_str) + bootfs_size_mb = IMG_DEFAULT_SIZE_BOOT_MB if encryption is None: encryption = profile.encryption @@ -145,7 +136,7 @@ def cmd_build( pkgbuilds |= set(filter_pkgbuilds(packages_extra, arch=arch, allow_empty_results=True, use_paths=False)) build_packages(pkgbuilds, arch, try_download=not no_download_pkgs) - sector_size = sector_size or device.get_image_sectorsize() + sector_size = sector_size or device.get_image_sectorsize() or 512 image_path = block_target or get_image_path(device, flavour.name) @@ -156,7 +147,7 @@ def cmd_build( loop_device = losetup_setup_image(image_path, sector_size or device.get_image_sectorsize_default()) - partition_device(loop_device, boot_partition_size=bootfs_size_str) + partition_device(loop_device, sector_size=sector_size, boot_partition_size_mb=bootfs_size_mb) partprobe(loop_device) boot_dev: str diff --git a/image/image.py b/image/image.py index 5b5412a..75985d5 100644 --- a/image/image.py +++ b/image/image.py @@ -23,9 +23,7 @@ from .cryptsetup import is_luks, get_luks_offset, luks_close, luks_open MAPPER_DIR = '/dev/mapper/' -# image files need to be slightly smaller than partitions to fit -IMG_FILE_ROOT_DEFAULT_SIZE = "1800M" -IMG_FILE_BOOT_DEFAULT_SIZE = "90M" +IMG_DEFAULT_SIZE_BOOT_MB = 90 def dd_image(input: str, output: str, blocksize='1M') -> CompletedProcess: @@ -356,14 +354,12 @@ def create_img_file(image_path: str, size_str: str): return image_path -def partition_device(device: str, boot_partition_size: Optional[str] = None): - if boot_partition_size is None: - boot_partition_size = IMG_FILE_BOOT_DEFAULT_SIZE - if boot_partition_size and boot_partition_size[-1] in ['M', 'G', 'K']: - boot_partition_size = f'{boot_partition_size}iB' +def partition_device(device: str, sector_size: int, boot_partition_size_mb: int = IMG_DEFAULT_SIZE_BOOT_MB): + initial_offset = 1048576 // sector_size # 2048 for 512, 256 for 4096 + boot_partition_size: int = align_bytes(boot_partition_size_mb * 1024 * 1024, 4096) create_partition_table = ['mklabel', 'msdos'] - create_boot_partition = ['mkpart', 'primary', 'ext2', '0%', boot_partition_size] - create_root_partition = ['mkpart', 'primary', boot_partition_size, '100%'] + create_boot_partition = ['mkpart', 'primary', 'ext2', f'{initial_offset}s', f'{boot_partition_size}b'] + create_root_partition = ['mkpart', 'primary', f'{bytes_to_sectors(boot_partition_size, sector_size) + initial_offset}s', '100%'] enable_boot = ['set', '1', 'boot', 'on'] result = run_root_cmd([ 'parted', From 4792eafe8015ef29d05561c8b90544e00019c108 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Mon, 10 Jul 2023 04:24:30 +0200 Subject: [PATCH 13/30] image: bump /boot default size to 200MB --- image/image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/image/image.py b/image/image.py index 75985d5..029954b 100644 --- a/image/image.py +++ b/image/image.py @@ -23,7 +23,7 @@ from .cryptsetup import is_luks, get_luks_offset, luks_close, luks_open MAPPER_DIR = '/dev/mapper/' -IMG_DEFAULT_SIZE_BOOT_MB = 90 +IMG_DEFAULT_SIZE_BOOT_MB = 200 def dd_image(input: str, output: str, blocksize='1M') -> CompletedProcess: From 4c5609423e8a307650de8d14f556df2b61a8e572 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Sun, 9 Jul 2023 22:20:48 +0200 Subject: [PATCH 14/30] constants: change default luks label to kupfer_cryptroot --- constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/constants.py b/constants.py index 29b7bad..9e63cc9 100644 --- a/constants.py +++ b/constants.py @@ -173,5 +173,5 @@ SRCINFO_TARBALL_URL = f'{KUPFER_HTTPS_BASE}/{SRCINFO_TARBALL_FILE}' FLAVOUR_INFO_FILE = 'flavourinfo.json' FLAVOUR_DESCRIPTION_PREFIX = 'kupfer flavour:' -LUKS_LABEL_DEFAULT = 'kupfer_crypt' +LUKS_LABEL_DEFAULT = 'kupfer_cryptroot' LUKS_MAPPER_DEFAULT = 'kupfer-crypt' From a0c40363903c5e1d89d6c799aa8abd125b9c2e15 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Sun, 29 Oct 2023 16:32:36 +0100 Subject: [PATCH 15/30] packages: try_download_package(): check pacman cache if file in db but doesn't exist in db folder --- packages/build.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/build.py b/packages/build.py index 72c9152..190e588 100644 --- a/packages/build.py +++ b/packages/build.py @@ -290,7 +290,8 @@ def try_download_package(dest_file_path: str, package: Pkgbuild, arch: Arch) -> return None repo_pkg: RemotePackage = repo.packages[pkgname] if repo_pkg.version != package.version: - logging.debug(f"Package {pkgname} versions differ: local: {package.version}, remote: {repo_pkg.version}. Building instead.") + logging.debug(f"Package {pkgname} versions differ: local: {package.version}, " + f"remote: {repo_pkg.version}. Building instead.") return None if repo_pkg.filename != filename: versions_str = f"local: {filename}, remote: {repo_pkg.filename}" @@ -298,6 +299,19 @@ def try_download_package(dest_file_path: str, package: Pkgbuild, arch: Arch) -> logging.debug(f"package filenames don't match: {versions_str}") return None logging.debug(f"ignoring compression extension difference: {versions_str}") + cache_file = os.path.join(config.get_path('pacman'), arch, repo_pkg.filename) + if os.path.exists(cache_file): + if not repo_pkg._desc or 'SHA256SUM' not in repo_pkg._desc: + cache_matches = False + extra_msg = ". However, we can't validate it, as the https repo doesnt provide a SHA256SUM for it." + else: + cache_matches = sha256sum(cache_file) == repo_pkg._desc['SHA256SUM'] + extra_msg = (". However its checksum doesn't match." if not cache_matches else " and its checksum matches.") + logging.debug(f"While checking the HTTPS repo DB, we found a matching filename in the pacman cache{extra_msg}") + if cache_matches: + logging.info(f'copying cache file {cache_file} to repo as verified by remote checksum') + shutil.move(cache_file, dest_file_path) + return dest_file_path url = repo_pkg.resolved_url assert url try: From 2e504b7b00f40b33cd583d5c246cda751fcdd8a0 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Mon, 11 Dec 2023 12:49:28 +0100 Subject: [PATCH 16/30] dictscheme: fix type hinting --- dictscheme.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dictscheme.py b/dictscheme.py index c5537d3..ca7c12c 100644 --- a/dictscheme.py +++ b/dictscheme.py @@ -52,7 +52,7 @@ class DictScheme(Munch): _sparse: ClassVar[bool] = False def __init__(self, d: Mapping = {}, validate: bool = True, **kwargs): - self.update(d | kwargs, validate=validate) + self.update(dict(d) | kwargs, validate=validate) @classmethod def transform( @@ -269,10 +269,13 @@ class DictScheme(Munch): ) -> str: import yaml yaml_args = {'sort_keys': False} | yaml_args - return yaml.dump( + dumped = yaml.dump( self.toDict(strip_hidden=strip_hidden, sparse=sparse), **yaml_args, ) + if dumped is None: + raise Exception(f"Failed to yaml-serialse {self}") + return dumped def toToml( self, From ff8a529690da1732f1b6bffd67c2b101b2cbf6ff Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Mon, 11 Dec 2023 16:37:48 +0100 Subject: [PATCH 17/30] docs: move usage guides to usage/, add quickstart and porting --- docs/source/index.md | 3 +- docs/source/{ => usage}/config.md | 6 +- docs/source/usage/faq.md | 39 +++++++++++++ docs/source/usage/index.md | 9 +++ docs/source/{ => usage}/install.md | 0 docs/source/usage/porting.md | 94 ++++++++++++++++++++++++++++++ docs/source/usage/quickstart.md | 9 +++ 7 files changed, 155 insertions(+), 5 deletions(-) rename docs/source/{ => usage}/config.md (94%) create mode 100644 docs/source/usage/faq.md create mode 100644 docs/source/usage/index.md rename docs/source/{ => usage}/install.md (100%) create mode 100644 docs/source/usage/porting.md create mode 100644 docs/source/usage/quickstart.md diff --git a/docs/source/index.md b/docs/source/index.md index 82a3e72..2cde0d6 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -6,7 +6,6 @@ a tool to build and flash packages and images for the [Kupfer](https://gitlab.co ## Documentation pages ```{toctree} -install -config +usage/index cli ``` diff --git a/docs/source/config.md b/docs/source/usage/config.md similarity index 94% rename from docs/source/config.md rename to docs/source/usage/config.md index 561d24c..c850cc7 100644 --- a/docs/source/config.md +++ b/docs/source/usage/config.md @@ -2,7 +2,7 @@ Kupferbootstrap uses [toml](https://en.wikipedia.org/wiki/TOML) for its configuration file. -The file can either be edited manually or managed via the {doc}`cli/config` subcommand. +The file can either be edited manually or managed via the {doc}`../cli/config` subcommand. You can quickly generate a default config by running {code}`kupferbootstrap config init -N`. @@ -54,7 +54,7 @@ This allows you to easily keep a number of slight variations of the same target without the need to constantly modify your Kupferbootstrap configuration file. You can easily create new profiles with -[kupferbootstrap config profile init](../cli/config/#kupferbootstrap-config-profile-init). +[kupferbootstrap config profile init](/cli/config/#kupferbootstrap-config-profile-init). Here's an example: @@ -97,7 +97,7 @@ hostname = "pocof1" The `current` key in the `profiles` section controlls which profile gets used by Kupferbootstrap by default. The first subsection (`profiles.default`) describes the `default` profile -which gets created by [config init](../cli/config/#kupferbootstrap-config-init). +which gets created by [config init](/cli/config/#kupferbootstrap-config-init). Next, we have a `graphical` profile that defines a couple of graphical programs for all but the `recovery` profile, since that doesn't have a GUI. diff --git a/docs/source/usage/faq.md b/docs/source/usage/faq.md new file mode 100644 index 0000000..441bef2 --- /dev/null +++ b/docs/source/usage/faq.md @@ -0,0 +1,39 @@ +# FAQ + + +```{contents} Table of Contents +:class: this-will-duplicate-information-and-it-is-still-useful-here +:depth: 3 +``` + + +## Which devices are currently supported? + +Currently very few! +See [the `devices` repo](https://gitlab.com/kupfer/packages/pkgbuilds/-/tree/dev/device). We use the same codenames as [postmarketOS](https://wiki.postmarketos.org/wiki/Devices) (although we prefix them with the SoC) + + +## How to port a new device or package? + +See [Porting](../porting) + +## How to build a specific package + +See also: The full [`kupferbootstrap packages build` docs](/cli/packages#kupferbootstrap-packages-build) + +### Example + +For rebuilding `kupfer-config` and `crossdirect`, defaulting to your device's architecture + +```sh +kupferbootstrap packages build [--force] [--arch $target_arch] kupfer-config crossdirect +``` + + +### By package path +You can also use the a path snippet (`$repo/$pkgbase`) to the PKGBUILD folder as seen inside your pkgbuilds.git: + +```sh +kupferbootstrap packages build [--force] main/kupfer-config cross/crossdirect +``` + diff --git a/docs/source/usage/index.md b/docs/source/usage/index.md new file mode 100644 index 0000000..d21c193 --- /dev/null +++ b/docs/source/usage/index.md @@ -0,0 +1,9 @@ +# Usage + +```{toctree} +quickstart +faq +install +config +porting +``` diff --git a/docs/source/install.md b/docs/source/usage/install.md similarity index 100% rename from docs/source/install.md rename to docs/source/usage/install.md diff --git a/docs/source/usage/porting.md b/docs/source/usage/porting.md new file mode 100644 index 0000000..b99303a --- /dev/null +++ b/docs/source/usage/porting.md @@ -0,0 +1,94 @@ +# Porting +## Porting devices + +### Homework +Before you can get started porting a device, you'll need to do some research: + +1. Familiarize yourself with git basics. +1. Familiarize yourself with Arch Linux packaging, i.e. `PKGBUILD`s and `makepkg` +1. Familiarize yourself with the postmarketOS port of the device. + ```{warning} + If there is no postmarketOS port yet, you'll probably need to get deep into kernel development. + We suggest [starting with a port to pmOS](https://wiki.postmarketos.org/wiki/Porting_to_a_new_device) then, especially if you're not familiar with the process already. + ``` + +### Porting +1. Navigate to your pkgbuilds checkout +1. Follow the [general package porting guidelines](#porting-packages) to create a device-, kernel- and probably also a firmware-package for the device and SoC. Usually this roughly means porting the postmarketOS APKBUILDs to our PKGBUILD scheme. + You can get inspiration by comparing existing Kupfer ports (e.g. one of the SDM845 devices) to the [postmarketOS packages](https://gitlab.com/postmarketOS/pmaports/-/tree/master/device) for that device. + Usually you should start out by copying and then customizing the Kupfer packages for a device that's as similar to yours as possible, i.e. uses the same or a related SoC, if something like that is already available in Kupfer. + ```{hint} Package Repos: + Device packages belong into `device/`, kernels into `linux/` and firmware into `firmware/`. + ``` +1. When submitting your MR, please include some information: + - what you have found to be working, broken, and not tested (and why) + - any necessary instructions for testing + - whether you'd be willing to maintain the device long-term (test kernel upgrades, submit device package updates, etc.) + + +### Gotchas + +Please be aware of these gotchas: +- As of now, Kupfer only really supports platforms using Android's `aboot` bootloader, i.e. ex-Android phones. In order to support other boot modes (e.g. uboot on the Librem5 and Pine devices), we'll need to port and switch to postmarketOS's [boot-deploy](https://gitlab.com/postmarketOS/boot-deploy) first and add support for EFI setups to Kupferbootstrap. + + +## Porting packages + +### Homework +Before you can get started, you'll need to do some research: + +1. Familiarize yourself with git basics. +1. Familiarize yourself with Arch Linux packaging, i.e. `PKGBUILD`s and `makepkg` + +### Development + +```{warning} +Throughout the process, use git to version your changes. +- Don't procrastinate using git or committing until you're "done" or "have got something working", you'll regret it. +- Don't worry about a "clean" git history while you're developing; we can squash it up later. +- \[Force-]Push your changes regularly, just like committing. Don't wait for perfection. +``` +1. Create a new git branch for your package locally. + ```{hint} + It might be a good ideaa to get into the habit of prefixing branch names with \[a part of] your username and a slash like so: + `myNickname/myFeatureNme` + This makes it easier to work in the same remote repo with multiple people. + ``` +1. + ```{note} + The pkgbuilds git repo contains multiple package repositories, represented by folders at the top level (`main`, `cross`, `phosh`, etc.). + ``` + Try to choose a sensible package repo for your new packages and create new folders for each `pkgbase` inside the repo folder. +1. Navigate into the folder of the new package and create a new `PKGBUILD`; fill it with life! +1. **`_mode`**: Add the build mode at the top of the PKGBUILD. + ```{hint} + If you're unsure what to pick, go with `_mode=host`. It'll use `crossdirect` to get speeds close to proper cross-compiling. + ``` + This determines whether it's built using a foreign-arch chroot (`_mode=host`) executed with qemu-user, or using real cross-compilation (`_mode=cross`) from a host-architecture chroot, but the package's build tooling has to specifically support the latter, so it's mostly useful for kernels and uncompiled packages. +1. **`_nodeps`**: (Optional) If your package doesn't require its listed dependencies to build + (usually because you're packaging a meta-package or only configs or scripts) + you can add `_nodeps=true` as the next line after the `_mode=` line to speed up packaging. + `makedeps` are still installed anyway. +1. Test building it with `kupferbootstrap packages build $pkgbname` +1. For any files and git repos downloaded by your PKGBUILD, + add them to a new `.gitignore` file in the same directory as your `PKGBUILD`. + ```{hint} + Don't forget to `git add` the new `.gitignore` file! + ``` +1. Run `kupferbootstrap packages check` to make sure the formatting for your PKGBUILDs is okay. + ```{warning} + This is **not** optional. MRs with failing CI will **not** be merged. + ``` + +### Pushing +1. Fork the Kupfer pkgbuilds repo on Gitlab using the Fork button +1. Add your fork's **SSH** URI to your local git repo as a **new remote**: `git remote add fork git@gitlab...` +1. `git push -u fork $branchname` it + +### Submitting the MR +When you're ready, open a Merge Request on the Kupfer pkgbuilds repo. + +```{hint} +Prefix the MR title with `Draft: ` to indicate a Work In Progress state. +``` + diff --git a/docs/source/usage/quickstart.md b/docs/source/usage/quickstart.md new file mode 100644 index 0000000..fc5a4d5 --- /dev/null +++ b/docs/source/usage/quickstart.md @@ -0,0 +1,9 @@ +# Quickstart + +1. [Install](../install) Kupferbootstrap +1. [Configure](../config) it: `kuperbootstrap config init` +1. [Update your PKGBUILDs + SRCINFO cache](/cli/packages#kupferbootstrap-packages-update): `kupferbootstrap packages update` +1. [Build an image](/cli/image#kupferbootstrap-image-build): `kupferbootstrap image build` +1. [Flash the image](/cli/image#kupferbootstrap-image-flash): `kupferbootstrap image flash abootimg && kupferbootstrap image flash full userdata` + +See also: [Frequently Asked Questions](../faq) From e783ec66325b254d92ec2733325b53b64eb6e9db Mon Sep 17 00:00:00 2001 From: Syboxez Blank <@Syboxez:matrix.org> Date: Thu, 14 Dec 2023 21:00:06 -0600 Subject: [PATCH 18/30] image/image.py: Fix off-by-one error when creating boot partition in full image --- image/image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/image/image.py b/image/image.py index 029954b..123ef6a 100644 --- a/image/image.py +++ b/image/image.py @@ -356,7 +356,7 @@ def create_img_file(image_path: str, size_str: str): def partition_device(device: str, sector_size: int, boot_partition_size_mb: int = IMG_DEFAULT_SIZE_BOOT_MB): initial_offset = 1048576 // sector_size # 2048 for 512, 256 for 4096 - boot_partition_size: int = align_bytes(boot_partition_size_mb * 1024 * 1024, 4096) + boot_partition_size: int = align_bytes((boot_partition_size_mb + 1) * 1024 * 1024, 4096) create_partition_table = ['mklabel', 'msdos'] create_boot_partition = ['mkpart', 'primary', 'ext2', f'{initial_offset}s', f'{boot_partition_size}b'] create_root_partition = ['mkpart', 'primary', f'{bytes_to_sectors(boot_partition_size, sector_size) + initial_offset}s', '100%'] From 95147ceceac6d8417f55df9bcbd881f8e85577d1 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Tue, 19 Dec 2023 23:34:33 +0100 Subject: [PATCH 19/30] docs: convert absolute links to relative --- docs/source/usage/config.md | 10 +++++++--- docs/source/usage/faq.md | 2 +- docs/source/usage/quickstart.md | 6 +++--- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/source/usage/config.md b/docs/source/usage/config.md index c850cc7..9c259c0 100644 --- a/docs/source/usage/config.md +++ b/docs/source/usage/config.md @@ -2,10 +2,14 @@ Kupferbootstrap uses [toml](https://en.wikipedia.org/wiki/TOML) for its configuration file. -The file can either be edited manually or managed via the {doc}`../cli/config` subcommand. +The file can either be edited manually or managed via the [`kupferbootstrap config`](../../cli/config) subcommand. +```{hint} You can quickly generate a default config by running {code}`kupferbootstrap config init -N`. +For an interactive dialogue, omit the `-N`. +``` + ## File Location The configuration is stored in `~/.config/kupfer/kupferbootstrap.toml`, where `~` is your user's home folder. @@ -54,7 +58,7 @@ This allows you to easily keep a number of slight variations of the same target without the need to constantly modify your Kupferbootstrap configuration file. You can easily create new profiles with -[kupferbootstrap config profile init](/cli/config/#kupferbootstrap-config-profile-init). +[kupferbootstrap config profile init](../../cli/config/#kupferbootstrap-config-profile-init). Here's an example: @@ -97,7 +101,7 @@ hostname = "pocof1" The `current` key in the `profiles` section controlls which profile gets used by Kupferbootstrap by default. The first subsection (`profiles.default`) describes the `default` profile -which gets created by [config init](/cli/config/#kupferbootstrap-config-init). +which gets created by [`kupferbootstrap config init`](../../cli/config/#kupferbootstrap-config-init). Next, we have a `graphical` profile that defines a couple of graphical programs for all but the `recovery` profile, since that doesn't have a GUI. diff --git a/docs/source/usage/faq.md b/docs/source/usage/faq.md index 441bef2..53b1818 100644 --- a/docs/source/usage/faq.md +++ b/docs/source/usage/faq.md @@ -19,7 +19,7 @@ See [Porting](../porting) ## How to build a specific package -See also: The full [`kupferbootstrap packages build` docs](/cli/packages#kupferbootstrap-packages-build) +See also: The full [`kupferbootstrap packages build` docs](../../cli/packages#kupferbootstrap-packages-build) ### Example diff --git a/docs/source/usage/quickstart.md b/docs/source/usage/quickstart.md index fc5a4d5..0076b58 100644 --- a/docs/source/usage/quickstart.md +++ b/docs/source/usage/quickstart.md @@ -2,8 +2,8 @@ 1. [Install](../install) Kupferbootstrap 1. [Configure](../config) it: `kuperbootstrap config init` -1. [Update your PKGBUILDs + SRCINFO cache](/cli/packages#kupferbootstrap-packages-update): `kupferbootstrap packages update` -1. [Build an image](/cli/image#kupferbootstrap-image-build): `kupferbootstrap image build` -1. [Flash the image](/cli/image#kupferbootstrap-image-flash): `kupferbootstrap image flash abootimg && kupferbootstrap image flash full userdata` +1. [Update your PKGBUILDs + SRCINFO cache](../../cli/packages#kupferbootstrap-packages-update): `kupferbootstrap packages update` +1. [Build an image](../../cli/image#kupferbootstrap-image-build): `kupferbootstrap image build` +1. [Flash the image](../../cli/image#kupferbootstrap-image-flash): `kupferbootstrap image flash abootimg && kupferbootstrap image flash full userdata` See also: [Frequently Asked Questions](../faq) From 4cce7e57aea2f4a0904943b9d57c6460ee210b44 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Wed, 20 Dec 2023 00:28:26 +0100 Subject: [PATCH 20/30] constants: use ALARM's aarch64 gcc that we package --- constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/constants.py b/constants.py index 723dcc0..2ddd686 100644 --- a/constants.py +++ b/constants.py @@ -89,7 +89,7 @@ COMPILE_ARCHES: dict[Arch, str] = { GCC_HOSTSPECS: dict[DistroArch, dict[TargetArch, str]] = { 'x86_64': { 'x86_64': 'x86_64-pc-linux-gnu', - 'aarch64': 'aarch64-linux-gnu', + 'aarch64': 'aarch64-unknown-linux-gnu', 'armv7h': 'arm-unknown-linux-gnueabihf' }, 'aarch64': { From a75f32b4b159eb65fd3b26158333e13530b272c8 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Wed, 20 Dec 2023 01:55:16 +0100 Subject: [PATCH 21/30] chroot/build: mount_crossdirect(): fix symlink creation if link exists --- chroot/build.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/chroot/build.py b/chroot/build.py index 2520d6a..40b123d 100644 --- a/chroot/build.py +++ b/chroot/build.py @@ -82,6 +82,7 @@ class BuildChroot(Chroot): native_chroot.mount_pacman_cache() native_chroot.mount_packages() native_chroot.activate() + logging.debug(f"Installing {CROSSDIRECT_PKGS=} + {gcc=}") results = dict(native_chroot.try_install_packages( CROSSDIRECT_PKGS + [gcc], refresh=True, @@ -103,8 +104,8 @@ class BuildChroot(Chroot): target_include_dir = os.path.join(self.path, 'include') for target, source in {cc_path: gcc, target_lib_dir: 'lib', target_include_dir: 'usr/include'}.items(): - if not os.path.exists(target): - logging.debug(f'Symlinking {source} at {target}') + if not (os.path.exists(target) or os.path.islink(target)): + logging.debug(f'Symlinking {source=} at {target=}') symlink(source, target) ld_so = os.path.basename(glob(f"{os.path.join(native_chroot.path, 'usr', 'lib', 'ld-linux-')}*")[0]) ld_so_target = os.path.join(target_lib_dir, ld_so) From c074fbe42c7c05c25e89df96224a66bcef645519 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Wed, 20 Dec 2023 03:33:28 +0100 Subject: [PATCH 22/30] packages/pkgbuild: parse_pkgbuild(): inherit depends, makedepends, provides, replaces from pkgbase unless overriden --- packages/pkgbuild.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/packages/pkgbuild.py b/packages/pkgbuild.py index 65722ba..baeea83 100644 --- a/packages/pkgbuild.py +++ b/packages/pkgbuild.py @@ -310,8 +310,11 @@ class SubPkgbuild(Pkgbuild): self.sources_refreshed = False self.update(pkgbase) - self.provides = {} - self.replaces = [] + # set to None - will be replaced with base_pkg if still None after parsing + self.depends = None # type: ignore[assignment] + self.makedepends = None # type: ignore[assignment] + self.provides = None # type: ignore[assignment] + self.replaces = None # type: ignore[assignment] def refresh_sources(self, lazy: bool = True): assert self.pkgbase @@ -383,13 +386,21 @@ def parse_pkgbuild( elif line.startswith('arch'): current.arches.append(splits[1]) elif line.startswith('provides'): + if not current.provides: + current.provides = {} current.provides = get_version_specs(splits[1], current.provides) elif line.startswith('replaces'): + if not current.replaces: + current.replaces = [] current.replaces.append(splits[1]) elif splits[0] in ['depends', 'makedepends', 'checkdepends', 'optdepends']: spec = splits[1].split(': ', 1)[0] + if not current.depends: + current.depends = {} current.depends = get_version_specs(spec, current.depends) if splits[0] == 'makedepends': + if not current.makedepends: + current.makedepends = {} current.makedepends = get_version_specs(spec, current.makedepends) results: list[Pkgbuild] = list(base_package.subpackages) @@ -402,6 +413,15 @@ def parse_pkgbuild( pkg.update_version() if not (pkg.version == base_package.version): raise Exception(f'Subpackage malformed! Versions differ! base: {base_package}, subpackage: {pkg}') + if isinstance(pkg, SubPkgbuild): + if pkg.depends is None: + pkg.depends = base_package.depends + if pkg.makedepends is None: + pkg.makedepends = base_package.makedepends + if pkg.replaces is None: + pkg.replaces = base_package.replaces + if pkg.provides is None: + pkg.provides = base_package.provides return results From eaac9195ea584964785055b4b8e66fe46b5a596b Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Wed, 20 Dec 2023 03:36:13 +0100 Subject: [PATCH 23/30] packages/build: build_enable_qemu_binfmt(): also build gcc package if available --- packages/build.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/packages/build.py b/packages/build.py index 190e588..455126b 100644 --- a/packages/build.py +++ b/packages/build.py @@ -438,10 +438,11 @@ def setup_build_chroot( extra_packages: list[str] = [], add_kupfer_repos: bool = True, clean_chroot: bool = False, + repo: Optional[dict[str, Pkgbuild]] = None, ) -> BuildChroot: assert config.runtime.arch if arch != config.runtime.arch: - build_enable_qemu_binfmt(arch, lazy=False) + build_enable_qemu_binfmt(arch, repo=repo or discover_pkgbuilds(), lazy=False) init_prebuilts(arch) chroot = get_build_chroot(arch, add_kupfer_repos=add_kupfer_repos) chroot.mount_packages() @@ -510,6 +511,7 @@ def build_package( enable_ccache: bool = True, clean_chroot: bool = False, build_user: str = 'kupfer', + repo: Optional[dict[str, Pkgbuild]] = None, ): makepkg_compile_opts = ['--holdver'] makepkg_conf_path = 'etc/makepkg.conf' @@ -529,6 +531,7 @@ def build_package( arch=arch, extra_packages=deps, clean_chroot=clean_chroot, + repo=repo, ) assert config.runtime.arch native_chroot = target_chroot @@ -538,6 +541,7 @@ def build_package( arch=config.runtime.arch, extra_packages=['base-devel'] + CROSSDIRECT_PKGS, clean_chroot=clean_chroot, + repo=repo, ) if not package.mode: logging.warning(f'Package {package.path} has no _mode set, assuming "host"') @@ -756,6 +760,7 @@ def build_packages( enable_crossdirect=enable_crossdirect, enable_ccache=enable_ccache, clean_chroot=clean_chroot, + repo=repo, ) files += add_package_to_repo(package, arch) updated_repos.add(package.repo) @@ -830,8 +835,20 @@ def build_enable_qemu_binfmt(arch: Arch, repo: Optional[dict[str, Pkgbuild]] = N logging.info('Installing qemu-user (building if necessary)') check_programs_wrap(['pacman', 'makepkg', 'pacstrap']) # build qemu-user, binfmt, crossdirect + packages = list(CROSSDIRECT_PKGS) + hostspec = GCC_HOSTSPECS[arch][arch] + cross_gcc = f"{hostspec}-gcc" + if repo: + for pkg in repo.values(): + if (pkg.name == cross_gcc or cross_gcc in pkg.provides): + if config.runtime.arch not in pkg.arches: + logging.debug(f"Package {pkg.path} matches {cross_gcc=} name but not arch: {pkg.arches=}") + continue + packages.append(pkg.path) + logging.debug(f"Adding gcc package {pkg.path} to the necessary crosscompilation tools") + break build_packages_by_paths( - CROSSDIRECT_PKGS, + packages, native, repo=repo, try_download=True, From 4b2150940d0fcedcc1a86d63e4c8ece7a54af519 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Fri, 22 Dec 2023 05:07:55 +0100 Subject: [PATCH 24/30] packages/build: use copy && remove_file() instead of shutil.move() --- packages/build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/build.py b/packages/build.py index 455126b..bcd5403 100644 --- a/packages/build.py +++ b/packages/build.py @@ -310,7 +310,8 @@ def try_download_package(dest_file_path: str, package: Pkgbuild, arch: Arch) -> logging.debug(f"While checking the HTTPS repo DB, we found a matching filename in the pacman cache{extra_msg}") if cache_matches: logging.info(f'copying cache file {cache_file} to repo as verified by remote checksum') - shutil.move(cache_file, dest_file_path) + shutil.copy(cache_file, dest_file_path) + remove_file(cache_file) return dest_file_path url = repo_pkg.resolved_url assert url From b006cd8f4da8c621370b2bf73762b248ddd74dbc Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Mon, 8 Jan 2024 02:30:57 +0100 Subject: [PATCH 25/30] packages/pkgbuild: support new key "_crossdirect" to enable/disable crossdirect for single packages --- packages/build.py | 2 +- packages/cli.py | 4 +++- packages/pkgbuild.py | 7 +++++++ packages/srcinfo_cache.py | 20 +++++++++++++++----- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/packages/build.py b/packages/build.py index bcd5403..b388221 100644 --- a/packages/build.py +++ b/packages/build.py @@ -575,7 +575,7 @@ def build_package( build_root = target_chroot makepkg_compile_opts += ['--nodeps' if package.nodeps else '--syncdeps'] env = deepcopy(get_makepkg_env(arch)) - if foreign_arch and enable_crossdirect and package.name not in CROSSDIRECT_PKGS: + if foreign_arch and package.crossdirect and enable_crossdirect and package.name not in CROSSDIRECT_PKGS: env['PATH'] = f"/native/usr/lib/crossdirect/{arch}:{env['PATH']}" target_chroot.mount_crossdirect(native_chroot) else: diff --git a/packages/cli.py b/packages/cli.py index c300111..3878ba4 100644 --- a/packages/cli.py +++ b/packages/cli.py @@ -313,7 +313,7 @@ def cmd_list(): logging.info(f'Done! {len(packages)} Pkgbuilds:') for name in sorted(packages.keys()): p = packages[name] - print(f'name: {p.name}; ver: {p.version}; mode: {p.mode}; provides: {p.provides}; replaces: {p.replaces};' + print(f'name: {p.name}; ver: {p.version}; mode: {p.mode}; crossdirect: {p.crossdirect} provides: {p.provides}; replaces: {p.replaces};' f'local_depends: {p.local_depends}; depends: {p.depends}') @@ -346,6 +346,7 @@ def cmd_check(paths): mode_key = '_mode' nodeps_key = '_nodeps' + crossdirect_key = '_crossdirect' pkgbase_key = 'pkgbase' pkgname_key = 'pkgname' arches_key = '_arches' @@ -356,6 +357,7 @@ def cmd_check(paths): required = { mode_key: True, nodeps_key: False, + crossdirect_key: False, pkgbase_key: False, pkgname_key: True, 'pkgdesc': False, diff --git a/packages/pkgbuild.py b/packages/pkgbuild.py index baeea83..3b89558 100644 --- a/packages/pkgbuild.py +++ b/packages/pkgbuild.py @@ -156,6 +156,7 @@ class Pkgbuild(PackageInfo): repo: str mode: str nodeps: bool + crossdirect: bool path: str pkgver: str pkgrel: str @@ -190,6 +191,7 @@ class Pkgbuild(PackageInfo): self.repo = repo or '' self.mode = '' self.nodeps = False + self.crossdirect = True self.path = relative_path self.pkgver = '' self.pkgrel = '' @@ -223,6 +225,7 @@ class Pkgbuild(PackageInfo): self.repo = pkg.repo self.mode = pkg.mode self.nodeps = pkg.nodeps + self.crossdirect = pkg.crossdirect self.path = pkg.path self.pkgver = pkg.pkgver self.pkgrel = pkg.pkgrel @@ -357,7 +360,11 @@ def parse_pkgbuild( else: raise Exception(msg) + # if _crossdirect is unset (None), it defaults to True + crossdirect_enabled = srcinfo_cache.build_crossdirect in (None, True) + base_package = Pkgbase(relative_pkg_dir, sources_refreshed=sources_refreshed, srcinfo_cache=srcinfo_cache) + base_package.crossdirect = crossdirect_enabled base_package.mode = mode base_package.nodeps = nodeps base_package.repo = relative_pkg_dir.split('/')[0] diff --git a/packages/srcinfo_cache.py b/packages/srcinfo_cache.py index 5cb2373..3d9737b 100644 --- a/packages/srcinfo_cache.py +++ b/packages/srcinfo_cache.py @@ -68,11 +68,19 @@ class SrcInitialisedFile(JsonFile): raise ex +srcinfo_meta_defaults = { + 'build_mode': None, + "build_nodeps": None, + "build_crossdirect": None, +} + + class SrcinfoMetaFile(JsonFile): checksums: dict[str, str] build_mode: Optional[str] build_nodeps: Optional[bool] + build_crossdirect: Optional[bool] _changed: bool _filename: ClassVar[str] = SRCINFO_METADATA_FILE @@ -92,9 +100,8 @@ class SrcinfoMetaFile(JsonFile): s = SrcinfoMetaFile({ '_relative_path': relative_pkg_dir, '_changed': True, - 'build_mode': '', - 'build_nodeps': None, 'checksums': {}, + **srcinfo_meta_defaults, }) return s, s.refresh_all() @@ -120,9 +127,11 @@ class SrcinfoMetaFile(JsonFile): if not force_refresh: logging.debug(f'{metadata._relative_path}: srcinfo checksums match!') lines = lines or metadata.read_srcinfo_file() - for build_field in ['build_mode', 'build_nodeps']: + for build_field in srcinfo_meta_defaults.keys(): if build_field not in metadata: metadata.refresh_build_fields() + if write: + metadata.write() break else: lines = metadata.refresh_all(write=write) @@ -143,8 +152,7 @@ class SrcinfoMetaFile(JsonFile): self._changed = True def refresh_build_fields(self): - self['build_mode'] = None - self['build_nodeps'] = None + self.update(srcinfo_meta_defaults) with open(os.path.join(config.get_path('pkgbuilds'), self._relative_path, 'PKGBUILD'), 'r') as file: lines = file.read().split('\n') for line in lines: @@ -156,6 +164,8 @@ class SrcinfoMetaFile(JsonFile): self.build_mode = val elif key == '_nodeps': self.build_nodeps = val.lower() == 'true' + elif key == '_crossdirect': + self.build_crossdirect = val.lower() == 'true' else: continue From f05de7738ae67e2a2b393b4437dbe43384159825 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Mon, 8 Jan 2024 04:25:42 +0100 Subject: [PATCH 26/30] integration_tests: test importing main.cli --- integration_tests.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/integration_tests.py b/integration_tests.py index 2b6c526..bc4eeb7 100644 --- a/integration_tests.py +++ b/integration_tests.py @@ -37,6 +37,11 @@ def ctx() -> click.Context: return click.Context(click.Command('integration_tests')) +def test_main_import(): + from main import cli + assert cli + + def test_config_load(ctx: click.Context): path = config.runtime.config_file assert path From cebac831864a2eaeaa86e3586f778e9aafb43d6f Mon Sep 17 00:00:00 2001 From: Syboxez Blank Date: Sat, 23 Mar 2024 17:48:38 +0000 Subject: [PATCH 27/30] packages/pkgbuild: parse_pkgbuild(): Reuse pkgbase's `makedepends` as dependencies Authored-by: InsanePrawn --- packages/pkgbuild.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pkgbuild.py b/packages/pkgbuild.py index 3b89558..9be89c3 100644 --- a/packages/pkgbuild.py +++ b/packages/pkgbuild.py @@ -403,7 +403,7 @@ def parse_pkgbuild( elif splits[0] in ['depends', 'makedepends', 'checkdepends', 'optdepends']: spec = splits[1].split(': ', 1)[0] if not current.depends: - current.depends = {} + current.depends = (base_package.makedepends or {}).copy() current.depends = get_version_specs(spec, current.depends) if splits[0] == 'makedepends': if not current.makedepends: From a4cfc3c3e51dc21e2e19729154afa7869e0a1698 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Wed, 20 Mar 2024 16:42:19 +0100 Subject: [PATCH 28/30] exec/file: makedir(): add mode=None arg --- exec/file.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/exec/file.py b/exec/file.py index 852ad48..00653aa 100644 --- a/exec/file.py +++ b/exec/file.py @@ -144,7 +144,13 @@ def remove_file(path: str, recursive=False): raise Exception(f"Unable to remove {path}: cmd returned {rc}") -def makedir(path, user: Optional[Union[str, int]] = None, group: Optional[Union[str, int]] = None, parents: bool = True): +def makedir( + path, + user: Optional[Union[str, int]] = None, + group: Optional[Union[str, int]] = None, + parents: bool = True, + mode: Optional[Union[int, str]] = None, +): if not root_check_exists(path): try: if parents: @@ -153,6 +159,8 @@ def makedir(path, user: Optional[Union[str, int]] = None, group: Optional[Union[ os.mkdir(path) except: run_root_cmd(['mkdir'] + (['-p'] if parents else []) + [path]) + if mode is not None: + chmod(path, mode=mode) chown(path, user, group) From a176fad05a358bcbb83d4e2207f8ae2a1c7abbee Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Wed, 20 Mar 2024 16:43:08 +0100 Subject: [PATCH 29/30] net/ssh: copy_ssh_keys(): pass chroot for uid resolution --- image/image.py | 2 +- net/ssh.py | 52 ++++++++++++++++++++++++++++++-------------------- 2 files changed, 32 insertions(+), 22 deletions(-) diff --git a/image/image.py b/image/image.py index 0cb0bcc..afb3ddb 100644 --- a/image/image.py +++ b/image/image.py @@ -333,7 +333,7 @@ def install_rootfs( ) chroot.add_sudo_config(config_name='wheel', privilegee='%wheel', password_required=True) copy_ssh_keys( - chroot.path, + chroot, user=user, ) files = { diff --git a/net/ssh.py b/net/ssh.py index 2a5ef7f..cf1ed37 100644 --- a/net/ssh.py +++ b/net/ssh.py @@ -6,7 +6,9 @@ import click from config.state import config from constants import SSH_COMMON_OPTIONS, SSH_DEFAULT_HOST, SSH_DEFAULT_PORT +from chroot.abstract import Chroot from exec.cmd import run_cmd +from exec.file import write_file from wrapper import check_programs_wrap @@ -83,21 +85,16 @@ def find_ssh_keys(): return keys -def copy_ssh_keys(root_dir: str, user: str): +def copy_ssh_keys(chroot: Chroot, user: str): check_programs_wrap(['ssh-keygen']) - authorized_keys_file = os.path.join( - root_dir, - 'home', - user, - '.ssh', - 'authorized_keys', - ) - if os.path.exists(authorized_keys_file): - os.unlink(authorized_keys_file) + ssh_dir_relative = os.path.join('/home', user, '.ssh') + ssh_dir = chroot.get_path(ssh_dir_relative) + authorized_keys_file_rel = os.path.join(ssh_dir_relative, 'authorized_keys') + authorized_keys_file = chroot.get_path(authorized_keys_file_rel) keys = find_ssh_keys() if len(keys) == 0: - logging.info("Could not find any ssh key to copy") + logging.warning("Could not find any ssh key to copy") create = click.confirm("Do you want me to generate an ssh key for you?", True) if not create: return @@ -116,15 +113,28 @@ def copy_ssh_keys(root_dir: str, user: str): logging.fatal("Failed to generate ssh key") keys = find_ssh_keys() - ssh_dir = os.path.join(root_dir, 'home', user, '.ssh') - if not os.path.exists(ssh_dir): - os.makedirs(ssh_dir, exist_ok=True, mode=0o700) + if not keys: + logging.warning("No SSH keys to be copied. Skipping.") + return - with open(authorized_keys_file, 'a') as authorized_keys: - for key in keys: - pub = f'{key}.pub' - if not os.path.exists(pub): - logging.debug(f'Skipping key {key}: {pub} not found') - continue + auth_key_lines = [] + for key in keys: + pub = f'{key}.pub' + if not os.path.exists(pub): + logging.debug(f'Skipping key {key}: {pub} not found') + continue + try: with open(pub, 'r') as file: - authorized_keys.write(file.read()) + contents = file.read() + if not contents.strip(): + continue + auth_key_lines.append(contents) + except Exception as ex: + logging.warning(f"Could not read ssh pub key {pub}", exc_info=ex) + continue + + if not os.path.exists(ssh_dir): + logging.info(f"Creating {ssh_dir_relative} dir in chroot {chroot.path}") + chroot.run_cmd(["mkdir", "-p", "-m", "700", ssh_dir_relative], switch_user=user) + logging.info(f"Writing SSH pub keys to {authorized_keys_file}") + write_file(authorized_keys_file, "\n".join(auth_key_lines), user=chroot.get_uid(user), mode="644") From a28550825f89673635e36456da06c5a23c43ece8 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Wed, 20 Mar 2024 20:56:17 +0100 Subject: [PATCH 30/30] image/image: tolerate pub-key copying to fail during image build --- image/image.py | 1 + net/ssh.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/image/image.py b/image/image.py index afb3ddb..6532af7 100644 --- a/image/image.py +++ b/image/image.py @@ -335,6 +335,7 @@ def install_rootfs( copy_ssh_keys( chroot, user=user, + allow_fail=True, ) files = { 'etc/pacman.conf': get_base_distro(arch).get_pacman_conf( diff --git a/net/ssh.py b/net/ssh.py index cf1ed37..6eb7294 100644 --- a/net/ssh.py +++ b/net/ssh.py @@ -85,7 +85,7 @@ def find_ssh_keys(): return keys -def copy_ssh_keys(chroot: Chroot, user: str): +def copy_ssh_keys(chroot: Chroot, user: str, allow_fail: bool = False): check_programs_wrap(['ssh-keygen']) ssh_dir_relative = os.path.join('/home', user, '.ssh') ssh_dir = chroot.get_path(ssh_dir_relative) @@ -134,7 +134,13 @@ def copy_ssh_keys(chroot: Chroot, user: str): continue if not os.path.exists(ssh_dir): - logging.info(f"Creating {ssh_dir_relative} dir in chroot {chroot.path}") + logging.info(f"Creating {ssh_dir_relative!r} dir in chroot {chroot.path!r}") chroot.run_cmd(["mkdir", "-p", "-m", "700", ssh_dir_relative], switch_user=user) logging.info(f"Writing SSH pub keys to {authorized_keys_file}") - write_file(authorized_keys_file, "\n".join(auth_key_lines), user=chroot.get_uid(user), mode="644") + try: + write_file(authorized_keys_file, "\n".join(auth_key_lines), user=str(chroot.get_uid(user)), mode="644") + except Exception as ex: + logging.error(f"Failed to write SSH authorized_keys_file at {authorized_keys_file!r}:", exc_info=ex) + if allow_fail: + return + raise ex from ex