From a0c40363903c5e1d89d6c799aa8abd125b9c2e15 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Sun, 29 Oct 2023 16:32:36 +0100 Subject: [PATCH 01/15] packages: try_download_package(): check pacman cache if file in db but doesn't exist in db folder --- packages/build.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/build.py b/packages/build.py index 72c9152..190e588 100644 --- a/packages/build.py +++ b/packages/build.py @@ -290,7 +290,8 @@ def try_download_package(dest_file_path: str, package: Pkgbuild, arch: Arch) -> return None repo_pkg: RemotePackage = repo.packages[pkgname] if repo_pkg.version != package.version: - logging.debug(f"Package {pkgname} versions differ: local: {package.version}, remote: {repo_pkg.version}. Building instead.") + logging.debug(f"Package {pkgname} versions differ: local: {package.version}, " + f"remote: {repo_pkg.version}. Building instead.") return None if repo_pkg.filename != filename: versions_str = f"local: {filename}, remote: {repo_pkg.filename}" @@ -298,6 +299,19 @@ def try_download_package(dest_file_path: str, package: Pkgbuild, arch: Arch) -> logging.debug(f"package filenames don't match: {versions_str}") return None logging.debug(f"ignoring compression extension difference: {versions_str}") + cache_file = os.path.join(config.get_path('pacman'), arch, repo_pkg.filename) + if os.path.exists(cache_file): + if not repo_pkg._desc or 'SHA256SUM' not in repo_pkg._desc: + cache_matches = False + extra_msg = ". However, we can't validate it, as the https repo doesnt provide a SHA256SUM for it." + else: + cache_matches = sha256sum(cache_file) == repo_pkg._desc['SHA256SUM'] + extra_msg = (". However its checksum doesn't match." if not cache_matches else " and its checksum matches.") + logging.debug(f"While checking the HTTPS repo DB, we found a matching filename in the pacman cache{extra_msg}") + if cache_matches: + logging.info(f'copying cache file {cache_file} to repo as verified by remote checksum') + shutil.move(cache_file, dest_file_path) + return dest_file_path url = repo_pkg.resolved_url assert url try: From 2e504b7b00f40b33cd583d5c246cda751fcdd8a0 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Mon, 11 Dec 2023 12:49:28 +0100 Subject: [PATCH 02/15] dictscheme: fix type hinting --- dictscheme.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dictscheme.py b/dictscheme.py index c5537d3..ca7c12c 100644 --- a/dictscheme.py +++ b/dictscheme.py @@ -52,7 +52,7 @@ class DictScheme(Munch): _sparse: ClassVar[bool] = False def __init__(self, d: Mapping = {}, validate: bool = True, **kwargs): - self.update(d | kwargs, validate=validate) + self.update(dict(d) | kwargs, validate=validate) @classmethod def transform( @@ -269,10 +269,13 @@ class DictScheme(Munch): ) -> str: import yaml yaml_args = {'sort_keys': False} | yaml_args - return yaml.dump( + dumped = yaml.dump( self.toDict(strip_hidden=strip_hidden, sparse=sparse), **yaml_args, ) + if dumped is None: + raise Exception(f"Failed to yaml-serialse {self}") + return dumped def toToml( self, From ff8a529690da1732f1b6bffd67c2b101b2cbf6ff Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Mon, 11 Dec 2023 16:37:48 +0100 Subject: [PATCH 03/15] docs: move usage guides to usage/, add quickstart and porting --- docs/source/index.md | 3 +- docs/source/{ => usage}/config.md | 6 +- docs/source/usage/faq.md | 39 +++++++++++++ docs/source/usage/index.md | 9 +++ docs/source/{ => usage}/install.md | 0 docs/source/usage/porting.md | 94 ++++++++++++++++++++++++++++++ docs/source/usage/quickstart.md | 9 +++ 7 files changed, 155 insertions(+), 5 deletions(-) rename docs/source/{ => usage}/config.md (94%) create mode 100644 docs/source/usage/faq.md create mode 100644 docs/source/usage/index.md rename docs/source/{ => usage}/install.md (100%) create mode 100644 docs/source/usage/porting.md create mode 100644 docs/source/usage/quickstart.md diff --git a/docs/source/index.md b/docs/source/index.md index 82a3e72..2cde0d6 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -6,7 +6,6 @@ a tool to build and flash packages and images for the [Kupfer](https://gitlab.co ## Documentation pages ```{toctree} -install -config +usage/index cli ``` diff --git a/docs/source/config.md b/docs/source/usage/config.md similarity index 94% rename from docs/source/config.md rename to docs/source/usage/config.md index 561d24c..c850cc7 100644 --- a/docs/source/config.md +++ b/docs/source/usage/config.md @@ -2,7 +2,7 @@ Kupferbootstrap uses [toml](https://en.wikipedia.org/wiki/TOML) for its configuration file. -The file can either be edited manually or managed via the {doc}`cli/config` subcommand. +The file can either be edited manually or managed via the {doc}`../cli/config` subcommand. You can quickly generate a default config by running {code}`kupferbootstrap config init -N`. @@ -54,7 +54,7 @@ This allows you to easily keep a number of slight variations of the same target without the need to constantly modify your Kupferbootstrap configuration file. You can easily create new profiles with -[kupferbootstrap config profile init](../cli/config/#kupferbootstrap-config-profile-init). +[kupferbootstrap config profile init](/cli/config/#kupferbootstrap-config-profile-init). Here's an example: @@ -97,7 +97,7 @@ hostname = "pocof1" The `current` key in the `profiles` section controlls which profile gets used by Kupferbootstrap by default. The first subsection (`profiles.default`) describes the `default` profile -which gets created by [config init](../cli/config/#kupferbootstrap-config-init). +which gets created by [config init](/cli/config/#kupferbootstrap-config-init). Next, we have a `graphical` profile that defines a couple of graphical programs for all but the `recovery` profile, since that doesn't have a GUI. diff --git a/docs/source/usage/faq.md b/docs/source/usage/faq.md new file mode 100644 index 0000000..441bef2 --- /dev/null +++ b/docs/source/usage/faq.md @@ -0,0 +1,39 @@ +# FAQ + + +```{contents} Table of Contents +:class: this-will-duplicate-information-and-it-is-still-useful-here +:depth: 3 +``` + + +## Which devices are currently supported? + +Currently very few! +See [the `devices` repo](https://gitlab.com/kupfer/packages/pkgbuilds/-/tree/dev/device). We use the same codenames as [postmarketOS](https://wiki.postmarketos.org/wiki/Devices) (although we prefix them with the SoC) + + +## How to port a new device or package? + +See [Porting](../porting) + +## How to build a specific package + +See also: The full [`kupferbootstrap packages build` docs](/cli/packages#kupferbootstrap-packages-build) + +### Example + +For rebuilding `kupfer-config` and `crossdirect`, defaulting to your device's architecture + +```sh +kupferbootstrap packages build [--force] [--arch $target_arch] kupfer-config crossdirect +``` + + +### By package path +You can also use the a path snippet (`$repo/$pkgbase`) to the PKGBUILD folder as seen inside your pkgbuilds.git: + +```sh +kupferbootstrap packages build [--force] main/kupfer-config cross/crossdirect +``` + diff --git a/docs/source/usage/index.md b/docs/source/usage/index.md new file mode 100644 index 0000000..d21c193 --- /dev/null +++ b/docs/source/usage/index.md @@ -0,0 +1,9 @@ +# Usage + +```{toctree} +quickstart +faq +install +config +porting +``` diff --git a/docs/source/install.md b/docs/source/usage/install.md similarity index 100% rename from docs/source/install.md rename to docs/source/usage/install.md diff --git a/docs/source/usage/porting.md b/docs/source/usage/porting.md new file mode 100644 index 0000000..b99303a --- /dev/null +++ b/docs/source/usage/porting.md @@ -0,0 +1,94 @@ +# Porting +## Porting devices + +### Homework +Before you can get started porting a device, you'll need to do some research: + +1. Familiarize yourself with git basics. +1. Familiarize yourself with Arch Linux packaging, i.e. `PKGBUILD`s and `makepkg` +1. Familiarize yourself with the postmarketOS port of the device. + ```{warning} + If there is no postmarketOS port yet, you'll probably need to get deep into kernel development. + We suggest [starting with a port to pmOS](https://wiki.postmarketos.org/wiki/Porting_to_a_new_device) then, especially if you're not familiar with the process already. + ``` + +### Porting +1. Navigate to your pkgbuilds checkout +1. Follow the [general package porting guidelines](#porting-packages) to create a device-, kernel- and probably also a firmware-package for the device and SoC. Usually this roughly means porting the postmarketOS APKBUILDs to our PKGBUILD scheme. + You can get inspiration by comparing existing Kupfer ports (e.g. one of the SDM845 devices) to the [postmarketOS packages](https://gitlab.com/postmarketOS/pmaports/-/tree/master/device) for that device. + Usually you should start out by copying and then customizing the Kupfer packages for a device that's as similar to yours as possible, i.e. uses the same or a related SoC, if something like that is already available in Kupfer. + ```{hint} Package Repos: + Device packages belong into `device/`, kernels into `linux/` and firmware into `firmware/`. + ``` +1. When submitting your MR, please include some information: + - what you have found to be working, broken, and not tested (and why) + - any necessary instructions for testing + - whether you'd be willing to maintain the device long-term (test kernel upgrades, submit device package updates, etc.) + + +### Gotchas + +Please be aware of these gotchas: +- As of now, Kupfer only really supports platforms using Android's `aboot` bootloader, i.e. ex-Android phones. In order to support other boot modes (e.g. uboot on the Librem5 and Pine devices), we'll need to port and switch to postmarketOS's [boot-deploy](https://gitlab.com/postmarketOS/boot-deploy) first and add support for EFI setups to Kupferbootstrap. + + +## Porting packages + +### Homework +Before you can get started, you'll need to do some research: + +1. Familiarize yourself with git basics. +1. Familiarize yourself with Arch Linux packaging, i.e. `PKGBUILD`s and `makepkg` + +### Development + +```{warning} +Throughout the process, use git to version your changes. +- Don't procrastinate using git or committing until you're "done" or "have got something working", you'll regret it. +- Don't worry about a "clean" git history while you're developing; we can squash it up later. +- \[Force-]Push your changes regularly, just like committing. Don't wait for perfection. +``` +1. Create a new git branch for your package locally. + ```{hint} + It might be a good ideaa to get into the habit of prefixing branch names with \[a part of] your username and a slash like so: + `myNickname/myFeatureNme` + This makes it easier to work in the same remote repo with multiple people. + ``` +1. + ```{note} + The pkgbuilds git repo contains multiple package repositories, represented by folders at the top level (`main`, `cross`, `phosh`, etc.). + ``` + Try to choose a sensible package repo for your new packages and create new folders for each `pkgbase` inside the repo folder. +1. Navigate into the folder of the new package and create a new `PKGBUILD`; fill it with life! +1. **`_mode`**: Add the build mode at the top of the PKGBUILD. + ```{hint} + If you're unsure what to pick, go with `_mode=host`. It'll use `crossdirect` to get speeds close to proper cross-compiling. + ``` + This determines whether it's built using a foreign-arch chroot (`_mode=host`) executed with qemu-user, or using real cross-compilation (`_mode=cross`) from a host-architecture chroot, but the package's build tooling has to specifically support the latter, so it's mostly useful for kernels and uncompiled packages. +1. **`_nodeps`**: (Optional) If your package doesn't require its listed dependencies to build + (usually because you're packaging a meta-package or only configs or scripts) + you can add `_nodeps=true` as the next line after the `_mode=` line to speed up packaging. + `makedeps` are still installed anyway. +1. Test building it with `kupferbootstrap packages build $pkgbname` +1. For any files and git repos downloaded by your PKGBUILD, + add them to a new `.gitignore` file in the same directory as your `PKGBUILD`. + ```{hint} + Don't forget to `git add` the new `.gitignore` file! + ``` +1. Run `kupferbootstrap packages check` to make sure the formatting for your PKGBUILDs is okay. + ```{warning} + This is **not** optional. MRs with failing CI will **not** be merged. + ``` + +### Pushing +1. Fork the Kupfer pkgbuilds repo on Gitlab using the Fork button +1. Add your fork's **SSH** URI to your local git repo as a **new remote**: `git remote add fork git@gitlab...` +1. `git push -u fork $branchname` it + +### Submitting the MR +When you're ready, open a Merge Request on the Kupfer pkgbuilds repo. + +```{hint} +Prefix the MR title with `Draft: ` to indicate a Work In Progress state. +``` + diff --git a/docs/source/usage/quickstart.md b/docs/source/usage/quickstart.md new file mode 100644 index 0000000..fc5a4d5 --- /dev/null +++ b/docs/source/usage/quickstart.md @@ -0,0 +1,9 @@ +# Quickstart + +1. [Install](../install) Kupferbootstrap +1. [Configure](../config) it: `kuperbootstrap config init` +1. [Update your PKGBUILDs + SRCINFO cache](/cli/packages#kupferbootstrap-packages-update): `kupferbootstrap packages update` +1. [Build an image](/cli/image#kupferbootstrap-image-build): `kupferbootstrap image build` +1. [Flash the image](/cli/image#kupferbootstrap-image-flash): `kupferbootstrap image flash abootimg && kupferbootstrap image flash full userdata` + +See also: [Frequently Asked Questions](../faq) From 95147ceceac6d8417f55df9bcbd881f8e85577d1 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Tue, 19 Dec 2023 23:34:33 +0100 Subject: [PATCH 04/15] docs: convert absolute links to relative --- docs/source/usage/config.md | 10 +++++++--- docs/source/usage/faq.md | 2 +- docs/source/usage/quickstart.md | 6 +++--- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/source/usage/config.md b/docs/source/usage/config.md index c850cc7..9c259c0 100644 --- a/docs/source/usage/config.md +++ b/docs/source/usage/config.md @@ -2,10 +2,14 @@ Kupferbootstrap uses [toml](https://en.wikipedia.org/wiki/TOML) for its configuration file. -The file can either be edited manually or managed via the {doc}`../cli/config` subcommand. +The file can either be edited manually or managed via the [`kupferbootstrap config`](../../cli/config) subcommand. +```{hint} You can quickly generate a default config by running {code}`kupferbootstrap config init -N`. +For an interactive dialogue, omit the `-N`. +``` + ## File Location The configuration is stored in `~/.config/kupfer/kupferbootstrap.toml`, where `~` is your user's home folder. @@ -54,7 +58,7 @@ This allows you to easily keep a number of slight variations of the same target without the need to constantly modify your Kupferbootstrap configuration file. You can easily create new profiles with -[kupferbootstrap config profile init](/cli/config/#kupferbootstrap-config-profile-init). +[kupferbootstrap config profile init](../../cli/config/#kupferbootstrap-config-profile-init). Here's an example: @@ -97,7 +101,7 @@ hostname = "pocof1" The `current` key in the `profiles` section controlls which profile gets used by Kupferbootstrap by default. The first subsection (`profiles.default`) describes the `default` profile -which gets created by [config init](/cli/config/#kupferbootstrap-config-init). +which gets created by [`kupferbootstrap config init`](../../cli/config/#kupferbootstrap-config-init). Next, we have a `graphical` profile that defines a couple of graphical programs for all but the `recovery` profile, since that doesn't have a GUI. diff --git a/docs/source/usage/faq.md b/docs/source/usage/faq.md index 441bef2..53b1818 100644 --- a/docs/source/usage/faq.md +++ b/docs/source/usage/faq.md @@ -19,7 +19,7 @@ See [Porting](../porting) ## How to build a specific package -See also: The full [`kupferbootstrap packages build` docs](/cli/packages#kupferbootstrap-packages-build) +See also: The full [`kupferbootstrap packages build` docs](../../cli/packages#kupferbootstrap-packages-build) ### Example diff --git a/docs/source/usage/quickstart.md b/docs/source/usage/quickstart.md index fc5a4d5..0076b58 100644 --- a/docs/source/usage/quickstart.md +++ b/docs/source/usage/quickstart.md @@ -2,8 +2,8 @@ 1. [Install](../install) Kupferbootstrap 1. [Configure](../config) it: `kuperbootstrap config init` -1. [Update your PKGBUILDs + SRCINFO cache](/cli/packages#kupferbootstrap-packages-update): `kupferbootstrap packages update` -1. [Build an image](/cli/image#kupferbootstrap-image-build): `kupferbootstrap image build` -1. [Flash the image](/cli/image#kupferbootstrap-image-flash): `kupferbootstrap image flash abootimg && kupferbootstrap image flash full userdata` +1. [Update your PKGBUILDs + SRCINFO cache](../../cli/packages#kupferbootstrap-packages-update): `kupferbootstrap packages update` +1. [Build an image](../../cli/image#kupferbootstrap-image-build): `kupferbootstrap image build` +1. [Flash the image](../../cli/image#kupferbootstrap-image-flash): `kupferbootstrap image flash abootimg && kupferbootstrap image flash full userdata` See also: [Frequently Asked Questions](../faq) From 4cce7e57aea2f4a0904943b9d57c6460ee210b44 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Wed, 20 Dec 2023 00:28:26 +0100 Subject: [PATCH 05/15] constants: use ALARM's aarch64 gcc that we package --- constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/constants.py b/constants.py index 723dcc0..2ddd686 100644 --- a/constants.py +++ b/constants.py @@ -89,7 +89,7 @@ COMPILE_ARCHES: dict[Arch, str] = { GCC_HOSTSPECS: dict[DistroArch, dict[TargetArch, str]] = { 'x86_64': { 'x86_64': 'x86_64-pc-linux-gnu', - 'aarch64': 'aarch64-linux-gnu', + 'aarch64': 'aarch64-unknown-linux-gnu', 'armv7h': 'arm-unknown-linux-gnueabihf' }, 'aarch64': { From a75f32b4b159eb65fd3b26158333e13530b272c8 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Wed, 20 Dec 2023 01:55:16 +0100 Subject: [PATCH 06/15] chroot/build: mount_crossdirect(): fix symlink creation if link exists --- chroot/build.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/chroot/build.py b/chroot/build.py index 2520d6a..40b123d 100644 --- a/chroot/build.py +++ b/chroot/build.py @@ -82,6 +82,7 @@ class BuildChroot(Chroot): native_chroot.mount_pacman_cache() native_chroot.mount_packages() native_chroot.activate() + logging.debug(f"Installing {CROSSDIRECT_PKGS=} + {gcc=}") results = dict(native_chroot.try_install_packages( CROSSDIRECT_PKGS + [gcc], refresh=True, @@ -103,8 +104,8 @@ class BuildChroot(Chroot): target_include_dir = os.path.join(self.path, 'include') for target, source in {cc_path: gcc, target_lib_dir: 'lib', target_include_dir: 'usr/include'}.items(): - if not os.path.exists(target): - logging.debug(f'Symlinking {source} at {target}') + if not (os.path.exists(target) or os.path.islink(target)): + logging.debug(f'Symlinking {source=} at {target=}') symlink(source, target) ld_so = os.path.basename(glob(f"{os.path.join(native_chroot.path, 'usr', 'lib', 'ld-linux-')}*")[0]) ld_so_target = os.path.join(target_lib_dir, ld_so) From c074fbe42c7c05c25e89df96224a66bcef645519 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Wed, 20 Dec 2023 03:33:28 +0100 Subject: [PATCH 07/15] packages/pkgbuild: parse_pkgbuild(): inherit depends, makedepends, provides, replaces from pkgbase unless overriden --- packages/pkgbuild.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/packages/pkgbuild.py b/packages/pkgbuild.py index 65722ba..baeea83 100644 --- a/packages/pkgbuild.py +++ b/packages/pkgbuild.py @@ -310,8 +310,11 @@ class SubPkgbuild(Pkgbuild): self.sources_refreshed = False self.update(pkgbase) - self.provides = {} - self.replaces = [] + # set to None - will be replaced with base_pkg if still None after parsing + self.depends = None # type: ignore[assignment] + self.makedepends = None # type: ignore[assignment] + self.provides = None # type: ignore[assignment] + self.replaces = None # type: ignore[assignment] def refresh_sources(self, lazy: bool = True): assert self.pkgbase @@ -383,13 +386,21 @@ def parse_pkgbuild( elif line.startswith('arch'): current.arches.append(splits[1]) elif line.startswith('provides'): + if not current.provides: + current.provides = {} current.provides = get_version_specs(splits[1], current.provides) elif line.startswith('replaces'): + if not current.replaces: + current.replaces = [] current.replaces.append(splits[1]) elif splits[0] in ['depends', 'makedepends', 'checkdepends', 'optdepends']: spec = splits[1].split(': ', 1)[0] + if not current.depends: + current.depends = {} current.depends = get_version_specs(spec, current.depends) if splits[0] == 'makedepends': + if not current.makedepends: + current.makedepends = {} current.makedepends = get_version_specs(spec, current.makedepends) results: list[Pkgbuild] = list(base_package.subpackages) @@ -402,6 +413,15 @@ def parse_pkgbuild( pkg.update_version() if not (pkg.version == base_package.version): raise Exception(f'Subpackage malformed! Versions differ! base: {base_package}, subpackage: {pkg}') + if isinstance(pkg, SubPkgbuild): + if pkg.depends is None: + pkg.depends = base_package.depends + if pkg.makedepends is None: + pkg.makedepends = base_package.makedepends + if pkg.replaces is None: + pkg.replaces = base_package.replaces + if pkg.provides is None: + pkg.provides = base_package.provides return results From eaac9195ea584964785055b4b8e66fe46b5a596b Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Wed, 20 Dec 2023 03:36:13 +0100 Subject: [PATCH 08/15] packages/build: build_enable_qemu_binfmt(): also build gcc package if available --- packages/build.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/packages/build.py b/packages/build.py index 190e588..455126b 100644 --- a/packages/build.py +++ b/packages/build.py @@ -438,10 +438,11 @@ def setup_build_chroot( extra_packages: list[str] = [], add_kupfer_repos: bool = True, clean_chroot: bool = False, + repo: Optional[dict[str, Pkgbuild]] = None, ) -> BuildChroot: assert config.runtime.arch if arch != config.runtime.arch: - build_enable_qemu_binfmt(arch, lazy=False) + build_enable_qemu_binfmt(arch, repo=repo or discover_pkgbuilds(), lazy=False) init_prebuilts(arch) chroot = get_build_chroot(arch, add_kupfer_repos=add_kupfer_repos) chroot.mount_packages() @@ -510,6 +511,7 @@ def build_package( enable_ccache: bool = True, clean_chroot: bool = False, build_user: str = 'kupfer', + repo: Optional[dict[str, Pkgbuild]] = None, ): makepkg_compile_opts = ['--holdver'] makepkg_conf_path = 'etc/makepkg.conf' @@ -529,6 +531,7 @@ def build_package( arch=arch, extra_packages=deps, clean_chroot=clean_chroot, + repo=repo, ) assert config.runtime.arch native_chroot = target_chroot @@ -538,6 +541,7 @@ def build_package( arch=config.runtime.arch, extra_packages=['base-devel'] + CROSSDIRECT_PKGS, clean_chroot=clean_chroot, + repo=repo, ) if not package.mode: logging.warning(f'Package {package.path} has no _mode set, assuming "host"') @@ -756,6 +760,7 @@ def build_packages( enable_crossdirect=enable_crossdirect, enable_ccache=enable_ccache, clean_chroot=clean_chroot, + repo=repo, ) files += add_package_to_repo(package, arch) updated_repos.add(package.repo) @@ -830,8 +835,20 @@ def build_enable_qemu_binfmt(arch: Arch, repo: Optional[dict[str, Pkgbuild]] = N logging.info('Installing qemu-user (building if necessary)') check_programs_wrap(['pacman', 'makepkg', 'pacstrap']) # build qemu-user, binfmt, crossdirect + packages = list(CROSSDIRECT_PKGS) + hostspec = GCC_HOSTSPECS[arch][arch] + cross_gcc = f"{hostspec}-gcc" + if repo: + for pkg in repo.values(): + if (pkg.name == cross_gcc or cross_gcc in pkg.provides): + if config.runtime.arch not in pkg.arches: + logging.debug(f"Package {pkg.path} matches {cross_gcc=} name but not arch: {pkg.arches=}") + continue + packages.append(pkg.path) + logging.debug(f"Adding gcc package {pkg.path} to the necessary crosscompilation tools") + break build_packages_by_paths( - CROSSDIRECT_PKGS, + packages, native, repo=repo, try_download=True, From 4b2150940d0fcedcc1a86d63e4c8ece7a54af519 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Fri, 22 Dec 2023 05:07:55 +0100 Subject: [PATCH 09/15] packages/build: use copy && remove_file() instead of shutil.move() --- packages/build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/build.py b/packages/build.py index 455126b..bcd5403 100644 --- a/packages/build.py +++ b/packages/build.py @@ -310,7 +310,8 @@ def try_download_package(dest_file_path: str, package: Pkgbuild, arch: Arch) -> logging.debug(f"While checking the HTTPS repo DB, we found a matching filename in the pacman cache{extra_msg}") if cache_matches: logging.info(f'copying cache file {cache_file} to repo as verified by remote checksum') - shutil.move(cache_file, dest_file_path) + shutil.copy(cache_file, dest_file_path) + remove_file(cache_file) return dest_file_path url = repo_pkg.resolved_url assert url From b006cd8f4da8c621370b2bf73762b248ddd74dbc Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Mon, 8 Jan 2024 02:30:57 +0100 Subject: [PATCH 10/15] packages/pkgbuild: support new key "_crossdirect" to enable/disable crossdirect for single packages --- packages/build.py | 2 +- packages/cli.py | 4 +++- packages/pkgbuild.py | 7 +++++++ packages/srcinfo_cache.py | 20 +++++++++++++++----- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/packages/build.py b/packages/build.py index bcd5403..b388221 100644 --- a/packages/build.py +++ b/packages/build.py @@ -575,7 +575,7 @@ def build_package( build_root = target_chroot makepkg_compile_opts += ['--nodeps' if package.nodeps else '--syncdeps'] env = deepcopy(get_makepkg_env(arch)) - if foreign_arch and enable_crossdirect and package.name not in CROSSDIRECT_PKGS: + if foreign_arch and package.crossdirect and enable_crossdirect and package.name not in CROSSDIRECT_PKGS: env['PATH'] = f"/native/usr/lib/crossdirect/{arch}:{env['PATH']}" target_chroot.mount_crossdirect(native_chroot) else: diff --git a/packages/cli.py b/packages/cli.py index c300111..3878ba4 100644 --- a/packages/cli.py +++ b/packages/cli.py @@ -313,7 +313,7 @@ def cmd_list(): logging.info(f'Done! {len(packages)} Pkgbuilds:') for name in sorted(packages.keys()): p = packages[name] - print(f'name: {p.name}; ver: {p.version}; mode: {p.mode}; provides: {p.provides}; replaces: {p.replaces};' + print(f'name: {p.name}; ver: {p.version}; mode: {p.mode}; crossdirect: {p.crossdirect} provides: {p.provides}; replaces: {p.replaces};' f'local_depends: {p.local_depends}; depends: {p.depends}') @@ -346,6 +346,7 @@ def cmd_check(paths): mode_key = '_mode' nodeps_key = '_nodeps' + crossdirect_key = '_crossdirect' pkgbase_key = 'pkgbase' pkgname_key = 'pkgname' arches_key = '_arches' @@ -356,6 +357,7 @@ def cmd_check(paths): required = { mode_key: True, nodeps_key: False, + crossdirect_key: False, pkgbase_key: False, pkgname_key: True, 'pkgdesc': False, diff --git a/packages/pkgbuild.py b/packages/pkgbuild.py index baeea83..3b89558 100644 --- a/packages/pkgbuild.py +++ b/packages/pkgbuild.py @@ -156,6 +156,7 @@ class Pkgbuild(PackageInfo): repo: str mode: str nodeps: bool + crossdirect: bool path: str pkgver: str pkgrel: str @@ -190,6 +191,7 @@ class Pkgbuild(PackageInfo): self.repo = repo or '' self.mode = '' self.nodeps = False + self.crossdirect = True self.path = relative_path self.pkgver = '' self.pkgrel = '' @@ -223,6 +225,7 @@ class Pkgbuild(PackageInfo): self.repo = pkg.repo self.mode = pkg.mode self.nodeps = pkg.nodeps + self.crossdirect = pkg.crossdirect self.path = pkg.path self.pkgver = pkg.pkgver self.pkgrel = pkg.pkgrel @@ -357,7 +360,11 @@ def parse_pkgbuild( else: raise Exception(msg) + # if _crossdirect is unset (None), it defaults to True + crossdirect_enabled = srcinfo_cache.build_crossdirect in (None, True) + base_package = Pkgbase(relative_pkg_dir, sources_refreshed=sources_refreshed, srcinfo_cache=srcinfo_cache) + base_package.crossdirect = crossdirect_enabled base_package.mode = mode base_package.nodeps = nodeps base_package.repo = relative_pkg_dir.split('/')[0] diff --git a/packages/srcinfo_cache.py b/packages/srcinfo_cache.py index 5cb2373..3d9737b 100644 --- a/packages/srcinfo_cache.py +++ b/packages/srcinfo_cache.py @@ -68,11 +68,19 @@ class SrcInitialisedFile(JsonFile): raise ex +srcinfo_meta_defaults = { + 'build_mode': None, + "build_nodeps": None, + "build_crossdirect": None, +} + + class SrcinfoMetaFile(JsonFile): checksums: dict[str, str] build_mode: Optional[str] build_nodeps: Optional[bool] + build_crossdirect: Optional[bool] _changed: bool _filename: ClassVar[str] = SRCINFO_METADATA_FILE @@ -92,9 +100,8 @@ class SrcinfoMetaFile(JsonFile): s = SrcinfoMetaFile({ '_relative_path': relative_pkg_dir, '_changed': True, - 'build_mode': '', - 'build_nodeps': None, 'checksums': {}, + **srcinfo_meta_defaults, }) return s, s.refresh_all() @@ -120,9 +127,11 @@ class SrcinfoMetaFile(JsonFile): if not force_refresh: logging.debug(f'{metadata._relative_path}: srcinfo checksums match!') lines = lines or metadata.read_srcinfo_file() - for build_field in ['build_mode', 'build_nodeps']: + for build_field in srcinfo_meta_defaults.keys(): if build_field not in metadata: metadata.refresh_build_fields() + if write: + metadata.write() break else: lines = metadata.refresh_all(write=write) @@ -143,8 +152,7 @@ class SrcinfoMetaFile(JsonFile): self._changed = True def refresh_build_fields(self): - self['build_mode'] = None - self['build_nodeps'] = None + self.update(srcinfo_meta_defaults) with open(os.path.join(config.get_path('pkgbuilds'), self._relative_path, 'PKGBUILD'), 'r') as file: lines = file.read().split('\n') for line in lines: @@ -156,6 +164,8 @@ class SrcinfoMetaFile(JsonFile): self.build_mode = val elif key == '_nodeps': self.build_nodeps = val.lower() == 'true' + elif key == '_crossdirect': + self.build_crossdirect = val.lower() == 'true' else: continue From f05de7738ae67e2a2b393b4437dbe43384159825 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Mon, 8 Jan 2024 04:25:42 +0100 Subject: [PATCH 11/15] integration_tests: test importing main.cli --- integration_tests.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/integration_tests.py b/integration_tests.py index 2b6c526..bc4eeb7 100644 --- a/integration_tests.py +++ b/integration_tests.py @@ -37,6 +37,11 @@ def ctx() -> click.Context: return click.Context(click.Command('integration_tests')) +def test_main_import(): + from main import cli + assert cli + + def test_config_load(ctx: click.Context): path = config.runtime.config_file assert path From cebac831864a2eaeaa86e3586f778e9aafb43d6f Mon Sep 17 00:00:00 2001 From: Syboxez Blank Date: Sat, 23 Mar 2024 17:48:38 +0000 Subject: [PATCH 12/15] packages/pkgbuild: parse_pkgbuild(): Reuse pkgbase's `makedepends` as dependencies Authored-by: InsanePrawn --- packages/pkgbuild.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pkgbuild.py b/packages/pkgbuild.py index 3b89558..9be89c3 100644 --- a/packages/pkgbuild.py +++ b/packages/pkgbuild.py @@ -403,7 +403,7 @@ def parse_pkgbuild( elif splits[0] in ['depends', 'makedepends', 'checkdepends', 'optdepends']: spec = splits[1].split(': ', 1)[0] if not current.depends: - current.depends = {} + current.depends = (base_package.makedepends or {}).copy() current.depends = get_version_specs(spec, current.depends) if splits[0] == 'makedepends': if not current.makedepends: From a4cfc3c3e51dc21e2e19729154afa7869e0a1698 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Wed, 20 Mar 2024 16:42:19 +0100 Subject: [PATCH 13/15] exec/file: makedir(): add mode=None arg --- exec/file.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/exec/file.py b/exec/file.py index 852ad48..00653aa 100644 --- a/exec/file.py +++ b/exec/file.py @@ -144,7 +144,13 @@ def remove_file(path: str, recursive=False): raise Exception(f"Unable to remove {path}: cmd returned {rc}") -def makedir(path, user: Optional[Union[str, int]] = None, group: Optional[Union[str, int]] = None, parents: bool = True): +def makedir( + path, + user: Optional[Union[str, int]] = None, + group: Optional[Union[str, int]] = None, + parents: bool = True, + mode: Optional[Union[int, str]] = None, +): if not root_check_exists(path): try: if parents: @@ -153,6 +159,8 @@ def makedir(path, user: Optional[Union[str, int]] = None, group: Optional[Union[ os.mkdir(path) except: run_root_cmd(['mkdir'] + (['-p'] if parents else []) + [path]) + if mode is not None: + chmod(path, mode=mode) chown(path, user, group) From a176fad05a358bcbb83d4e2207f8ae2a1c7abbee Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Wed, 20 Mar 2024 16:43:08 +0100 Subject: [PATCH 14/15] net/ssh: copy_ssh_keys(): pass chroot for uid resolution --- image/image.py | 2 +- net/ssh.py | 52 ++++++++++++++++++++++++++++++-------------------- 2 files changed, 32 insertions(+), 22 deletions(-) diff --git a/image/image.py b/image/image.py index 0cb0bcc..afb3ddb 100644 --- a/image/image.py +++ b/image/image.py @@ -333,7 +333,7 @@ def install_rootfs( ) chroot.add_sudo_config(config_name='wheel', privilegee='%wheel', password_required=True) copy_ssh_keys( - chroot.path, + chroot, user=user, ) files = { diff --git a/net/ssh.py b/net/ssh.py index 2a5ef7f..cf1ed37 100644 --- a/net/ssh.py +++ b/net/ssh.py @@ -6,7 +6,9 @@ import click from config.state import config from constants import SSH_COMMON_OPTIONS, SSH_DEFAULT_HOST, SSH_DEFAULT_PORT +from chroot.abstract import Chroot from exec.cmd import run_cmd +from exec.file import write_file from wrapper import check_programs_wrap @@ -83,21 +85,16 @@ def find_ssh_keys(): return keys -def copy_ssh_keys(root_dir: str, user: str): +def copy_ssh_keys(chroot: Chroot, user: str): check_programs_wrap(['ssh-keygen']) - authorized_keys_file = os.path.join( - root_dir, - 'home', - user, - '.ssh', - 'authorized_keys', - ) - if os.path.exists(authorized_keys_file): - os.unlink(authorized_keys_file) + ssh_dir_relative = os.path.join('/home', user, '.ssh') + ssh_dir = chroot.get_path(ssh_dir_relative) + authorized_keys_file_rel = os.path.join(ssh_dir_relative, 'authorized_keys') + authorized_keys_file = chroot.get_path(authorized_keys_file_rel) keys = find_ssh_keys() if len(keys) == 0: - logging.info("Could not find any ssh key to copy") + logging.warning("Could not find any ssh key to copy") create = click.confirm("Do you want me to generate an ssh key for you?", True) if not create: return @@ -116,15 +113,28 @@ def copy_ssh_keys(root_dir: str, user: str): logging.fatal("Failed to generate ssh key") keys = find_ssh_keys() - ssh_dir = os.path.join(root_dir, 'home', user, '.ssh') - if not os.path.exists(ssh_dir): - os.makedirs(ssh_dir, exist_ok=True, mode=0o700) + if not keys: + logging.warning("No SSH keys to be copied. Skipping.") + return - with open(authorized_keys_file, 'a') as authorized_keys: - for key in keys: - pub = f'{key}.pub' - if not os.path.exists(pub): - logging.debug(f'Skipping key {key}: {pub} not found') - continue + auth_key_lines = [] + for key in keys: + pub = f'{key}.pub' + if not os.path.exists(pub): + logging.debug(f'Skipping key {key}: {pub} not found') + continue + try: with open(pub, 'r') as file: - authorized_keys.write(file.read()) + contents = file.read() + if not contents.strip(): + continue + auth_key_lines.append(contents) + except Exception as ex: + logging.warning(f"Could not read ssh pub key {pub}", exc_info=ex) + continue + + if not os.path.exists(ssh_dir): + logging.info(f"Creating {ssh_dir_relative} dir in chroot {chroot.path}") + chroot.run_cmd(["mkdir", "-p", "-m", "700", ssh_dir_relative], switch_user=user) + logging.info(f"Writing SSH pub keys to {authorized_keys_file}") + write_file(authorized_keys_file, "\n".join(auth_key_lines), user=chroot.get_uid(user), mode="644") From a28550825f89673635e36456da06c5a23c43ece8 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Wed, 20 Mar 2024 20:56:17 +0100 Subject: [PATCH 15/15] image/image: tolerate pub-key copying to fail during image build --- image/image.py | 1 + net/ssh.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/image/image.py b/image/image.py index afb3ddb..6532af7 100644 --- a/image/image.py +++ b/image/image.py @@ -335,6 +335,7 @@ def install_rootfs( copy_ssh_keys( chroot, user=user, + allow_fail=True, ) files = { 'etc/pacman.conf': get_base_distro(arch).get_pacman_conf( diff --git a/net/ssh.py b/net/ssh.py index cf1ed37..6eb7294 100644 --- a/net/ssh.py +++ b/net/ssh.py @@ -85,7 +85,7 @@ def find_ssh_keys(): return keys -def copy_ssh_keys(chroot: Chroot, user: str): +def copy_ssh_keys(chroot: Chroot, user: str, allow_fail: bool = False): check_programs_wrap(['ssh-keygen']) ssh_dir_relative = os.path.join('/home', user, '.ssh') ssh_dir = chroot.get_path(ssh_dir_relative) @@ -134,7 +134,13 @@ def copy_ssh_keys(chroot: Chroot, user: str): continue if not os.path.exists(ssh_dir): - logging.info(f"Creating {ssh_dir_relative} dir in chroot {chroot.path}") + logging.info(f"Creating {ssh_dir_relative!r} dir in chroot {chroot.path!r}") chroot.run_cmd(["mkdir", "-p", "-m", "700", ssh_dir_relative], switch_user=user) logging.info(f"Writing SSH pub keys to {authorized_keys_file}") - write_file(authorized_keys_file, "\n".join(auth_key_lines), user=chroot.get_uid(user), mode="644") + try: + write_file(authorized_keys_file, "\n".join(auth_key_lines), user=str(chroot.get_uid(user)), mode="644") + except Exception as ex: + logging.error(f"Failed to write SSH authorized_keys_file at {authorized_keys_file!r}:", exc_info=ex) + if allow_fail: + return + raise ex from ex