kupferbootstrap: add typed marker and convert non-relative imports

Authored-by: InsanePrawn <insane.prawny@gmail.com>

.dockerignore

@@ -0,0 +1,5 @@
+/venv
+/build
+__pycache__
+.mypy_cache
+*.xml

.gitignore

@@ -1,5 +1,7 @@
 *.kate-swp
-venv/
+/venv
+/build
 __pycache__/
 .coverage*
 *.xml
+*.egg-info

.gitlab-ci.yml

@@ -7,15 +7,19 @@ format:
   stage: check
   image: python
   before_script:
-    - pip install yapf autoflake --break-system-packages
+    - python3 -m venv venv
+    - venv/bin/pip3 install yapf autoflake
   script:
+    - source venv/bin/activate
     - ./format.sh --check
 
 typecheck:
   stage: check
   image: python
   before_script:
-    - pip install mypy --break-system-packages
+    - python3 -m venv venv
+    - source venv/bin/activate
+    - pip install mypy
   script:
     - ./typecheck.sh --non-interactive --junit-xml mypy-report.xml
   artifacts:
@@ -27,12 +31,13 @@ pytest:
   image: archlinux
   before_script:
     - pacman -Sy --noconfirm --needed archlinux-keyring && pacman -Su --noconfirm python python-pip sudo git base-devel arch-install-scripts rsync
-    - pip install -r test_requirements.txt -r requirements.txt --break-system-packages
+    - python3 -m venv venv
+    - venv/bin/pip3 install -r test_requirements.txt -r requirements.txt
     - 'echo "kupfer ALL = (ALL) NOPASSWD: ALL" > /etc/sudoers.d/kupfer_all'
     - useradd -m kupfer
     - chmod 777 .
   script:
-    - script -e -c 'su kupfer -s /bin/bash -c "INTEGRATION_TESTS_USE_GLOBAL_CONFIG=TRUE KUPFERBOOTSTRAP_WRAPPED=DOCKER ./pytest.sh --junit-xml=pytest-report.xml --cov-report=xml:coverage.xml integration_tests.py"'
+    - script -e -c 'su kupfer -s /bin/bash -c ". venv/bin/activate && INTEGRATION_TESTS_USE_GLOBAL_CONFIG=TRUE KUPFERBOOTSTRAP_WRAPPED=DOCKER ./pytest.sh --junit-xml=pytest-report.xml --cov-report=xml:coverage.xml integration_tests.py"'
   coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
   artifacts:
     reports:
@@ -51,6 +56,7 @@ build_docker:
     DOCKER_TLS_CERTDIR: ""
   script:
     - 'docker build --pull -t "${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA}" -t "${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_SLUG}" .'
+    - 'echo "running sanity check" && docker run -it --rm "${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA}" kupferbootstrap --help'
   only:
     - branches
   except:
@@ -77,7 +83,9 @@ push_docker:
     DOCS_MAKE_TARGET: "html"
     DOCS_MAKE_THREADS: 6
   before_script: &docs_before_script
-    - pip install -r requirements.txt -r docs/requirements.txt --break-system-packages
+    - python3 -m venv venv
+    - source venv/bin/activate
+    - pip install -r requirements.txt -r docs/requirements.txt
   script: &docs_script
     - make -C docs -j$DOCS_MAKE_THREADS SPHINXARGS="$DOCS_SPHINXARGS" $DOCS_MAKE_TARGET
     - mv "docs/$DOCS_MAKE_TARGET" public

Dockerfile

@@ -17,16 +17,14 @@ RUN yes | pacman -Scc
 RUN sed -i "s/SigLevel.*/SigLevel = Never/g" /etc/pacman.conf
 
 ENV KUPFERBOOTSTRAP_WRAPPED=DOCKER
-ENV PATH=/app/bin:/app/local/bin:$PATH
+ENV PATH=/app/bin:/app/local/bin:/app/venv/bin:$PATH
 WORKDIR /app
 
-COPY requirements.txt .
-# TODO: pip packaging so we don't need --break-system-packages
-RUN pip install -r requirements.txt --break-system-packages
-
 COPY . .
+RUN python3 -m venv /app/venv
+RUN /app/venv/bin/pip3 install -r requirements.txt
 
-RUN python -c "from distro import distro; distro.get_kupfer_local(arch=None,in_chroot=False).repos_config_snippet()" | tee -a /etc/pacman.conf
+RUN /app/venv/bin/python3 -c "from kupferbootstrap.distro import distro; distro.get_kupfer_local(arch=None,in_chroot=False).repos_config_snippet()" | tee -a /etc/pacman.conf
 RUN useradd -m -g users kupfer
 RUN echo "kupfer ALL=(ALL) NOPASSWD: ALL" | tee /etc/sudoers.d/kupfer
 

README.md

@@ -18,14 +18,26 @@ This will run a webserver on localhost:9999. Access it like `firefox http://loca
 
 
 ## Installation
-Install Docker, Python 3 with the libraries from `requirements.txt` and put `bin/` into your `PATH`.
-Then use `kupferbootstrap`.
+0. If you're not on ArchLinux (i.e. don't have `pacman`, `makepkg`, etc. available in your $PATH), install Docker and add yourself to the docker group.
+1. Craate a python venv: `python3 -m venv venv`
+1. Activate it: `source venv/bin/activate`
+1. Install KBS: `pip3 install .`
+
+Then run `kupferbootstrap`.
+
+### Pro Tip:
+
+- You can add a shell alias for `$(PWD)/venv/bin/kupferbootstrap` or create a symlink to it at `/usr/local/bin/kuperbootstrap` for quick access without needing to manually source the venv script every time.
+- It is recommended to abbreviate `kupferbootstrap` to `kbs` for even less typing.
 
 
 ## Quickstart
 1. Initialize config with defaults, configure your device and flavour: `kupferbootstrap config init`
+1. Initialize PKGBUILDs and caches: `kupferbootstrap packages init`
 1. Build an image and packages along the way: `kupferbootstrap image build`
 
 
 ## Development
-Put `dev` into `version.txt` to always rebuild kupferboostrap from this directory and use `kupferbootstrap` as normal.
+
+### Docker
+Put `BUILD` (the default) into `docker_version.txt` to always rebuild kupferboostrap from this directory; otherwise the image is pulled from `registry.gitlab.com/kupfer/kupferbootstrap:$VERSION`, where `$VERSION` is the contents of `docker_version.txt`.

bin/kupferbootstrap

@@ -1,4 +0,0 @@
-#!/bin/bash
-
-# shellcheck disable=SC2068
-python3 "$(dirname "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")/main.py" $@

docker_version.txt

@@ -0,0 +1 @@
+BUILD

docs/.gitignore

@@ -2,6 +2,7 @@
 .doctrees
 html
 source/cli
+source/code
 checkouts
 versions
 archived

docs/Makefile

@@ -14,7 +14,7 @@ cleanbuild:
 	@$(MAKE) html
 
 clean:
-	rm -rf html source/cli .buildinfo .doctrees versions checkouts
+	rm -rf html source/cli source/code .buildinfo .doctrees versions checkouts
 
 html:
 	sphinx-build $(SPHINXARGS) $(buildargs) html

docs/source/cli.md

@@ -1,7 +1,7 @@
 # CLI Interface
 
 ```{eval-rst}
-.. click:: main:cli
+.. click:: kupferbootstrap.main:cli
    :nested: none
    :prog: kupferbootstrap
 

docs/source/cmd.md

@@ -6,6 +6,7 @@ orphan: true
 only used to trigger builds of the submodule docs!
 
 ```{eval-rst}
+.. currentmodule:: kupferbootstrap
 .. autosummary::
    :toctree: cli
    :template: command.rst

docs/source/code.md

@@ -0,0 +1,9 @@
+# Code
+
+Code documentation is available here
+
+```{toctree}
+:glob: true
+
+code/kupferbootstrap
+```

docs/source/codegen.rst

@@ -0,0 +1,8 @@
+:nosearch:
+:orphan:
+
+.. autosummary::
+   :toctree: code
+   :recursive:
+
+  kupferbootstrap

docs/source/conf.py

@@ -1,10 +1,14 @@
+import logging
 import os
-import sys
+from sphinx.config import getenv
+from kupferbootstrap.utils import git
 
-sys.path.insert(0, os.path.abspath('../..'))
+#sys.path.insert(0, os.path.abspath('../..'))
 extensions = [
     'sphinx_click',
-    'sphinx.ext.autosummary',  # Create neat summary tables
+    "sphinx.ext.autodoc",
+    'sphinx.ext.autosummary',
+    "sphinx.ext.linkcode",
     'myst_parser'
 ]
 myst_all_links_external = True
@@ -29,4 +33,45 @@ html_theme_options = {
         "color-brand-content": "#eba38d",
         "color-problematic": "#ff7564",
     },
+    "source_repository": "https://gitlab.com/kupfer/kupferbootstrap",
+    "source_directory": "docs/source/",
 }
+
+
+autosummary_generate = True
+autodoc_default_options = {
+    "members": True,
+    "undoc-members": True,
+    "show-inheritance": True,
+    "inherited-members": True,
+}
+autodoc_preserve_defaults = True
+
+
+def get_version():
+    try:
+        res = git(
+            ["rev-parse", "HEAD"],
+            dir=os.path.join(os.path.dirname(__file__), "../.."),
+            use_git_dir=True,
+            capture_output=True,
+        )
+        res.check_returncode()
+        ver = res.stdout.decode().strip()
+        logging.info(f"Detected git {ver=}")
+        return ver
+    except Exception as ex:
+        logging.warning("Couldn't get git branch:", exc_info=ex)
+        return "HEAD"
+
+
+version = getenv("version") or get_version()
+
+
+def linkcode_resolve(domain, info):
+    if domain != 'py':
+        return None
+    if not info['module']:
+        return None
+    filename = info['module'].replace('.', '/')
+    return "%s/-/blob/%s/src/%s.py" % (html_theme_options["source_repository"], version, filename)

docs/source/genindex.rst

@@ -0,0 +1,2 @@
+Module Index
+============

docs/source/index.md

@@ -6,7 +6,8 @@ a tool to build and flash packages and images for the [Kupfer](https://gitlab.co
 ## Documentation pages
 
 ```{toctree}
-install
-config
+usage/index
 cli
+code
+genindex
 ```

docs/source/templates/code.rst

@@ -0,0 +1,36 @@
+{% set reduced_name = fullname.split(".", 1)[-1] if fullname.startswith("kupferbootstrap.") else fullname %}
+
+{{ fullname | escape | underline }}
+
+.. rubric:: Description
+
+.. automodule:: {{ fullname }}
+   :members:
+   :undoc-members:
+
+.. currentmodule:: {{ fullname }}
+
+
+
+
+{% if classes %}
+.. rubric:: Classes
+
+.. autosummary::
+    :toctree: .
+    {% for class in classes %}
+    {{ class }}
+    {% endfor %}
+
+{% endif %}
+
+{% if functions %}
+.. rubric:: Functions
+
+.. autosummary::
+    :toctree: .
+    {% for function in functions %}
+    {{ function }}
+    {% endfor %}
+
+{% endif %}

docs/source/templates/command.rst

@@ -1,6 +1,9 @@
-.. title: {{fullname}}
+{% set reduced_name = fullname.split(".", 1)[-1] if fullname.startswith("kupferbootstrap.") else fullname %}
+.. title: {{reduced_name}}
 
-.. click:: {% if fullname == 'main' %}main:cli{% else %}{{fullname}}.cli:cmd_{{fullname}}{% endif %}
-  :prog: kupferbootstrap {{fullname}}
+
+.. currentmodule:: {{ fullname }}
+.. click:: {% if fullname == 'main' %}kupferbootstrap.main:cli{% else %}{{fullname}}.cli:cmd_{{reduced_name}}{% endif %}
+  :prog: kupferbootstrap {{reduced_name}}
   :nested: full
 

docs/source/usage/config.md

@@ -2,10 +2,14 @@
 
 Kupferbootstrap uses [toml](https://en.wikipedia.org/wiki/TOML) for its configuration file.
 
-The file can either be edited manually or managed via the {doc}`cli/config` subcommand.
+The file can either be edited manually or managed via the [`kupferbootstrap config`](../../cli/config) subcommand.
 
+```{hint}
 You can quickly generate a default config by running {code}`kupferbootstrap config init -N`.
 
+For an interactive dialogue, omit the `-N`.
+```
+
 ## File Location
 
 The configuration is stored in `~/.config/kupfer/kupferbootstrap.toml`, where `~` is your user's home folder.
@@ -54,7 +58,7 @@ This allows you to easily keep a number of slight variations of the same target
 without the need to constantly modify your Kupferbootstrap configuration file.
 
 You can easily create new profiles with
-[kupferbootstrap config profile init](../cli/config/#kupferbootstrap-config-profile-init).
+[kupferbootstrap config profile init](../../cli/config/#kupferbootstrap-config-profile-init).
 
 Here's an example:
 
@@ -64,7 +68,7 @@ current = "graphical"
 
 [profiles.default]
 parent = ""
-device = "oneplus-enchilada"
+device = "sdm845-oneplus-enchilada"
 flavour = "barebone"
 pkgs_include = [ "wget", "rsync", "nano", "tmux", "zsh", "pv", ]
 pkgs_exclude = []
@@ -89,7 +93,7 @@ flavour = "debug-shell"
 
 [profiles.beryllium]
 parent = "graphical"
-device = "xiaomi-beryllium-ebbg"
+device = "sdm845-xiaomi-beryllium-ebbg"
 flavour = "gnome"
 hostname = "pocof1"
 ```
@@ -97,7 +101,7 @@ hostname = "pocof1"
 The `current` key in the `profiles` section controlls which profile gets used by Kupferbootstrap by default.
 
 The first subsection (`profiles.default`) describes the `default` profile
-which gets created by [config init](../cli/config/#kupferbootstrap-config-init).
+which gets created by [`kupferbootstrap config init`](../../cli/config/#kupferbootstrap-config-init).
 
 Next, we have a `graphical` profile that defines a couple of graphical programs for all but the `recovery` profile,
 since that doesn't have a GUI.

docs/source/usage/faq.md

@@ -0,0 +1,39 @@
+# FAQ
+
+
+```{contents} Table of Contents
+:class: this-will-duplicate-information-and-it-is-still-useful-here
+:depth: 3
+```
+
+
+## Which devices are currently supported?
+
+Currently very few!  
+See [the `devices` repo](https://gitlab.com/kupfer/packages/pkgbuilds/-/tree/dev/device). We use the same codenames as [postmarketOS](https://wiki.postmarketos.org/wiki/Devices) (although we prefix them with the SoC)
+
+
+## How to port a new device or package?
+
+See [Porting](../porting)
+
+## How to build a specific package
+
+See also: The full [`kupferbootstrap packages build` docs](../../cli/packages#kupferbootstrap-packages-build)
+
+### Example
+
+For rebuilding `kupfer-config` and `crossdirect`, defaulting to your device's architecture
+
+```sh
+kupferbootstrap packages build [--force] [--arch $target_arch] kupfer-config crossdirect
+```
+
+
+### By package path
+You can also use the a path snippet (`$repo/$pkgbase`) to the PKGBUILD folder as seen inside your pkgbuilds.git:
+
+```sh
+kupferbootstrap packages build [--force] main/kupfer-config cross/crossdirect
+```
+

docs/source/usage/index.md

@@ -0,0 +1,9 @@
+# Usage
+
+```{toctree}
+quickstart
+faq
+install
+config
+porting
+```

docs/source/usage/porting.md

@@ -0,0 +1,94 @@
+# Porting
+## Porting devices
+
+### Homework
+Before you can get started porting a device, you'll need to do some research:
+
+1. Familiarize yourself with git basics.
+1. Familiarize yourself with Arch Linux packaging, i.e. `PKGBUILD`s and `makepkg`
+1. Familiarize yourself with the postmarketOS port of the device.
+   ```{warning}
+    If there is no postmarketOS port yet, you'll probably need to get deep into kernel development.  
+    We suggest [starting with a port to pmOS](https://wiki.postmarketos.org/wiki/Porting_to_a_new_device) then, especially if you're not familiar with the process already.
+    ```
+
+### Porting
+1. Navigate to your pkgbuilds checkout
+1. Follow the [general package porting guidelines](#porting-packages) to create a device-, kernel- and probably also a firmware-package for the device and SoC. Usually this roughly means porting the postmarketOS APKBUILDs to our PKGBUILD scheme.  
+   You can get inspiration by comparing existing Kupfer ports (e.g. one of the SDM845 devices) to the [postmarketOS packages](https://gitlab.com/postmarketOS/pmaports/-/tree/master/device) for that device.
+   Usually you should start out by copying and then customizing the Kupfer packages for a device that's as similar to yours as possible, i.e. uses the same or a related SoC, if something like that is already available in Kupfer.
+   ```{hint} Package Repos:
+   Device packages belong into `device/`, kernels into `linux/` and firmware into `firmware/`.
+   ```
+1. When submitting your MR, please include some information:
+    - what you have found to be working, broken, and not tested (and why)
+    - any necessary instructions for testing
+    - whether you'd be willing to maintain the device long-term (test kernel upgrades, submit device package updates, etc.)
+
+
+### Gotchas
+
+Please be aware of these gotchas:
+- As of now, Kupfer only really supports platforms using Android's `aboot` bootloader, i.e. ex-Android phones. In order to support other boot modes (e.g. uboot on the Librem5 and Pine devices), we'll need to port and switch to postmarketOS's [boot-deploy](https://gitlab.com/postmarketOS/boot-deploy) first and add support for EFI setups to Kupferbootstrap.
+
+
+## Porting packages
+
+### Homework
+Before you can get started, you'll need to do some research:
+
+1. Familiarize yourself with git basics.
+1. Familiarize yourself with Arch Linux packaging, i.e. `PKGBUILD`s and `makepkg`
+
+### Development
+
+```{warning}
+Throughout the process, use git to version your changes.  
+- Don't procrastinate using git or committing until you're "done" or "have got something working", you'll regret it.  
+- Don't worry about a "clean" git history while you're developing; we can squash it up later.
+- \[Force-]Push your changes regularly, just like committing. Don't wait for perfection.
+```
+1. Create a new git branch for your package locally.
+   ```{hint}
+   It might be a good ideaa to get into the habit of prefixing branch names with \[a part of] your username and a slash like so:  
+   `myNickname/myFeatureNme`  
+   This makes it easier to work in the same remote repo with multiple people.
+   ```
+1. 
+   ```{note}
+   The pkgbuilds git repo contains multiple package repositories, represented by folders at the top level (`main`, `cross`, `phosh`, etc.).  
+   ```
+   Try to choose a sensible package repo for your new packages and create new folders for each `pkgbase` inside the repo folder.
+1. Navigate into the folder of the new package and create a new `PKGBUILD`; fill it with life!
+1. **`_mode`**: Add the build mode at the top of the PKGBUILD.
+   ```{hint}
+   If you're unsure what to pick, go with `_mode=host`. It'll use `crossdirect` to get speeds close to proper cross-compiling.
+   ```
+   This determines whether it's built using a foreign-arch chroot (`_mode=host`) executed with qemu-user, or using real cross-compilation (`_mode=cross`) from a host-architecture chroot, but the package's build tooling has to specifically support the latter, so it's mostly useful for kernels and uncompiled packages.
+1. **`_nodeps`**: (Optional) If your package doesn't require its listed dependencies to build
+   (usually because you're packaging a meta-package or only configs or scripts)
+   you can add `_nodeps=true` as the next line after the `_mode=` line to speed up packaging.
+   `makedeps` are still installed anyway.
+1. Test building it with `kupferbootstrap packages build $pkgbname`
+1. For any files and git repos downloaded by your PKGBUILD,
+   add them to a new `.gitignore` file in the same directory as your `PKGBUILD`.  
+   ```{hint}
+   Don't forget to `git add` the new `.gitignore` file!
+   ```
+1. Run `kupferbootstrap packages check` to make sure the formatting for your PKGBUILDs is okay.
+   ```{warning}
+   This is **not** optional. MRs with failing CI will **not** be merged.
+   ```
+
+### Pushing
+1. Fork the Kupfer pkgbuilds repo on Gitlab using the Fork button
+1. Add your fork's **SSH** URI to your local git repo as a **new remote**: `git remote add fork git@gitlab...`
+1. `git push -u fork $branchname` it
+
+### Submitting the MR
+When you're ready, open a Merge Request on the Kupfer pkgbuilds repo.
+
+```{hint}
+Prefix the MR title with `Draft: ` to indicate a Work In Progress state.
+```
+

docs/source/usage/quickstart.md

@@ -0,0 +1,9 @@
+# Quickstart
+
+1. [Install](../install) Kupferbootstrap
+1. [Configure](../config) it: `kuperbootstrap config init`
+1. [Update your PKGBUILDs + SRCINFO cache](../../cli/packages#kupferbootstrap-packages-update): `kupferbootstrap packages update`
+1. [Build an image](../../cli/image#kupferbootstrap-image-build): `kupferbootstrap image build`
+1. [Flash the image](../../cli/image#kupferbootstrap-image-flash): `kupferbootstrap image flash abootimg && kupferbootstrap image flash full userdata`
+
+See also: [Frequently Asked Questions](../faq)

format.sh

@@ -6,7 +6,7 @@ autoflake_args=('--recursive' '--remove-unused-variables' '--remove-all-unused-i
 format() {
     files=("$@")
     if [[ -z "${files[*]}" ]]; then
-        files=(".")
+        files=(*.py "src")
     fi
 
     yapf "${yapf_args[@]}"  "${files[@]}"

integration_tests.py

@@ -5,13 +5,13 @@ import pytest
 from glob import glob
 from subprocess import CompletedProcess
 
-from config.state import config, CONFIG_DEFAULTS
-from constants import SRCINFO_METADATA_FILE
-from exec.cmd import run_cmd
-from exec.file import get_temp_dir
-from logger import setup_logging
-from packages.cli import SRCINFO_CACHE_FILES, cmd_build, cmd_clean, cmd_init, cmd_update
-from utils import git_get_branch
+from kupferbootstrap.config.state import config, CONFIG_DEFAULTS
+from kupferbootstrap.constants import SRCINFO_METADATA_FILE
+from kupferbootstrap.exec.cmd import run_cmd
+from kupferbootstrap.exec.file import get_temp_dir
+from kupferbootstrap.logger import setup_logging
+from kupferbootstrap.packages.cli import SRCINFO_CACHE_FILES, cmd_build, cmd_clean, cmd_init, cmd_update
+from kupferbootstrap.utils import git_get_branch
 
 tempdir = None
 config.try_load_file()
@@ -37,6 +37,11 @@ def ctx() -> click.Context:
     return click.Context(click.Command('integration_tests'))
 
 
+def test_main_import():
+    from main import cli
+    assert cli
+
+
 def test_config_load(ctx: click.Context):
     path = config.runtime.config_file
     assert path
@@ -47,7 +52,8 @@ def test_config_load(ctx: click.Context):
 
 def test_packages_update(ctx: click.Context):
     pkgbuilds_path = config.get_path('pkgbuilds')
-    kbs_branch = git_get_branch(config.runtime.script_source_dir)
+    assert config.runtime.script_source_dir
+    kbs_branch = git_get_branch(os.path.join(config.runtime.script_source_dir, "../.."))
     # Gitlab CI integration: the CI checks out a detached commit, branch comes back empty.
     if not kbs_branch and os.environ.get('CI', 'false') == 'true':
         kbs_branch = os.environ.get('CI_COMMIT_BRANCH', '')

pyproject.toml

@@ -0,0 +1,29 @@
+[project]
+name = "kupferbootstrap"
+dependencies = [
+    "click>=8.0.1",
+    "appdirs>=1.4.4",
+    "joblib>=1.0.1",
+    "toml",
+    "typing_extensions",
+    "coloredlogs",
+    "munch",
+    "requests",
+    "python-dateutil",
+    "enlighten",
+    "PyYAML",
+]
+dynamic = ["version"]
+
+[project.scripts]
+kupferbootstrap = "kupferbootstrap.main:main"
+
+[tool.setuptools.package-data]
+"*" = ["version.txt"]
+
+[build-system]
+requires = [ "setuptools>=41", "wheel", "setuptools-git-versioning<2", ]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools-git-versioning]
+enabled = true

pytest.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
 sudo -v
-python -m pytest -v --cov=. --cov-branch --cov-report=term "$@" ./*/test_*.py
+python -m pytest -v --cov=. --cov-branch --cov-report=term "$@" src/kupferbootstrap

requirements.txt

@@ -1,12 +1 @@
-click>=8.0.1
-appdirs>=1.4.4
-joblib>=1.0.1
-toml
-typing_extensions
-coloredlogs
-munch
-setuptools # required by munch
-requests
-python-dateutil
-enlighten
-PyYAML
+-e .

src/kupferbootstrap/binfmt/binfmt.py

@@ -5,10 +5,10 @@ import logging
 
 from typing import Optional
 
-from chroot.abstract import Chroot
-from constants import Arch, QEMU_ARCHES
-from exec.cmd import run_root_cmd, CompletedProcess
-from utils import mount
+from kupferbootstrap.chroot.abstract import Chroot
+from kupferbootstrap.constants import Arch, QEMU_ARCHES
+from kupferbootstrap.exec.cmd import run_root_cmd, CompletedProcess
+from kupferbootstrap.utils import mount
 
 
 def binfmt_info(chroot: Optional[Chroot] = None):

src/kupferbootstrap/binfmt/cli.py

@@ -3,7 +3,7 @@ import os
 
 from typing import Optional
 
-from constants import Arch, ARCHES
+from kupferbootstrap.constants import Arch, ARCHES
 
 from .binfmt import binfmt_unregister, binfmt_is_registered
 
@@ -15,7 +15,7 @@ arches_arg_optional = click.argument('arches', type=click.Choice(ARCHES), nargs=
 @cmd_binfmt.command('register', help='Register a binfmt handler with the kernel')
 @arches_arg
 def cmd_register(arches: list[Arch], disable_chroot: bool = False):
-    from packages.build import build_enable_qemu_binfmt
+    from ..packages.build import build_enable_qemu_binfmt
     for arch in arches:
         build_enable_qemu_binfmt(arch)
 

src/kupferbootstrap/cache/cli.py

@@ -2,11 +2,11 @@ import click
 import os
 import logging
 
-from config.state import config
-from constants import CHROOT_PATHS
-from exec.file import remove_file
-from packages.cli import cmd_clean as cmd_clean_pkgbuilds
-from wrapper import enforce_wrap
+from kupferbootstrap.config.state import config
+from kupferbootstrap.constants import CHROOT_PATHS
+from kupferbootstrap.exec.file import remove_file
+from kupferbootstrap.packages.cli import cmd_clean as cmd_clean_pkgbuilds
+from kupferbootstrap.wrapper import enforce_wrap
 
 PATHS = list(CHROOT_PATHS.keys())
 

src/kupferbootstrap/chroot/abstract.py

@@ -9,13 +9,13 @@ from shlex import quote as shell_quote
 from typing import ClassVar, Iterable, Protocol, Union, Optional, Mapping
 from uuid import uuid4
 
-from config.state import config
-from constants import Arch, CHROOT_PATHS, GCC_HOSTSPECS
-from distro.distro import get_base_distro, get_kupfer_local, RepoInfo
-from exec.cmd import FileDescriptor, run_root_cmd, generate_env_cmd, flatten_shell_script, wrap_in_bash, generate_cmd_su
-from exec.file import makedir, root_makedir, root_write_file, write_file
-from generator import generate_makepkg_conf
-from utils import mount, umount, check_findmnt, log_or_exception
+from kupferbootstrap.config.state import config
+from kupferbootstrap.constants import Arch, CHROOT_PATHS, GCC_HOSTSPECS
+from kupferbootstrap.distro.distro import get_base_distro, get_kupfer_local, RepoInfo
+from kupferbootstrap.exec.cmd import FileDescriptor, run_root_cmd, generate_env_cmd, flatten_shell_script, wrap_in_bash, generate_cmd_su
+from kupferbootstrap.exec.file import makedir, root_makedir, root_write_file, write_file
+from kupferbootstrap.generator import generate_makepkg_conf
+from kupferbootstrap.utils import mount, umount, check_findmnt, log_or_exception
 
 from .helpers import BASE_CHROOT_PREFIX, BASIC_MOUNTS, base_chroot_name, make_abs_path
 

src/kupferbootstrap/chroot/base.py

@@ -6,10 +6,10 @@ from glob import glob
 from shutil import rmtree
 from typing import ClassVar
 
-from constants import Arch
-from exec.cmd import run_root_cmd
-from exec.file import makedir, root_makedir
-from config.state import config
+from kupferbootstrap.constants import Arch
+from kupferbootstrap.exec.cmd import run_root_cmd
+from kupferbootstrap.exec.file import makedir, root_makedir
+from kupferbootstrap.config.state import config
 
 from .abstract import Chroot, get_chroot
 from .helpers import base_chroot_name

src/kupferbootstrap/chroot/build.py

@@ -4,11 +4,11 @@ import subprocess
 from glob import glob
 from typing import ClassVar, Optional
 
-from config.state import config
-from constants import Arch, GCC_HOSTSPECS, CROSSDIRECT_PKGS, CHROOT_PATHS
-from distro.distro import get_kupfer_local
-from exec.cmd import run_root_cmd
-from exec.file import makedir, remove_file, root_makedir, root_write_file, symlink
+from kupferbootstrap.config.state import config
+from kupferbootstrap.constants import Arch, GCC_HOSTSPECS, CROSSDIRECT_PKGS, CHROOT_PATHS
+from kupferbootstrap.distro.distro import get_kupfer_local
+from kupferbootstrap.exec.cmd import run_root_cmd
+from kupferbootstrap.exec.file import makedir, remove_file, root_makedir, root_write_file, symlink
 
 from .abstract import Chroot, get_chroot
 from .helpers import build_chroot_name
@@ -82,6 +82,7 @@ class BuildChroot(Chroot):
         native_chroot.mount_pacman_cache()
         native_chroot.mount_packages()
         native_chroot.activate()
+        logging.debug(f"Installing {CROSSDIRECT_PKGS=} + {gcc=}")
         results = dict(native_chroot.try_install_packages(
             CROSSDIRECT_PKGS + [gcc],
             refresh=True,
@@ -103,8 +104,8 @@ class BuildChroot(Chroot):
         target_include_dir = os.path.join(self.path, 'include')
 
         for target, source in {cc_path: gcc, target_lib_dir: 'lib', target_include_dir: 'usr/include'}.items():
-            if not os.path.exists(target):
-                logging.debug(f'Symlinking {source} at {target}')
+            if not (os.path.exists(target) or os.path.islink(target)):
+                logging.debug(f'Symlinking {source=} at {target=}')
                 symlink(source, target)
         ld_so = os.path.basename(glob(f"{os.path.join(native_chroot.path, 'usr', 'lib', 'ld-linux-')}*")[0])
         ld_so_target = os.path.join(target_lib_dir, ld_so)

src/kupferbootstrap/chroot/cli.py

@@ -4,9 +4,9 @@ import os
 
 from typing import Optional
 
-from config.state import config
-from wrapper import enforce_wrap
-from devices.device import get_profile_device
+from kupferbootstrap.config.state import config
+from kupferbootstrap.wrapper import enforce_wrap
+from kupferbootstrap.devices.device import get_profile_device
 
 from .abstract import Chroot
 from .base import get_base_chroot
@@ -30,7 +30,7 @@ def cmd_chroot(ctx: click.Context, type: str = 'build', name: Optional[str] = No
         raise Exception(f'Unknown chroot type: "{type}"')
 
     if type == 'rootfs':
-        from image.image import cmd_inspect
+        from ..image.image import cmd_inspect
         assert isinstance(cmd_inspect, click.Command)
         ctx.invoke(cmd_inspect, profile=name, shell=True)
         return

src/kupferbootstrap/chroot/device.py

@@ -3,12 +3,12 @@ import os
 
 from typing import ClassVar, Optional
 
-from config.state import config
-from constants import Arch, BASE_PACKAGES
-from distro.repo import RepoInfo
-from distro.distro import get_kupfer_local, get_kupfer_https
-from exec.file import get_temp_dir, makedir, root_makedir
-from utils import check_findmnt
+from kupferbootstrap.config.state import config
+from kupferbootstrap.constants import Arch, BASE_PACKAGES
+from kupferbootstrap.distro.repo import RepoInfo
+from kupferbootstrap.distro.distro import get_kupfer_local, get_kupfer_https
+from kupferbootstrap.exec.file import get_temp_dir, makedir, root_makedir
+from kupferbootstrap.utils import check_findmnt
 
 from .base import BaseChroot
 from .build import BuildChroot

src/kupferbootstrap/chroot/helpers.py

@@ -1,8 +1,8 @@
 import os
 from typing import Optional, TypedDict
 
-from config.state import config
-from constants import Arch
+from kupferbootstrap.config.state import config
+from kupferbootstrap.constants import Arch
 
 BIND_BUILD_DIRS = 'BINDBUILDDIRS'
 BASE_CHROOT_PREFIX = 'base_'

src/kupferbootstrap/config/cli.py

@@ -4,10 +4,10 @@ import logging
 from copy import deepcopy
 from typing import Any, Callable, Iterable, Mapping, Optional, Union
 
-from devices.device import get_devices, sanitize_device_name
-from flavours.flavour import get_flavours
-from utils import color_bold, colors_supported, color_mark_selected
-from wrapper import execute_without_exit
+from kupferbootstrap.devices.device import get_devices, sanitize_device_name
+from kupferbootstrap.flavours.flavour import get_flavours
+from kupferbootstrap.utils import color_bold, colors_supported, color_mark_selected
+from kupferbootstrap.wrapper import execute_without_exit
 
 from .scheme import Profile
 from .profile import PROFILE_EMPTY, PROFILE_DEFAULTS, resolve_profile_attr, SparseProfile
@@ -129,7 +129,7 @@ def prompt_choice(current: Optional[Any], key: str, choices: Iterable[Any], allo
     res, _ = prompt_config(text=key, default=current, field_type=click.Choice(choices), show_choices=show_choices)
     if allow_none and res == '':
         res = None
-    return res, res == current
+    return res, res != current
 
 
 def resolve_profile_field(current: Any, *kargs):
@@ -205,7 +205,7 @@ def prompt_for_save(retry_ctx: Optional[click.Context] = None):
     If `retry_ctx` is passed, the context's command will be reexecuted with the same arguments if the user chooses to retry.
     False will still be returned as the retry is expected to either save, perform another retry or arbort.
     """
-    from wrapper import is_wrapped
+    from ..wrapper import is_wrapped
     if click.confirm(f'Do you want to save your changes to {config.runtime.config_file}?', default=True):
         if is_wrapped():
             logging.warning("Writing to config file inside wrapper."

src/kupferbootstrap/config/scheme.py

@@ -3,8 +3,8 @@ from __future__ import annotations
 from munch import Munch
 from typing import Any, Optional, Mapping, Union
 
-from dictscheme import DictScheme
-from constants import Arch
+from kupferbootstrap.dictscheme import DictScheme
+from kupferbootstrap.constants import Arch
 
 
 class SparseProfile(DictScheme):

src/kupferbootstrap/config/state.py

@@ -5,7 +5,7 @@ import toml
 from copy import deepcopy
 from typing import Mapping, Optional
 
-from constants import DEFAULT_PACKAGE_BRANCH
+from kupferbootstrap.constants import DEFAULT_PACKAGE_BRANCH
 
 from .scheme import Config, ConfigLoadState, DictScheme, Profile, RuntimeConfiguration
 from .profile import PROFILE_DEFAULTS, PROFILE_DEFAULTS_DICT, resolve_profile

src/kupferbootstrap/config/test_config.py

@@ -7,9 +7,9 @@ import toml
 from tempfile import mktemp, gettempdir as get_system_tempdir
 from typing import Any, Optional
 
-from config.profile import PROFILE_DEFAULTS
-from config.scheme import Config, Profile
-from config.state import CONFIG_DEFAULTS, ConfigStateHolder
+from kupferbootstrap.config.profile import PROFILE_DEFAULTS
+from kupferbootstrap.config.scheme import Config, Profile
+from kupferbootstrap.config.state import CONFIG_DEFAULTS, ConfigStateHolder
 
 
 def get_filename():

src/kupferbootstrap/constants.py

@@ -1,4 +1,4 @@
-from typehelpers import TypeAlias
+from .typehelpers import TypeAlias
 
 FASTBOOT = 'fastboot'
 FLASH_PARTS = {
@@ -89,7 +89,7 @@ COMPILE_ARCHES: dict[Arch, str] = {
 GCC_HOSTSPECS: dict[DistroArch, dict[TargetArch, str]] = {
     'x86_64': {
         'x86_64': 'x86_64-pc-linux-gnu',
-        'aarch64': 'aarch64-linux-gnu',
+        'aarch64': 'aarch64-unknown-linux-gnu',
         'armv7h': 'arm-unknown-linux-gnueabihf'
     },
     'aarch64': {

src/kupferbootstrap/devices/cli.py

@@ -4,9 +4,9 @@ import logging
 from json import dumps as json_dump
 from typing import Optional
 
-from config.state import config
-from config.cli import resolve_profile_field
-from utils import color_mark_selected, colors_supported
+from kupferbootstrap.config.state import config
+from kupferbootstrap.config.cli import resolve_profile_field
+from kupferbootstrap.utils import color_mark_selected, colors_supported
 
 from .device import get_devices, get_device
 

src/kupferbootstrap/devices/device.py

@@ -3,13 +3,13 @@ import os
 
 from typing import Optional
 
-from config.state import config
-from constants import Arch, ARCHES
-from dictscheme import DictScheme
-from distro.distro import get_kupfer_local
-from distro.package import LocalPackage
-from packages.pkgbuild import Pkgbuild, _pkgbuilds_cache, discover_pkgbuilds, get_pkgbuild_by_path, init_pkgbuilds
-from utils import read_files_from_tar, color_str
+from kupferbootstrap.config.state import config
+from kupferbootstrap.constants import Arch, ARCHES
+from kupferbootstrap.dictscheme import DictScheme
+from kupferbootstrap.distro.distro import get_kupfer_local
+from kupferbootstrap.distro.package import LocalPackage
+from kupferbootstrap.packages.pkgbuild import Pkgbuild, _pkgbuilds_cache, discover_pkgbuilds, get_pkgbuild_by_path, init_pkgbuilds
+from kupferbootstrap.utils import read_files_from_tar, color_str
 
 from .deviceinfo import DEFAULT_IMAGE_SECTOR_SIZE, DeviceInfo, parse_deviceinfo
 
@@ -72,7 +72,7 @@ class Device(DictScheme):
     def parse_deviceinfo(self, try_download: bool = True, lazy: bool = True) -> DeviceInfo:
         if not lazy or 'deviceinfo' not in self or self.deviceinfo is None:
             # avoid import loop
-            from packages.build import check_package_version_built
+            from kupferbootstrap.packages.build import check_package_version_built
             is_built = check_package_version_built(self.package, self.arch, try_download=try_download)
             if not is_built:
                 raise Exception(f"device package {self.package.name} for device {self.name} couldn't be acquired!")

src/kupferbootstrap/devices/deviceinfo.py

@@ -7,9 +7,9 @@ import os
 
 from typing import Mapping, Optional
 
-from config.state import config
-from constants import Arch
-from dictscheme import DictScheme
+from kupferbootstrap.config.state import config
+from kupferbootstrap.constants import Arch
+from kupferbootstrap.dictscheme import DictScheme
 
 PMOS_ARCHES_OVERRIDES: dict[str, Arch] = {
     "armv7": 'armv7h',
@@ -205,15 +205,12 @@ def parse_kernel_suffix(deviceinfo: dict[str, Optional[str]], kernel: str = 'mai
     https://wiki.postmarketos.org/wiki/Device_specific_package#Multiple_kernels
 
     :param info: deviceinfo dict, e.g.:
-                 {"a": "first",
-                  "b_mainline": "second",
-                  "b_downstream": "third"}
+        {"a": "first", "b_mainline": "second", "b_downstream": "third"}
     :param device: which device info belongs to
     :param kernel: which kernel suffix to remove (e.g. "mainline")
     :returns: info, but with the configured kernel suffix removed, e.g:
-              {"a": "first",
-               "b": "second",
-               "b_downstream": "third"}
+        {"a": "first", "b": "second", "b_downstream": "third"}
+
     """
     # Do nothing if the configured kernel isn't available in the kernel (e.g.
     # after switching from device with multiple kernels to device with only one

src/kupferbootstrap/devices/test_device.py

@@ -4,8 +4,8 @@ import os
 
 from copy import copy
 
-from config.state import ConfigStateHolder, config
-from packages.pkgbuild import init_pkgbuilds, discover_pkgbuilds, Pkgbuild, parse_pkgbuild
+from kupferbootstrap.config.state import ConfigStateHolder, config
+from kupferbootstrap.packages.pkgbuild import init_pkgbuilds, discover_pkgbuilds, Pkgbuild, parse_pkgbuild
 from .device import Device, DEVICE_DEPRECATIONS, get_device, get_devices, parse_device_pkg, check_devicepkg_name
 
 

src/kupferbootstrap/devices/test_deviceinfo.py

@@ -1,4 +1,4 @@
-from config.state import config
+from kupferbootstrap.config.state import config
 
 from .deviceinfo import DeviceInfo, parse_deviceinfo
 from .device import get_device

src/kupferbootstrap/dictscheme.py

@@ -7,7 +7,7 @@ from munch import Munch
 from toml.encoder import TomlEncoder, TomlPreserveInlineDictEncoder
 from typing import ClassVar, Generator, Optional, Union, Mapping, Any, get_type_hints, get_origin, get_args, Iterable
 
-from typehelpers import UnionType, NoneType
+from .typehelpers import UnionType, NoneType
 
 
 def resolve_type_hint(hint: type, ignore_origins: list[type] = []) -> Iterable[type]:
@@ -52,7 +52,7 @@ class DictScheme(Munch):
     _sparse: ClassVar[bool] = False
 
     def __init__(self, d: Mapping = {}, validate: bool = True, **kwargs):
-        self.update(d | kwargs, validate=validate)
+        self.update(dict(d) | kwargs, validate=validate)
 
     @classmethod
     def transform(
@@ -269,10 +269,13 @@ class DictScheme(Munch):
     ) -> str:
         import yaml
         yaml_args = {'sort_keys': False} | yaml_args
-        return yaml.dump(
+        dumped = yaml.dump(
             self.toDict(strip_hidden=strip_hidden, sparse=sparse),
             **yaml_args,
         )
+        if dumped is None:
+            raise Exception(f"Failed to yaml-serialse {self}")
+        return dumped
 
     def toToml(
             self,

src/kupferbootstrap/distro/distro.py

@@ -3,9 +3,9 @@ import logging
 from enum import IntFlag
 from typing import Generic, Mapping, Optional, TypeVar
 
-from constants import Arch, ARCHES, REPOSITORIES, KUPFER_BRANCH_MARKER, KUPFER_HTTPS, CHROOT_PATHS
-from generator import generate_pacman_conf_body
-from config.state import config
+from kupferbootstrap.constants import Arch, ARCHES, REPOSITORIES, KUPFER_BRANCH_MARKER, KUPFER_HTTPS, CHROOT_PATHS
+from kupferbootstrap.generator import generate_pacman_conf_body
+from kupferbootstrap.config.state import config
 
 from .repo import BinaryPackageType, RepoInfo, Repo, LocalRepo, RemoteRepo
 from .repo_config import AbstrRepoConfig, BaseDistro, ReposConfigFile, REPOS_CONFIG_DEFAULT, get_repo_config as _get_repo_config

src/kupferbootstrap/distro/package.py

@@ -5,7 +5,7 @@ from shutil import copyfileobj
 from typing import Optional, Union
 from urllib.request import urlopen
 
-from exec.file import get_temp_dir, makedir
+from kupferbootstrap.exec.file import get_temp_dir, makedir
 
 
 class PackageInfo:

src/kupferbootstrap/distro/repo.py

@@ -5,9 +5,9 @@ import tarfile
 
 from typing import Generic, TypeVar
 
-from config.state import config
-from exec.file import get_temp_dir
-from utils import download_file
+from kupferbootstrap.config.state import config
+from kupferbootstrap.exec.file import get_temp_dir
+from kupferbootstrap.utils import download_file
 
 from .package import BinaryPackage, LocalPackage, RemotePackage
 

src/kupferbootstrap/distro/repo_config.py

@@ -8,10 +8,10 @@ import yaml
 from copy import deepcopy
 from typing import ClassVar, Optional, Mapping, Union
 
-from config.state import config
-from constants import Arch, BASE_DISTROS, KUPFER_HTTPS, REPOS_CONFIG_FILE, REPOSITORIES
-from dictscheme import DictScheme, toml_inline_dicts, TomlPreserveInlineDictEncoder
-from utils import sha256sum
+from ..config.state import config
+from ..constants import Arch, BASE_DISTROS, KUPFER_HTTPS, REPOS_CONFIG_FILE, REPOSITORIES
+from ..dictscheme import DictScheme, toml_inline_dicts, TomlPreserveInlineDictEncoder
+from ..utils import sha256sum
 
 REPOS_KEY = 'repos'
 REMOTEURL_KEY = 'remote_url'
@@ -142,7 +142,7 @@ def get_repo_config(
     config_exists = os.path.exists(repo_config_file_path)
     if not config_exists and _current_config is None:
         if initialize_pkgbuilds:
-            from packages.pkgbuild import init_pkgbuilds
+            from ..packages.pkgbuild import init_pkgbuilds
             init_pkgbuilds(update=False)
             return get_repo_config(initialize_pkgbuilds=False, repo_config_file=repo_config_file)
         if repo_config_file is not None:

src/kupferbootstrap/exec/cmd.py

@@ -7,7 +7,7 @@ from subprocess import CompletedProcess  # make it easy for users of this module
 from shlex import quote as shell_quote
 from typing import IO, Optional, Union
 
-from typehelpers import TypeAlias
+from kupferbootstrap.typehelpers import TypeAlias
 
 ElevationMethod: TypeAlias = str
 

src/kupferbootstrap/exec/file.py

@@ -9,7 +9,7 @@ from tempfile import mkdtemp
 from typing import Optional, Union
 
 from .cmd import run_cmd, run_root_cmd, elevation_noop, generate_cmd_su, wrap_in_bash, shell_quote
-from utils import get_user_name, get_group_name
+from kupferbootstrap.utils import get_user_name, get_group_name
 
 
 def try_native_filewrite(path: str, content: Union[str, bytes], chmod: Optional[str] = None) -> Optional[Exception]:
@@ -144,7 +144,13 @@ def remove_file(path: str, recursive=False):
             raise Exception(f"Unable to remove {path}: cmd returned {rc}")
 
 
-def makedir(path, user: Optional[Union[str, int]] = None, group: Optional[Union[str, int]] = None, parents: bool = True):
+def makedir(
+    path,
+    user: Optional[Union[str, int]] = None,
+    group: Optional[Union[str, int]] = None,
+    parents: bool = True,
+    mode: Optional[Union[int, str]] = None,
+):
     if not root_check_exists(path):
         try:
             if parents:
@@ -153,6 +159,8 @@ def makedir(path, user: Optional[Union[str, int]] = None, group: Optional[Union[
                 os.mkdir(path)
         except:
             run_root_cmd(['mkdir'] + (['-p'] if parents else []) + [path])
+    if mode is not None:
+        chmod(path, mode=mode)
     chown(path, user, group)
 
 

src/kupferbootstrap/exec/test_file.py

@@ -8,7 +8,7 @@ from dataclasses import dataclass
 
 from .cmd import run_root_cmd
 from .file import chmod, chown, get_temp_dir, write_file
-from utils import get_gid, get_uid
+from kupferbootstrap.utils import get_gid, get_uid
 
 TEMPDIR_MODE = 0o755
 

src/kupferbootstrap/flavours/cli.py

@@ -4,9 +4,9 @@ import logging
 from json import dumps as json_dump
 from typing import Optional
 
-from config.cli import resolve_profile_field
-from config.state import config
-from utils import color_mark_selected, colors_supported
+from kupferbootstrap.config.cli import resolve_profile_field
+from kupferbootstrap.config.state import config
+from kupferbootstrap.utils import color_mark_selected, colors_supported
 
 from .flavour import get_flavours, get_flavour
 

src/kupferbootstrap/flavours/flavour.py

@@ -6,11 +6,11 @@ import os
 
 from typing import Optional
 
-from config.state import config
-from constants import FLAVOUR_DESCRIPTION_PREFIX, FLAVOUR_INFO_FILE
-from dictscheme import DictScheme
-from packages.pkgbuild import discover_pkgbuilds, get_pkgbuild_by_name, init_pkgbuilds, Pkgbuild
-from utils import color_str
+from kupferbootstrap.config.state import config
+from kupferbootstrap.constants import FLAVOUR_DESCRIPTION_PREFIX, FLAVOUR_INFO_FILE
+from kupferbootstrap.dictscheme import DictScheme
+from kupferbootstrap.packages.pkgbuild import discover_pkgbuilds, get_pkgbuild_by_name, init_pkgbuilds, Pkgbuild
+from kupferbootstrap.utils import color_str
 
 
 class FlavourInfo(DictScheme):

src/kupferbootstrap/generator.py

@@ -1,7 +1,7 @@
 from typing import Optional
 
-from constants import Arch, CFLAGS_ARCHES, CFLAGS_GENERAL, COMPILE_ARCHES, GCC_HOSTSPECS
-from config.state import config
+from .constants import Arch, CFLAGS_ARCHES, CFLAGS_GENERAL, COMPILE_ARCHES, GCC_HOSTSPECS
+from .config.state import config
 
 
 def generate_makepkg_conf(arch: Arch, cross: bool = False, chroot: Optional[str] = None) -> str:

src/kupferbootstrap/image/boot.py

@@ -4,13 +4,13 @@ import click
 
 from typing import Optional
 
-from config.state import config
-from constants import FLASH_PARTS, FASTBOOT, JUMPDRIVE, JUMPDRIVE_VERSION
-from exec.file import makedir
-from devices.device import get_profile_device
-from flavours.flavour import get_profile_flavour
-from flavours.cli import profile_option
-from wrapper import enforce_wrap
+from kupferbootstrap.config.state import config
+from kupferbootstrap.constants import FLASH_PARTS, FASTBOOT, JUMPDRIVE, JUMPDRIVE_VERSION
+from kupferbootstrap.exec.file import makedir
+from kupferbootstrap.devices.device import get_profile_device
+from kupferbootstrap.flavours.flavour import get_profile_flavour
+from kupferbootstrap.flavours.cli import profile_option
+from kupferbootstrap.wrapper import enforce_wrap
 
 from .fastboot import fastboot_boot, fastboot_erase
 from .image import get_device_name, losetup_rootfs_image, get_image_path, dump_aboot, dump_lk2nd

src/kupferbootstrap/image/fastboot.py

@@ -1,7 +1,7 @@
 import click
 import logging
 
-from exec.cmd import run_cmd, CompletedProcess
+from kupferbootstrap.exec.cmd import run_cmd, CompletedProcess
 from typing import Optional
 
 

src/kupferbootstrap/image/flash.py

@@ -5,13 +5,13 @@ import logging
 
 from typing import Optional
 
-from constants import FLASH_PARTS, LOCATIONS, FASTBOOT, JUMPDRIVE
-from exec.cmd import run_root_cmd
-from exec.file import get_temp_dir
-from devices.device import get_profile_device
-from flavours.flavour import get_profile_flavour
-from flavours.cli import profile_option
-from wrapper import enforce_wrap
+from kupferbootstrap.constants import FLASH_PARTS, LOCATIONS, FASTBOOT, JUMPDRIVE
+from kupferbootstrap.exec.cmd import run_root_cmd
+from kupferbootstrap.exec.file import get_temp_dir
+from kupferbootstrap.devices.device import get_profile_device
+from kupferbootstrap.flavours.flavour import get_profile_flavour
+from kupferbootstrap.flavours.cli import profile_option
+from kupferbootstrap.wrapper import enforce_wrap
 
 from .fastboot import fastboot_flash
 from .image import dd_image, dump_aboot, dump_lk2nd, dump_qhypstub, get_image_path, losetup_destroy, losetup_rootfs_image, partprobe, shrink_fs

src/kupferbootstrap/image/image.py

@@ -9,17 +9,17 @@ from signal import pause
 from subprocess import CompletedProcess
 from typing import Optional, Union
 
-from config.state import config, Profile
-from chroot.device import DeviceChroot, get_device_chroot
-from constants import Arch, BASE_LOCAL_PACKAGES, BASE_PACKAGES, POST_INSTALL_CMDS
-from distro.distro import get_base_distro, get_kupfer_https
-from devices.device import Device, get_profile_device
-from exec.cmd import run_root_cmd, generate_cmd_su
-from exec.file import get_temp_dir, root_write_file, root_makedir, makedir
-from flavours.flavour import Flavour, get_profile_flavour
-from net.ssh import copy_ssh_keys
-from packages.build import build_enable_qemu_binfmt, build_packages, filter_pkgbuilds
-from wrapper import enforce_wrap
+from kupferbootstrap.config.state import config, Profile
+from kupferbootstrap.chroot.device import DeviceChroot, get_device_chroot
+from kupferbootstrap.constants import Arch, BASE_LOCAL_PACKAGES, BASE_PACKAGES, POST_INSTALL_CMDS
+from kupferbootstrap.distro.distro import get_base_distro, get_kupfer_https
+from kupferbootstrap.devices.device import Device, get_profile_device
+from kupferbootstrap.exec.cmd import run_root_cmd, generate_cmd_su
+from kupferbootstrap.exec.file import get_temp_dir, root_write_file, root_makedir, makedir
+from kupferbootstrap.flavours.flavour import Flavour, get_profile_flavour
+from kupferbootstrap.net.ssh import copy_ssh_keys
+from kupferbootstrap.packages.build import build_enable_qemu_binfmt, build_packages, filter_pkgbuilds
+from kupferbootstrap.wrapper import enforce_wrap
 
 # image files need to be slightly smaller than partitions to fit
 IMG_FILE_ROOT_DEFAULT_SIZE = "1800M"
@@ -333,8 +333,9 @@ def install_rootfs(
     )
     chroot.add_sudo_config(config_name='wheel', privilegee='%wheel', password_required=True)
     copy_ssh_keys(
-        chroot.path,
+        chroot,
         user=user,
+        allow_fail=True,
     )
     files = {
         'etc/pacman.conf': get_base_distro(arch).get_pacman_conf(

src/kupferbootstrap/main.py

@@ -7,19 +7,19 @@ from os import isatty
 from traceback import format_exc, format_exception_only, format_tb
 from typing import Optional
 
-from logger import color_option, logging, quiet_option, setup_logging, verbose_option
-from wrapper import get_wrapper_type, enforce_wrap, nowrapper_option
-from progressbar import progress_bars_option
+from .logger import color_option, logging, quiet_option, setup_logging, verbose_option
+from .wrapper import get_wrapper_type, enforce_wrap, nowrapper_option
+from .progressbar import progress_bars_option
 
-from binfmt.cli import cmd_binfmt
-from config.cli import config, config_option, cmd_config
-from packages.cli import cmd_packages
-from flavours.cli import cmd_flavours
-from devices.cli import cmd_devices
-from net.cli import cmd_net
-from chroot.cli import cmd_chroot
-from cache.cli import cmd_cache
-from image.cli import cmd_image
+from .binfmt.cli import cmd_binfmt
+from .config.cli import config, config_option, cmd_config
+from .packages.cli import cmd_packages
+from .flavours.cli import cmd_flavours
+from .devices.cli import cmd_devices
+from .net.cli import cmd_net
+from .chroot.cli import cmd_chroot
+from .cache.cli import cmd_cache
+from .image.cli import cmd_image
 
 
 @click.group()

src/kupferbootstrap/net/forwarding.py

@@ -1,8 +1,8 @@
 import click
 import logging
 
-from exec.cmd import run_root_cmd
-from wrapper import check_programs_wrap
+from kupferbootstrap.exec.cmd import run_root_cmd
+from kupferbootstrap.wrapper import check_programs_wrap
 
 from .ssh import run_ssh_command
 

src/kupferbootstrap/net/ssh.py

@@ -4,10 +4,11 @@ import os
 import pathlib
 import click
 
-from config.state import config
-from constants import SSH_COMMON_OPTIONS, SSH_DEFAULT_HOST, SSH_DEFAULT_PORT
-from exec.cmd import run_cmd
-from wrapper import check_programs_wrap
+from kupferbootstrap.config.state import config
+from kupferbootstrap.constants import SSH_COMMON_OPTIONS, SSH_DEFAULT_HOST, SSH_DEFAULT_PORT
+from kupferbootstrap.exec.cmd import run_cmd
+from kupferbootstrap.exec.file import write_file
+from kupferbootstrap.wrapper import check_programs_wrap
 
 
 @click.command(name='ssh')
@@ -83,21 +84,16 @@ def find_ssh_keys():
     return keys
 
 
-def copy_ssh_keys(root_dir: str, user: str):
+def copy_ssh_keys(chroot: Chroot, user: str, allow_fail: bool = False):
     check_programs_wrap(['ssh-keygen'])
-    authorized_keys_file = os.path.join(
-        root_dir,
-        'home',
-        user,
-        '.ssh',
-        'authorized_keys',
-    )
-    if os.path.exists(authorized_keys_file):
-        os.unlink(authorized_keys_file)
+    ssh_dir_relative = os.path.join('/home', user, '.ssh')
+    ssh_dir = chroot.get_path(ssh_dir_relative)
+    authorized_keys_file_rel = os.path.join(ssh_dir_relative, 'authorized_keys')
+    authorized_keys_file = chroot.get_path(authorized_keys_file_rel)
 
     keys = find_ssh_keys()
     if len(keys) == 0:
-        logging.info("Could not find any ssh key to copy")
+        logging.warning("Could not find any ssh key to copy")
         create = click.confirm("Do you want me to generate an ssh key for you?", True)
         if not create:
             return
@@ -116,15 +112,34 @@ def copy_ssh_keys(root_dir: str, user: str):
             logging.fatal("Failed to generate ssh key")
         keys = find_ssh_keys()
 
-    ssh_dir = os.path.join(root_dir, 'home', user, '.ssh')
-    if not os.path.exists(ssh_dir):
-        os.makedirs(ssh_dir, exist_ok=True, mode=0o700)
+    if not keys:
+        logging.warning("No SSH keys to be copied. Skipping.")
+        return
 
-    with open(authorized_keys_file, 'a') as authorized_keys:
-        for key in keys:
-            pub = f'{key}.pub'
-            if not os.path.exists(pub):
-                logging.debug(f'Skipping key {key}: {pub} not found')
-                continue
+    auth_key_lines = []
+    for key in keys:
+        pub = f'{key}.pub'
+        if not os.path.exists(pub):
+            logging.debug(f'Skipping key {key}: {pub} not found')
+            continue
+        try:
             with open(pub, 'r') as file:
-                authorized_keys.write(file.read())
+                contents = file.read()
+                if not contents.strip():
+                    continue
+                auth_key_lines.append(contents)
+        except Exception as ex:
+            logging.warning(f"Could not read ssh pub key {pub}", exc_info=ex)
+            continue
+
+    if not os.path.exists(ssh_dir):
+        logging.info(f"Creating {ssh_dir_relative!r} dir in chroot {chroot.path!r}")
+        chroot.run_cmd(["mkdir", "-p", "-m", "700", ssh_dir_relative], switch_user=user)
+    logging.info(f"Writing SSH pub keys to {authorized_keys_file}")
+    try:
+        write_file(authorized_keys_file, "\n".join(auth_key_lines), user=str(chroot.get_uid(user)), mode="644")
+    except Exception as ex:
+        logging.error(f"Failed to write SSH authorized_keys_file at {authorized_keys_file!r}:", exc_info=ex)
+        if allow_fail:
+            return
+        raise ex from ex

src/kupferbootstrap/net/telnet.py

@@ -1,6 +1,6 @@
 import subprocess
 import click
-from wrapper import check_programs_wrap
+from kupferbootstrap.wrapper import check_programs_wrap
 
 
 @click.command(name='telnet')

src/kupferbootstrap/packages/build.py

@@ -9,18 +9,18 @@ from copy import deepcopy
 from urllib.error import HTTPError
 from typing import Iterable, Iterator, Optional
 
-from binfmt.binfmt import binfmt_is_registered, binfmt_register
-from constants import CROSSDIRECT_PKGS, QEMU_BINFMT_PKGS, GCC_HOSTSPECS, ARCHES, Arch, CHROOT_PATHS, MAKEPKG_CMD
-from config.state import config
-from exec.cmd import run_cmd, run_root_cmd
-from exec.file import makedir, remove_file, symlink
-from chroot.build import get_build_chroot, BuildChroot
-from distro.distro import get_kupfer_https, get_kupfer_local, get_kupfer_repo_names
-from distro.package import RemotePackage, LocalPackage
-from distro.repo import LocalRepo
-from progressbar import BAR_PADDING, get_levels_bar
-from wrapper import check_programs_wrap, is_wrapped
-from utils import ellipsize, sha256sum
+from kupferbootstrap.binfmt.binfmt import binfmt_is_registered, binfmt_register
+from kupferbootstrap.constants import CROSSDIRECT_PKGS, QEMU_BINFMT_PKGS, GCC_HOSTSPECS, ARCHES, Arch, CHROOT_PATHS, MAKEPKG_CMD
+from kupferbootstrap.config.state import config
+from kupferbootstrap.exec.cmd import run_cmd, run_root_cmd
+from kupferbootstrap.exec.file import makedir, remove_file, symlink
+from kupferbootstrap.chroot.build import get_build_chroot, BuildChroot
+from kupferbootstrap.distro.distro import get_kupfer_https, get_kupfer_local, get_kupfer_repo_names
+from kupferbootstrap.distro.package import RemotePackage, LocalPackage
+from kupferbootstrap.distro.repo import LocalRepo
+from kupferbootstrap.progressbar import BAR_PADDING, get_levels_bar
+from kupferbootstrap.wrapper import check_programs_wrap, is_wrapped
+from kupferbootstrap.utils import ellipsize, sha256sum
 
 from .pkgbuild import discover_pkgbuilds, filter_pkgbuilds, Pkgbase, Pkgbuild, SubPkgbuild
 
@@ -290,7 +290,8 @@ def try_download_package(dest_file_path: str, package: Pkgbuild, arch: Arch) ->
         return None
     repo_pkg: RemotePackage = repo.packages[pkgname]
     if repo_pkg.version != package.version:
-        logging.debug(f"Package {pkgname} versions differ: local: {package.version}, remote: {repo_pkg.version}. Building instead.")
+        logging.debug(f"Package {pkgname} versions differ: local: {package.version}, "
+                      f"remote: {repo_pkg.version}. Building instead.")
         return None
     if repo_pkg.filename != filename:
         versions_str = f"local: {filename}, remote: {repo_pkg.filename}"
@@ -298,6 +299,20 @@ def try_download_package(dest_file_path: str, package: Pkgbuild, arch: Arch) ->
             logging.debug(f"package filenames don't match: {versions_str}")
             return None
         logging.debug(f"ignoring compression extension difference: {versions_str}")
+    cache_file = os.path.join(config.get_path('pacman'), arch, repo_pkg.filename)
+    if os.path.exists(cache_file):
+        if not repo_pkg._desc or 'SHA256SUM' not in repo_pkg._desc:
+            cache_matches = False
+            extra_msg = ". However, we can't validate it, as the https repo doesnt provide a SHA256SUM for it."
+        else:
+            cache_matches = sha256sum(cache_file) == repo_pkg._desc['SHA256SUM']
+            extra_msg = (". However its checksum doesn't match." if not cache_matches else " and its checksum matches.")
+        logging.debug(f"While checking the HTTPS repo DB, we found a matching filename in the pacman cache{extra_msg}")
+        if cache_matches:
+            logging.info(f'copying cache file {cache_file} to repo as verified by remote checksum')
+            shutil.copy(cache_file, dest_file_path)
+            remove_file(cache_file)
+            return dest_file_path
     url = repo_pkg.resolved_url
     assert url
     try:
@@ -424,10 +439,11 @@ def setup_build_chroot(
     extra_packages: list[str] = [],
     add_kupfer_repos: bool = True,
     clean_chroot: bool = False,
+    repo: Optional[dict[str, Pkgbuild]] = None,
 ) -> BuildChroot:
     assert config.runtime.arch
     if arch != config.runtime.arch:
-        build_enable_qemu_binfmt(arch, lazy=False)
+        build_enable_qemu_binfmt(arch, repo=repo or discover_pkgbuilds(), lazy=False)
     init_prebuilts(arch)
     chroot = get_build_chroot(arch, add_kupfer_repos=add_kupfer_repos)
     chroot.mount_packages()
@@ -496,6 +512,7 @@ def build_package(
     enable_ccache: bool = True,
     clean_chroot: bool = False,
     build_user: str = 'kupfer',
+    repo: Optional[dict[str, Pkgbuild]] = None,
 ):
     makepkg_compile_opts = ['--holdver']
     makepkg_conf_path = 'etc/makepkg.conf'
@@ -515,6 +532,7 @@ def build_package(
         arch=arch,
         extra_packages=deps,
         clean_chroot=clean_chroot,
+        repo=repo,
     )
     assert config.runtime.arch
     native_chroot = target_chroot
@@ -524,6 +542,7 @@ def build_package(
             arch=config.runtime.arch,
             extra_packages=['base-devel'] + CROSSDIRECT_PKGS,
             clean_chroot=clean_chroot,
+            repo=repo,
         )
     if not package.mode:
         logging.warning(f'Package {package.path} has no _mode set, assuming "host"')
@@ -556,7 +575,7 @@ def build_package(
         build_root = target_chroot
         makepkg_compile_opts += ['--nodeps' if package.nodeps else '--syncdeps']
         env = deepcopy(get_makepkg_env(arch))
-        if foreign_arch and enable_crossdirect and package.name not in CROSSDIRECT_PKGS:
+        if foreign_arch and package.crossdirect and enable_crossdirect and package.name not in CROSSDIRECT_PKGS:
             env['PATH'] = f"/native/usr/lib/crossdirect/{arch}:{env['PATH']}"
             target_chroot.mount_crossdirect(native_chroot)
         else:
@@ -742,6 +761,7 @@ def build_packages(
                 enable_crossdirect=enable_crossdirect,
                 enable_ccache=enable_ccache,
                 clean_chroot=clean_chroot,
+                repo=repo,
             )
             files += add_package_to_repo(package, arch)
             updated_repos.add(package.repo)
@@ -816,8 +836,20 @@ def build_enable_qemu_binfmt(arch: Arch, repo: Optional[dict[str, Pkgbuild]] = N
     logging.info('Installing qemu-user (building if necessary)')
     check_programs_wrap(['pacman', 'makepkg', 'pacstrap'])
     # build qemu-user, binfmt, crossdirect
+    packages = list(CROSSDIRECT_PKGS)
+    hostspec = GCC_HOSTSPECS[arch][arch]
+    cross_gcc = f"{hostspec}-gcc"
+    if repo:
+        for pkg in repo.values():
+            if (pkg.name == cross_gcc or cross_gcc in pkg.provides):
+                if config.runtime.arch not in pkg.arches:
+                    logging.debug(f"Package {pkg.path} matches {cross_gcc=} name but not arch: {pkg.arches=}")
+                    continue
+                packages.append(pkg.path)
+                logging.debug(f"Adding gcc package {pkg.path} to the  necessary crosscompilation tools")
+                break
     build_packages_by_paths(
-        CROSSDIRECT_PKGS,
+        packages,
         native,
         repo=repo,
         try_download=True,

src/kupferbootstrap/packages/cli.py

@@ -6,16 +6,16 @@ import os
 from glob import glob
 from typing import Iterable, Optional
 
-from config.state import config
-from constants import Arch, ARCHES, SRCINFO_FILE, SRCINFO_INITIALISED_FILE, SRCINFO_METADATA_FILE, SRCINFO_TARBALL_FILE, SRCINFO_TARBALL_URL
-from exec.cmd import run_cmd, shell_quote, CompletedProcess
-from exec.file import get_temp_dir, makedir, remove_file
-from devices.device import get_profile_device
-from distro.distro import get_kupfer_local, get_kupfer_url, get_kupfer_repo_names
-from distro.package import LocalPackage
-from net.ssh import run_ssh_command, scp_put_files
-from utils import download_file, git, sha256sum
-from wrapper import check_programs_wrap, enforce_wrap
+from kupferbootstrap.config.state import config
+from kupferbootstrap.constants import Arch, ARCHES, SRCINFO_FILE, SRCINFO_INITIALISED_FILE, SRCINFO_METADATA_FILE, SRCINFO_TARBALL_FILE, SRCINFO_TARBALL_URL
+from kupferbootstrap.exec.cmd import run_cmd, shell_quote, CompletedProcess
+from kupferbootstrap.exec.file import get_temp_dir, makedir, remove_file
+from kupferbootstrap.devices.device import get_profile_device
+from kupferbootstrap.distro.distro import get_kupfer_local, get_kupfer_url, get_kupfer_repo_names
+from kupferbootstrap.distro.package import LocalPackage
+from kupferbootstrap.net.ssh import run_ssh_command, scp_put_files
+from kupferbootstrap.utils import download_file, git, sha256sum
+from kupferbootstrap.wrapper import check_programs_wrap, enforce_wrap
 
 from .build import build_packages_by_paths, init_prebuilts
 from .pkgbuild import discover_pkgbuilds, filter_pkgbuilds, get_pkgbuild_dirs, init_pkgbuilds
@@ -313,7 +313,7 @@ def cmd_list():
     logging.info(f'Done! {len(packages)} Pkgbuilds:')
     for name in sorted(packages.keys()):
         p = packages[name]
-        print(f'name: {p.name}; ver: {p.version}; mode: {p.mode}; provides: {p.provides}; replaces: {p.replaces};'
+        print(f'name: {p.name}; ver: {p.version}; mode: {p.mode}; crossdirect: {p.crossdirect} provides: {p.provides}; replaces: {p.replaces};'
               f'local_depends: {p.local_depends}; depends: {p.depends}')
 
 
@@ -346,6 +346,7 @@ def cmd_check(paths):
 
         mode_key = '_mode'
         nodeps_key = '_nodeps'
+        crossdirect_key = '_crossdirect'
         pkgbase_key = 'pkgbase'
         pkgname_key = 'pkgname'
         arches_key = '_arches'
@@ -356,6 +357,7 @@ def cmd_check(paths):
         required = {
             mode_key: True,
             nodeps_key: False,
+            crossdirect_key: False,
             pkgbase_key: False,
             pkgname_key: True,
             'pkgdesc': False,

src/kupferbootstrap/packages/pkgbuild.py

@@ -8,15 +8,15 @@ import os
 from joblib import Parallel, delayed
 from typing import Iterable, Optional
 
-from config.state import config, ConfigStateHolder
-from constants import Arch
-from distro.distro import get_kupfer_repo_names
-from distro.package import PackageInfo
-from exec.file import remove_file
-from logger import setup_logging
-from utils import git, git_get_branch
-from wrapper import check_programs_wrap
-from typehelpers import TypeAlias
+from kupferbootstrap.config.state import config, ConfigStateHolder
+from kupferbootstrap.constants import Arch
+from kupferbootstrap.distro.distro import get_kupfer_repo_names
+from kupferbootstrap.distro.package import PackageInfo
+from kupferbootstrap.exec.file import remove_file
+from kupferbootstrap.logger import setup_logging
+from kupferbootstrap.utils import git, git_get_branch
+from kupferbootstrap.wrapper import check_programs_wrap
+from kupferbootstrap.typehelpers import TypeAlias
 
 from .srcinfo_cache import SrcinfoMetaFile
 
@@ -156,6 +156,7 @@ class Pkgbuild(PackageInfo):
     repo: str
     mode: str
     nodeps: bool
+    crossdirect: bool
     path: str
     pkgver: str
     pkgrel: str
@@ -190,6 +191,7 @@ class Pkgbuild(PackageInfo):
         self.repo = repo or ''
         self.mode = ''
         self.nodeps = False
+        self.crossdirect = True
         self.path = relative_path
         self.pkgver = ''
         self.pkgrel = ''
@@ -223,6 +225,7 @@ class Pkgbuild(PackageInfo):
         self.repo = pkg.repo
         self.mode = pkg.mode
         self.nodeps = pkg.nodeps
+        self.crossdirect = pkg.crossdirect
         self.path = pkg.path
         self.pkgver = pkg.pkgver
         self.pkgrel = pkg.pkgrel
@@ -310,8 +313,11 @@ class SubPkgbuild(Pkgbuild):
         self.sources_refreshed = False
         self.update(pkgbase)
 
-        self.provides = {}
-        self.replaces = []
+        # set to None - will be replaced with base_pkg if still None after parsing
+        self.depends = None  # type: ignore[assignment]
+        self.makedepends = None  # type: ignore[assignment]
+        self.provides = None  # type: ignore[assignment]
+        self.replaces = None  # type: ignore[assignment]
 
     def refresh_sources(self, lazy: bool = True):
         assert self.pkgbase
@@ -354,7 +360,11 @@ def parse_pkgbuild(
         else:
             raise Exception(msg)
 
+    # if _crossdirect is unset (None), it defaults to True
+    crossdirect_enabled = srcinfo_cache.build_crossdirect in (None, True)
+
     base_package = Pkgbase(relative_pkg_dir, sources_refreshed=sources_refreshed, srcinfo_cache=srcinfo_cache)
+    base_package.crossdirect = crossdirect_enabled
     base_package.mode = mode
     base_package.nodeps = nodeps
     base_package.repo = relative_pkg_dir.split('/')[0]
@@ -383,13 +393,21 @@ def parse_pkgbuild(
         elif line.startswith('arch'):
             current.arches.append(splits[1])
         elif line.startswith('provides'):
+            if not current.provides:
+                current.provides = {}
             current.provides = get_version_specs(splits[1], current.provides)
         elif line.startswith('replaces'):
+            if not current.replaces:
+                current.replaces = []
             current.replaces.append(splits[1])
         elif splits[0] in ['depends', 'makedepends', 'checkdepends', 'optdepends']:
             spec = splits[1].split(': ', 1)[0]
+            if not current.depends:
+                current.depends = (base_package.makedepends or {}).copy()
             current.depends = get_version_specs(spec, current.depends)
             if splits[0] == 'makedepends':
+                if not current.makedepends:
+                    current.makedepends = {}
                 current.makedepends = get_version_specs(spec, current.makedepends)
 
     results: list[Pkgbuild] = list(base_package.subpackages)
@@ -402,6 +420,15 @@ def parse_pkgbuild(
         pkg.update_version()
         if not (pkg.version == base_package.version):
             raise Exception(f'Subpackage malformed! Versions differ! base: {base_package}, subpackage: {pkg}')
+        if isinstance(pkg, SubPkgbuild):
+            if pkg.depends is None:
+                pkg.depends = base_package.depends
+            if pkg.makedepends is None:
+                pkg.makedepends = base_package.makedepends
+            if pkg.replaces is None:
+                pkg.replaces = base_package.replaces
+            if pkg.provides is None:
+                pkg.provides = base_package.provides
     return results
 
 

src/kupferbootstrap/packages/srcinfo_cache.py

@@ -7,11 +7,11 @@ import subprocess
 
 from typing import Any, ClassVar, Optional
 
-from config.state import config
-from constants import MAKEPKG_CMD, SRCINFO_FILE, SRCINFO_METADATA_FILE, SRCINFO_INITIALISED_FILE
-from dictscheme import DictScheme
-from exec.cmd import run_cmd
-from utils import sha256sum
+from kupferbootstrap.config.state import config
+from kupferbootstrap.constants import MAKEPKG_CMD, SRCINFO_FILE, SRCINFO_METADATA_FILE, SRCINFO_INITIALISED_FILE
+from kupferbootstrap.dictscheme import DictScheme
+from kupferbootstrap.exec.cmd import run_cmd
+from kupferbootstrap.utils import sha256sum
 
 SRCINFO_CHECKSUM_FILES = ['PKGBUILD', SRCINFO_FILE]
 
@@ -68,11 +68,19 @@ class SrcInitialisedFile(JsonFile):
                 raise ex
 
 
+srcinfo_meta_defaults = {
+    'build_mode': None,
+    "build_nodeps": None,
+    "build_crossdirect": None,
+}
+
+
 class SrcinfoMetaFile(JsonFile):
 
     checksums: dict[str, str]
     build_mode: Optional[str]
     build_nodeps: Optional[bool]
+    build_crossdirect: Optional[bool]
 
     _changed: bool
     _filename: ClassVar[str] = SRCINFO_METADATA_FILE
@@ -92,9 +100,8 @@ class SrcinfoMetaFile(JsonFile):
         s = SrcinfoMetaFile({
             '_relative_path': relative_pkg_dir,
             '_changed': True,
-            'build_mode': '',
-            'build_nodeps': None,
             'checksums': {},
+            **srcinfo_meta_defaults,
         })
         return s, s.refresh_all()
 
@@ -120,9 +127,11 @@ class SrcinfoMetaFile(JsonFile):
         if not force_refresh:
             logging.debug(f'{metadata._relative_path}: srcinfo checksums match!')
             lines = lines or metadata.read_srcinfo_file()
-            for build_field in ['build_mode', 'build_nodeps']:
+            for build_field in srcinfo_meta_defaults.keys():
                 if build_field not in metadata:
                     metadata.refresh_build_fields()
+                    if write:
+                        metadata.write()
                     break
         else:
             lines = metadata.refresh_all(write=write)
@@ -143,8 +152,7 @@ class SrcinfoMetaFile(JsonFile):
             self._changed = True
 
     def refresh_build_fields(self):
-        self['build_mode'] = None
-        self['build_nodeps'] = None
+        self.update(srcinfo_meta_defaults)
         with open(os.path.join(config.get_path('pkgbuilds'), self._relative_path, 'PKGBUILD'), 'r') as file:
             lines = file.read().split('\n')
         for line in lines:
@@ -156,6 +164,8 @@ class SrcinfoMetaFile(JsonFile):
                 self.build_mode = val
             elif key == '_nodeps':
                 self.build_nodeps = val.lower() == 'true'
+            elif key == '_crossdirect':
+                self.build_crossdirect = val.lower() == 'true'
             else:
                 continue
 

src/kupferbootstrap/progressbar.py

@@ -4,7 +4,7 @@ import sys
 from enlighten import Counter, Manager, get_manager as _getmanager
 from typing import Hashable, Optional
 
-from config.state import config
+from .config.state import config
 
 BAR_PADDING = 25
 DEFAULT_OUTPUT = sys.stderr

src/kupferbootstrap/utils.py

@@ -14,7 +14,7 @@ from dateutil.parser import parse as parsedate
 from shutil import which
 from typing import Any, Generator, IO, Optional, Union, Sequence
 
-from exec.cmd import run_cmd, run_root_cmd
+from .exec.cmd import run_cmd, run_root_cmd
 
 _programs_available = dict[str, bool]()
 
@@ -232,12 +232,12 @@ def color_mark_selected(
 
     marker_full = color_str(marker, use_colors=use_colors, **marker_config)
 
-    msg_items = (color_str(profile_name, use_colors=use_colors, **msg_item_colors),)
+    msg_items = [color_str(profile_name, use_colors=use_colors, **msg_item_colors), '']
     if inherited_from and inherited_from != profile_name:
-        msg_items = msg_items + (''.join([
+        msg_items[1] = ''.join([
             bold(' (inherited from profile "'),
             green(inherited_from, bold=True),
             bold('")'),
-        ]),)  # type: ignore[assignment]
-    output = f'{item}{suffix}{msg_fmt % msg_items}'
+        ])
+    output = f'{item}{suffix}{msg_fmt % tuple(msg_items)}'
     return '\n'.join([(marker_full + o) for o in output.split(split_on)])

src/kupferbootstrap/wrapper/__init__.py

@@ -3,9 +3,9 @@ import logging
 
 from typing import Optional, Sequence, Union
 
-from config.state import config
-from constants import Arch
-from utils import programs_available
+from kupferbootstrap.config.state import config
+from kupferbootstrap.constants import Arch
+from kupferbootstrap.utils import programs_available
 from .docker import DockerWrapper
 from .wrapper import Wrapper
 
@@ -57,10 +57,12 @@ def wrap_if_foreign_arch(arch: Arch):
 
 
 def execute_without_exit(f, argv_override: Optional[list[str]], *args, **kwargs):
-    """If no wrap is needed, executes and returns f(*args, **kwargs).
+    """
+    If no wrap is needed, executes and returns `f(*args, **kwargs)`.
     If a wrap is determined to be necessary, force a wrap with argv_override applied.
     If a wrap was forced, None is returned.
-    WARNING: No protection against f() returning None is taken."""
+    WARNING: No protection against f() returning None is taken.
+    """
     if not needs_wrap():
         return f(*args, **kwargs)
     assert get_wrapper_type() != 'none', "needs_wrap() should've returned False"

src/kupferbootstrap/wrapper/docker.py

@@ -4,11 +4,13 @@ import pathlib
 import subprocess
 import sys
 
-from config.state import config
-from exec.file import makedir
+from kupferbootstrap.config.state import config
+from kupferbootstrap.exec.file import makedir
 
 from .wrapper import Wrapper, WRAPPER_PATHS
 
+VERSION_FILE = "docker_version.txt"
+DOCKER_FILE = "Dockerfile"
 DOCKER_PATHS = WRAPPER_PATHS.copy()
 
 
@@ -26,10 +28,39 @@ class DockerWrapper(Wrapper):
         super().wrap()
         script_path = config.runtime.script_source_dir
         assert script_path
-        with open(os.path.join(script_path, 'version.txt')) as version_file:
-            version = version_file.read().replace('\n', '')
+        docker_path = script_path
+        tried = [docker_path]
+        if not os.path.exists(os.path.join(docker_path, DOCKER_FILE)):
+            docker_path = os.path.realpath(os.path.join(script_path, "../.."))
+            tried.append(docker_path)
+        if not os.path.exists(os.path.join(docker_path, DOCKER_FILE)):
+            _par_dir = os.path.dirname(script_path)
+            # handle venv
+            if os.path.basename(_par_dir) == "site-packages":
+                _path = os.path.join(_par_dir, "../../../..")
+                docker_path = os.path.realpath(_path)
+                tried.append(f"{_path} => {docker_path}")
+                logging.debug(f"{DOCKER_FILE!r} not found at {script_path!r}, trying {docker_path!r}")
+        version_file = os.path.join(script_path, '../..', VERSION_FILE)
+        if not os.path.exists(version_file):
+            _vfile = os.path.join(docker_path, VERSION_FILE)
+            logging.warning(f"{VERSION_FILE} not found at {version_file!r}."
+                            f"\nTrying {_vfile!r}"
+                            "\nDid you use `pip install .` instead of `pip install -e .`?")
+            if os.path.exists(_vfile):
+                version_file = _vfile
+        if os.path.exists(version_file):
+            with open(version_file) as fd:
+                version = fd.read().replace('\n', '').strip()
+            logging.debug(f"Read docker tag {version} from {version_file}")
+        else:
+            version = "BUILD"
+            logging.error(f"'{script_path}/{VERSION_FILE}' doesn't exist, defaulting docker tag to {version}!"
+                          "\nThis installation is potentially broken!"
+                          "\nDid you use `pip install .` instead of `pip install -e .` to install kupferboostrap?"
+                          f"Tried locations: {[version_file, _vfile]}")
         tag = f'registry.gitlab.com/kupfer/kupferbootstrap:{version}'
-        if version == 'dev':
+        if version == 'BUILD':
             logging.info(f'Building docker image "{tag}"')
             cmd = [
                 'docker',
@@ -38,11 +69,15 @@ class DockerWrapper(Wrapper):
                 '-t',
                 tag,
             ] + (['-q'] if not config.runtime.verbose else [])
-            logging.debug('Running docker cmd: ' + ' '.join(cmd))
+            _dfile = os.path.join(docker_path, DOCKER_FILE)
+            if not os.path.exists(_dfile):
+                _sep = "\n -"
+                raise Exception(f'{DOCKER_FILE!r} not found. Tried locations:' + (_sep.join(["", *[repr(f"{p}/{DOCKER_FILE}") for p in tried]])))
+            logging.debug(f'Running docker cmd (chdir={script_path!r}) : ' + ' '.join(cmd))
             mute_docker = not config.runtime.verbose
             result = subprocess.run(
                 cmd,
-                cwd=script_path,
+                cwd=docker_path,
                 capture_output=mute_docker,
             )
             if result.returncode != 0:

src/kupferbootstrap/wrapper/wrapper.py

@@ -5,9 +5,9 @@ import pathlib
 
 from typing import Optional, Protocol
 
-from config.state import config
-from config.state import dump_file as dump_config_file
-from constants import CHROOT_PATHS, WRAPPER_ENV_VAR
+from kupferbootstrap.config.state import config
+from kupferbootstrap.config.state import dump_file as dump_config_file
+from kupferbootstrap.constants import CHROOT_PATHS, WRAPPER_ENV_VAR
 
 WRAPPER_PATHS = CHROOT_PATHS | {
     'ccache': '/ccache',

version.txt

@@ -1 +0,0 @@
-dev

wrapper_su_helper.py

@@ -1,14 +1,14 @@
-#!/bin/python3
+#!/usr/bin/env python3
 
 import click
 import pwd
 import os
 
-from logger import logging, setup_logging
+from kupferbootstrap.logger import logging, setup_logging
 
-from constants import WRAPPER_ENV_VAR
-from exec.cmd import run_cmd, flatten_shell_script
-from exec.file import chown
+from kupferbootstrap.constants import WRAPPER_ENV_VAR
+from kupferbootstrap.exec.cmd import run_cmd, flatten_shell_script
+from kupferbootstrap.exec.file import chown
 
 
 @click.command('kupferbootstrap_su')