Add separate boot partition

Dockerfile

@@ -5,7 +5,8 @@ RUN pacman -Syu --noconfirm \
     arch-install-scripts rsync \
     aarch64-linux-gnu-gcc aarch64-linux-gnu-binutils aarch64-linux-gnu-glibc aarch64-linux-gnu-linux-api-headers \
     git \
-    android-tools openssh inetutils
+    android-tools openssh inetutils \
+    parted
 
 RUN sed -i "s/EUID == 0/EUID == -1/g" $(which makepkg)
 

constants.py

@@ -6,9 +6,8 @@ FLASH_PARTS = {
     'QHYPSTUB': 'qhypstub',
 }
 EMMC = 'emmc'
-EMMCFILE = 'emmc-file'
 MICROSD = 'microsd'
-LOCATIONS = [EMMC, EMMCFILE, MICROSD]
+LOCATIONS = [EMMC, MICROSD]
 
 JUMPDRIVE = 'jumpdrive'
 JUMPDRIVE_VERSION = '0.8'

flash.py

@@ -6,12 +6,10 @@ import click
 import tempfile
 
 from constants import FLASH_PARTS, LOCATIONS
-from fastboot import fastboot_flash
 from chroot import get_device_chroot
-from image import dump_bootimg, dump_lk2nd, dump_qhypstub, get_device_and_flavour, get_image_name, get_image_path
+from fastboot import fastboot_flash
+from image import shrink_fs, losetup_rootfs_image, dump_bootimg, dump_lk2nd, dump_qhypstub, get_device_and_flavour, get_image_name, get_image_path
 from wrapper import enforce_wrap
-from image import resize_fs
-from utils import mount
 
 BOOTIMG = FLASH_PARTS['BOOTIMG']
 LK2ND = FLASH_PARTS['LK2ND']
@@ -29,16 +27,23 @@ def cmd_flash(what, location):
     device_image_name = get_image_name(chroot)
     device_image_path = get_image_path(chroot)
 
+    # TODO: PARSE DEVICE SECTOR SIZE
+    sector_size = 4096
+
     if what not in FLASH_PARTS.values():
         raise Exception(f'Unknown what "{what}", must be one of {", ".join(FLASH_PARTS.values())}')
 
     if what == ROOTFS:
         if location is None:
             raise Exception(f'You need to specify a location to flash {what} to')
+
+        path = ''
+        if location.startswith("/dev/"):
+            path = location
+        else:
             if location not in LOCATIONS:
                 raise Exception(f'Invalid location {location}. Choose one of {", ".join(LOCATIONS)}')
 
-        path = ''
             dir = '/dev/disk/by-id'
             for file in os.listdir(dir):
                 sanitized_file = file.replace('-', '').replace('_', '').lower()
@@ -64,34 +69,9 @@ def cmd_flash(what, location):
 
         shutil.copyfile(device_image_path, minimal_image_path)
 
-        resize_fs(minimal_image_path, shrink=True)
+        loop_device = losetup_rootfs_image(minimal_image_path, sector_size)
+        shrink_fs(loop_device, minimal_image_path, sector_size)
 
-        if location.endswith('-file'):
-            part_mount = '/mnt/kupfer/fs'
-            if not os.path.exists(part_mount):
-                os.makedirs(part_mount)
-
-            result = mount(path, part_mount, options=[])
-            if result.returncode != 0:
-                raise Exception(f'Failed to mount {path} to {part_mount}')
-
-            dir = os.path.join(part_mount, '.stowaways')
-            if not os.path.exists(dir):
-                os.makedirs(dir)
-
-            result = subprocess.run([
-                'rsync',
-                '--archive',
-                '--inplace',
-                '--partial',
-                '--progress',
-                '--human-readable',
-                minimal_image_path,
-                os.path.join(dir, 'kupfer.img'),
-            ])
-            if result.returncode != 0:
-                raise Exception(f'Failed to mount {path} to {part_mount}')
-        else:
         result = subprocess.run([
             'dd',
             f'if={minimal_image_path}',
@@ -104,15 +84,16 @@ def cmd_flash(what, location):
         ])
         if result.returncode != 0:
             raise Exception(f'Failed to flash {minimal_image_path} to {path}')
-
+    else:
-    elif what == BOOTIMG:
+        loop_device = losetup_rootfs_image(device_image_path, sector_size)
-        path = dump_bootimg(device_image_path)
+        if what == BOOTIMG:
+            path = dump_bootimg(f'{loop_device}p1')
             fastboot_flash('boot', path)
         elif what == LK2ND:
-        path = dump_lk2nd(device_image_path)
+            path = dump_lk2nd(f'{loop_device}p1')
             fastboot_flash('lk2nd', path)
         elif what == QHYPSTUB:
-        path = dump_qhypstub(device_image_path)
+            path = dump_qhypstub(f'{loop_device}p1')
             fastboot_flash('qhypstub', path)
         else:
             raise Exception(f'Unknown what "{what}", this must be a bug in kupferbootstrap!')

image.py

@@ -1,9 +1,12 @@
+import atexit
+import json
 import os
+import re
 import subprocess
 import click
 from logger import logging
 from chroot import Chroot, get_device_chroot
-from constants import BASE_PACKAGES, DEVICES, FLAVOURS, Arch
+from constants import BASE_PACKAGES, DEVICES, FLAVOURS
 from config import config
 from distro import get_base_distro, get_kupfer_https, get_kupfer_local
 from ssh import copy_ssh_keys
@@ -11,24 +14,76 @@ from wrapper import enforce_wrap
 from signal import pause
 
 
-def resize_fs(image_path: str, shrink: bool = False):
+def shrink_fs(loop_device: str, file: str, sector_size: int):
-    result = subprocess.run([
+    # 8: 512 bytes sectors
-        'e2fsck',
+    # 1: 4096 bytes sectors
-        '-fy',
+    sectors_blocks_factor = 4096 // sector_size
-        image_path,
-    ])
-    # https://man7.org/linux/man-pages/man8/e2fsck.8.html#EXIT_CODE
-    if result.returncode > 2:
-        print(result.returncode)
-        msg = f'Failed to e2fsck {image_path}'
-        if shrink:
-            raise Exception(msg)
-        else:
-            logging.warning(msg)
 
-    result = subprocess.run(['resize2fs'] + (['-M'] if shrink else []) + [image_path])
+    logging.debug(f"Checking filesystem at {loop_device}p2")
+    result = subprocess.run(['e2fsck', '-fy', f'{loop_device}p2'])
+    if result.returncode > 2:
+        # https://man7.org/linux/man-pages/man8/e2fsck.8.html#EXIT_CODE
+        raise Exception(f'Failed to e2fsck {loop_device}p2 with exit code {result.returncode}')
+
+    logging.debug(f'Shrinking filesystem at {loop_device}p2')
+    result = subprocess.run(['resize2fs', '-M', f'{loop_device}p2'], capture_output=True)
     if result.returncode != 0:
-        raise Exception(f'Failed to resize2fs {image_path}')
+        print(result.stdout)
+        print(result.stderr)
+        raise Exception(f'Failed to resize2fs {loop_device}p2')
+
+    logging.debug(f'Finding end block of shrunken filesystem on {loop_device}p2')
+    blocks = int(re.search('is now [0-9]+', result.stdout.decode('utf-8')).group(0).split(' ')[2])
+    sectors = blocks * sectors_blocks_factor  #+ 157812 - 25600
+
+    logging.debug(f'Shrinking partition at {loop_device}p2 to {sectors} sectors')
+    child_proccess = subprocess.Popen(
+        ['fdisk', '-b', str(sector_size), loop_device],
+        stdin=subprocess.PIPE,
+    )
+    child_proccess.stdin.write('\n'.join([
+        'd',
+        '2',
+        'n',
+        'p',
+        '2',
+        '',
+        f'+{sectors}',
+        'w',
+        'q',
+    ]).encode('utf-8'))
+
+    child_proccess.communicate()
+
+    returncode = child_proccess.wait()
+    if returncode == 1:
+        # For some reason re-reading the partition table fails, but that is not a problem
+        subprocess.run(['partprobe'])
+    if returncode > 1:
+        raise Exception(f'Failed to shrink partition size of {loop_device}p2 with fdisk')
+
+    logging.debug(f'Finding end sector of partition at {loop_device}p2')
+    result = subprocess.run(['fdisk', '-b', str(sector_size), '-l', loop_device], capture_output=True)
+    if result.returncode != 0:
+        print(result.stdout)
+        print(result.stderr)
+        raise Exception(f'Failed to fdisk -l {loop_device}')
+
+    end_sector = 0
+    for line in result.stdout.decode('utf-8').split('\n'):
+        if line.startswith(f'{loop_device}p2'):
+            parts = list(filter(lambda part: part != '', line.split(' ')))
+            end_sector = int(parts[2])
+
+    if end_sector == 0:
+        raise Exception(f'Failed to find end sector of {loop_device}p2')
+
+    end_block = end_sector // sectors_blocks_factor
+
+    logging.debug(f'Truncating {file} to {end_block} blocks')
+    result = subprocess.run(['truncate', '-o', '-s', str(end_block), file])
+    if result.returncode != 0:
+        raise Exception(f'Failed to truncate {file}')
 
 
 def get_device_and_flavour(profile: str = None) -> tuple[str, str]:
@@ -51,13 +106,73 @@ def get_image_path(device_chroot: Chroot) -> str:
     return os.path.join(config.get_path('images'), get_image_name(device_chroot))
 
 
+def losetup_rootfs_image(image_path: str, sector_size: int) -> str:
+    logging.debug(f'Creating loop device for {image_path}')
+    result = subprocess.run([
+        'losetup',
+        '-f',
+        '-b',
+        str(sector_size),
+        image_path,
+    ])
+    if result.returncode != 0:
+        logging.fatal(f'Failed create loop device for {image_path}')
+        exit(1)
+
+    logging.debug(f'Finding loop device for {image_path}')
+
+    result = subprocess.run(['losetup', '-J'], capture_output=True)
+    if result.returncode != 0:
+        print(result.stdout)
+        print(result.stderr)
+        logging.fatal('Failed to list loop devices')
+        exit(1)
+
+    data = json.loads(result.stdout.decode('utf-8'))
+    loop_device = ''
+    for d in data['loopdevices']:
+        if d['back-file'] == image_path:
+            loop_device = d['name']
+            break
+
+    if loop_device == '':
+        raise Exception(f'Failed to find loop device for {image_path}')
+
+    def losetup_destroy():
+        logging.debug(f'Destroying loop device {loop_device} for {image_path}')
+        subprocess.run(
+            [
+                'losetup',
+                '-d',
+                loop_device,
+            ],
+            stderr=subprocess.DEVNULL,
+        )
+
+    atexit.register(losetup_destroy)
+
+    return loop_device
+
+
+def mount_rootfs_loop_device(loop_device, chroot: Chroot):
+    logging.debug(f'Mounting {loop_device}p2 at {chroot.path}')
+
+    chroot.mount_rootfs(loop_device + 'p2')
+
+    if not os.path.exists(f'{chroot.path}/boot'):
+        os.makedirs(f'{chroot.path}/boot')
+
+    logging.debug(f'Mounting {loop_device}p1 at {chroot.path}/boot')
+    chroot.mount(loop_device + 'p1', '/boot')
+
+
 def dump_bootimg(image_path: str) -> str:
     path = '/tmp/boot.img'
     result = subprocess.run([
         'debugfs',
         image_path,
         '-R',
-        f'dump /boot/boot.img {path}',
+        f'dump /boot.img {path}',
     ])
     if result.returncode != 0:
         logging.fatal('Failed to dump boot.img')
@@ -74,7 +189,7 @@ def dump_lk2nd(image_path: str) -> str:
         'debugfs',
         image_path,
         '-R',
-        f'dump /boot/lk2nd.img {path}',
+        f'dump /lk2nd.img {path}',
     ])
     if result.returncode != 0:
         logging.fatal('Failed to dump lk2nd.img')
@@ -88,7 +203,7 @@ def dump_qhypstub(image_path: str) -> str:
         'debugfs',
         image_path,
         '-R',
-        f'dump /boot/qhypstub.bin {path}',
+        f'dump /qhypstub.bin {path}',
     ])
     if result.returncode != 0:
         logging.fatal('Failed to dump qhypstub.bin')
@@ -108,8 +223,9 @@ def cmd_build():
     device, flavour = get_device_and_flavour()
     post_cmds = FLAVOURS[flavour].get('post_cmds', [])
 
-    # TODO: PARSE DEVICE ARCH
+    # TODO: PARSE DEVICE ARCH AND SECTOR SIZE
-    arch: Arch = 'aarch64'
+    arch = 'aarch64'
+    sector_size = 4096
 
     packages_dir = config.get_package_dir(arch)
     if os.path.exists(os.path.join(packages_dir, 'main')):
@@ -121,26 +237,58 @@ def cmd_build():
     chroot = get_device_chroot(device=device, flavour=flavour, arch=arch, packages=packages, extra_repos=extra_repos)
     image_path = get_image_path(chroot)
 
-    if not os.path.exists(image_path):
+    new_image = not os.path.exists(image_path)
+    if new_image:
         result = subprocess.run([
-            'fallocate',
+            'truncate',
-            '-l',
+            '-s',
             f"{FLAVOURS[flavour].get('size',2)}G",
             image_path,
         ])
         if result.returncode != 0:
             raise Exception(f'Failed to allocate {image_path}')
 
+    loop_device = losetup_rootfs_image(image_path, sector_size)
+
+    if new_image:
+        boot_partition_size = '100MiB'
+        create_partition_table = ['mklabel', 'msdos']
+        create_boot_partition = ['mkpart', 'primary', 'ext2', '0%', boot_partition_size]
+        create_root_partition = ['mkpart', 'primary', boot_partition_size, '100%']
+        enable_boot = ['set', '1', 'boot', 'on']
         result = subprocess.run([
-            'mkfs.ext4',
+            'parted',
+            '--script',
+            loop_device,
+        ] + create_partition_table + create_boot_partition + create_root_partition + enable_boot)
+        if result.returncode != 0:
+            raise Exception(f'Failed to create partitions on {loop_device}')
+
+        result = subprocess.run([
+            'mkfs.ext2',
+            '-F',
             '-L',
-            'kupfer',
+            'kupfer_boot',
-            image_path,
+            f'{loop_device}p1',
         ])
         if result.returncode != 0:
-            raise Exception(f'Failed to create ext4 filesystem on {image_path}')
+            raise Exception(f'Failed to create ext2 filesystem on {loop_device}p1')
-    else:
+
-        resize_fs(image_path=image_path)
+        result = subprocess.run([
+            'mkfs.ext4',
+            '-O',
+            '^metadata_csum',
+            '-F',
+            '-L',
+            'kupfer_root',
+            '-N',
+            '100000',
+            f'{loop_device}p2',
+        ])
+        if result.returncode != 0:
+            raise Exception(f'Failed to create ext4 filesystem on {loop_device}p2')
+
+    mount_rootfs_loop_device(loop_device, chroot)
 
     chroot.mount_rootfs(image_path)
     chroot.initialize()
@@ -168,10 +316,13 @@ def cmd_inspect(shell: bool = False):
     device, flavour = get_device_and_flavour()
     # TODO: get arch from profile
     arch = 'aarch64'
+    # TODO: PARSE DEVICE SECTOR SIZE
+    sector_size = 4096
     chroot = get_device_chroot(device, flavour, arch)
     image_path = get_image_path(chroot)
+    loop_device = losetup_rootfs_image(image_path, sector_size)
 
-    chroot.mount_rootfs(image_path)
+    mount_rootfs_loop_device(loop_device, chroot)
 
     logging.info(f'Inspect the rootfs image at {chroot.path}')