Add separate boot partition

Dockerfile

@@ -5,7 +5,8 @@ RUN pacman -Syu --noconfirm \
     arch-install-scripts rsync \
     aarch64-linux-gnu-gcc aarch64-linux-gnu-binutils aarch64-linux-gnu-glibc aarch64-linux-gnu-linux-api-headers \
     git \
-    android-tools openssh inetutils
+    android-tools openssh inetutils \
+    parted
 
 RUN sed -i "s/EUID == 0/EUID == -1/g" $(which makepkg)
 

boot.py

@@ -28,9 +28,9 @@ def cmd_boot(type):
             if not os.path.exists(path):
                 urllib.request.urlretrieve(f'https://github.com/dreemurrs-embedded/Jumpdrive/releases/download/{JUMPDRIVE_VERSION}/{file}', path)
         elif type == LK2ND:
-            path = dump_lk2nd(image_name)
+            path = dump_lk2nd(os.path.join('/images', image_name))
         elif type == BOOTIMG:
-            path = dump_bootimg(image_name)
+            path = dump_bootimg(os.path.join('/images', image_name))
         else:
             raise Exception(f'Unknown boot image type {type}')
         fastboot_erase_dtbo()

cache.py

@@ -5,7 +5,7 @@ from config import config
 from wrapper import enforce_wrap
 import logging
 
-PATHS = ['chroots', 'pacman', 'jumpdrive', 'packages']
+PATHS = ['chroots', 'pacman', 'jumpdrive', 'packages', 'images']
 
 
 @click.group(name='cache')

config.py

@@ -43,6 +43,7 @@ CONFIG_DEFAULTS = {
         'packages': os.path.join('%cache_dir%', 'packages'),
         'pkgbuilds': os.path.join('%cache_dir%', 'pkgbuilds'),
         'jumpdrive': os.path.join('%cache_dir%', 'jumpdrive'),
+        'images': os.path.join('%cache_dir%', 'images'),
     },
     'profiles': {
         'current': 'default',

constants.py

@@ -6,9 +6,8 @@ FLASH_PARTS = {
     'QHYPSTUB': 'qhypstub',
 }
 EMMC = 'emmc'
-EMMCFILE = 'emmc-file'
 MICROSD = 'microsd'
-LOCATIONS = [EMMC, EMMCFILE, MICROSD]
+LOCATIONS = [EMMC, MICROSD]
 
 JUMPDRIVE = 'jumpdrive'
 JUMPDRIVE_VERSION = '0.8'

flash.py

@@ -2,13 +2,13 @@ import atexit
 from constants import FLASH_PARTS, LOCATIONS
 from fastboot import fastboot_flash
 import shutil
-from image import dump_bootimg, dump_lk2nd, dump_qhypstub, get_device_and_flavour, get_image_name
+from image import dump_bootimg, dump_lk2nd, dump_qhypstub, get_device_and_flavour, get_image_name, losetup_rootfs_image
 import os
 import subprocess
 import click
 import tempfile
 from wrapper import enforce_wrap
-from image import resize_fs
+from image import shrink_fs
 
 BOOTIMG = FLASH_PARTS['BOOTIMG']
 LK2ND = FLASH_PARTS['LK2ND']
@@ -24,30 +24,37 @@ def cmd_flash(what, location):
     device, flavour = get_device_and_flavour()
     image_name = get_image_name(device, flavour)
 
+    # TODO: PARSE DEVICE SECTOR SIZE
+    sector_size = 4096
+
     if what not in FLASH_PARTS.values():
         raise Exception(f'Unknown what "{what}", must be one of {", ".join(FLASH_PARTS.values())}')
 
     if what == ROOTFS:
         if location is None:
             raise Exception(f'You need to specify a location to flash {what} to')
-        if location not in LOCATIONS:
-            raise Exception(f'Invalid location {location}. Choose one of {", ".join(LOCATIONS)}')
 
         path = ''
-        dir = '/dev/disk/by-id'
+        if location.startswith("/dev/"):
-        for file in os.listdir(dir):
+            path = location
-            sanitized_file = file.replace('-', '').replace('_', '').lower()
+        else:
-            if f'jumpdrive{location.split("-")[0]}' in sanitized_file:
+            if location not in LOCATIONS:
-                path = os.path.realpath(os.path.join(dir, file))
+                raise Exception(f'Invalid location {location}. Choose one of {", ".join(LOCATIONS)}')
-                result = subprocess.run(['lsblk', path, '-o', 'SIZE'], capture_output=True)
+
-                if result.returncode != 0:
+            dir = '/dev/disk/by-id'
-                    raise Exception(f'Failed to lsblk {path}')
+            for file in os.listdir(dir):
-                if result.stdout == b'SIZE\n  0B\n':
+                sanitized_file = file.replace('-', '').replace('_', '').lower()
-                    raise Exception(
+                if f'jumpdrive{location.split("-")[0]}' in sanitized_file:
-                        f'Disk {path} has a size of 0B. That probably means it is not available (e.g. no microSD inserted or no microSD card slot installed in the device) or corrupt or defect'
+                    path = os.path.realpath(os.path.join(dir, file))
-                    )
+                    result = subprocess.run(['lsblk', path, '-o', 'SIZE'], capture_output=True)
-        if path == '':
+                    if result.returncode != 0:
-            raise Exception('Unable to discover Jumpdrive')
+                        raise Exception(f'Failed to lsblk {path}')
+                    if result.stdout == b'SIZE\n  0B\n':
+                        raise Exception(
+                            f'Disk {path} has a size of 0B. That probably means it is not available (e.g. no microSD inserted or no microSD card slot installed in the device) or corrupt or defect'
+                        )
+            if path == '':
+                raise Exception('Unable to discover Jumpdrive')
 
         image_dir = tempfile.gettempdir()
         image_path = os.path.join(image_dir, f'minimal-{image_name}')
@@ -57,73 +64,33 @@ def cmd_flash(what, location):
 
         atexit.register(clean_dir)
 
-        shutil.copyfile(image_name, image_path)
+        shutil.copyfile(os.path.join('/images', image_name), image_path)
 
-        resize_fs(image_path, shrink=True)
+        loop_device = losetup_rootfs_image(image_path, sector_size)
+        shrink_fs(loop_device, image_path, sector_size)
 
-        if location.endswith('-file'):
+        result = subprocess.run([
-            part_mount = '/mnt/kupfer/fs'
+            'dd',
-            if not os.path.exists(part_mount):
+            f'if={image_path}',
-                os.makedirs(part_mount)
+            f'of={path}',
-
+            'bs=20M',
-            def umount():
+            'iflag=direct',
-                subprocess.run(
+            'oflag=direct',
-                    [
+            'status=progress',
-                        'umount',
+            'conv=sync,noerror',
-                        '-lc',
+        ])
-                        part_mount,
+        if result.returncode != 0:
-                    ],
+            raise Exception(f'Failed to flash {image_path} to {path}')
-                    stderr=subprocess.DEVNULL,
-                )
-
-            atexit.register(umount)
-
-            result = subprocess.run([
-                'mount',
-                path,
-                part_mount,
-            ])
-            if result.returncode != 0:
-                raise Exception(f'Failed to mount {path} to {part_mount}')
-
-            dir = os.path.join(part_mount, '.stowaways')
-            if not os.path.exists(dir):
-                os.makedirs(dir)
-
-            result = subprocess.run([
-                'rsync',
-                '--archive',
-                '--inplace',
-                '--partial',
-                '--progress',
-                '--human-readable',
-                image_path,
-                os.path.join(dir, 'kupfer.img'),
-            ])
-            if result.returncode != 0:
-                raise Exception(f'Failed to mount {path} to {part_mount}')
-        else:
-            result = subprocess.run([
-                'dd',
-                f'if={image_path}',
-                f'of={path}',
-                'bs=20M',
-                'iflag=direct',
-                'oflag=direct',
-                'status=progress',
-                'conv=sync,noerror',
-            ])
-            if result.returncode != 0:
-                raise Exception(f'Failed to flash {image_path} to {path}')
-
-    elif what == BOOTIMG:
-        path = dump_bootimg(image_name)
-        fastboot_flash('boot', path)
-    elif what == LK2ND:
-        path = dump_lk2nd(image_name)
-        fastboot_flash('lk2nd', path)
-    elif what == QHYPSTUB:
-        path = dump_qhypstub(image_name)
-        fastboot_flash('qhypstub', path)
     else:
-        raise Exception(f'Unknown what "{what}", this must be a bug in kupferbootstrap!')
+        loop_device = losetup_rootfs_image(os.path.join('/images', image_name), sector_size)
+        if what == BOOTIMG:
+            path = dump_bootimg(f'{loop_device}p1')
+            fastboot_flash('boot', path)
+        elif what == LK2ND:
+            path = dump_lk2nd(f'{loop_device}p1')
+            fastboot_flash('lk2nd', path)
+        elif what == QHYPSTUB:
+            path = dump_qhypstub(f'{loop_device}p1')
+            fastboot_flash('qhypstub', path)
+        else:
+            raise Exception(f'Unknown what "{what}", this must be a bug in kupferbootstrap!')

image.py

@@ -1,5 +1,7 @@
 import atexit
+import json
 import os
+import re
 import subprocess
 import click
 from logger import logging
@@ -12,24 +14,76 @@ from wrapper import enforce_wrap
 from signal import pause
 
 
-def resize_fs(image_path: str, shrink: bool = False):
+def shrink_fs(loop_device: str, file: str, sector_size: int):
-    result = subprocess.run([
+    # 8: 512 bytes sectors
-        'e2fsck',
+    # 1: 4096 bytes sectors
-        '-fy',
+    sectors_blocks_factor = 4096 // sector_size
-        image_path,
-    ])
-    # https://man7.org/linux/man-pages/man8/e2fsck.8.html#EXIT_CODE
-    if result.returncode > 2:
-        print(result.returncode)
-        msg = f'Failed to e2fsck {image_path}'
-        if shrink:
-            raise Exception(msg)
-        else:
-            logging.warning(msg)
 
-    result = subprocess.run(['resize2fs'] + (['-M'] if shrink else []) + [image_path])
+    logging.debug(f"Checking filesystem at {loop_device}p2")
+    result = subprocess.run(['e2fsck', '-fy', f'{loop_device}p2'])
+    if result.returncode > 2:
+        # https://man7.org/linux/man-pages/man8/e2fsck.8.html#EXIT_CODE
+        raise Exception(f'Failed to e2fsck {loop_device}p2 with exit code {result.returncode}')
+
+    logging.debug(f'Shrinking filesystem at {loop_device}p2')
+    result = subprocess.run(['resize2fs', '-M', f'{loop_device}p2'], capture_output=True)
     if result.returncode != 0:
-        raise Exception(f'Failed to resize2fs {image_path}')
+        print(result.stdout)
+        print(result.stderr)
+        raise Exception(f'Failed to resize2fs {loop_device}p2')
+
+    logging.debug(f'Finding end block of shrunken filesystem on {loop_device}p2')
+    blocks = int(re.search('is now [0-9]+', result.stdout.decode('utf-8')).group(0).split(' ')[2])
+    sectors = blocks * sectors_blocks_factor  #+ 157812 - 25600
+
+    logging.debug(f'Shrinking partition at {loop_device}p2 to {sectors} sectors')
+    child_proccess = subprocess.Popen(
+        ['fdisk', '-b', str(sector_size), loop_device],
+        stdin=subprocess.PIPE,
+    )
+    child_proccess.stdin.write('\n'.join([
+        'd',
+        '2',
+        'n',
+        'p',
+        '2',
+        '',
+        f'+{sectors}',
+        'w',
+        'q',
+    ]).encode('utf-8'))
+
+    child_proccess.communicate()
+
+    returncode = child_proccess.wait()
+    if returncode == 1:
+        # For some reason re-reading the partition table fails, but that is not a problem
+        subprocess.run(['partprobe'])
+    if returncode > 1:
+        raise Exception(f'Failed to shrink partition size of {loop_device}p2 with fdisk')
+
+    logging.debug(f'Finding end sector of partition at {loop_device}p2')
+    result = subprocess.run(['fdisk', '-b', str(sector_size), '-l', loop_device], capture_output=True)
+    if result.returncode != 0:
+        print(result.stdout)
+        print(result.stderr)
+        raise Exception(f'Failed to fdisk -l {loop_device}')
+
+    end_sector = 0
+    for line in result.stdout.decode('utf-8').split('\n'):
+        if line.startswith(f'{loop_device}p2'):
+            parts = list(filter(lambda part: part != '', line.split(' ')))
+            end_sector = int(parts[2])
+
+    if end_sector == 0:
+        raise Exception(f'Failed to find end sector of {loop_device}p2')
+
+    end_block = end_sector // sectors_blocks_factor
+
+    logging.debug(f'Truncating {file} to {end_block} blocks')
+    result = subprocess.run(['truncate', '-o', '-s', str(end_block), file])
+    if result.returncode != 0:
+        raise Exception(f'Failed to truncate {file}')
 
 
 def get_device_and_flavour(profile: str = None) -> tuple[str, str]:
@@ -48,11 +102,65 @@ def get_image_name(device, flavour) -> str:
     return f'{device}-{flavour}-rootfs.img'
 
 
-def mount_rootfs_image(image_path, mount_path):
+def losetup_rootfs_image(image_path: str, sector_size: int) -> str:
-    if not os.path.exists(mount_path):
+    logging.debug(f'Creating loop device for {image_path}')
-        os.makedirs(mount_path)
+    result = subprocess.run([
+        'losetup',
+        '-f',
+        '-b',
+        str(sector_size),
+        image_path,
+    ])
+    if result.returncode != 0:
+        logging.fatal(f'Failed create loop device for {image_path}')
+        exit(1)
+
+    logging.debug(f'Finding loop device for {image_path}')
+
+    result = subprocess.run(['losetup', '-J'], capture_output=True)
+    if result.returncode != 0:
+        print(result.stdout)
+        print(result.stderr)
+        logging.fatal('Failed to list loop devices')
+        exit(1)
+
+    data = json.loads(result.stdout.decode('utf-8'))
+    loop_device = ''
+    for d in data['loopdevices']:
+        if d['back-file'] == image_path:
+            loop_device = d['name']
+            break
+
+    if loop_device == '':
+        raise Exception(f'Failed to find loop device for {image_path}')
+
+    def losetup_destroy():
+        logging.debug(f'Destroying loop device {loop_device} for {image_path}')
+        subprocess.run(
+            [
+                'losetup',
+                '-d',
+                loop_device,
+            ],
+            stderr=subprocess.DEVNULL,
+        )
+
+    atexit.register(losetup_destroy)
+
+    return loop_device
+
+
+def mount_rootfs_loop_device(loop_device, mount_path):
 
     def umount():
+        subprocess.run(
+            [
+                'umount',
+                '-lc',
+                f'{mount_path}/boot',
+            ],
+            stderr=subprocess.DEVNULL,
+        )
         subprocess.run(
             [
                 'umount',
@@ -64,15 +172,36 @@ def mount_rootfs_image(image_path, mount_path):
 
     atexit.register(umount)
 
+    if not os.path.exists(mount_path):
+        os.makedirs(mount_path)
+
+    logging.debug(f'Mounting {loop_device}p2 at {mount_path}')
+
     result = subprocess.run([
         'mount',
         '-o',
         'loop',
-        image_path,
+        f'{loop_device}p2',
         mount_path,
     ])
     if result.returncode != 0:
-        logging.fatal(f'Failed to loop mount {image_path} to {mount_path}')
+        logging.fatal(f'Failed to loop mount {loop_device}p2 to {mount_path}')
+        exit(1)
+
+    if not os.path.exists(f'{mount_path}/boot'):
+        os.makedirs(f'{mount_path}/boot')
+
+    logging.debug(f'Mounting {loop_device}p1 at {mount_path}/boot')
+
+    result = subprocess.run([
+        'mount',
+        '-o',
+        'loop',
+        f'{loop_device}p1',
+        f'{mount_path}/boot',
+    ])
+    if result.returncode != 0:
+        logging.fatal(f'Failed to loop mount {loop_device}p1 to {mount_path}/boot')
         exit(1)
 
 
@@ -82,7 +211,7 @@ def dump_bootimg(image_name: str) -> str:
         'debugfs',
         image_name,
         '-R',
-        f'dump /boot/boot.img {path}',
+        f'dump /boot.img {path}',
     ])
     if result.returncode != 0:
         logging.fatal('Failed to dump boot.img')
@@ -99,7 +228,7 @@ def dump_lk2nd(image_name: str) -> str:
         'debugfs',
         image_name,
         '-R',
-        f'dump /boot/lk2nd.img {path}',
+        f'dump /lk2nd.img {path}',
     ])
     if result.returncode != 0:
         logging.fatal('Failed to dump lk2nd.img')
@@ -113,7 +242,7 @@ def dump_qhypstub(image_name: str) -> str:
         'debugfs',
         image_name,
         '-R',
-        f'dump /boot/qhypstub.bin {path}',
+        f'dump /qhypstub.bin {path}',
     ])
     if result.returncode != 0:
         logging.fatal('Failed to dump qhypstub.bin')
@@ -132,35 +261,66 @@ def cmd_build():
     profile = config.get_profile()
     device, flavour = get_device_and_flavour()
     post_cmds = FLAVOURS[flavour].get('post_cmds', [])
-    image_name = get_image_name(device, flavour)
+    image_name = os.path.join('/images', get_image_name(device, flavour))
 
-    # TODO: PARSE DEVICE ARCH
+    # TODO: PARSE DEVICE ARCH AND SECTOR SIZE
     arch = 'aarch64'
+    sector_size = 4096
 
-    if not os.path.exists(image_name):
+    new_image = not os.path.exists(image_name)
+    if new_image:
         result = subprocess.run([
-            'fallocate',
+            'truncate',
-            '-l',
+            '-s',
             f"{FLAVOURS[flavour].get('size',2)}G",
             image_name,
         ])
         if result.returncode != 0:
             raise Exception(f'Failed to allocate {image_name}')
 
+    loop_device = losetup_rootfs_image(image_name, sector_size)
+
+    if new_image:
+        boot_partition_size = '100MiB'
+        create_partition_table = ['mklabel', 'msdos']
+        create_boot_partition = ['mkpart', 'primary', 'ext2', '0%', boot_partition_size]
+        create_root_partition = ['mkpart', 'primary', boot_partition_size, '100%']
+        enable_boot = ['set', '1', 'boot', 'on']
         result = subprocess.run([
-            'mkfs.ext4',
+            'parted',
+            '--script',
+            loop_device,
+        ] + create_partition_table + create_boot_partition + create_root_partition + enable_boot)
+        if result.returncode != 0:
+            raise Exception(f'Failed to create partitions on {loop_device}')
+
+        result = subprocess.run([
+            'mkfs.ext2',
+            '-F',
             '-L',
-            'kupfer',
+            'kupfer_boot',
-            image_name,
+            f'{loop_device}p1',
         ])
         if result.returncode != 0:
-            raise Exception(f'Failed to create ext4 filesystem on {image_name}')
+            raise Exception(f'Failed to create ext2 filesystem on {loop_device}p1')
-    else:
+
-        resize_fs(image_path=image_name)
+        result = subprocess.run([
+            'mkfs.ext4',
+            '-O',
+            '^metadata_csum',
+            '-F',
+            '-L',
+            'kupfer_root',
+            '-N',
+            '100000',
+            f'{loop_device}p2',
+        ])
+        if result.returncode != 0:
+            raise Exception(f'Failed to create ext4 filesystem on {loop_device}p2')
 
     chroot_name = f'rootfs_{device}-{flavour}'
     rootfs_mount = get_chroot_path(chroot_name)
-    mount_rootfs_image(image_name, rootfs_mount)
+    mount_rootfs_loop_device(loop_device, rootfs_mount)
 
     packages_dir = config.get_package_dir(arch)
     if os.path.exists(os.path.join(packages_dir, 'main')):
@@ -199,8 +359,12 @@ def cmd_inspect():
     device, flavour = get_device_and_flavour()
     image_name = get_image_name(device, flavour)
 
+    # TODO: PARSE DEVICE SECTOR SIZE
+    sector_size = 4096
+
     rootfs_mount = get_chroot_path(f'rootfs_{device}-{flavour}')
-    mount_rootfs_image(image_name, rootfs_mount)
+    loop_device = losetup_rootfs_image(image_name, sector_size)
+    mount_rootfs_loop_device(loop_device, rootfs_mount)
 
     logging.info(f'Inspect the rootfs image at {rootfs_mount}')
 

wrapper.py

@@ -15,6 +15,7 @@ DOCKER_PATHS = {
     'pacman': '/var/cache/pacman',
     'packages': '/prebuilts',
     'pkgbuilds': '/pkgbuilds',
+    'images': '/images',
 }