From 955546c9186394853ce9dc2bedddfeee57ccd747 Mon Sep 17 00:00:00 2001 From: jld3103 Date: Fri, 22 Oct 2021 17:07:05 +0200 Subject: [PATCH] Add separate boot partition --- Dockerfile | 3 +- constants.py | 3 +- flash.py | 121 ++++++++++++----------------- image.py | 215 +++++++++++++++++++++++++++++++++++++++++++-------- 4 files changed, 237 insertions(+), 105 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4b6ef46..86c3ca8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,8 @@ RUN pacman -Syu --noconfirm \ arch-install-scripts rsync \ aarch64-linux-gnu-gcc aarch64-linux-gnu-binutils aarch64-linux-gnu-glibc aarch64-linux-gnu-linux-api-headers \ git \ - android-tools openssh inetutils + android-tools openssh inetutils \ + parted RUN sed -i "s/EUID == 0/EUID == -1/g" $(which makepkg) diff --git a/constants.py b/constants.py index 85b9ea3..8ce4a26 100644 --- a/constants.py +++ b/constants.py @@ -6,9 +6,8 @@ FLASH_PARTS = { 'QHYPSTUB': 'qhypstub', } EMMC = 'emmc' -EMMCFILE = 'emmc-file' MICROSD = 'microsd' -LOCATIONS = [EMMC, EMMCFILE, MICROSD] +LOCATIONS = [EMMC, MICROSD] JUMPDRIVE = 'jumpdrive' JUMPDRIVE_VERSION = '0.8' diff --git a/flash.py b/flash.py index e53022e..eeecc0f 100644 --- a/flash.py +++ b/flash.py @@ -6,12 +6,10 @@ import click import tempfile from constants import FLASH_PARTS, LOCATIONS -from fastboot import fastboot_flash from chroot import get_device_chroot -from image import dump_bootimg, dump_lk2nd, dump_qhypstub, get_device_and_flavour, get_image_name, get_image_path +from fastboot import fastboot_flash +from image import shrink_fs, losetup_rootfs_image, dump_bootimg, dump_lk2nd, dump_qhypstub, get_device_and_flavour, get_image_name, get_image_path from wrapper import enforce_wrap -from image import resize_fs -from utils import mount BOOTIMG = FLASH_PARTS['BOOTIMG'] LK2ND = FLASH_PARTS['LK2ND'] @@ -29,30 +27,37 @@ def cmd_flash(what, location): device_image_name = get_image_name(chroot) device_image_path = get_image_path(chroot) + # TODO: PARSE DEVICE SECTOR SIZE + sector_size = 4096 + if what not in FLASH_PARTS.values(): raise Exception(f'Unknown what "{what}", must be one of {", ".join(FLASH_PARTS.values())}') if what == ROOTFS: if location is None: raise Exception(f'You need to specify a location to flash {what} to') - if location not in LOCATIONS: - raise Exception(f'Invalid location {location}. Choose one of {", ".join(LOCATIONS)}') path = '' - dir = '/dev/disk/by-id' - for file in os.listdir(dir): - sanitized_file = file.replace('-', '').replace('_', '').lower() - if f'jumpdrive{location.split("-")[0]}' in sanitized_file: - path = os.path.realpath(os.path.join(dir, file)) - result = subprocess.run(['lsblk', path, '-o', 'SIZE'], capture_output=True) - if result.returncode != 0: - raise Exception(f'Failed to lsblk {path}') - if result.stdout == b'SIZE\n 0B\n': - raise Exception( - f'Disk {path} has a size of 0B. That probably means it is not available (e.g. no microSD inserted or no microSD card slot installed in the device) or corrupt or defect' - ) - if path == '': - raise Exception('Unable to discover Jumpdrive') + if location.startswith("/dev/"): + path = location + else: + if location not in LOCATIONS: + raise Exception(f'Invalid location {location}. Choose one of {", ".join(LOCATIONS)}') + + dir = '/dev/disk/by-id' + for file in os.listdir(dir): + sanitized_file = file.replace('-', '').replace('_', '').lower() + if f'jumpdrive{location.split("-")[0]}' in sanitized_file: + path = os.path.realpath(os.path.join(dir, file)) + result = subprocess.run(['lsblk', path, '-o', 'SIZE'], capture_output=True) + if result.returncode != 0: + raise Exception(f'Failed to lsblk {path}') + if result.stdout == b'SIZE\n 0B\n': + raise Exception( + f'Disk {path} has a size of 0B. That probably means it is not available (e.g. no microSD inserted or no microSD card slot installed in the device) or corrupt or defect' + ) + if path == '': + raise Exception('Unable to discover Jumpdrive') minimal_image_dir = tempfile.gettempdir() minimal_image_path = os.path.join(minimal_image_dir, f'minimal-{device_image_name}') @@ -64,55 +69,31 @@ def cmd_flash(what, location): shutil.copyfile(device_image_path, minimal_image_path) - resize_fs(minimal_image_path, shrink=True) + loop_device = losetup_rootfs_image(minimal_image_path, sector_size) + shrink_fs(loop_device, minimal_image_path, sector_size) - if location.endswith('-file'): - part_mount = '/mnt/kupfer/fs' - if not os.path.exists(part_mount): - os.makedirs(part_mount) - - result = mount(path, part_mount, options=[]) - if result.returncode != 0: - raise Exception(f'Failed to mount {path} to {part_mount}') - - dir = os.path.join(part_mount, '.stowaways') - if not os.path.exists(dir): - os.makedirs(dir) - - result = subprocess.run([ - 'rsync', - '--archive', - '--inplace', - '--partial', - '--progress', - '--human-readable', - minimal_image_path, - os.path.join(dir, 'kupfer.img'), - ]) - if result.returncode != 0: - raise Exception(f'Failed to mount {path} to {part_mount}') - else: - result = subprocess.run([ - 'dd', - f'if={minimal_image_path}', - f'of={path}', - 'bs=20M', - 'iflag=direct', - 'oflag=direct', - 'status=progress', - 'conv=sync,noerror', - ]) - if result.returncode != 0: - raise Exception(f'Failed to flash {minimal_image_path} to {path}') - - elif what == BOOTIMG: - path = dump_bootimg(device_image_path) - fastboot_flash('boot', path) - elif what == LK2ND: - path = dump_lk2nd(device_image_path) - fastboot_flash('lk2nd', path) - elif what == QHYPSTUB: - path = dump_qhypstub(device_image_path) - fastboot_flash('qhypstub', path) + result = subprocess.run([ + 'dd', + f'if={minimal_image_path}', + f'of={path}', + 'bs=20M', + 'iflag=direct', + 'oflag=direct', + 'status=progress', + 'conv=sync,noerror', + ]) + if result.returncode != 0: + raise Exception(f'Failed to flash {minimal_image_path} to {path}') else: - raise Exception(f'Unknown what "{what}", this must be a bug in kupferbootstrap!') + loop_device = losetup_rootfs_image(device_image_path, sector_size) + if what == BOOTIMG: + path = dump_bootimg(f'{loop_device}p1') + fastboot_flash('boot', path) + elif what == LK2ND: + path = dump_lk2nd(f'{loop_device}p1') + fastboot_flash('lk2nd', path) + elif what == QHYPSTUB: + path = dump_qhypstub(f'{loop_device}p1') + fastboot_flash('qhypstub', path) + else: + raise Exception(f'Unknown what "{what}", this must be a bug in kupferbootstrap!') diff --git a/image.py b/image.py index 0b1679a..79f8902 100644 --- a/image.py +++ b/image.py @@ -1,9 +1,12 @@ +import atexit +import json import os +import re import subprocess import click from logger import logging from chroot import Chroot, get_device_chroot -from constants import BASE_PACKAGES, DEVICES, FLAVOURS, Arch +from constants import BASE_PACKAGES, DEVICES, FLAVOURS from config import config from distro import get_base_distro, get_kupfer_https, get_kupfer_local from ssh import copy_ssh_keys @@ -11,24 +14,76 @@ from wrapper import enforce_wrap from signal import pause -def resize_fs(image_path: str, shrink: bool = False): - result = subprocess.run([ - 'e2fsck', - '-fy', - image_path, - ]) - # https://man7.org/linux/man-pages/man8/e2fsck.8.html#EXIT_CODE - if result.returncode > 2: - print(result.returncode) - msg = f'Failed to e2fsck {image_path}' - if shrink: - raise Exception(msg) - else: - logging.warning(msg) +def shrink_fs(loop_device: str, file: str, sector_size: int): + # 8: 512 bytes sectors + # 1: 4096 bytes sectors + sectors_blocks_factor = 4096 // sector_size - result = subprocess.run(['resize2fs'] + (['-M'] if shrink else []) + [image_path]) + logging.debug(f"Checking filesystem at {loop_device}p2") + result = subprocess.run(['e2fsck', '-fy', f'{loop_device}p2']) + if result.returncode > 2: + # https://man7.org/linux/man-pages/man8/e2fsck.8.html#EXIT_CODE + raise Exception(f'Failed to e2fsck {loop_device}p2 with exit code {result.returncode}') + + logging.debug(f'Shrinking filesystem at {loop_device}p2') + result = subprocess.run(['resize2fs', '-M', f'{loop_device}p2'], capture_output=True) if result.returncode != 0: - raise Exception(f'Failed to resize2fs {image_path}') + print(result.stdout) + print(result.stderr) + raise Exception(f'Failed to resize2fs {loop_device}p2') + + logging.debug(f'Finding end block of shrunken filesystem on {loop_device}p2') + blocks = int(re.search('is now [0-9]+', result.stdout.decode('utf-8')).group(0).split(' ')[2]) + sectors = blocks * sectors_blocks_factor #+ 157812 - 25600 + + logging.debug(f'Shrinking partition at {loop_device}p2 to {sectors} sectors') + child_proccess = subprocess.Popen( + ['fdisk', '-b', str(sector_size), loop_device], + stdin=subprocess.PIPE, + ) + child_proccess.stdin.write('\n'.join([ + 'd', + '2', + 'n', + 'p', + '2', + '', + f'+{sectors}', + 'w', + 'q', + ]).encode('utf-8')) + + child_proccess.communicate() + + returncode = child_proccess.wait() + if returncode == 1: + # For some reason re-reading the partition table fails, but that is not a problem + subprocess.run(['partprobe']) + if returncode > 1: + raise Exception(f'Failed to shrink partition size of {loop_device}p2 with fdisk') + + logging.debug(f'Finding end sector of partition at {loop_device}p2') + result = subprocess.run(['fdisk', '-b', str(sector_size), '-l', loop_device], capture_output=True) + if result.returncode != 0: + print(result.stdout) + print(result.stderr) + raise Exception(f'Failed to fdisk -l {loop_device}') + + end_sector = 0 + for line in result.stdout.decode('utf-8').split('\n'): + if line.startswith(f'{loop_device}p2'): + parts = list(filter(lambda part: part != '', line.split(' '))) + end_sector = int(parts[2]) + + if end_sector == 0: + raise Exception(f'Failed to find end sector of {loop_device}p2') + + end_block = end_sector // sectors_blocks_factor + + logging.debug(f'Truncating {file} to {end_block} blocks') + result = subprocess.run(['truncate', '-o', '-s', str(end_block), file]) + if result.returncode != 0: + raise Exception(f'Failed to truncate {file}') def get_device_and_flavour(profile: str = None) -> tuple[str, str]: @@ -51,13 +106,73 @@ def get_image_path(device_chroot: Chroot) -> str: return os.path.join(config.get_path('images'), get_image_name(device_chroot)) +def losetup_rootfs_image(image_path: str, sector_size: int) -> str: + logging.debug(f'Creating loop device for {image_path}') + result = subprocess.run([ + 'losetup', + '-f', + '-b', + str(sector_size), + image_path, + ]) + if result.returncode != 0: + logging.fatal(f'Failed create loop device for {image_path}') + exit(1) + + logging.debug(f'Finding loop device for {image_path}') + + result = subprocess.run(['losetup', '-J'], capture_output=True) + if result.returncode != 0: + print(result.stdout) + print(result.stderr) + logging.fatal('Failed to list loop devices') + exit(1) + + data = json.loads(result.stdout.decode('utf-8')) + loop_device = '' + for d in data['loopdevices']: + if d['back-file'] == image_path: + loop_device = d['name'] + break + + if loop_device == '': + raise Exception(f'Failed to find loop device for {image_path}') + + def losetup_destroy(): + logging.debug(f'Destroying loop device {loop_device} for {image_path}') + subprocess.run( + [ + 'losetup', + '-d', + loop_device, + ], + stderr=subprocess.DEVNULL, + ) + + atexit.register(losetup_destroy) + + return loop_device + + +def mount_rootfs_loop_device(loop_device, chroot: Chroot): + logging.debug(f'Mounting {loop_device}p2 at {chroot.path}') + + chroot.mount_rootfs(loop_device + 'p2') + + if not os.path.exists(f'{chroot.path}/boot'): + os.makedirs(f'{chroot.path}/boot') + + logging.debug(f'Mounting {loop_device}p1 at {chroot.path}/boot') + chroot.mount(loop_device + 'p1', '/boot') + + def dump_bootimg(image_path: str) -> str: path = '/tmp/boot.img' result = subprocess.run([ 'debugfs', image_path, '-R', - f'dump /boot/boot.img {path}', + f'dump /boot.img {path}', ]) if result.returncode != 0: logging.fatal('Failed to dump boot.img') @@ -74,7 +189,7 @@ def dump_lk2nd(image_path: str) -> str: 'debugfs', image_path, '-R', - f'dump /boot/lk2nd.img {path}', + f'dump /lk2nd.img {path}', ]) if result.returncode != 0: logging.fatal('Failed to dump lk2nd.img') @@ -88,7 +203,7 @@ def dump_qhypstub(image_path: str) -> str: 'debugfs', image_path, '-R', - f'dump /boot/qhypstub.bin {path}', + f'dump /qhypstub.bin {path}', ]) if result.returncode != 0: logging.fatal('Failed to dump qhypstub.bin') @@ -108,8 +223,9 @@ def cmd_build(): device, flavour = get_device_and_flavour() post_cmds = FLAVOURS[flavour].get('post_cmds', []) - # TODO: PARSE DEVICE ARCH - arch: Arch = 'aarch64' + # TODO: PARSE DEVICE ARCH AND SECTOR SIZE + arch = 'aarch64' + sector_size = 4096 packages_dir = config.get_package_dir(arch) if os.path.exists(os.path.join(packages_dir, 'main')): @@ -121,26 +237,58 @@ def cmd_build(): chroot = get_device_chroot(device=device, flavour=flavour, arch=arch, packages=packages, extra_repos=extra_repos) image_path = get_image_path(chroot) - if not os.path.exists(image_path): + new_image = not os.path.exists(image_path) + if new_image: result = subprocess.run([ - 'fallocate', - '-l', + 'truncate', + '-s', f"{FLAVOURS[flavour].get('size',2)}G", image_path, ]) if result.returncode != 0: raise Exception(f'Failed to allocate {image_path}') + loop_device = losetup_rootfs_image(image_path, sector_size) + + if new_image: + boot_partition_size = '100MiB' + create_partition_table = ['mklabel', 'msdos'] + create_boot_partition = ['mkpart', 'primary', 'ext2', '0%', boot_partition_size] + create_root_partition = ['mkpart', 'primary', boot_partition_size, '100%'] + enable_boot = ['set', '1', 'boot', 'on'] result = subprocess.run([ - 'mkfs.ext4', + 'parted', + '--script', + loop_device, + ] + create_partition_table + create_boot_partition + create_root_partition + enable_boot) + if result.returncode != 0: + raise Exception(f'Failed to create partitions on {loop_device}') + + result = subprocess.run([ + 'mkfs.ext2', + '-F', '-L', - 'kupfer', - image_path, + 'kupfer_boot', + f'{loop_device}p1', ]) if result.returncode != 0: - raise Exception(f'Failed to create ext4 filesystem on {image_path}') - else: - resize_fs(image_path=image_path) + raise Exception(f'Failed to create ext2 filesystem on {loop_device}p1') + + result = subprocess.run([ + 'mkfs.ext4', + '-O', + '^metadata_csum', + '-F', + '-L', + 'kupfer_root', + '-N', + '100000', + f'{loop_device}p2', + ]) + if result.returncode != 0: + raise Exception(f'Failed to create ext4 filesystem on {loop_device}p2') + + mount_rootfs_loop_device(loop_device, chroot) chroot.mount_rootfs(image_path) chroot.initialize() @@ -168,10 +316,13 @@ def cmd_inspect(shell: bool = False): device, flavour = get_device_and_flavour() # TODO: get arch from profile arch = 'aarch64' + # TODO: PARSE DEVICE SECTOR SIZE + sector_size = 4096 chroot = get_device_chroot(device, flavour, arch) image_path = get_image_path(chroot) + loop_device = losetup_rootfs_image(image_path, sector_size) - chroot.mount_rootfs(image_path) + mount_rootfs_loop_device(loop_device, chroot) logging.info(f'Inspect the rootfs image at {chroot.path}')