diff --git a/Dockerfile b/Dockerfile index 4b6ef46..86c3ca8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,8 @@ RUN pacman -Syu --noconfirm \ arch-install-scripts rsync \ aarch64-linux-gnu-gcc aarch64-linux-gnu-binutils aarch64-linux-gnu-glibc aarch64-linux-gnu-linux-api-headers \ git \ - android-tools openssh inetutils + android-tools openssh inetutils \ + parted RUN sed -i "s/EUID == 0/EUID == -1/g" $(which makepkg) diff --git a/boot.py b/boot.py index 988a194..528d232 100644 --- a/boot.py +++ b/boot.py @@ -28,9 +28,9 @@ def cmd_boot(type): if not os.path.exists(path): urllib.request.urlretrieve(f'https://github.com/dreemurrs-embedded/Jumpdrive/releases/download/{JUMPDRIVE_VERSION}/{file}', path) elif type == LK2ND: - path = dump_lk2nd(image_name) + path = dump_lk2nd(os.path.join('/images', image_name)) elif type == BOOTIMG: - path = dump_bootimg(image_name) + path = dump_bootimg(os.path.join('/images', image_name)) else: raise Exception(f'Unknown boot image type {type}') fastboot_erase_dtbo() diff --git a/cache.py b/cache.py index bc9fc67..26cb568 100644 --- a/cache.py +++ b/cache.py @@ -5,7 +5,7 @@ from config import config from wrapper import enforce_wrap import logging -PATHS = ['chroots', 'pacman', 'jumpdrive', 'packages'] +PATHS = ['chroots', 'pacman', 'jumpdrive', 'packages', 'images'] @click.group(name='cache') diff --git a/config.py b/config.py index 70ce4bc..47f3b9f 100644 --- a/config.py +++ b/config.py @@ -43,6 +43,7 @@ CONFIG_DEFAULTS = { 'packages': os.path.join('%cache_dir%', 'packages'), 'pkgbuilds': os.path.join('%cache_dir%', 'pkgbuilds'), 'jumpdrive': os.path.join('%cache_dir%', 'jumpdrive'), + 'images': os.path.join('%cache_dir%', 'images'), }, 'profiles': { 'current': 'default', diff --git a/constants.py b/constants.py index 6706835..b52a8e9 100644 --- a/constants.py +++ b/constants.py @@ -6,9 +6,8 @@ FLASH_PARTS = { 'QHYPSTUB': 'qhypstub', } EMMC = 'emmc' -EMMCFILE = 'emmc-file' MICROSD = 'microsd' -LOCATIONS = [EMMC, EMMCFILE, MICROSD] +LOCATIONS = [EMMC, MICROSD] JUMPDRIVE = 'jumpdrive' JUMPDRIVE_VERSION = '0.8' diff --git a/flash.py b/flash.py index a34cd9a..8b34931 100644 --- a/flash.py +++ b/flash.py @@ -2,13 +2,13 @@ import atexit from constants import FLASH_PARTS, LOCATIONS from fastboot import fastboot_flash import shutil -from image import dump_bootimg, dump_lk2nd, dump_qhypstub, get_device_and_flavour, get_image_name +from image import dump_bootimg, dump_lk2nd, dump_qhypstub, get_device_and_flavour, get_image_name, losetup_rootfs_image import os import subprocess import click import tempfile from wrapper import enforce_wrap -from image import resize_fs +from image import shrink_fs BOOTIMG = FLASH_PARTS['BOOTIMG'] LK2ND = FLASH_PARTS['LK2ND'] @@ -24,30 +24,37 @@ def cmd_flash(what, location): device, flavour = get_device_and_flavour() image_name = get_image_name(device, flavour) + # TODO: PARSE DEVICE SECTOR SIZE + sector_size = 4096 + if what not in FLASH_PARTS.values(): raise Exception(f'Unknown what "{what}", must be one of {", ".join(FLASH_PARTS.values())}') if what == ROOTFS: if location is None: raise Exception(f'You need to specify a location to flash {what} to') - if location not in LOCATIONS: - raise Exception(f'Invalid location {location}. Choose one of {", ".join(LOCATIONS)}') path = '' - dir = '/dev/disk/by-id' - for file in os.listdir(dir): - sanitized_file = file.replace('-', '').replace('_', '').lower() - if f'jumpdrive{location.split("-")[0]}' in sanitized_file: - path = os.path.realpath(os.path.join(dir, file)) - result = subprocess.run(['lsblk', path, '-o', 'SIZE'], capture_output=True) - if result.returncode != 0: - raise Exception(f'Failed to lsblk {path}') - if result.stdout == b'SIZE\n 0B\n': - raise Exception( - f'Disk {path} has a size of 0B. That probably means it is not available (e.g. no microSD inserted or no microSD card slot installed in the device) or corrupt or defect' - ) - if path == '': - raise Exception('Unable to discover Jumpdrive') + if location.startswith("/dev/"): + path = location + else: + if location not in LOCATIONS: + raise Exception(f'Invalid location {location}. Choose one of {", ".join(LOCATIONS)}') + + dir = '/dev/disk/by-id' + for file in os.listdir(dir): + sanitized_file = file.replace('-', '').replace('_', '').lower() + if f'jumpdrive{location.split("-")[0]}' in sanitized_file: + path = os.path.realpath(os.path.join(dir, file)) + result = subprocess.run(['lsblk', path, '-o', 'SIZE'], capture_output=True) + if result.returncode != 0: + raise Exception(f'Failed to lsblk {path}') + if result.stdout == b'SIZE\n 0B\n': + raise Exception( + f'Disk {path} has a size of 0B. That probably means it is not available (e.g. no microSD inserted or no microSD card slot installed in the device) or corrupt or defect' + ) + if path == '': + raise Exception('Unable to discover Jumpdrive') image_dir = tempfile.gettempdir() image_path = os.path.join(image_dir, f'minimal-{image_name}') @@ -57,73 +64,33 @@ def cmd_flash(what, location): atexit.register(clean_dir) - shutil.copyfile(image_name, image_path) + shutil.copyfile(os.path.join('/images', image_name), image_path) - resize_fs(image_path, shrink=True) + loop_device = losetup_rootfs_image(image_path, sector_size) + shrink_fs(loop_device, image_path, sector_size) - if location.endswith('-file'): - part_mount = '/mnt/kupfer/fs' - if not os.path.exists(part_mount): - os.makedirs(part_mount) - - def umount(): - subprocess.run( - [ - 'umount', - '-lc', - part_mount, - ], - stderr=subprocess.DEVNULL, - ) - - atexit.register(umount) - - result = subprocess.run([ - 'mount', - path, - part_mount, - ]) - if result.returncode != 0: - raise Exception(f'Failed to mount {path} to {part_mount}') - - dir = os.path.join(part_mount, '.stowaways') - if not os.path.exists(dir): - os.makedirs(dir) - - result = subprocess.run([ - 'rsync', - '--archive', - '--inplace', - '--partial', - '--progress', - '--human-readable', - image_path, - os.path.join(dir, 'kupfer.img'), - ]) - if result.returncode != 0: - raise Exception(f'Failed to mount {path} to {part_mount}') - else: - result = subprocess.run([ - 'dd', - f'if={image_path}', - f'of={path}', - 'bs=20M', - 'iflag=direct', - 'oflag=direct', - 'status=progress', - 'conv=sync,noerror', - ]) - if result.returncode != 0: - raise Exception(f'Failed to flash {image_path} to {path}') - - elif what == BOOTIMG: - path = dump_bootimg(image_name) - fastboot_flash('boot', path) - elif what == LK2ND: - path = dump_lk2nd(image_name) - fastboot_flash('lk2nd', path) - elif what == QHYPSTUB: - path = dump_qhypstub(image_name) - fastboot_flash('qhypstub', path) + result = subprocess.run([ + 'dd', + f'if={image_path}', + f'of={path}', + 'bs=20M', + 'iflag=direct', + 'oflag=direct', + 'status=progress', + 'conv=sync,noerror', + ]) + if result.returncode != 0: + raise Exception(f'Failed to flash {image_path} to {path}') else: - raise Exception(f'Unknown what "{what}", this must be a bug in kupferbootstrap!') + loop_device = losetup_rootfs_image(os.path.join('/images', image_name), sector_size) + if what == BOOTIMG: + path = dump_bootimg(f'{loop_device}p1') + fastboot_flash('boot', path) + elif what == LK2ND: + path = dump_lk2nd(f'{loop_device}p1') + fastboot_flash('lk2nd', path) + elif what == QHYPSTUB: + path = dump_qhypstub(f'{loop_device}p1') + fastboot_flash('qhypstub', path) + else: + raise Exception(f'Unknown what "{what}", this must be a bug in kupferbootstrap!') diff --git a/image.py b/image.py index 59d80dc..ae7965f 100644 --- a/image.py +++ b/image.py @@ -1,5 +1,7 @@ import atexit +import json import os +import re import subprocess import click from logger import logging @@ -12,24 +14,76 @@ from wrapper import enforce_wrap from signal import pause -def resize_fs(image_path: str, shrink: bool = False): - result = subprocess.run([ - 'e2fsck', - '-fy', - image_path, - ]) - # https://man7.org/linux/man-pages/man8/e2fsck.8.html#EXIT_CODE - if result.returncode > 2: - print(result.returncode) - msg = f'Failed to e2fsck {image_path}' - if shrink: - raise Exception(msg) - else: - logging.warning(msg) +def shrink_fs(loop_device: str, file: str, sector_size: int): + # 8: 512 bytes sectors + # 1: 4096 bytes sectors + sectors_blocks_factor = 4096 // sector_size - result = subprocess.run(['resize2fs'] + (['-M'] if shrink else []) + [image_path]) + logging.debug(f"Checking filesystem at {loop_device}p2") + result = subprocess.run(['e2fsck', '-fy', f'{loop_device}p2']) + if result.returncode > 2: + # https://man7.org/linux/man-pages/man8/e2fsck.8.html#EXIT_CODE + raise Exception(f'Failed to e2fsck {loop_device}p2 with exit code {result.returncode}') + + logging.debug(f'Shrinking filesystem at {loop_device}p2') + result = subprocess.run(['resize2fs', '-M', f'{loop_device}p2'], capture_output=True) if result.returncode != 0: - raise Exception(f'Failed to resize2fs {image_path}') + print(result.stdout) + print(result.stderr) + raise Exception(f'Failed to resize2fs {loop_device}p2') + + logging.debug(f'Finding end block of shrunken filesystem on {loop_device}p2') + blocks = int(re.search('is now [0-9]+', result.stdout.decode('utf-8')).group(0).split(' ')[2]) + sectors = blocks * sectors_blocks_factor #+ 157812 - 25600 + + logging.debug(f'Shrinking partition at {loop_device}p2 to {sectors} sectors') + child_proccess = subprocess.Popen( + ['fdisk', '-b', str(sector_size), loop_device], + stdin=subprocess.PIPE, + ) + child_proccess.stdin.write('\n'.join([ + 'd', + '2', + 'n', + 'p', + '2', + '', + f'+{sectors}', + 'w', + 'q', + ]).encode('utf-8')) + + child_proccess.communicate() + + returncode = child_proccess.wait() + if returncode == 1: + # For some reason re-reading the partition table fails, but that is not a problem + subprocess.run(['partprobe']) + if returncode > 1: + raise Exception(f'Failed to shrink partition size of {loop_device}p2 with fdisk') + + logging.debug(f'Finding end sector of partition at {loop_device}p2') + result = subprocess.run(['fdisk', '-b', str(sector_size), '-l', loop_device], capture_output=True) + if result.returncode != 0: + print(result.stdout) + print(result.stderr) + raise Exception(f'Failed to fdisk -l {loop_device}') + + end_sector = 0 + for line in result.stdout.decode('utf-8').split('\n'): + if line.startswith(f'{loop_device}p2'): + parts = list(filter(lambda part: part != '', line.split(' '))) + end_sector = int(parts[2]) + + if end_sector == 0: + raise Exception(f'Failed to find end sector of {loop_device}p2') + + end_block = end_sector // sectors_blocks_factor + + logging.debug(f'Truncating {file} to {end_block} blocks') + result = subprocess.run(['truncate', '-o', '-s', str(end_block), file]) + if result.returncode != 0: + raise Exception(f'Failed to truncate {file}') def get_device_and_flavour(profile: str = None) -> tuple[str, str]: @@ -48,11 +102,65 @@ def get_image_name(device, flavour) -> str: return f'{device}-{flavour}-rootfs.img' -def mount_rootfs_image(image_path, mount_path): - if not os.path.exists(mount_path): - os.makedirs(mount_path) +def losetup_rootfs_image(image_path: str, sector_size: int) -> str: + logging.debug(f'Creating loop device for {image_path}') + result = subprocess.run([ + 'losetup', + '-f', + '-b', + str(sector_size), + image_path, + ]) + if result.returncode != 0: + logging.fatal(f'Failed create loop device for {image_path}') + exit(1) + + logging.debug(f'Finding loop device for {image_path}') + + result = subprocess.run(['losetup', '-J'], capture_output=True) + if result.returncode != 0: + print(result.stdout) + print(result.stderr) + logging.fatal('Failed to list loop devices') + exit(1) + + data = json.loads(result.stdout.decode('utf-8')) + loop_device = '' + for d in data['loopdevices']: + if d['back-file'] == image_path: + loop_device = d['name'] + break + + if loop_device == '': + raise Exception(f'Failed to find loop device for {image_path}') + + def losetup_destroy(): + logging.debug(f'Destroying loop device {loop_device} for {image_path}') + subprocess.run( + [ + 'losetup', + '-d', + loop_device, + ], + stderr=subprocess.DEVNULL, + ) + + atexit.register(losetup_destroy) + + return loop_device + + +def mount_rootfs_loop_device(loop_device, mount_path): def umount(): + subprocess.run( + [ + 'umount', + '-lc', + f'{mount_path}/boot', + ], + stderr=subprocess.DEVNULL, + ) subprocess.run( [ 'umount', @@ -64,15 +172,36 @@ def mount_rootfs_image(image_path, mount_path): atexit.register(umount) + if not os.path.exists(mount_path): + os.makedirs(mount_path) + + logging.debug(f'Mounting {loop_device}p2 at {mount_path}') + result = subprocess.run([ 'mount', '-o', 'loop', - image_path, + f'{loop_device}p2', mount_path, ]) if result.returncode != 0: - logging.fatal(f'Failed to loop mount {image_path} to {mount_path}') + logging.fatal(f'Failed to loop mount {loop_device}p2 to {mount_path}') + exit(1) + + if not os.path.exists(f'{mount_path}/boot'): + os.makedirs(f'{mount_path}/boot') + + logging.debug(f'Mounting {loop_device}p1 at {mount_path}/boot') + + result = subprocess.run([ + 'mount', + '-o', + 'loop', + f'{loop_device}p1', + f'{mount_path}/boot', + ]) + if result.returncode != 0: + logging.fatal(f'Failed to loop mount {loop_device}p1 to {mount_path}/boot') exit(1) @@ -82,7 +211,7 @@ def dump_bootimg(image_name: str) -> str: 'debugfs', image_name, '-R', - f'dump /boot/boot.img {path}', + f'dump /boot.img {path}', ]) if result.returncode != 0: logging.fatal('Failed to dump boot.img') @@ -99,7 +228,7 @@ def dump_lk2nd(image_name: str) -> str: 'debugfs', image_name, '-R', - f'dump /boot/lk2nd.img {path}', + f'dump /lk2nd.img {path}', ]) if result.returncode != 0: logging.fatal('Failed to dump lk2nd.img') @@ -113,7 +242,7 @@ def dump_qhypstub(image_name: str) -> str: 'debugfs', image_name, '-R', - f'dump /boot/qhypstub.bin {path}', + f'dump /qhypstub.bin {path}', ]) if result.returncode != 0: logging.fatal('Failed to dump qhypstub.bin') @@ -132,35 +261,66 @@ def cmd_build(): profile = config.get_profile() device, flavour = get_device_and_flavour() post_cmds = FLAVOURS[flavour].get('post_cmds', []) - image_name = get_image_name(device, flavour) + image_name = os.path.join('/images', get_image_name(device, flavour)) - # TODO: PARSE DEVICE ARCH + # TODO: PARSE DEVICE ARCH AND SECTOR SIZE arch = 'aarch64' + sector_size = 4096 - if not os.path.exists(image_name): + new_image = not os.path.exists(image_name) + if new_image: result = subprocess.run([ - 'fallocate', - '-l', + 'truncate', + '-s', f"{FLAVOURS[flavour].get('size',2)}G", image_name, ]) if result.returncode != 0: raise Exception(f'Failed to allocate {image_name}') + loop_device = losetup_rootfs_image(image_name, sector_size) + + if new_image: + boot_partition_size = '100MiB' + create_partition_table = ['mklabel', 'msdos'] + create_boot_partition = ['mkpart', 'primary', 'ext2', '0%', boot_partition_size] + create_root_partition = ['mkpart', 'primary', boot_partition_size, '100%'] + enable_boot = ['set', '1', 'boot', 'on'] result = subprocess.run([ - 'mkfs.ext4', + 'parted', + '--script', + loop_device, + ] + create_partition_table + create_boot_partition + create_root_partition + enable_boot) + if result.returncode != 0: + raise Exception(f'Failed to create partitions on {loop_device}') + + result = subprocess.run([ + 'mkfs.ext2', + '-F', '-L', - 'kupfer', - image_name, + 'kupfer_boot', + f'{loop_device}p1', ]) if result.returncode != 0: - raise Exception(f'Failed to create ext4 filesystem on {image_name}') - else: - resize_fs(image_path=image_name) + raise Exception(f'Failed to create ext2 filesystem on {loop_device}p1') + + result = subprocess.run([ + 'mkfs.ext4', + '-O', + '^metadata_csum', + '-F', + '-L', + 'kupfer_root', + '-N', + '100000', + f'{loop_device}p2', + ]) + if result.returncode != 0: + raise Exception(f'Failed to create ext4 filesystem on {loop_device}p2') chroot_name = f'rootfs_{device}-{flavour}' rootfs_mount = get_chroot_path(chroot_name) - mount_rootfs_image(image_name, rootfs_mount) + mount_rootfs_loop_device(loop_device, rootfs_mount) packages_dir = config.get_package_dir(arch) if os.path.exists(os.path.join(packages_dir, 'main')): @@ -199,8 +359,12 @@ def cmd_inspect(): device, flavour = get_device_and_flavour() image_name = get_image_name(device, flavour) + # TODO: PARSE DEVICE SECTOR SIZE + sector_size = 4096 + rootfs_mount = get_chroot_path(f'rootfs_{device}-{flavour}') - mount_rootfs_image(image_name, rootfs_mount) + loop_device = losetup_rootfs_image(image_name, sector_size) + mount_rootfs_loop_device(loop_device, rootfs_mount) logging.info(f'Inspect the rootfs image at {rootfs_mount}') diff --git a/wrapper.py b/wrapper.py index 55a5db1..9d44080 100644 --- a/wrapper.py +++ b/wrapper.py @@ -15,6 +15,7 @@ DOCKER_PATHS = { 'pacman': '/var/cache/pacman', 'packages': '/prebuilts', 'pkgbuilds': '/pkgbuilds', + 'images': '/images', }