parchlinux-mobile/parchbootstrap
1
0
Fork 0
forked from parchlinux-mobile/kupferbootstrap
Code Issues Pull requests Activity

packages: clean up check_package_built() and use RemotePackage.acquire() in try_download_package()

Browse source
This commit is contained in:
InsanePrawn 2022-09-04 02:38:31 +02:00
parent a9edbfd07d
commit 81f88dd636
1 changed files with 54 additions and 49 deletions
Download patch file Download diff file Expand all files Collapse all files

77
packages/build.py
View file

@ -6,8 +6,6 @@ import subprocess
from copy import deepcopy from copy import deepcopy
from urllib.error import HTTPError from urllib.error import HTTPError
from urllib.request import urlopen
from shutil import copyfileobj
from typing import Iterable, Iterator, Optional from typing import Iterable, Iterator, Optional
from binfmt import register as binfmt_register, QEMU_ARCHES from binfmt import register as binfmt_register, QEMU_ARCHES
@ -16,7 +14,8 @@ from config import config
from exec.cmd import run_cmd, run_root_cmd from exec.cmd import run_cmd, run_root_cmd
from exec.file import makedir, remove_file, symlink from exec.file import makedir, remove_file, symlink
from chroot.build import get_build_chroot, BuildChroot from chroot.build import get_build_chroot, BuildChroot
from distro.distro import BinaryPackage, get_kupfer_https, get_kupfer_local from distro.distro import get_kupfer_https, get_kupfer_local
from distro.package import RemotePackage
from wrapper import check_programs_wrap, wrap_if_foreign_arch from wrapper import check_programs_wrap, wrap_if_foreign_arch
from .pkgbuild import discover_pkgbuilds, filter_pkgbuilds, Pkgbuild from .pkgbuild import discover_pkgbuilds, filter_pkgbuilds, Pkgbuild
@ -271,7 +270,7 @@ def add_package_to_repo(package: Pkgbuild, arch: Arch):
return files return files
def try_download_package(dest_file_path: str, package: Pkgbuild, arch: Arch) -> bool: def try_download_package(dest_file_path: str, package: Pkgbuild, arch: Arch) -> Optional[str]:
logging.debug(f"checking if we can download {package.name}") logging.debug(f"checking if we can download {package.name}")
filename = os.path.basename(dest_file_path) filename = os.path.basename(dest_file_path)
pkgname = package.name pkgname = package.name
@ -279,33 +278,33 @@ def try_download_package(dest_file_path: str, package: Pkgbuild, arch: Arch) ->
repos = get_kupfer_https(arch, scan=True).repos repos = get_kupfer_https(arch, scan=True).repos
if repo_name not in repos: if repo_name not in repos:
logging.warning(f"Repository {repo_name} is not a known HTTPS repo") logging.warning(f"Repository {repo_name} is not a known HTTPS repo")
return False return None
repo = repos[repo_name] repo = repos[repo_name]
if pkgname not in repo.packages: if pkgname not in repo.packages:
logging.warning(f"Package {pkgname} not found in remote repos, building instead.") logging.warning(f"Package {pkgname} not found in remote repos, building instead.")
return False return None
repo_pkg: BinaryPackage = repo.packages[pkgname] repo_pkg: RemotePackage = repo.packages[pkgname]
if repo_pkg.version != package.version: if repo_pkg.version != package.version:
logging.debug(f"Package {pkgname} versions differ: local: {package.version}, remote: {repo_pkg.version}. Building instead.") logging.debug(f"Package {pkgname} versions differ: local: {package.version}, remote: {repo_pkg.version}. Building instead.")
return False return None
if repo_pkg.filename != filename: if repo_pkg.filename != filename:
logging.debug(f"package filenames don't match: local: {filename}, remote: {repo_pkg.filename}") versions_str = f"local: {filename}, remote: {repo_pkg.filename}"
return False if strip_compression_extension(repo_pkg.filename) != strip_compression_extension(filename):
url = f"{repo.resolve_url()}/{filename}" logging.debug(f"package filenames don't match: {versions_str}")
return None
logging.debug(f"ignoring compression extension difference: {versions_str}")
url = repo_pkg.resolved_url
assert url assert url
try: try:
logging.info(f"Trying to download package {url}") path = repo_pkg.acquire()
makedir(os.path.dirname(dest_file_path)) assert os.path.exists(path)
with urlopen(url) as fsrc, open(dest_file_path, 'wb') as fdst: return path
copyfileobj(fsrc, fdst)
logging.info(f"{filename} downloaded from repos")
return True
except HTTPError as e: except HTTPError as e:
if e.code == 404: if e.code == 404:
logging.debug(f"remote package {filename} nonexistant on server: {url}") logging.debug(f"remote package {filename} missing on server: {url}")
else: else:
logging.error(f"remote package {filename} failed to download ({e.code}): {url}: {e}") logging.error(f"remote package {filename} failed to download ({e.code}): {url}: {e}")
return False return None
def check_package_version_built(package: Pkgbuild, arch: Arch, try_download: bool = False) -> bool: def check_package_version_built(package: Pkgbuild, arch: Arch, try_download: bool = False) -> bool:
@ -313,42 +312,48 @@ def check_package_version_built(package: Pkgbuild, arch: Arch, try_download: boo
filename = package.get_filename(arch) filename = package.get_filename(arch)
filename_stripped = strip_compression_extension(filename) filename_stripped = strip_compression_extension(filename)
logging.debug(f'Checking if {filename_stripped} is built') logging.debug(f'Checking if {filename_stripped} is built')
any_arch = filename_stripped.endswith('any.pkg.tar')
if any_arch:
logging.debug("any-arch pkg detected")
for ext in ['xz', 'zst']: for ext in ['xz', 'zst']:
file = os.path.join(config.get_package_dir(arch), package.repo, f'{filename_stripped}.{ext}') file = os.path.join(config.get_package_dir(arch), package.repo, f'{filename_stripped}.{ext}')
if not filename_stripped.endswith('.pkg.tar'): if not filename_stripped.endswith('.pkg.tar'):
raise Exception(f'stripped filename has unknown extension. {filename}') raise Exception(f'stripped filename has unknown extension. {filename}')
if os.path.exists(file) or (try_download and try_download_package(file, package, arch)): if not os.path.exists(file):
missing = False # look for 'any' arch packages in other repos
add_file_to_repo(file, repo_name=package.repo, arch=arch) if any_arch:
# copy arch=(any) packages to all arches
if filename_stripped.endswith('any.pkg.tar'):
logging.debug("any-arch pkg detected")
target_repo_file = os.path.join(config.get_package_dir(arch), package.repo, filename) target_repo_file = os.path.join(config.get_package_dir(arch), package.repo, filename)
if os.path.exists(target_repo_file): if os.path.exists(target_repo_file):
file = target_repo_file
missing = False missing = False
else: else:
# we have to check if another arch's repo holds our any-arch pkg # we have to check if another arch's repo holds our any-arch pkg
for repo_arch in ARCHES: for repo_arch in ARCHES:
if repo_arch == arch: if repo_arch == arch:
continue # we already checked that continue # we already checked that
other_repo_path = os.path.join(config.get_package_dir(repo_arch), package.repo, filename) other_repo_file = os.path.join(config.get_package_dir(repo_arch), package.repo, filename)
if os.path.exists(other_repo_path): if os.path.exists(other_repo_file):
logging.info(f"package {file} found in {repo_arch} repo, copying to {arch}")
file = other_repo_file
missing = False missing = False
logging.info(f"package {file} found in {repo_arch} repos, copying to {arch}") if try_download and missing:
shutil.copyfile(other_repo_path, target_repo_file) downloaded = try_download_package(file, package, arch)
add_file_to_repo(target_repo_file, package.repo, arch) if downloaded:
break file = downloaded
missing = False
if os.path.exists(target_repo_file): if os.path.exists(file):
missing = False
add_file_to_repo(file, repo_name=package.repo, arch=arch, remove_original=False)
# copy arch=(any) packages to all arches
if any_arch and not missing:
# copy to other arches if they don't have it # copy to other arches if they don't have it
for repo_arch in ARCHES: for repo_arch in ARCHES:
if repo_arch == arch: if repo_arch == arch:
continue # we already have that continue # we already have that
copy_target = os.path.join(config.get_package_dir(repo_arch), package.repo, filename) copy_target = os.path.join(config.get_package_dir(repo_arch), package.repo, filename)
if not os.path.exists(copy_target): if not os.path.exists(copy_target):
logging.info(f"copying to {copy_target}") logging.info(f"copying any-arch package {package.name} to {repo_arch} repo: {copy_target}")
shutil.copyfile(target_repo_file, copy_target) add_file_to_repo(file, package.repo, repo_arch, remove_original=False)
add_file_to_repo(copy_target, package.repo, repo_arch)
if not missing: if not missing:
return True return True
return False return False