diff --git a/README.md b/README.md index fbe13b1..e842b4f 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ Expected local files: - `offline/downloads/magisk.version` - `offline/images/redroid-redroid-13.0.0-latest.tar` -`redroid.py` now auto-loads the base image tar when `redroid/redroid:13.0.0-latest` is not already present in the local container runtime. +`redroid.py` now tries to pull `redroid/redroid:13.0.0-latest` first. If that fails, it falls back to the local image cache, then to `offline/images/redroid-redroid-13.0.0-latest.tar`. ## Dependencies diff --git a/offline/README.md b/offline/README.md index 044c7b2..f8b6eac 100644 --- a/offline/README.md +++ b/offline/README.md @@ -27,5 +27,5 @@ python verify_offline_bundle.py 1. Make sure your container runtime can load local image archives. 2. Populate `downloads/` and `images/` with the files listed in `manifest.json`. 3. Run `python redroid.py -mtg -i -m`. -4. If the base image tag is missing, `redroid.py` will automatically load `images/redroid-redroid-13.0.0-latest.tar`. -5. The build modules read the zip/apk files from `downloads/` first and do not need the network when those files are present and valid. +4. The build first tries to pull the base image and download dependency archives from their online sources. +5. If an online pull/download fails, the build falls back to the validated files in `downloads/` and `images/`. diff --git a/redroid.py b/redroid.py index 63e8188..c7dcbfb 100644 --- a/redroid.py +++ b/redroid.py @@ -22,19 +22,37 @@ def ensure_base_image(container_runtime, android_version): stderr=subprocess.DEVNULL, check=False, ) - if inspect.returncode == 0: + local_present = inspect.returncode == 0 + + helper.print_color( + "Trying to pull base image {} first ...".format(base_image), + helper.bcolors.GREEN, + ) + pull = subprocess.run([container_runtime, "pull", base_image], check=False) + if pull.returncode == 0: + return "pulled" + + if local_present: + helper.print_color( + "Online pull failed, falling back to local image cache ...", + helper.bcolors.YELLOW, + ) return "present" tar_name = "redroid-redroid-{}-latest.tar".format(android_version) tar_path = os.path.join(helper.get_image_dir(), tar_name) if not os.path.isfile(tar_path): raise FileNotFoundError( - "Base image {} is missing and offline archive {} was not found".format( + "Online pull for {} failed, local image cache is missing, and offline archive {} was not found".format( base_image, tar_path, ) ) + helper.print_color( + "Online pull failed, loading local image archive {} ...".format(tar_path), + helper.bcolors.YELLOW, + ) subprocess.run([container_runtime, "load", "-i", tar_path], check=True) return tar_path diff --git a/stuff/general.py b/stuff/general.py index 13395b4..81610ab 100644 --- a/stuff/general.py +++ b/stuff/general.py @@ -1,24 +1,49 @@ - import os import zipfile -import hashlib -from tools.helper import bcolors, download_file, print_color +from tools.helper import bcolors, download_file, file_md5, print_color class General: + def _local_md5(self): + if not os.path.isfile(self.dl_file_name): + return "" + return file_md5(self.dl_file_name) + + def _has_valid_local_file(self): + return os.path.isfile(self.dl_file_name) and self._local_md5() == self.act_md5 + def download(self): - loc_md5 = "" - if os.path.isfile(self.dl_file_name): - with open(self.dl_file_name,"rb") as f: - bytes = f.read() - loc_md5 = hashlib.md5(bytes).hexdigest() - while not os.path.isfile(self.dl_file_name) or loc_md5 != self.act_md5: - if os.path.isfile(self.dl_file_name): - os.remove(self.dl_file_name) - print_color("md5 mismatches, redownloading now ....",bcolors.YELLOW) - loc_md5 = download_file(self.dl_link, self.dl_file_name) - + temp_path = self.dl_file_name + ".tmp" + local_valid = self._has_valid_local_file() + + try: + if os.path.isfile(temp_path): + os.remove(temp_path) + print_color("Trying online download first ...", bcolors.GREEN) + remote_md5 = download_file(self.dl_link, temp_path) + if remote_md5 != self.act_md5: + raise ValueError( + "Downloaded file md5 mismatch: expected {}, got {}".format( + self.act_md5, + remote_md5, + ) + ) + os.replace(temp_path, self.dl_file_name) + except Exception as exc: + if os.path.isfile(temp_path): + os.remove(temp_path) + if local_valid: + print_color( + "Online download failed, falling back to local {} ({})".format( + self.dl_file_name, + exc, + ), + bcolors.YELLOW, + ) + return + raise + def extract(self): print_color("Extracting archive...", bcolors.GREEN) print(self.dl_file_name) diff --git a/stuff/magisk.py b/stuff/magisk.py index 3f84fa8..79b37cf 100644 --- a/stuff/magisk.py +++ b/stuff/magisk.py @@ -137,13 +137,41 @@ on property:init.svc.zygote=stopped with open(self.release_meta_file, "w", encoding="utf-8") as meta_file: meta_file.write(self.release_version) - def _validate_archive(self): - if not zipfile.is_zipfile(self.dl_file_name): + def _validate_archive(self, path=None): + archive_path = path or self.dl_file_name + if not zipfile.is_zipfile(archive_path): raise ValueError( "Downloaded Magisk/Kitsune archive is not a valid APK/ZIP. " "Set REDROID_MAGISK_APK to a local APK or REDROID_MAGISK_URL to a working direct APK URL." ) + def _has_valid_local_archive(self): + if not os.path.isfile(self.dl_file_name): + return False + try: + self._validate_archive() + except ValueError: + return False + return True + + def _download_remote_archive(self): + temp_path = self.dl_file_name + ".tmp" + try: + if os.path.isfile(temp_path): + os.remove(temp_path) + print_color( + "Trying online download for Kitsune Mask {} ...".format(self.release_version), + bcolors.GREEN, + ) + download_file(self.dl_link, temp_path) + self._validate_archive(temp_path) + os.replace(temp_path, self.dl_file_name) + self._write_version() + except Exception: + if os.path.isfile(temp_path): + os.remove(temp_path) + raise + def resolve_release(self): print_color("Resolving Kitsune Mask stable release ...", bcolors.GREEN) response = requests.get(self.stable_json, timeout=30) @@ -174,41 +202,42 @@ on property:init.svc.zygote=stopped self._validate_archive() return - if not self.custom_url and os.path.isfile(self.dl_file_name): - self.release_version = self._cached_version() or "bundled" - print_color( - "Using bundled Kitsune Mask {} ...".format(self.release_version), - bcolors.GREEN, - ) - self._validate_archive() - return + local_valid = self._has_valid_local_archive() + local_version = self._cached_version() or "bundled" - if self.custom_url: - self.dl_link = self.custom_url - self.release_version = "url:" + os.path.basename(self.custom_url) - else: - self.resolve_release() + try: + if self.custom_url: + self.dl_link = self.custom_url + self.release_version = "url:" + os.path.basename(self.custom_url) + else: + self.resolve_release() + except Exception as exc: + if local_valid: + self.release_version = local_version + print_color( + "Resolving Kitsune Mask online failed, falling back to local {} ({})".format( + self.release_version, + exc, + ), + bcolors.YELLOW, + ) + return + raise - if ( - os.path.isfile(self.dl_file_name) - and self._cached_version() == self.release_version - ): - print_color( - "Using cached Kitsune Mask {} ...".format(self.release_version), - bcolors.GREEN, - ) - self._validate_archive() - return - - print_color( - "Downloading Kitsune Mask {} now .....".format(self.release_version), - bcolors.GREEN, - ) - if os.path.isfile(self.dl_file_name): - os.remove(self.dl_file_name) - download_file(self.dl_link, self.dl_file_name) - self._validate_archive() - self._write_version() + try: + self._download_remote_archive() + except Exception as exc: + if local_valid: + self.release_version = local_version + print_color( + "Downloading Kitsune Mask online failed, falling back to local {} ({})".format( + self.release_version, + exc, + ), + bcolors.YELLOW, + ) + return + raise def extract(self): print_color("Extracting archive...", bcolors.GREEN) diff --git a/tools/helper.py b/tools/helper.py index 7db8049..2fc9865 100644 --- a/tools/helper.py +++ b/tools/helper.py @@ -25,6 +25,15 @@ def get_download_dir(): def get_image_dir(): return ensure_dir(IMAGE_ROOT) + +def file_md5(path, block_size=1024 * 1024): + hasher = hashlib.md5() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(block_size), b""): + hasher.update(chunk) + return hasher.hexdigest() + + def run(args): result = subprocess.run(args=args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if result.stderr: @@ -36,23 +45,31 @@ def run(args): ) return result -def download_file(url, f_name): - md5 = "" - response = requests.get(url, stream=True) - total_size_in_bytes = int(response.headers.get('content-length', 0)) - block_size = 1024 # 1 Kibibyte - progress_bar = tqdm(total=total_size_in_bytes, unit='iB', unit_scale=True) - with open(f_name, 'wb') as file: - for data in response.iter_content(block_size): - progress_bar.update(len(data)) - file.write(data) - progress_bar.close() - with open(f_name, "rb") as f: - bytes = f.read() - md5 = hashlib.md5(bytes).hexdigest() + +def download_file(url, f_name, timeout=60): + response = requests.get(url, stream=True, timeout=timeout) + response.raise_for_status() + + total_size_in_bytes = int(response.headers.get("content-length", 0)) + block_size = 1024 * 1024 + progress_bar = tqdm(total=total_size_in_bytes, unit="iB", unit_scale=True) + hasher = hashlib.md5() + + try: + with open(f_name, "wb") as file: + for data in response.iter_content(block_size): + if not data: + continue + progress_bar.update(len(data)) + file.write(data) + hasher.update(data) + finally: + progress_bar.close() + response.close() + if total_size_in_bytes != 0 and progress_bar.n != total_size_in_bytes: raise ValueError("Something went wrong while downloading") - return md5 + return hasher.hexdigest() def host(): machine = platform.machine().lower()