diff --git a/build_files/utils/make_update.py b/build_files/utils/make_update.py index b8b6a9132c7..2eb2334d66a 100755 --- a/build_files/utils/make_update.py +++ b/build_files/utils/make_update.py @@ -131,7 +131,7 @@ def ensure_git_lfs(args: argparse.Namespace) -> None: call((args.git_command, "lfs", "install", "--skip-repo"), exit_on_error=True) -def update_precompiled_libraries(args: argparse.Namespace) -> None: +def update_precompiled_libraries(args: argparse.Namespace) -> str: """ Configure and update submodule for precompiled libraries @@ -153,21 +153,24 @@ def update_precompiled_libraries(args: argparse.Namespace) -> None: if sys.platform == "linux" and not args.use_linux_libraries: print("Skipping Linux libraries configuration") - return + return "" submodule_dir = f"lib/{platform}_{arch}" submodule_directories = get_submodule_directories(args) if Path(submodule_dir) not in submodule_directories: - print("Skipping libraries update: no configured submodule") - return + return "Skipping libraries update: no configured submodule\n" make_utils.git_enable_submodule(args.git_command, submodule_dir) - make_utils.git_update_submodule(args.git_command, submodule_dir) + + if not make_utils.git_update_submodule(args.git_command, submodule_dir): + return "Error updating precompiled libraries\n" + + return "" -def update_tests_data_files(args: argparse.Namespace) -> None: +def update_tests_data_files(args: argparse.Namespace) -> str: """ Configure and update submodule with files used by regression tests """ @@ -177,7 +180,11 @@ def update_tests_data_files(args: argparse.Namespace) -> None: submodule_dir = "tests/data" make_utils.git_enable_submodule(args.git_command, submodule_dir) - make_utils.git_update_submodule(args.git_command, submodule_dir) + + if not make_utils.git_update_submodule(args.git_command, submodule_dir): + return "Error updating test data\n" + + return "" def git_update_skip(args: argparse.Namespace, check_remote_exists: bool = True) -> str: @@ -566,9 +573,7 @@ def submodules_update(args: argparse.Namespace, branch: Optional[str]) -> str: print(f"Skipping tests submodule {submodule_path}") continue - exitcode = call((args.git_command, "submodule", "update", "--init", submodule_path), - exit_on_error=False) - if exitcode != 0: + if not make_utils.git_update_submodule(args.git_command, submodule_path): msg += f"Error updating Git submodule {submodule_path}\n" add_submodule_push_url(args) @@ -579,6 +584,7 @@ def submodules_update(args: argparse.Namespace, branch: Optional[str]) -> str: if __name__ == "__main__": args = parse_arguments() blender_skip_msg = "" + libraries_skip_msg = "" submodules_skip_msg = "" blender_version = make_utils. parse_blender_version() @@ -600,19 +606,18 @@ if __name__ == "__main__": blender_skip_msg = "Blender repository skipped: " + blender_skip_msg + "\n" if not args.no_libraries: - update_precompiled_libraries(args) + libraries_skip_msg += update_precompiled_libraries(args) if args.use_tests: - update_tests_data_files(args) + libraries_skip_msg += update_tests_data_files(args) if not args.no_submodules: submodules_skip_msg = submodules_update(args, branch) # Report any skipped repositories at the end, so it's not as easy to miss. - skip_msg = blender_skip_msg + submodules_skip_msg + skip_msg = blender_skip_msg + libraries_skip_msg + submodules_skip_msg if skip_msg: - print() + print_stage("Update finished with the following messages") print(skip_msg.strip()) - print() # For failed submodule update we throw an error, since not having correct # submodules can make Blender throw errors. diff --git a/build_files/utils/make_utils.py b/build_files/utils/make_utils.py index b44d8acaad8..9fd8bad4a41 100755 --- a/build_files/utils/make_utils.py +++ b/build_files/utils/make_utils.py @@ -8,6 +8,7 @@ Utility functions for make update and make tests. """ import re +import os import shutil import subprocess import sys @@ -19,18 +20,30 @@ from typing import ( ) -def call(cmd: Sequence[str], exit_on_error: bool = True, silent: bool = False) -> int: +def call(cmd: Sequence[str], exit_on_error: bool = True, silent: bool = False, env=None) -> int: if not silent: - print(" ".join([str(x) for x in cmd])) + cmd_str = "" + if env: + cmd_str += " ".join([f"{item[0]}={item[1]}" for item in env.items()]) + cmd_str += " " + cmd_str += " ".join([str(x) for x in cmd]) + print(cmd_str) + + env_full = None + if env: + env_full = os.environ.copy() + for key, value in env.items(): + env_full[key] = value # Flush to ensure correct order output on Windows. sys.stdout.flush() sys.stderr.flush() if silent: - retcode = subprocess.call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + retcode = subprocess.call( + cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=env_full) else: - retcode = subprocess.call(cmd) + retcode = subprocess.call(cmd, env=env_full) if exit_on_error and retcode != 0: sys.exit(retcode) @@ -127,15 +140,43 @@ def git_enable_submodule(git_command: str, submodule_dir: str): call(command, exit_on_error=True, silent=False) -def git_update_submodule(git_command: str, submodule_dir: str): +def git_update_submodule(git_command: str, submodule_dir: str) -> bool: """ Update the given submodule. The submodule is denoted by its path within the repository. This function will initialize the submodule if it has not been initialized. + + Returns true if the update succeeded """ - call((git_command, "submodule", "update", "--init", submodule_dir)) + # Use the two stage update process: + # - Step 1: checkout the submodule to the desired (by the parent repository) hash, but + # skip the LFS smudging. + # - Step 2: Fetch LFS files, if needed. + # + # This allows to show download progress, potentially allowing resuming the download + # progress, and even recovering from partial/corrupted checkout of submodules. + # + # This bypasses the limitation of submodules which are configured as "update=checkout" + # with regular `git submodule update` which, depending on the Git version will not report + # any progress. This is because submodule--helper.c configures Git checkout process with + # the "quiet" flag, so that there is no detached head information printed after submodule + # update, and since Git 2.33 the LFS messages "Filtering contents..." is suppressed by + # + # https://github.com/git/git/commit/7a132c628e57b9bceeb88832ea051395c0637b16 + # + # Doing "git lfs pull" after checkout with GIT_LFS_SKIP_SMUDGE=true seems to be the + # valid process. For example, https://www.mankier.com/7/git-lfs-faq + + env = {"GIT_LFS_SKIP_SMUDGE": "1"} + + if call((git_command, "submodule", "update", "--init", "--progress", submodule_dir), + exit_on_error=False, env=env) != 0: + return False + + return call((git_command, "-C", submodule_dir, "lfs", "pull"), + exit_on_error=False) == 0 def command_missing(command: str) -> bool: