From cb7cebe4d85b7935fb037f92c05f895a7842e03b Mon Sep 17 00:00:00 2001 From: Campbell Barton Date: Wed, 31 Jul 2024 11:34:44 +1000 Subject: [PATCH] PyAPI: add bpy.app.python_args, use when calling Python from Blender Expose arguments to use when creating a Python sub-process. Python could fail to start when loaded in a customized environment, with PYTHONPATH set for e.g. Blender ignores these and loads but a Python sub-process attempts to use these environment variables which may point to incompatible Python versions. Resolve the root cause of #124731. --- .../addons_core/bl_pkg/bl_extension_notify.py | 1 + .../addons_core/bl_pkg/bl_extension_ops.py | 9 +++++ .../addons_core/bl_pkg/bl_extension_utils.py | 39 ++++++++++++------- scripts/modules/_bpy_internal/freedesktop.py | 25 ++++++++++-- source/blender/python/BPY_extern_python.h | 1 + .../blender/python/generic/py_capi_utils.cc | 17 ++++++++ source/blender/python/generic/py_capi_utils.h | 5 +++ source/blender/python/intern/bpy_app.cc | 19 +++++++++ source/blender/python/intern/bpy_interface.cc | 5 +++ 9 files changed, 104 insertions(+), 17 deletions(-) diff --git a/scripts/addons_core/bl_pkg/bl_extension_notify.py b/scripts/addons_core/bl_pkg/bl_extension_notify.py index 55af9dc5714..e9a367bb26f 100644 --- a/scripts/addons_core/bl_pkg/bl_extension_notify.py +++ b/scripts/addons_core/bl_pkg/bl_extension_notify.py @@ -220,6 +220,7 @@ def sync_status_generator(repos_and_do_online): timeout=network_timeout, # Never sleep while there is no input, as this blocks Blender. use_idle=False, + python_args=bpy.app.python_args, # Needed so the user can exit blender without warnings about a broken pipe. # TODO: write to a temporary location, once done: # There is no chance of corrupt data as the data isn't written directly to the target JSON. diff --git a/scripts/addons_core/bl_pkg/bl_extension_ops.py b/scripts/addons_core/bl_pkg/bl_extension_ops.py index feccabeb6f5..fe6db77227b 100644 --- a/scripts/addons_core/bl_pkg/bl_extension_ops.py +++ b/scripts/addons_core/bl_pkg/bl_extension_ops.py @@ -1356,6 +1356,7 @@ class EXTENSIONS_OT_dummy_progress(Operator, _ExtCmdMixIn): partial( bl_extension_utils.dummy_progress, use_idle=is_modal, + python_args=bpy.app.python_args, ), ], batch_job_limit=1, @@ -1410,6 +1411,7 @@ class EXTENSIONS_OT_repo_sync(Operator, _ExtCmdMixIn): access_token=repo_item.access_token, timeout=prefs.system.network_timeout, use_idle=is_modal, + python_args=bpy.app.python_args, ) ) repos_lock.append(repo_item.directory) @@ -1509,6 +1511,7 @@ class EXTENSIONS_OT_repo_sync_all(Operator, _ExtCmdMixIn): access_token=repo_item.access_token, timeout=prefs.system.network_timeout, use_idle=is_modal, + python_args=bpy.app.python_args, )) repos_lock.append(repo_item.directory) @@ -1876,6 +1879,7 @@ class EXTENSIONS_OT_package_upgrade_all(Operator, _ExtCmdMixIn): timeout=prefs.system.network_timeout, use_cache=repo_item.use_cache, use_idle=is_modal, + python_args=bpy.app.python_args, )) self._repo_directories.add(repo_item.directory) @@ -1997,6 +2001,7 @@ class EXTENSIONS_OT_package_install_marked(Operator, _ExtCmdMixIn): timeout=prefs.system.network_timeout, use_cache=repo_item.use_cache, use_idle=is_modal, + python_args=bpy.app.python_args, )) self._repo_directories.add(repo_item.directory) @@ -2149,6 +2154,7 @@ class EXTENSIONS_OT_package_uninstall_marked(Operator, _ExtCmdMixIn): user_directory=repo_user_directory(repo_item.module), pkg_id_sequence=pkg_id_sequence, use_idle=is_modal, + python_args=bpy.app.python_args, )) self._repo_directories.add(repo_item.directory) package_count += len(pkg_id_sequence) @@ -2379,6 +2385,7 @@ class EXTENSIONS_OT_package_install_files(Operator, _ExtCmdMixIn): files=pkg_files, blender_version=bpy.app.version, use_idle=is_modal, + python_args=bpy.app.python_args, ) ], batch_job_limit=1, @@ -2760,6 +2767,7 @@ class EXTENSIONS_OT_package_install(Operator, _ExtCmdMixIn): timeout=prefs.system.network_timeout, use_cache=repo_item.use_cache, use_idle=is_modal, + python_args=bpy.app.python_args, ) ], batch_job_limit=1, @@ -3217,6 +3225,7 @@ class EXTENSIONS_OT_package_uninstall(Operator, _ExtCmdMixIn): user_directory=repo_user_directory(repo_item.module), pkg_id_sequence=(pkg_id, ), use_idle=is_modal, + python_args=bpy.app.python_args, ), ], batch_job_limit=1, diff --git a/scripts/addons_core/bl_pkg/bl_extension_utils.py b/scripts/addons_core/bl_pkg/bl_extension_utils.py index 8cf7df9ceb6..c6ce7be5000 100644 --- a/scripts/addons_core/bl_pkg/bl_extension_utils.py +++ b/scripts/addons_core/bl_pkg/bl_extension_utils.py @@ -80,12 +80,6 @@ from typing import ( BASE_DIR = os.path.abspath(os.path.dirname(__file__)) -BLENDER_EXT_CMD = ( - # When run from within Blender, it will point to Blender's local Python binary. - sys.executable, - os.path.normpath(os.path.join(BASE_DIR, "cli", "blender_ext.py")), -) - # This directory is in the local repository. REPO_LOCAL_PRIVATE_DIR = ".blender_ext" # Locate inside `REPO_LOCAL_PRIVATE_DIR`. @@ -213,6 +207,14 @@ def rmtree_with_fallback_or_error( return result +def blender_ext_cmd(python_args: Sequence[str]) -> Sequence[str]: + return ( + sys.executable, + *python_args, + os.path.normpath(os.path.join(BASE_DIR, "cli", "blender_ext.py")), + ) + + # ----------------------------------------------------------------------------- # Call JSON. # @@ -230,8 +232,10 @@ def non_blocking_call(cmd: Sequence[str]) -> subprocess.Popen[bytes]: def command_output_from_json_0( args: Sequence[str], use_idle: bool, + *, + python_args: Sequence[str], ) -> Generator[InfoItemSeq, bool, None]: - cmd = [*BLENDER_EXT_CMD, *args, "--output-type=JSON_0"] + cmd = [*blender_ext_cmd(python_args), *args, "--output-type=JSON_0"] ps = non_blocking_call(cmd) stdout = ps.stdout assert stdout is not None @@ -538,6 +542,7 @@ def repo_sync( access_token: str, timeout: float, use_idle: bool, + python_args: Sequence[str], force_exit_ok: bool = False, dry_run: bool = False, demote_connection_errors_to_status: bool = False, @@ -562,7 +567,7 @@ def repo_sync( *(("--force-exit-ok",) if force_exit_ok else ()), *(("--demote-connection-errors-to-status",) if demote_connection_errors_to_status else ()), *(("--extension-override", extension_override) if extension_override else ()), - ], use_idle=use_idle) + ], use_idle=use_idle, python_args=python_args) yield [COMPLETE_ITEM] @@ -573,6 +578,7 @@ def repo_upgrade( online_user_agent: str, access_token: str, use_idle: bool, + python_args: Sequence[str], ) -> Generator[InfoItemSeq, None, None]: """ Implementation: @@ -584,7 +590,7 @@ def repo_upgrade( "--remote-url", remote_url, "--online-user-agent", online_user_agent, "--access-token", access_token, - ], use_idle=use_idle) + ], use_idle=use_idle, python_args=python_args) yield [COMPLETE_ITEM] @@ -613,6 +619,7 @@ def pkg_install_files( files: Sequence[str], blender_version: Tuple[int, int, int], use_idle: bool, + python_args: Sequence[str], ) -> Generator[InfoItemSeq, None, None]: """ Implementation: @@ -622,7 +629,7 @@ def pkg_install_files( "install-files", *files, "--local-dir", directory, "--blender-version", "{:d}.{:d}.{:d}".format(*blender_version), - ], use_idle=use_idle) + ], use_idle=use_idle, python_args=python_args) yield [COMPLETE_ITEM] @@ -637,6 +644,7 @@ def pkg_install( timeout: float, use_cache: bool, use_idle: bool, + python_args: Sequence[str], ) -> Generator[InfoItemSeq, None, None]: """ Implementation: @@ -651,7 +659,7 @@ def pkg_install( "--access-token", access_token, "--local-cache", str(int(use_cache)), "--timeout", "{:g}".format(timeout), - ], use_idle=use_idle) + ], use_idle=use_idle, python_args=python_args) yield [COMPLETE_ITEM] @@ -661,6 +669,7 @@ def pkg_uninstall( user_directory: str, pkg_id_sequence: Sequence[str], use_idle: bool, + python_args: Sequence[str], ) -> Generator[InfoItemSeq, None, None]: """ Implementation: @@ -670,7 +679,7 @@ def pkg_uninstall( "uninstall", ",".join(pkg_id_sequence), "--local-dir", directory, "--user-dir", user_directory, - ], use_idle=use_idle) + ], use_idle=use_idle, python_args=python_args) yield [COMPLETE_ITEM] @@ -681,12 +690,16 @@ def pkg_uninstall( def dummy_progress( *, use_idle: bool, + python_args: Sequence[str], ) -> Generator[InfoItemSeq, bool, None]: """ Implementation: ``bpy.ops.extensions.dummy_progress()``. """ - yield from command_output_from_json_0(["dummy-progress", "--time-duration=1.0"], use_idle=use_idle) + yield from command_output_from_json_0([ + "dummy-progress", + "--time-duration=1.0", + ], use_idle=use_idle, python_args=python_args) yield [COMPLETE_ITEM] diff --git a/scripts/modules/_bpy_internal/freedesktop.py b/scripts/modules/_bpy_internal/freedesktop.py index d4a222b89cd..a9bd956ea8c 100644 --- a/scripts/modules/_bpy_internal/freedesktop.py +++ b/scripts/modules/_bpy_internal/freedesktop.py @@ -24,6 +24,7 @@ import tempfile from typing import ( Callable, + Tuple, Optional, ) @@ -393,7 +394,11 @@ def handle_icon(do_register: bool, all_users: bool) -> Optional[str]: # ----------------------------------------------------------------------------- # Escalate Privileges -def main_run_as_root(do_register: bool) -> Optional[str]: +def main_run_as_root( + do_register: bool, + *, + python_args: Tuple[str, ...], +) -> Optional[str]: # If the system prefix doesn't exist, fail with an error because it's highly likely that the # system won't use this when it has not been created. if not os.path.exists(SYSTEM_PREFIX): @@ -403,11 +408,20 @@ def main_run_as_root(do_register: bool) -> Optional[str]: if prog is None: return "Error: command \"pkexec\" not found" + python_args_extra = ( + # Skips users `site-packages` because they are only additional overhead for running this script. + "-s", + ) + + python_args = ( + *python_args, + *(arg for arg in python_args_extra if arg not in python_args) + ) + cmd = [ prog, sys.executable, - # Skips users `site-packages`. - "-s", + *python_args, __file__, BLENDER_BIN, "--action={:s}".format("register-allusers" if do_register else "unregister-allusers"), @@ -476,7 +490,10 @@ def register_impl(do_register: bool, all_users: bool) -> Optional[str]: if all_users: if os.geteuid() != 0: # Run this script with escalated privileges. - return main_run_as_root(do_register) + return main_run_as_root( + do_register, + python_args=__import__("bpy").app.python_args, + ) else: assert BLENDER_BIN != "" diff --git a/source/blender/python/BPY_extern_python.h b/source/blender/python/BPY_extern_python.h index c07de1e7089..c01fa3fdbba 100644 --- a/source/blender/python/BPY_extern_python.h +++ b/source/blender/python/BPY_extern_python.h @@ -26,6 +26,7 @@ void BPY_python_start(struct bContext *C, int argc, const char **argv); void BPY_python_end(bool do_python_exit); void BPY_python_reset(struct bContext *C); void BPY_python_use_system_env(void); +bool BPY_python_use_system_env_get(); void BPY_python_backtrace(FILE *fp); /* `bpy_app.cc` */ diff --git a/source/blender/python/generic/py_capi_utils.cc b/source/blender/python/generic/py_capi_utils.cc index 66bfe462263..c7d550be267 100644 --- a/source/blender/python/generic/py_capi_utils.cc +++ b/source/blender/python/generic/py_capi_utils.cc @@ -351,6 +351,23 @@ PyObject *PyC_Tuple_PackArray_Bool(const bool *array, uint len) return tuple; } +PyObject *PyC_Tuple_PackArray_String(const char **array, uint len) +{ + /* Not part of numeric array packing but useful none the less. */ + PyObject *tuple = PyTuple_New(len); + for (uint i = 0; i < len; i++) { + if (PyObject *value = PyUnicode_FromString(array[i])) { + PyTuple_SET_ITEM(tuple, i, value); + } + else { + Py_DECREF(tuple); + tuple = nullptr; + break; + } + } + return tuple; +} + /** \} */ /* -------------------------------------------------------------------- */ diff --git a/source/blender/python/generic/py_capi_utils.h b/source/blender/python/generic/py_capi_utils.h index 42c43b25c33..05d9125e8c1 100644 --- a/source/blender/python/generic/py_capi_utils.h +++ b/source/blender/python/generic/py_capi_utils.h @@ -102,6 +102,11 @@ PyObject *PyC_Tuple_PackArray_I32(const int *array, uint len); PyObject *PyC_Tuple_PackArray_I32FromBool(const int *array, uint len); PyObject *PyC_Tuple_PackArray_Bool(const bool *array, uint len); +/** + * \note Any errors converting strings will return null with the error left as-is. + */ +PyObject *PyC_Tuple_PackArray_String(const char **array, uint len); + PyObject *PyC_Tuple_PackArray_Multi_F32(const float *array, const int dims[], int dims_len); PyObject *PyC_Tuple_PackArray_Multi_F64(const double *array, const int dims[], int dims_len); PyObject *PyC_Tuple_PackArray_Multi_I32(const int *array, const int dims[], int dims_len); diff --git a/source/blender/python/intern/bpy_app.cc b/source/blender/python/intern/bpy_app.cc index d4e546a1650..6e5b7893306 100644 --- a/source/blender/python/intern/bpy_app.cc +++ b/source/blender/python/intern/bpy_app.cc @@ -371,6 +371,23 @@ static PyObject *bpy_app_autoexec_fail_message_get(PyObject * /*self*/, void * / return PyC_UnicodeFromBytes(G.autoexec_fail); } +PyDoc_STRVAR( + /* Wrap. */ + bpy_app_python_args_doc, + "Leading arguments to use when calling Python directly (via ``sys.executable``). " + "These arguments match settings Blender uses to " + "ensure Python runs with a compatible environment (read-only)."); +static PyObject *bpy_app_python_args_get(PyObject * /*self*/, void * /*closure*/) +{ + const char *args[1]; + int args_num = 0; + if (!BPY_python_use_system_env_get()) { + /* Isolated Python environment. */ + args[args_num++] = "-I"; + } + return PyC_Tuple_PackArray_String(args, args_num); +} + PyDoc_STRVAR( /* Wrap. */ bpy_app_binary_path_doc, @@ -520,6 +537,8 @@ static PyGetSetDef bpy_app_getsets[] = { (void *)G_FLAG_SCRIPT_AUTOEXEC_FAIL_QUIET}, {"autoexec_fail_message", bpy_app_autoexec_fail_message_get, nullptr, nullptr, nullptr}, + {"python_args", bpy_app_python_args_get, nullptr, bpy_app_python_args_doc, nullptr}, + /* Support script authors setting the Blender binary path to use, otherwise this value * is not known when built as a Python module. */ {"binary_path", diff --git a/source/blender/python/intern/bpy_interface.cc b/source/blender/python/intern/bpy_interface.cc index e14f4ce4e59..137b37120a3 100644 --- a/source/blender/python/intern/bpy_interface.cc +++ b/source/blender/python/intern/bpy_interface.cc @@ -639,6 +639,11 @@ void BPY_python_use_system_env() py_use_system_env = true; } +bool BPY_python_use_system_env_get() +{ + return py_use_system_env; +} + void BPY_python_backtrace(FILE *fp) { fputs("\n# Python backtrace\n", fp);