From 656fe6d3e40041616fbadd7a191011fe99a1ddbb Mon Sep 17 00:00:00 2001 From: Campbell Barton Date: Fri, 26 Jul 2024 16:05:34 +1000 Subject: [PATCH] Extensions: support "blocklist" in the remote repository Support blocking extensions so there is a way for the maintainers of the remote repository to notify the user if one of their installed extensions blocked along with a reason for blocking. Blocked extensions cannot be installed from the preferences or by dropping a URL. When an installed & blocked extension is found: - An icon int the status bar shows an alert, clicking on the icon shows the blocked extensions. - The extensions preferences show a warning. - The extensions & add-ons UI shows an alert icon and "details" section shows the reason. Details: - Blocked & installed extensions are shown first in the installed extensions panel. - The internal "install" logic prevents downloading & installing blocked extensions. - Blocked extensions can still be downloaded & installed from disk. - The "list" command includes an error message if any installed extensions are blocked. - The "server-generate" command can optionally take a configuration file that includes the blocklist for the generated JSON. See design #124954. --- scripts/addons_core/bl_pkg/__init__.py | 28 +++- .../addons_core/bl_pkg/bl_extension_cli.py | 21 +++ .../addons_core/bl_pkg/bl_extension_notify.py | 20 ++- .../addons_core/bl_pkg/bl_extension_ops.py | 30 ++++- scripts/addons_core/bl_pkg/bl_extension_ui.py | 125 ++++++++++++++---- .../addons_core/bl_pkg/bl_extension_utils.py | 100 ++++++++++++-- scripts/addons_core/bl_pkg/cli/blender_ext.py | 116 +++++++++++++++- .../bl_pkg/tests/test_cli_blender.py | 84 ++++++++++++ .../templates/interface_templates.cc | 23 ++++ .../makesdna/DNA_windowmanager_types.h | 3 +- source/blender/makesrna/intern/rna_wm.cc | 13 +- source/blender/windowmanager/intern/wm.cc | 1 + 12 files changed, 517 insertions(+), 47 deletions(-) diff --git a/scripts/addons_core/bl_pkg/__init__.py b/scripts/addons_core/bl_pkg/__init__.py index 70a244f23ad..ca8fca12a8a 100644 --- a/scripts/addons_core/bl_pkg/__init__.py +++ b/scripts/addons_core/bl_pkg/__init__.py @@ -183,6 +183,29 @@ def repo_stats_calc_outdated_for_repo_directory(repo_cache_store, repo_directory return package_count +def repo_stats_calc_blocked(repo_cache_store): + block_count = 0 + for ( + pkg_manifest_remote, + pkg_manifest_local, + ) in zip( + repo_cache_store.pkg_manifest_from_remote_ensure(error_fn=print), + repo_cache_store.pkg_manifest_from_local_ensure(error_fn=print), + ): + if (pkg_manifest_remote is None) or (pkg_manifest_local is None): + continue + + for pkg_id in pkg_manifest_local.keys(): + item_remote = pkg_manifest_remote.get(pkg_id) + if item_remote is None: + continue + + if item_remote.block: + block_count += 1 + + return block_count + + def repo_stats_calc(): # NOTE: if repositories get very large, this could be optimized to only check repositories that have changed. # Although this isn't called all that often - it's unlikely to be a bottleneck. @@ -213,7 +236,10 @@ def repo_stats_calc(): package_count += repo_stats_calc_outdated_for_repo_directory(repo_cache_store, repo_directory) - bpy.context.window_manager.extensions_updates = package_count + wm = bpy.context.window_manager + wm.extensions_updates = package_count + + wm.extensions_blocked = repo_stats_calc_blocked(repo_cache_store) def print_debug(*args, **kw): diff --git a/scripts/addons_core/bl_pkg/bl_extension_cli.py b/scripts/addons_core/bl_pkg/bl_extension_cli.py index ad28f378d84..32b6f82e875 100644 --- a/scripts/addons_core/bl_pkg/bl_extension_cli.py +++ b/scripts/addons_core/bl_pkg/bl_extension_cli.py @@ -242,6 +242,9 @@ class subcmd_query: colorize(item.tagline or "", "faint"), )) + if item_remote and item_remote.block: + print(" Blocked:", colorize(item_remote.block.reason, "red")) + if item_warnings: # Including all text on one line doesn't work well here, # add warnings below the package. @@ -266,6 +269,9 @@ class subcmd_query: extensions_warnings: Dict[str, List[str]] = addon_utils._extensions_warnings_get() assert isinstance(extensions_warnings, dict) + # Blocked and installed. + blocked_and_installed_count = 0 + for repo_index, ( pkg_manifest_local, pkg_manifest_remote, @@ -285,6 +291,21 @@ class subcmd_query: item_remote = pkg_manifest_remote.get(pkg_id) if (pkg_manifest_remote is not None) else None item_warnings = extensions_warnings.get("bl_ext.{:s}.{:s}".format(repo.module, pkg_id), []) list_item(pkg_id, item_local, item_remote, has_remote, item_warnings) + if item_local and item_remote and item_remote.block: + blocked_and_installed_count += 1 + sys.stdout.flush() + + if blocked_and_installed_count: + sys.stderr.write("\n") + sys.stderr.write( + " Warning: " + + colorize("{:d} installed extension(s) are blocked!\n".format(blocked_and_installed_count), "red") + ) + sys.stderr.write( + " " + + colorize("Uninstall them to remove this message!\n", "red") + ) + sys.stderr.write("\n") return True diff --git a/scripts/addons_core/bl_pkg/bl_extension_notify.py b/scripts/addons_core/bl_pkg/bl_extension_notify.py index b06df2a89e6..55af9dc5714 100644 --- a/scripts/addons_core/bl_pkg/bl_extension_notify.py +++ b/scripts/addons_core/bl_pkg/bl_extension_notify.py @@ -465,7 +465,7 @@ def _ui_refresh_apply(): _region_refresh_registered() -def _ui_refresh_timer(): +def _ui_refresh_timer_impl(): wm = bpy.context.window_manager # Ensure the first item is running, skipping any items that have no work. @@ -517,6 +517,24 @@ def _ui_refresh_timer(): return default_wait +def _ui_refresh_timer(): + result = _ui_refresh_timer_impl() + + # Ensure blocked packages are counted before finishing. + if result is None: + from . import ( + repo_cache_store_ensure, + repo_stats_calc_blocked, + ) + wm = bpy.context.window_manager + repo_cache_store = repo_cache_store_ensure() + extensions_blocked = repo_stats_calc_blocked(repo_cache_store) + if extensions_blocked != wm.extensions_blocked: + wm.extensions_blocked = extensions_blocked + + return result + + # ----------------------------------------------------------------------------- # Internal Region Updating diff --git a/scripts/addons_core/bl_pkg/bl_extension_ops.py b/scripts/addons_core/bl_pkg/bl_extension_ops.py index 83755cdce68..feccabeb6f5 100644 --- a/scripts/addons_core/bl_pkg/bl_extension_ops.py +++ b/scripts/addons_core/bl_pkg/bl_extension_ops.py @@ -1847,6 +1847,9 @@ class EXTENSIONS_OT_package_upgrade_all(Operator, _ExtCmdMixIn): if item_local is None: # Not installed. continue + if item_remote.block: + # Blocked, don't touch. + continue if item_remote.version != item_local.version: packages_to_upgrade[repo_index].append(pkg_id) @@ -3077,6 +3080,23 @@ class EXTENSIONS_OT_package_install(Operator, _ExtCmdMixIn): ) return False + if item_remote.block: + self._draw_override = ( + self._draw_override_errors, + { + "errors": [ + ( + "Repository \"{:s}\" has blocked \"{:s}\"\n" + "for the following reason:" + ).format(repo_name, pkg_id), + " " + item_remote.block.reason, + "If you wish to install the extensions anyway,\n" + "manually download and install the extension from disk." + ] + } + ) + return False + self._drop_variables = repo_index, repo_name, pkg_id, item_remote self.repo_index = repo_index @@ -3096,7 +3116,15 @@ class EXTENSIONS_OT_package_install(Operator, _ExtCmdMixIn): icon = 'ERROR' for error in errors: if isinstance(error, str): - layout.label(text=error, translate=False, icon=icon) + # Group text split by newlines more closely. + # Without this, lines have too much vertical space. + if "\n" in error: + layout_aligned = layout.column(align=True) + for error in error.split("\n"): + layout_aligned.label(text=error, translate=False, icon=icon) + icon = 'BLANK1' + else: + layout.label(text=error, translate=False, icon=icon) else: error(layout) icon = 'BLANK1' diff --git a/scripts/addons_core/bl_pkg/bl_extension_ui.py b/scripts/addons_core/bl_pkg/bl_extension_ui.py index 784694042b8..455736f3abd 100644 --- a/scripts/addons_core/bl_pkg/bl_extension_ui.py +++ b/scripts/addons_core/bl_pkg/bl_extension_ui.py @@ -432,6 +432,8 @@ def addons_panel_draw_items( addon_tags_exclude, # `Set[str]` enabled_only, # `bool` addon_extension_manifest_map, # `Dict[str, PkgManifest_Normalized]` + addon_extension_block_map, # `Dict[str, PkgBlock_Normalized]` + show_development, # `bool` ): # `-> Set[str]` # NOTE: this duplicates logic from `USERPREF_PT_addons` eventually this logic should be used instead. @@ -464,6 +466,9 @@ def addons_panel_draw_items( if is_extension: item_warnings = [] + if pkg_block := addon_extension_block_map.get(module_name): + item_warnings.append("Blocked: {:s}".format(pkg_block.reason)) + if value := extensions_warnings.get(module_name): item_warnings.extend(value) del value @@ -677,10 +682,17 @@ def addons_panel_draw_impl( local_ex = ex addon_extension_manifest_map = {} + addon_extension_block_map = {} - for repo_index, pkg_manifest_local in enumerate( - repo_cache_store.pkg_manifest_from_local_ensure(error_fn=error_fn_local) - ): + # The `pkg_manifest_remote` is only needed for `PkgBlock_Normalized` data. + for repo_index, ( + pkg_manifest_local, + pkg_manifest_remote, + ) in enumerate(zip( + repo_cache_store.pkg_manifest_from_local_ensure(error_fn=error_fn_local), + repo_cache_store.pkg_manifest_from_remote_ensure(error_fn=error_fn_local), + strict=True, + )): if pkg_manifest_local is None: continue @@ -691,6 +703,11 @@ def addons_panel_draw_impl( module_name = repo_module_prefix + pkg_id addon_extension_manifest_map[module_name] = item_local + if pkg_manifest_remote is not None: + if (item_remote := pkg_manifest_remote.get(pkg_id)) is not None: + if (pkg_block := item_remote.block) is not None: + addon_extension_block_map[module_name] = pkg_block + used_addon_module_name_map = {addon.module: addon for addon in prefs.addons} module_names = addons_panel_draw_items( @@ -702,6 +719,7 @@ def addons_panel_draw_impl( addon_tags_exclude=addon_tags_exclude, enabled_only=enabled_only, addon_extension_manifest_map=addon_extension_manifest_map, + addon_extension_block_map=addon_extension_block_map, show_development=show_development, ) @@ -1039,14 +1057,21 @@ class display_errors: box_header = layout.box() # Don't clip longer names. row = box_header.split(factor=0.9) - row.label(text="Repository Access Errors:", icon='ERROR') + row.label(text="Repository Alert:", icon='ERROR') rowsub = row.row(align=True) rowsub.alignment = 'RIGHT' rowsub.operator("extensions.status_clear_errors", text="", icon='X', emboss=False) box_contents = box_header.box() for err in display_errors.errors_curr: - box_contents.label(text=err) + # Group text split by newlines more closely. + # Without this, lines have too much vertical space. + if "\n" in err: + box_contents_align = box_contents.column(align=True) + for err in err.split("\n"): + box_contents_align.label(text=err) + else: + box_contents.label(text=err) class notify_info: @@ -1115,21 +1140,33 @@ class ExtensionUI_Section: # Label & panel property or None not to define a header, # in this case the previous panel is used. "panel_header", - "do_sort", + "ui_ext_sort_fn", "enabled", "extension_ui_list", ) - def __init__(self, *, panel_header, do_sort): + def __init__(self, *, panel_header, ui_ext_sort_fn): self.panel_header = panel_header - self.do_sort = do_sort + self.ui_ext_sort_fn = ui_ext_sort_fn self.enabled = True self.extension_ui_list = [] - def sort_by_name(self): - self.extension_ui_list.sort(key=lambda ext_ui: (ext_ui.item_local or ext_ui.item_remote).name.casefold()) + @staticmethod + def sort_by_blocked_and_name_fn(ext_ui): + item_local = ext_ui.item_local + item_remote = ext_ui.item_remote + return ( + # Not blocked. + not (item_remote is not None and item_remote.block is not None), + # Name. + (item_local or item_remote).name.casefold(), + ) + + @staticmethod + def sort_by_name_fn(ext_ui): + return (ext_ui.item_local or ext_ui.item_remote).name.casefold() def extensions_panel_draw_online_extensions_request_impl( @@ -1226,6 +1263,11 @@ def extension_draw_item( is_installed = item_local is not None has_remote = repo_item.remote_url != "" + if item_remote is not None: + pkg_block = item_remote.block + else: + pkg_block = None + if is_enabled: item_warnings = extensions_warnings.get(pkg_repo_module_prefix(repo_item) + pkg_id, []) else: @@ -1257,7 +1299,7 @@ def extension_draw_item( # Without checking `is_enabled` here, there is no way for the user to know if an extension # is enabled or not, which is useful to show - when they may be considering removing/updating # extensions based on them being used or not. - if item_warnings: + if pkg_block or item_warnings: sub.label(text=item.name, icon='ERROR', translate=False) else: sub.label(text=item.name, translate=False) @@ -1274,8 +1316,9 @@ def extension_draw_item( row_right.alignment = 'RIGHT' if has_remote and (item_remote is not None): - if is_installed: - # Include uninstall below. + if pkg_block is not None: + row_right.label(text="Blocked ") + elif is_installed: if is_outdated: props = row_right.operator("extensions.package_install", text="Update") props.repo_index = repo_index @@ -1328,6 +1371,10 @@ def extension_draw_item( col_b = split.column() col_a.alignment = "RIGHT" + if pkg_block is not None: + col_a.label(text="Blocked") + col_b.label(text=pkg_block.reason, translate=False) + if item_warnings: col_a.label(text="Warning") col_b.label(text=item_warnings[0]) @@ -1404,6 +1451,7 @@ def extensions_panel_draw_impl( from . import repo_cache_store_ensure + wm = context.window_manager prefs = context.preferences repo_cache_store = repo_cache_store_ensure() @@ -1486,16 +1534,31 @@ def extensions_panel_draw_impl( section_list = ( # Installed (upgrade, enabled). - ExtensionUI_Section(panel_header=(iface_("Installed"), "extension_show_panel_installed"), do_sort=True), + ExtensionUI_Section( + panel_header=(iface_("Installed"), "extension_show_panel_installed"), + ui_ext_sort_fn=ExtensionUI_Section.sort_by_blocked_and_name_fn, + ), # Installed (upgrade, disabled). Use the previous panel. - ExtensionUI_Section(panel_header=None, do_sort=True), + ExtensionUI_Section( + panel_header=None, + ui_ext_sort_fn=ExtensionUI_Section.sort_by_name_fn, + ), # Installed (up-to-date, enabled). Use the previous panel. - ExtensionUI_Section(panel_header=None, do_sort=True), + ExtensionUI_Section( + panel_header=None, + ui_ext_sort_fn=ExtensionUI_Section.sort_by_name_fn, + ), # Installed (up-to-date, disabled). - ExtensionUI_Section(panel_header=None, do_sort=True), + ExtensionUI_Section( + panel_header=None, + ui_ext_sort_fn=ExtensionUI_Section.sort_by_name_fn, + ), # Available (remaining). # NOTE: don't use A-Z here to prevent name manipulation to bring an extension up on the ranks. - ExtensionUI_Section(panel_header=(iface_("Available"), "extension_show_panel_available"), do_sort=False), + ExtensionUI_Section( + panel_header=(iface_("Available"), "extension_show_panel_available"), + ui_ext_sort_fn=None, + ), ) # The key is: (is_outdated, is_enabled) or None for the rest. section_table = { @@ -1586,12 +1649,24 @@ def extensions_panel_draw_impl( pkg_manifest_local, pkg_manifest_remote, ): - section = ( - section_available if ext_ui.item_local is None else - section_table[ext_ui.is_outdated, ext_ui.is_enabled] - ) + if ext_ui.item_local is None: + section = section_available + else: + if ((item_remote := ext_ui.item_remote) is not None) and (item_remote.block is not None): + # Blocked are always first. + section = section_installed + else: + section = section_table[ext_ui.is_outdated, ext_ui.is_enabled] + section.extension_ui_list.append(ext_ui) + if wm.extensions_blocked: + errors_on_draw.append(( + "Found {:d} extension(s) blocked by the remote repository!\n" + "Expand the extension to see the reason for blocking.\n" + "Uninstall these extensions to remove the warning." + ).format(wm.extensions_blocked)) + del repo_index, pkg_manifest_local, pkg_manifest_remote section_installed.enabled = (params.has_installed_enabled or params.has_installed_disabled) @@ -1605,7 +1680,7 @@ def extensions_panel_draw_impl( if section.panel_header: label, prop_id = section.panel_header - layout_header, layout_panel = layout.panel_prop(context.window_manager, prop_id) + layout_header, layout_panel = layout.panel_prop(wm, prop_id) layout_header.label(text=label, translate=False) del label, prop_id, layout_header @@ -1614,8 +1689,8 @@ def extensions_panel_draw_impl( if not section.extension_ui_list: continue - if section.do_sort: - section.sort_by_name() + if section.ui_ext_sort_fn is not None: + section.extension_ui_list.sort(key=section.ui_ext_sort_fn) for ext_ui in section.extension_ui_list: extension_draw_item( diff --git a/scripts/addons_core/bl_pkg/bl_extension_utils.py b/scripts/addons_core/bl_pkg/bl_extension_utils.py index 22af25ac516..8cf7df9ceb6 100644 --- a/scripts/addons_core/bl_pkg/bl_extension_utils.py +++ b/scripts/addons_core/bl_pkg/bl_extension_utils.py @@ -1121,6 +1121,29 @@ class CommandBatch: # Internal Repo Data Source # +class PkgBlock_Normalized(NamedTuple): + reason: str + + @staticmethod + def from_dict_with_error_fn( + block_dict: Dict[str, Any], + *, + # Only for useful error messages. + pkg_idname: str, + error_fn: Callable[[Exception], None], + ) -> Optional["PkgBlock_Normalized"]: + + try: + reason = block_dict["reason"] + except KeyError as ex: + error_fn(KeyError("{:s}: missing key {:s}".format(pkg_idname, str(ex)))) + return None + + return PkgBlock_Normalized( + reason=reason, + ) + + # See similar named tuple: `bl_pkg.cli.blender_ext.PkgManifest`. # This type is loaded from an external source and had it's valued parsed into a known "normalized" state. # Some transformation is performed to the purpose of displaying in the UI although this type isn't specifically for UI. @@ -1147,12 +1170,16 @@ class PkgManifest_Normalized(NamedTuple): archive_size: int archive_url: str + # Taken from the `blocklist`. + block: Optional[PkgBlock_Normalized] + @staticmethod def from_dict_with_error_fn( manifest_dict: Dict[str, Any], *, # Only for useful error messages. pkg_idname: str, + pkg_block: Optional[PkgBlock_Normalized], error_fn: Callable[[Exception], None], ) -> Optional["PkgManifest_Normalized"]: # NOTE: it is expected there are no errors here for typical usage. @@ -1264,6 +1291,8 @@ class PkgManifest_Normalized(NamedTuple): archive_size=field_archive_size, archive_url=field_archive_url, + + block=pkg_block, ) @@ -1343,11 +1372,47 @@ def pkg_manifest_params_compatible_or_error( return None +def repository_parse_blocklist( + data: List[Dict[str, Any]], + *, + repo_directory: str, + error_fn: Callable[[Exception], None], +) -> Dict[str, PkgBlock_Normalized]: + pkg_block_map = {} + + for item in data: + if not isinstance(item, dict): + error_fn(Exception("found non dict item in repository \"blocklist\", found {:s}".format(str(type(item))))) + continue + + if (pkg_idname := repository_id_with_error_fn( + item, + repo_directory=repo_directory, + error_fn=error_fn, + )) is None: + continue + if (value := PkgBlock_Normalized.from_dict_with_error_fn( + item, + pkg_idname=pkg_idname, + error_fn=error_fn, + )) is None: + # NOTE: typically we would skip invalid items + # however as it's known this ID is blocked, create a dummy item. + value = PkgBlock_Normalized( + reason="Unknown (parse error)", + ) + + pkg_block_map[pkg_idname] = value + + return pkg_block_map + + def repository_parse_data_filtered( data: List[Dict[str, Any]], *, repo_directory: str, filter_params: PkgManifest_FilterParams, + pkg_block_map: Dict[str, PkgBlock_Normalized], error_fn: Callable[[Exception], None], ) -> Dict[str, PkgManifest_Normalized]: pkg_manifest_map = {} @@ -1373,6 +1438,7 @@ def repository_parse_data_filtered( if (value := PkgManifest_Normalized.from_dict_with_error_fn( item, pkg_idname=pkg_idname, + pkg_block=pkg_block_map.get(pkg_idname), error_fn=error_fn, )) is None: continue @@ -1384,8 +1450,7 @@ def repository_parse_data_filtered( class RepoRemoteData(NamedTuple): version: str - blocklist: List[str] - # Converted from the `data` field. + # Converted from the `data` & `blocklist` fields. pkg_manifest_map: Dict[str, PkgManifest_Normalized] @@ -1532,15 +1597,29 @@ class _RepoDataSouce_JSON(_RepoDataSouce_ABC): # It's important to assign this value even if it's "empty", # otherwise corrupt files will be detected as unset and continuously attempt to load. + + repo_directory = os.path.dirname(self._filepath) + + # Useful for testing: + # `data_dict["blocklist"] = [{"id": "math_vis_console", "reason": "This is blocked"}]` + + pkg_block_map = repository_parse_blocklist( + data_dict.get("blocklist", []), + repo_directory=repo_directory, + error_fn=error_fn, + ) + + pkg_manifest_map = repository_parse_data_filtered( + data_dict.get("data", []), + repo_directory=repo_directory, + filter_params=self._filter_params, + pkg_block_map=pkg_block_map, + error_fn=error_fn, + ) + data = RepoRemoteData( version=data_dict.get("version", "v1"), - blocklist=data_dict.get("blocklist", []), - pkg_manifest_map=repository_parse_data_filtered( - data_dict.get("data", []), - repo_directory=os.path.dirname(self._filepath), - filter_params=self._filter_params, - error_fn=error_fn, - ), + pkg_manifest_map=pkg_manifest_map, ) self._data = data @@ -1641,6 +1720,7 @@ class _RepoDataSouce_TOML_FILES(_RepoDataSouce_ABC): if (value := PkgManifest_Normalized.from_dict_with_error_fn( item_local, pkg_idname=pkg_idname, + pkg_block=None, error_fn=error_fn, )) is None: continue @@ -1652,7 +1732,6 @@ class _RepoDataSouce_TOML_FILES(_RepoDataSouce_ABC): # to use the same structure as the actual JSON. data = RepoRemoteData( version="v1", - blocklist=[], pkg_manifest_map=pkg_manifest_map, ) # End: compatibility change. @@ -1858,6 +1937,7 @@ class _RepoCacheEntry: if (value := PkgManifest_Normalized.from_dict_with_error_fn( item_local, pkg_idname=pkg_idname, + pkg_block=None, error_fn=error_fn, )) is not None: pkg_manifest_local[pkg_idname] = value diff --git a/scripts/addons_core/bl_pkg/cli/blender_ext.py b/scripts/addons_core/bl_pkg/cli/blender_ext.py index 66744659f48..e2ccf4e303d 100755 --- a/scripts/addons_core/bl_pkg/cli/blender_ext.py +++ b/scripts/addons_core/bl_pkg/cli/blender_ext.py @@ -288,7 +288,7 @@ class CleanupPathsContext: class PkgRepoData(NamedTuple): version: str - blocklist: List[str] + blocklist: List[Dict[str, Any]] data: List[Dict[str, Any]] @@ -384,6 +384,12 @@ class PkgManifest_Archive(NamedTuple): archive_url: str +class PkgServerRepoConfig(NamedTuple): + """Server configuration (for generating repositories).""" + schema_version: str + blocklist: List[Dict[str, Any]] + + # ----------------------------------------------------------------------------- # Generic Functions @@ -835,6 +841,33 @@ def pkg_manifest_from_archive_and_validate( return pkg_manifest_from_zipfile_and_validate(zip_fh, archive_subdir, strict=strict) +def pkg_server_repo_config_from_toml_and_validate( + filepath: str, +) -> Union[PkgServerRepoConfig, str]: + + if isinstance(result := toml_from_filepath_or_error(filepath), str): + return result + + if not (field_schema_version := result.get("schema_version", "")): + return "missing \"schema_version\" field" + + if not (field_blocklist := result.get("blocklist", "")): + return "missing \"blocklist\" field" + + for item in field_blocklist: + if not isinstance(item, dict): + return "blocklist contains non dictionary item, found ({:s})".format(str(type(item))) + if not isinstance(value := item.get("id"), str): + return "blocklist items must have have a string typed \"id\" entry, found {:s}".format(str(type(value))) + if not isinstance(value := item.get("reason"), str): + return "blocklist items must have have a string typed \"reason\" entry, found {:s}".format(str(type(value))) + + return PkgServerRepoConfig( + schema_version=field_schema_version, + blocklist=field_blocklist, + ) + + def pkg_is_legacy_addon(filepath: str) -> bool: # Python file is legacy. if os.path.splitext(filepath)[1].lower() == ".py": @@ -2258,9 +2291,9 @@ def repo_json_is_valid_or_error(filepath: str) -> Optional[str]: if not isinstance(value, list): return "Expected \"blocklist\" to be a list, not a {:s}".format(str(type(value))) for item in value: - if isinstance(item, str): + if isinstance(item, dict): continue - return "Expected \"blocklist\" to be a list of strings, found {:s}".format(str(type(item))) + return "Expected \"blocklist\" to be a list of dictionaries, found {:s}".format(str(type(item))) if (value := result.get("data")) is None: return "Expected a \"data\" key which was not found" @@ -2605,6 +2638,9 @@ def pkg_repo_data_from_json_or_error(json_data: Dict[str, Any]) -> Union[PkgRepo if not isinstance((blocklist := json_data.get("blocklist", [])), list): return "expected \"blocklist\" to be a list" + for item in blocklist: + if not isinstance(item, dict): + return "expected \"blocklist\" contain dictionary items" if not isinstance((data := json_data.get("data", [])), list): return "expected \"data\" to be a list" @@ -2724,6 +2760,31 @@ def generic_arg_package_valid_tags(subparse: argparse.ArgumentParser) -> None: # ----------------------------------------------------------------------------- # Argument Handlers ("server-generate" command) +def generic_arg_server_generate_repo_config(subparse: argparse.ArgumentParser) -> None: + subparse.add_argument( + "--repo-config", + dest="repo_config", + default="", + metavar="REPO_CONFIG", + help=( + "An optional server configuration to include information which can't be detected.\n" + "Defaults to ``blender_repo.toml`` (in the repository directory).\n" + "\n" + "This can be used to defined blocked extensions, for example ::\n" + "\n" + " schema_version = \"1.0.0\"\n" + "\n" + " [[blocklist]]\n" + " id = \"my_example_package\"\n" + " reason = \"Explanation for why this extension was blocked\"\n" + " [[blocklist]]\n" + " id = \"other_extenison\"\n" + " reason = \"Another reason for why this is blocked\"\n" + "\n" + ), + ) + + def generic_arg_server_generate_html(subparse: argparse.ArgumentParser) -> None: subparse.add_argument( "--html", @@ -3185,6 +3246,7 @@ class subcmd_server: msglog: MessageLogger, *, repo_dir: str, + repo_config_filepath: str, html: bool, html_template: str, ) -> bool: @@ -3196,14 +3258,44 @@ class subcmd_server: msglog.fatal_error("Directory: {!r} not found!".format(repo_dir)) return False + # Server manifest (optional), use if found. + server_manifest_default = "blender_repo.toml" + if not repo_config_filepath: + server_manifest_test = os.path.join(repo_dir, server_manifest_default) + if os.path.exists(server_manifest_test): + repo_config_filepath = server_manifest_test + del server_manifest_test + del server_manifest_default + + repo_config = None + if repo_config_filepath: + repo_config = pkg_server_repo_config_from_toml_and_validate(repo_config_filepath) + if isinstance(repo_config, str): + msglog.fatal_error("parsing repository configuration {!r}, {:s}".format( + repo_config, + repo_config_filepath, + )) + return False + if repo_config.schema_version != "1.0.0": + msglog.fatal_error("unsupported schema version {!r} in {:s}, expected 1.0.0".format( + repo_config.schema_version, + repo_config_filepath, + )) + return False + assert repo_config is None or isinstance(repo_config, PkgServerRepoConfig) + repo_data_idname_map: Dict[str, List[PkgManifest]] = {} repo_data: List[Dict[str, Any]] = [] + # Write package meta-data into each directory. repo_gen_dict = { "version": "v1", - "blocklist": [], + "blocklist": [] if repo_config is None else repo_config.blocklist, "data": repo_data, } + + del repo_config + for entry in os.scandir(repo_dir): if not entry.name.endswith(PKG_EXT): continue @@ -3602,6 +3694,14 @@ class subcmd_client: for pkg_info in json_data_pkg_info: json_data_pkg_info_map[pkg_info["id"]].append(pkg_info) + # NOTE: we could have full validation as a separate function, + # currently install is the only place this is needed. + json_data_pkg_block_map = { + pkg_idname: pkg_block.get("reason", "Unknown") + for pkg_block in pkg_repo_data.blocklist + if (pkg_idname := pkg_block.get("id")) + } + platform_this = platform_from_this_system() has_fatal_error = False @@ -3612,6 +3712,11 @@ class subcmd_client: has_fatal_error = True continue + if (result := json_data_pkg_block_map.get(pkg_idname)) is not None: + msglog.fatal_error("Package \"{:s}\", is blocked: {:s}".format(pkg_idname, result)) + has_fatal_error = True + continue + pkg_info_list = [ pkg_info for pkg_info in pkg_info_list if not repository_filter_skip( @@ -4410,6 +4515,7 @@ def unregister(): if not subcmd_server.generate( MessageLogger(msg_fn_no_done), repo_dir=repo_dir, + repo_config_filepath="", html=True, html_template="", ): @@ -4466,6 +4572,7 @@ def argparse_create_server_generate( ) generic_arg_repo_dir(subparse) + generic_arg_server_generate_repo_config(subparse) generic_arg_server_generate_html(subparse) generic_arg_server_generate_html_template(subparse) if args_internal: @@ -4475,6 +4582,7 @@ def argparse_create_server_generate( func=lambda args: subcmd_server.generate( msglog_from_args(args), repo_dir=args.repo_dir, + repo_config_filepath=args.repo_config, html=args.html, html_template=args.html_template, ), diff --git a/scripts/addons_core/bl_pkg/tests/test_cli_blender.py b/scripts/addons_core/bl_pkg/tests/test_cli_blender.py index eb7f9038eb8..d78b668931e 100644 --- a/scripts/addons_core/bl_pkg/tests/test_cli_blender.py +++ b/scripts/addons_core/bl_pkg/tests/test_cli_blender.py @@ -861,6 +861,90 @@ class TestModuleViolation(TestWithTempBlenderUser_MixIn, unittest.TestCase): ) +class TestBlockList(TestWithTempBlenderUser_MixIn, unittest.TestCase): + + def test_blocked(self) -> None: + """ + Warn when: + - extensions add themselves to the ``sys.path``. + - extensions add top-level modules into ``sys.modules``. + """ + repo_id = "test_repo_blocklist" + repo_name = "MyTestRepoBlocked" + + self.repo_add(repo_id=repo_id, repo_name=repo_name) + + pkg_idnames = ( + "my_test_pkg_a", + "my_test_pkg_b", + "my_test_pkg_c", + ) + + # Create a package contents. + for pkg_idname in pkg_idnames: + self.build_package(pkg_idname=pkg_idname) + + repo_config_filepath = os.path.join(TEMP_DIR_REMOTE, "blender_repo.toml") + with open(repo_config_filepath, "w", encoding="utf8") as fh: + fh.write( + '''schema_version = "1.0.0"\n''' + '''[[blocklist]]\n''' + '''id = "my_test_pkg_a"\n''' + '''reason = "One example reason"\n''' + '''[[blocklist]]\n''' + '''id = "my_test_pkg_c"\n''' + '''reason = "Another example reason"\n''' + ) + + # Generate the repository. + stdout = run_blender_extensions_no_errors(( + "server-generate", + "--repo-dir", TEMP_DIR_REMOTE, + "--repo-config", repo_config_filepath, + )) + self.assertEqual(stdout, "found 3 packages.\n") + + stdout = run_blender_extensions_no_errors(( + "sync", + )) + self.assertEqual( + stdout.rstrip("\n").split("\n")[-1], + "STATUS Extensions list for \"{:s}\" updated".format(repo_name), + ) + + # List packages. + stdout = run_blender_extensions_no_errors(("list",)) + self.assertEqual( + stdout, + ( + '''Repository: "{:s}" (id={:s})\n''' + ''' my_test_pkg_a: "My Test Pkg A", This is a tagline\n''' + ''' Blocked: One example reason\n''' + ''' my_test_pkg_b: "My Test Pkg B", This is a tagline\n''' + ''' my_test_pkg_c: "My Test Pkg C", This is a tagline\n''' + ''' Blocked: Another example reason\n''' + ).format( + repo_name, + repo_id, + )) + + # Install the package into Blender. + stdout = run_blender_extensions_no_errors(("install", pkg_idnames[1], "--enable")) + self.assertEqual( + [line for line in stdout.split("\n") if line.startswith("STATUS ")][0], + "STATUS Installed \"{:s}\"".format(pkg_idnames[1]) + ) + + # Ensure blocking works, fail to install the package into Blender. + stdout = run_blender_extensions_no_errors(("install", pkg_idnames[0], "--enable")) + self.assertEqual( + [line for line in stdout.split("\n") if line.startswith("FATAL_ERROR ")][0], + "FATAL_ERROR Package \"{:s}\", is blocked: One example reason".format(pkg_idnames[0]) + ) + + # Install the package into Blender. + + def main() -> None: # pylint: disable-next=global-statement global TEMP_DIR_BLENDER_USER, TEMP_DIR_REMOTE, TEMP_DIR_LOCAL, TEMP_DIR_TMPDIR, TEMP_DIR_REMOTE_AS_URL diff --git a/source/blender/editors/interface/templates/interface_templates.cc b/source/blender/editors/interface/templates/interface_templates.cc index b7f3e15261c..0dfedbb9260 100644 --- a/source/blender/editors/interface/templates/interface_templates.cc +++ b/source/blender/editors/interface/templates/interface_templates.cc @@ -6469,6 +6469,29 @@ void uiTemplateStatusInfo(uiLayout *layout, bContext *C) if (U.statusbar_flag & STATUSBAR_SHOW_EXTENSIONS_UPDATES) { wmWindowManager *wm = CTX_wm_manager(C); + + /* Special case, always show an alert for any blocked extensions. */ + if (wm->extensions_blocked > 0) { + if (has_status_info) { + uiItemS_ex(row, -0.5f); + uiItemL(row, "|", ICON_NONE); + uiItemS_ex(row, -0.5f); + } + uiLayoutSetEmboss(row, UI_EMBOSS_NONE); + /* This operator also works fine for blocked extensions. */ + uiItemO(row, "", ICON_ERROR, "EXTENSIONS_OT_userpref_show_for_update"); + uiBut *but = static_cast(uiLayoutGetBlock(layout)->buttons.last); + uchar color[4]; + UI_GetThemeColor4ubv(TH_TEXT, color); + copy_v4_v4_uchar(but->col, color); + + BLI_str_format_integer_unit(but->icon_overlay_text.text, wm->extensions_blocked); + UI_but_icon_indicator_color_set(but, color); + + uiItemS_ex(row, 1.0f); + has_status_info = true; + } + if ((G.f & G_FLAG_INTERNET_ALLOW) == 0) { if (has_status_info) { uiItemS_ex(row, -0.5f); diff --git a/source/blender/makesdna/DNA_windowmanager_types.h b/source/blender/makesdna/DNA_windowmanager_types.h index ffb821fb1e0..bb08c4599d1 100644 --- a/source/blender/makesdna/DNA_windowmanager_types.h +++ b/source/blender/makesdna/DNA_windowmanager_types.h @@ -186,7 +186,8 @@ typedef struct wmWindowManager { /** Available/pending extensions updates. */ int extensions_updates; - int _pad3; + /** Number of blocked & installed extensions. */ + int extensions_blocked; /** Threaded jobs manager. */ ListBase jobs; diff --git a/source/blender/makesrna/intern/rna_wm.cc b/source/blender/makesrna/intern/rna_wm.cc index 16d07c61062..fee47042ceb 100644 --- a/source/blender/makesrna/intern/rna_wm.cc +++ b/source/blender/makesrna/intern/rna_wm.cc @@ -1197,9 +1197,9 @@ static void rna_WindowManager_active_keyconfig_set(PointerRNA *ptr, } } -static void rna_WindowManager_extensions_update(Main * /*bmain*/, - Scene * /*scene*/, - PointerRNA *ptr) +static void rna_WindowManager_extensions_statusbar_update(Main * /*bmain*/, + Scene * /*scene*/, + PointerRNA *ptr) { if ((U.statusbar_flag & STATUSBAR_SHOW_EXTENSIONS_UPDATES) == 0) { return; @@ -2706,7 +2706,12 @@ static void rna_def_windowmanager(BlenderRNA *brna) prop = RNA_def_property(srna, "extensions_updates", PROP_INT, PROP_NONE); RNA_def_property_ui_text( prop, "Extensions Updates", "Number of extensions with available update"); - RNA_def_property_update(prop, 0, "rna_WindowManager_extensions_update"); + RNA_def_property_update(prop, 0, "rna_WindowManager_extensions_statusbar_update"); + + prop = RNA_def_property(srna, "extensions_blocked", PROP_INT, PROP_NONE); + RNA_def_property_ui_text( + prop, "Extensions Blocked", "Number of installed extensions which are blocked"); + RNA_def_property_update(prop, 0, "rna_WindowManager_extensions_statusbar_update"); RNA_api_wm(srna); } diff --git a/source/blender/windowmanager/intern/wm.cc b/source/blender/windowmanager/intern/wm.cc index 6d687d23866..e90d0256b27 100644 --- a/source/blender/windowmanager/intern/wm.cc +++ b/source/blender/windowmanager/intern/wm.cc @@ -229,6 +229,7 @@ static void window_manager_blend_read_data(BlendDataReader *reader, ID *id) wm->init_flag = 0; wm->op_undo_depth = 0; wm->extensions_updates = WM_EXTENSIONS_UPDATE_UNSET; + wm->extensions_blocked = 0; BLI_assert(wm->runtime == nullptr); wm->runtime = MEM_new(__func__);