From a94f0ca1db258639d46df07cf7186f5fbf2b6d28 Mon Sep 17 00:00:00 2001 From: Cody Winchester Date: Tue, 28 Oct 2025 14:41:54 -0500 Subject: [PATCH] GOOENGINE: Fix shader compile freezing viewport Versions off the draw code so the viewport updates when shaders are done compiling. When in gooengine it uses the old version --- source/blender/draw/DRW_engine.hh | 3 + .../draw/intern/draw_manager_shader.cc | 366 +++++++++++++++++- source/blender/gpu/GPU_material.hh | 24 ++ source/blender/gpu/intern/gpu_material.cc | 49 ++- 4 files changed, 437 insertions(+), 5 deletions(-) diff --git a/source/blender/draw/DRW_engine.hh b/source/blender/draw/DRW_engine.hh index 3fb2518b2d7..ab1cba8e596 100644 --- a/source/blender/draw/DRW_engine.hh +++ b/source/blender/draw/DRW_engine.hh @@ -199,6 +199,9 @@ void DRW_blender_gpu_render_context_disable(void *re_gpu_context); void DRW_deferred_shader_remove(GPUMaterial *mat); void DRW_deferred_shader_optimize_remove(GPUMaterial *mat); +void GOO_deferred_shader_remove(GPUMaterial *mat); +void GOO_deferred_shader_optimize_remove(GPUMaterial *mat); + /** * Get DrawData from the given ID-block. In order for this to work, we assume that * the DrawData pointer is stored in the in the same fashion as in #IdDdtTemplate. diff --git a/source/blender/draw/intern/draw_manager_shader.cc b/source/blender/draw/intern/draw_manager_shader.cc index 8f64cd67bcf..9790f6ae8ac 100644 --- a/source/blender/draw/intern/draw_manager_shader.cc +++ b/source/blender/draw/intern/draw_manager_shader.cc @@ -63,6 +63,13 @@ struct DRWShaderCompiler { /** Optimization queue. */ Vector optimize_queue; + /** Default goo compilation queue. */ + ListBase goo_queue; /* GPUMaterial */ + SpinLock list_lock; + + /** Optimization queue. */ + ListBase goo_optimize_queue; /* GPUMaterial */ + std::mutex queue_mutex; std::condition_variable queue_cv; @@ -70,6 +77,7 @@ struct DRWShaderCompiler { GPUContext *blender_gpu_context; std::atomic stop; + std::atomic own_context; }; /** NOTE: While the `BLI_threads` API requires a List, @@ -191,6 +199,163 @@ static void *drw_deferred_shader_compilation_exec(void *) return nullptr; } +static void goo_deferred_shader_compilation_exec(void *custom_data, + wmJobWorkerStatus *worker_status) +{ + using namespace blender; + + GPU_render_begin(); + DRWShaderCompiler *comp = (DRWShaderCompiler *)custom_data; + void *system_gpu_context = comp->system_gpu_context; + GPUContext *blender_gpu_context = comp->blender_gpu_context; + + BLI_assert(system_gpu_context != nullptr); + BLI_assert(blender_gpu_context != nullptr); + + const bool use_main_context_workaround = GPU_use_main_context_workaround(); + if (use_main_context_workaround) { + BLI_assert(system_gpu_context == DST.system_gpu_context); + GPU_context_main_lock(); + } + + const bool use_parallel_compilation = GPU_use_parallel_compilation(); + + WM_system_gpu_context_activate(system_gpu_context); + GPU_context_active_set(blender_gpu_context); + + Vector next_batch; + Map> batches; + + while (true) { + if (worker_status->stop) { + break; + } + + BLI_spin_lock(&comp->list_lock); + /* Pop tail because it will be less likely to lock the main thread + * if all GPUMaterials are to be freed (see GOO_deferred_shader_remove()). */ + LinkData *link = (LinkData *)BLI_poptail(&comp->goo_queue); + GPUMaterial *mat = link ? (GPUMaterial *)link->data : nullptr; + if (mat) { + /* Avoid another thread freeing the material mid compilation. */ + GPU_material_acquire(mat); + MEM_freeN(link); + } + BLI_spin_unlock(&comp->list_lock); + + if (mat) { + /* We have a new material that must be compiled, + * we either compile it directly or add it to a parallel compilation batch. */ + if (use_parallel_compilation) { + next_batch.append(mat); + } + else { + GPU_material_compile(mat); + GPU_material_release(mat); + } + } + else if (!next_batch.is_empty()) { + /* (only if use_parallel_compilation == true) + * We ran out of pending materials. Request the compilation of the current batch. */ + BatchHandle batch_handle = GPU_material_batch_compile(next_batch); + batches.add(batch_handle, next_batch); + next_batch.clear(); + } + else if (!batches.is_empty()) { + /* (only if use_parallel_compilation == true) + * Keep querying the requested batches until all of them are ready. */ + Vector ready_handles; + for (BatchHandle handle : batches.keys()) { + if (GPU_material_batch_is_ready(handle)) { + ready_handles.append(handle); + } + } + for (BatchHandle handle : ready_handles) { + Vector batch = batches.pop(handle); + GPU_material_batch_finalize(handle, batch); + for (GPUMaterial *mat : batch) { + GPU_material_release(mat); + } + } + } + else { + /* Check for Material Optimization job once there are no more + * shaders to compile. */ + BLI_spin_lock(&comp->list_lock); + /* Pop tail because it will be less likely to lock the main thread + * if all GPUMaterials are to be freed (see GOO_deferred_shader_remove()). */ + LinkData *link = (LinkData *)BLI_poptail(&comp->goo_optimize_queue); + GPUMaterial *optimize_mat = link ? (GPUMaterial *)link->data : nullptr; + if (optimize_mat) { + /* Avoid another thread freeing the material during optimization. */ + GPU_material_acquire(optimize_mat); + } + BLI_spin_unlock(&comp->list_lock); + + if (optimize_mat) { + /* Compile optimized material shader. */ + GPU_material_optimize(optimize_mat); + GPU_material_release(optimize_mat); + MEM_freeN(link); + } + else { + /* No more materials to optimize, or shaders to compile. */ + break; + } + } + + if (GPU_type_matches_ex(GPU_DEVICE_ANY, GPU_OS_ANY, GPU_DRIVER_ANY, GPU_BACKEND_OPENGL)) { + GPU_flush(); + } + } + + /* We have to wait until all the requested batches are ready, + * even if worker_status->stop is true. */ + for (BatchHandle handle : batches.keys()) { + Vector &batch = batches.lookup(handle); + GPU_material_batch_finalize(handle, batch); + for (GPUMaterial *mat : batch) { + GPU_material_release(mat); + } + } + + GPU_context_active_set(nullptr); + WM_system_gpu_context_release(system_gpu_context); + if (use_main_context_workaround) { + GPU_context_main_unlock(); + } + GPU_render_end(); +} + +static void goo_deferred_shader_compilation_free(void *custom_data) +{ + DRWShaderCompiler *comp = (DRWShaderCompiler *)custom_data; + + BLI_spin_lock(&comp->list_lock); + LISTBASE_FOREACH (LinkData *, link, &comp->goo_queue) { + GPU_material_status_set(static_cast(link->data), GPU_MAT_CREATED); + } + LISTBASE_FOREACH (LinkData *, link, &comp->goo_optimize_queue) { + GPU_material_optimization_status_set(static_cast(link->data), + GPU_MAT_OPTIMIZATION_READY); + } + BLI_freelistN(&comp->goo_queue); + BLI_freelistN(&comp->goo_optimize_queue); + BLI_spin_unlock(&comp->list_lock); + + if (comp->own_context) { + /* Only destroy if the job owns the context. */ + WM_system_gpu_context_activate(comp->system_gpu_context); + GPU_context_active_set(comp->blender_gpu_context); + GPU_context_discard(comp->blender_gpu_context); + WM_system_gpu_context_dispose(comp->system_gpu_context); + + wm_window_reset_drawable(); + } + + MEM_freeN(comp); +} + void DRW_shader_init() { if (GPU_use_main_context_workaround()) { @@ -270,6 +435,85 @@ static void drw_deferred_queue_append(GPUMaterial *mat, bool is_optimization_job compiler_data().queue_cv.notify_one(); } +/** + * Append either shader compilation or optimization job to deferred goo_queue and + * ensure shader compilation worker is active. + * We keep two separate goo_queue's to ensure core compilations always complete before optimization. + */ +static void goo_deferred_queue_append(GPUMaterial *mat, bool is_optimization_job) +{ + const bool use_main_context = GPU_use_main_context_workaround(); + const bool job_own_context = !use_main_context; + + BLI_assert(DST.draw_ctx.evil_C); + wmWindowManager *wm = CTX_wm_manager(DST.draw_ctx.evil_C); + wmWindow *win = CTX_wm_window(DST.draw_ctx.evil_C); + + /* Get the running job or a new one if none is running. Can only have one job per type & owner. + */ + wmJob *wm_job = WM_jobs_get( + wm, win, wm, "Shaders Compilation", eWM_JobFlag(0), WM_JOB_TYPE_SHADER_COMPILATION); + + DRWShaderCompiler *old_comp = (DRWShaderCompiler *)WM_jobs_customdata_get(wm_job); + + DRWShaderCompiler *comp = static_cast( + MEM_callocN(sizeof(DRWShaderCompiler), "DRWShaderCompiler")); + BLI_spin_init(&comp->list_lock); + + if (old_comp) { + BLI_spin_lock(&old_comp->list_lock); + BLI_movelisttolist(&comp->goo_queue, &old_comp->goo_queue); + BLI_movelisttolist(&comp->goo_optimize_queue, &old_comp->goo_optimize_queue); + BLI_spin_unlock(&old_comp->list_lock); + /* Do not recreate context, just pass ownership. */ + if (old_comp->system_gpu_context) { + comp->system_gpu_context = old_comp->system_gpu_context; + comp->blender_gpu_context = old_comp->blender_gpu_context; + old_comp->own_context = false; + comp->own_context = job_own_context; + } + } + + /* Add to either compilation or optimization goo_queue. */ + if (is_optimization_job) { + BLI_assert(GPU_material_optimization_status(mat) != GPU_MAT_OPTIMIZATION_QUEUED); + GPU_material_optimization_status_set(mat, GPU_MAT_OPTIMIZATION_QUEUED); + LinkData *node = BLI_genericNodeN(mat); + BLI_addtail(&comp->goo_optimize_queue, node); + } + else { + GPU_material_status_set(mat, GPU_MAT_QUEUED); + LinkData *node = BLI_genericNodeN(mat); + BLI_addtail(&comp->goo_queue, node); + } + + /* Create only one context. */ + if (comp->system_gpu_context == nullptr) { + if (use_main_context) { + comp->system_gpu_context = DST.system_gpu_context; + comp->blender_gpu_context = DST.blender_gpu_context; + } + else { + comp->system_gpu_context = WM_system_gpu_context_create(); + comp->blender_gpu_context = GPU_context_create(nullptr, comp->system_gpu_context); + GPU_context_active_set(nullptr); + + WM_system_gpu_context_activate(DST.system_gpu_context); + GPU_context_active_set(DST.blender_gpu_context); + } + comp->own_context = job_own_context; + } + + WM_jobs_customdata_set(wm_job, comp, goo_deferred_shader_compilation_free); + WM_jobs_timer(wm_job, 0.1, NC_MATERIAL | ND_SHADING_DRAW, 0); + WM_jobs_delay_start(wm_job, 0.1); + WM_jobs_callbacks(wm_job, goo_deferred_shader_compilation_exec, nullptr, nullptr, nullptr); + + G.is_break = false; + + WM_jobs_start(wm, wm_job); +} + static void drw_deferred_shader_add(GPUMaterial *mat, bool deferred) { if (ELEM(GPU_material_status(mat), GPU_MAT_SUCCESS, GPU_MAT_FAILED)) { @@ -301,6 +545,46 @@ static void drw_deferred_shader_add(GPUMaterial *mat, bool deferred) drw_deferred_queue_append(mat, false); } +static void goo_deferred_shader_add(GPUMaterial *mat, bool deferred) +{ + if (ELEM(GPU_material_status(mat), GPU_MAT_SUCCESS, GPU_MAT_FAILED)) { + return; + } + + /* Do not defer the compilation if we are rendering for image. + * deferred rendering is only possible when `evil_C` is available */ + if (DST.draw_ctx.evil_C == nullptr || DRW_state_is_image_render() || !USE_DEFERRED_COMPILATION) { + deferred = false; + } + + /* Avoid crashes with RenderDoc on Windows + Nvidia. */ + if (G.debug & G_DEBUG_GPU_RENDERDOC && + GPU_type_matches(GPU_DEVICE_NVIDIA, GPU_OS_ANY, GPU_DRIVER_OFFICIAL)) + { + deferred = false; + } + + if (!deferred) { + GOO_deferred_shader_remove(mat); + /* Shaders could already be compiling. Have to wait for compilation to finish. */ + while (GPU_material_status(mat) == GPU_MAT_QUEUED) { + BLI_time_sleep_ms(20); + } + if (GPU_material_status(mat) == GPU_MAT_CREATED) { + GPU_material_compile(mat); + } + return; + } + + /* Don't add material to the goo_queue twice. */ + if (GPU_material_status(mat) == GPU_MAT_QUEUED) { + return; + } + + /* Add deferred shader compilation to goo_queue. */ + goo_deferred_queue_append(mat, false); +} + static void drw_register_shader_vlattrs(GPUMaterial *mat) { const ListBase *attrs = GPU_material_layer_attributes(mat); @@ -359,6 +643,40 @@ void DRW_deferred_shader_remove(GPUMaterial *mat) } } +void GOO_deferred_shader_remove(GPUMaterial *mat) +{ + LISTBASE_FOREACH (wmWindowManager *, wm, &G_MAIN->wm) { + LISTBASE_FOREACH (wmWindow *, win, &wm->windows) { + DRWShaderCompiler *comp = (DRWShaderCompiler *)WM_jobs_customdata_from_type( + wm, wm, WM_JOB_TYPE_SHADER_COMPILATION); + if (comp != nullptr) { + BLI_spin_lock(&comp->list_lock); + + /* Search for compilation job in goo_queue. */ + LinkData *link = (LinkData *)BLI_findptr(&comp->goo_queue, mat, offsetof(LinkData, data)); + if (link) { + BLI_remlink(&comp->goo_queue, link); + GPU_material_status_set(static_cast(link->data), GPU_MAT_CREATED); + } + + MEM_SAFE_FREE(link); + + /* Search for optimization job in goo_queue. */ + LinkData *opti_link = (LinkData *)BLI_findptr( + &comp->goo_optimize_queue, mat, offsetof(LinkData, data)); + if (opti_link) { + BLI_remlink(&comp->goo_optimize_queue, opti_link); + GPU_material_optimization_status_set(static_cast(opti_link->data), + GPU_MAT_OPTIMIZATION_READY); + } + BLI_spin_unlock(&comp->list_lock); + + MEM_SAFE_FREE(opti_link); + } + } + } +} + void DRW_deferred_shader_optimize_remove(GPUMaterial *mat) { if (GPU_use_main_context_workaround()) { @@ -375,6 +693,30 @@ void DRW_deferred_shader_optimize_remove(GPUMaterial *mat) } } +void GOO_deferred_shader_optimize_remove(GPUMaterial *mat) +{ + LISTBASE_FOREACH (wmWindowManager *, wm, &G_MAIN->wm) { + LISTBASE_FOREACH (wmWindow *, win, &wm->windows) { + DRWShaderCompiler *comp = (DRWShaderCompiler *)WM_jobs_customdata_from_type( + wm, wm, WM_JOB_TYPE_SHADER_COMPILATION); + if (comp != nullptr) { + BLI_spin_lock(&comp->list_lock); + /* Search for optimization job in goo_queue. */ + LinkData *opti_link = (LinkData *)BLI_findptr( + &comp->goo_optimize_queue, mat, offsetof(LinkData, data)); + if (opti_link) { + BLI_remlink(&comp->goo_optimize_queue, opti_link); + GPU_material_optimization_status_set(static_cast(opti_link->data), + GPU_MAT_OPTIMIZATION_READY); + } + BLI_spin_unlock(&comp->list_lock); + + MEM_SAFE_FREE(opti_link); + } + } + } +} + /** \} */ /* -------------------------------------------------------------------- */ @@ -410,7 +752,12 @@ GPUMaterial *DRW_shader_from_world(World *wo, deferred = false; } - drw_deferred_shader_add(mat, deferred); + if (STREQ(DST.draw_ctx.engine_type->idname, "BLENDER_EEVEE")) { + goo_deferred_shader_add(mat, deferred); + } else { + drw_deferred_shader_add(mat, deferred); + } + DRW_shader_queue_optimize_material(mat); return mat; } @@ -446,7 +793,12 @@ GPUMaterial *DRW_shader_from_material(Material *ma, deferred = false; } - drw_deferred_shader_add(mat, deferred); + if (STREQ(DST.draw_ctx.engine_type->idname, "BLENDER_EEVEE")) { + goo_deferred_shader_add(mat, deferred); + } else { + drw_deferred_shader_add(mat, deferred); + } + DRW_shader_queue_optimize_material(mat); return mat; } @@ -457,8 +809,14 @@ void DRW_shader_queue_optimize_material(GPUMaterial *mat) * De-queue any queued optimization jobs. */ if (DRW_state_is_image_render()) { if (GPU_material_optimization_status(mat) == GPU_MAT_OPTIMIZATION_QUEUED) { - /* Remove from pending optimization job queue. */ - DRW_deferred_shader_optimize_remove(mat); + + /* Remove from pending optimization job queue. */ + if (STREQ(DST.draw_ctx.engine_type->idname, "BLENDER_EEVEE")) { + GOO_deferred_shader_optimize_remove(mat); + } else { + DRW_deferred_shader_optimize_remove(mat); + } + /* If optimization job had already started, wait for it to complete. */ while (GPU_material_optimization_status(mat) == GPU_MAT_OPTIMIZATION_QUEUED) { BLI_time_sleep_ms(20); diff --git a/source/blender/gpu/GPU_material.hh b/source/blender/gpu/GPU_material.hh index a500c7bb5d0..bea960c157e 100644 --- a/source/blender/gpu/GPU_material.hh +++ b/source/blender/gpu/GPU_material.hh @@ -263,6 +263,30 @@ void GPU_material_async_compile(GPUMaterial *mat); /** Returns true if the material have finished its compilation. */ bool GPU_material_async_try_finalize(GPUMaterial *mat); +/** + * Request the creation of multiple `GPUMaterial`s at once, allowing the backend to use + * multithreaded compilation. + * Returns a handle that can be used to poll if all materials have been + * compiled, and to retrieve the compiled result. + * NOTE: This function is asynchronous on OpenGL, but it's blocking on Vulkan and Metal. + * WARNING: The material pointers and their pass->create_info should be valid until + * `GPU_material_batch_finalize` has returned. + */ +BatchHandle GPU_material_batch_compile(blender::Span mats); +/** + * Returns true if all the materials from the batch have finished their compilation. + */ +bool GPU_material_batch_is_ready(BatchHandle handle); +/** + * Assign the compiled shaders to their respective materials and flag their status. + * The materials list should have the same length and order as in the `GPU_material_batch_compile` + * call. + * If the compilation has not finished yet, this call will block the thread until all the + * shaders are ready. + * WARNING: The handle will be invalidated by this call, you can't process the same batch twice. + */ +void GPU_material_batch_finalize(BatchHandle &handle, blender::Span mats); + void GPU_material_acquire(GPUMaterial *mat); void GPU_material_release(GPUMaterial *mat); diff --git a/source/blender/gpu/intern/gpu_material.cc b/source/blender/gpu/intern/gpu_material.cc index 6603de967b8..377a2cb2683 100644 --- a/source/blender/gpu/intern/gpu_material.cc +++ b/source/blender/gpu/intern/gpu_material.cc @@ -294,7 +294,13 @@ void GPU_material_free(ListBase *gpumaterial) { LISTBASE_FOREACH (LinkData *, link, gpumaterial) { GPUMaterial *material = static_cast(link->data); - DRW_deferred_shader_remove(material); + + if (GPU_material_flag_get(material, GPU_MATFLAG_OBJECT_INFO) && GPU_material_gooengine_get(material)) { + GOO_deferred_shader_remove(material); + } else { + DRW_deferred_shader_remove(material); + } + GPU_material_free_single(material); } BLI_freelistN(gpumaterial); @@ -1083,6 +1089,47 @@ bool GPU_material_async_try_finalize(GPUMaterial *mat) return false; } +BatchHandle GPU_material_batch_compile(blender::Span mats) +{ + blender::Vector infos; + infos.reserve(mats.size()); + + for (GPUMaterial *mat : mats) { + BLI_assert(ELEM(mat->status, GPU_MAT_QUEUED, GPU_MAT_CREATED)); + BLI_assert(mat->pass); +#ifndef NDEBUG + const char *name = mat->name; +#else + const char *name = __func__; +#endif + mat->do_batch_compilation = false; + if (GPUShaderCreateInfo *info = GPU_pass_begin_compilation(mat->pass, name)) { + infos.append(info); + mat->do_batch_compilation = true; + } + } + + return GPU_shader_batch_create_from_infos(infos); +} + +bool GPU_material_batch_is_ready(BatchHandle handle) +{ + return GPU_shader_batch_is_ready(handle); +} + +void GPU_material_batch_finalize(BatchHandle &handle, blender::Span mats) +{ + blender::Vector shaders = GPU_shader_batch_finalize(handle); + int i = 0; + for (GPUMaterial *mat : mats) { + bool success = true; + if (mat->do_batch_compilation) { + success = GPU_pass_finalize_compilation(mat->pass, shaders[i++]); + } + gpu_material_finalize(mat, success); + } +} + void GPU_material_optimize(GPUMaterial *mat) { /* If shader is flagged for skipping optimization or has already been successfully