From d7d8cefd72a884f835e6dbf71a8040d5c9e867bf Mon Sep 17 00:00:00 2001
From: Jacques Lucke <jacques@blender.org>
Date: Mon, 22 Apr 2024 19:52:40 +0200
Subject: [PATCH] Fix: incorrect safe float4x4 to quaternion conversion

The issue was that when a 4x4 matrix is normalized, it does not
always mean that any inner 3x3 matrix is normalized too.
---
 source/blender/blenkernel/intern/type_conversions.cc        | 2 +-
 source/blender/blenlib/BLI_math_matrix.hh                   | 6 ------
 .../nodes/function/nodes/node_fn_separate_transform.cc      | 3 ++-
 3 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/source/blender/blenkernel/intern/type_conversions.cc b/source/blender/blenkernel/intern/type_conversions.cc
index 1b39c93ac06..eeee4658abf 100644
--- a/source/blender/blenkernel/intern/type_conversions.cc
+++ b/source/blender/blenkernel/intern/type_conversions.cc
@@ -351,7 +351,7 @@ static ColorGeometry4f byte_color_to_color(const ColorGeometry4b &a)
 
 static math::Quaternion float4x4_to_quaternion(const float4x4 &a)
 {
-  return math::normalized_to_quaternion_safe(math::normalize(a));
+  return math::normalized_to_quaternion_safe(math::normalize(float3x3(a)));
 }
 
 static float3 quaternion_to_float3(const math::Quaternion &a)
diff --git a/source/blender/blenlib/BLI_math_matrix.hh b/source/blender/blenlib/BLI_math_matrix.hh
index 6a97a2ebe9c..7ae7de75360 100644
--- a/source/blender/blenlib/BLI_math_matrix.hh
+++ b/source/blender/blenlib/BLI_math_matrix.hh
@@ -1234,12 +1234,6 @@ template<typename T>
   return to_quaternion(to_euler(mat));
 }
 
-template<typename T>
-[[nodiscard]] inline QuaternionBase<T> normalized_to_quaternion_safe(const MatBase<T, 4, 4> &mat)
-{
-  return to_quaternion(to_euler(mat));
-}
-
 template<bool AllowNegativeScale, typename T, int NumCol, int NumRow>
 [[nodiscard]] inline VecBase<T, 3> to_scale(const MatBase<T, NumCol, NumRow> &mat)
 {
diff --git a/source/blender/nodes/function/nodes/node_fn_separate_transform.cc b/source/blender/nodes/function/nodes/node_fn_separate_transform.cc
index d21e1af69f0..a78b3d764c5 100644
--- a/source/blender/nodes/function/nodes/node_fn_separate_transform.cc
+++ b/source/blender/nodes/function/nodes/node_fn_separate_transform.cc
@@ -52,7 +52,8 @@ class SeparateTransformFunction : public mf::MultiFunction {
     }
     else if (!rotation.is_empty() && scale.is_empty()) {
       mask.foreach_index([&](const int64_t i) {
-        rotation[i] = math::normalized_to_quaternion_safe(math::normalize(transforms[i]));
+        rotation[i] = math::normalized_to_quaternion_safe(
+            math::normalize(float3x3(transforms[i])));
       });
     }
     else if (!rotation.is_empty() && !scale.is_empty()) {