From 90269dfe14c02ead4bb6ab2ddb5f4ae2c68e10a9 Mon Sep 17 00:00:00 2001
From: Hans Goudey <hans@blender.org>
Date: Wed, 1 Nov 2023 18:33:01 +0100
Subject: [PATCH] Fix #114357: Dyntopo face set brush does not create face sets
 layer

Create the face set layer in the BMesh when the face set brush is first
used. This requires iterating through all faces to set the default value
of 1 because the default of 0 (SCULPT_FACE_SET_NONE) has different
behavior defined elsewhere.

Pair-reviewed in person with Sergey
---
 source/blender/blenkernel/intern/paint.cc | 61 ++++++++++++++++-------
 1 file changed, 43 insertions(+), 18 deletions(-)

diff --git a/source/blender/blenkernel/intern/paint.cc b/source/blender/blenkernel/intern/paint.cc
index 8b4405fc6de..25a5e5868e6 100644
--- a/source/blender/blenkernel/intern/paint.cc
+++ b/source/blender/blenkernel/intern/paint.cc
@@ -1949,28 +1949,53 @@ void BKE_sculpt_update_object_for_edit(
 
 int *BKE_sculpt_face_sets_ensure(Object *ob)
 {
-  SculptSession *ss = ob->sculpt;
-  Mesh *mesh = static_cast<Mesh *>(ob->data);
-
   using namespace blender;
   using namespace blender::bke;
-  MutableAttributeAccessor attributes = mesh->attributes_for_write();
-  if (!attributes.contains(".sculpt_face_set")) {
-    SpanAttributeWriter<int> face_sets = attributes.lookup_or_add_for_write_only_span<int>(
-        ".sculpt_face_set", ATTR_DOMAIN_FACE);
-    face_sets.span.fill(1);
-    mesh->face_sets_color_default = 1;
-    face_sets.finish();
+  Mesh *mesh = static_cast<Mesh *>(ob->data);
+  SculptSession *ss = ob->sculpt;
+  PBVH *pbvh = ss->pbvh;
+  if (!pbvh) {
+    BLI_assert_unreachable();
+    return nullptr;
+  }
+  const StringRefNull name = ".sculpt_face_set";
+
+  switch (BKE_pbvh_type(pbvh)) {
+    case PBVH_FACES:
+    case PBVH_GRIDS: {
+      MutableAttributeAccessor attributes = mesh->attributes_for_write();
+      if (!attributes.contains(name)) {
+        attributes.add<int>(name,
+                            ATTR_DOMAIN_FACE,
+                            AttributeInitVArray(VArray<int>::ForSingle(1, mesh->faces_num)));
+        mesh->face_sets_color_default = 1;
+      }
+
+      int *face_sets = static_cast<int *>(CustomData_get_layer_named_for_write(
+          &mesh->face_data, CD_PROP_INT32, name.c_str(), mesh->faces_num));
+      BKE_pbvh_face_sets_set(pbvh, face_sets);
+      return face_sets;
+    }
+    case PBVH_BMESH: {
+      BMesh *bm = BKE_pbvh_get_bmesh(pbvh);
+      if (!CustomData_has_layer_named(&bm->pdata, CD_PROP_INT32, name.c_str())) {
+        BM_data_layer_add_named(bm, &bm->pdata, CD_PROP_INT32, name.c_str());
+        const int offset = CustomData_get_offset_named(&bm->pdata, CD_PROP_INT32, name.c_str());
+        if (offset == -1) {
+          return nullptr;
+        }
+        BMIter iter;
+        BMFace *face;
+        BM_ITER_MESH (face, &iter, bm, BM_FACES_OF_MESH) {
+          BM_ELEM_CD_SET_INT(face, offset, 1);
+        }
+        mesh->face_sets_color_default = 1;
+      }
+      break;
+    }
   }
 
-  int *face_sets = static_cast<int *>(CustomData_get_layer_named_for_write(
-      &mesh->face_data, CD_PROP_INT32, ".sculpt_face_set", mesh->faces_num));
-
-  if (ss->pbvh && ELEM(BKE_pbvh_type(ss->pbvh), PBVH_FACES, PBVH_GRIDS)) {
-    BKE_pbvh_face_sets_set(ss->pbvh, face_sets);
-  }
-
-  return face_sets;
+  return nullptr;
 }
 
 bool *BKE_sculpt_hide_poly_ensure(Mesh *mesh)