Fix #115313: GPencil brush direction is not kept

The core of the issue was that `sculpt_flag` was used by three different enums (`eGP_Sculpt_Flag`, `eGP_Sculpt_Mode_Flag`, and `eBrushFlags`). This resulted in the flag getting overriden because `ENUM_OPERATORS` expected the maximum value of `eGP_Sculpt_Flag` to be `(1 << 3)` which it wasn't.

The `sculpt_flag` was exposed through python as `"direction"`.
In the UI this meant that it was effectively used as `brush.direction`. This fix replaces `brush.gpencil_settings.direction` with `brush.direction`.
It also makes sure `sculpt_flag` is only ever used with values from `eGP_Sculpt_Flag`.

Pull Request: https://projects.blender.org/blender/blender/pulls/119373

scripts/startup/bl_ui/properties_paint_common.py

@@ -1409,20 +1409,12 @@ def brush_basic_gpencil_sculpt_settings(layout, _context, brush, *, compact=Fals
     if compact:
         if tool in {'THICKNESS', 'STRENGTH', 'PINCH', 'TWIST'}:
             row.separator()
-            row.prop(gp_settings, "direction", expand=True, text="")
+            row.prop(brush, "direction", expand=True, text="")
     else:
         use_property_split_prev = layout.use_property_split
         layout.use_property_split = False
-        if tool in {'THICKNESS', 'STRENGTH'}:
-            layout.row().prop(gp_settings, "direction", expand=True)
-        elif tool == 'PINCH':
-            row = layout.row(align=True)
-            row.prop_enum(gp_settings, "direction", value='ADD', text="Pinch")
-            row.prop_enum(gp_settings, "direction", value='SUBTRACT', text="Inflate")
-        elif tool == 'TWIST':
-            row = layout.row(align=True)
-            row.prop_enum(gp_settings, "direction", value='ADD', text="CCW")
-            row.prop_enum(gp_settings, "direction", value='SUBTRACT', text="CW")
+        if tool in {'THICKNESS', 'STRENGTH', 'PINCH', 'TWIST'}:
+            layout.row().prop(brush, "direction", expand=True)
         layout.use_property_split = use_property_split_prev
 
 
@@ -1436,8 +1428,7 @@ def brush_basic_gpencil_weight_settings(layout, _context, brush, *, compact=Fals
     if brush.gpencil_weight_tool in {'WEIGHT'}:
         layout.prop(brush, "weight", slider=True)
 
-        gp_settings = brush.gpencil_settings
-        layout.prop(gp_settings, "direction", expand=True, text="" if compact else "Direction")
+        layout.prop(brush, "direction", expand=True, text="" if compact else "Direction")
 
 
 def brush_basic_gpencil_vertex_settings(layout, _context, brush, *, compact=False):

source/blender/blenkernel/BKE_blender_version.h

@@ -29,7 +29,7 @@ extern "C" {
 
 /* Blender file format version. */
 #define BLENDER_FILE_VERSION BLENDER_VERSION
-#define BLENDER_FILE_SUBVERSION 21
+#define BLENDER_FILE_SUBVERSION 22
 
 /* Minimum Blender version that supports reading file written with the current
  * version. Older Blender versions will test this and cancel loading the file, showing a warning to

source/blender/blenkernel/intern/brush.cc

@@ -1151,7 +1151,6 @@ void BKE_gpencil_brush_preset_set(Main *bmain, Brush *brush, const short type)
 
       brush->gpencil_settings->draw_strength = 0.3f;
       brush->gpencil_settings->flag |= GP_BRUSH_USE_STRENGTH_PRESSURE;
-      brush->gpencil_settings->sculpt_flag = GP_SCULPT_FLAGMODE_APPLY_THICKNESS;
       brush->gpencil_settings->sculpt_mode_flag |= GP_SCULPT_FLAGMODE_APPLY_POSITION;
 
       break;

source/blender/blenkernel/intern/paint.cc

@@ -587,6 +587,10 @@ PaintMode BKE_paintmode_get_active_from_context(const bContext *C)
       switch (obact->mode) {
         case OB_MODE_SCULPT:
           return PaintMode::Sculpt;
+        case OB_MODE_SCULPT_GPENCIL_LEGACY:
+          return PaintMode::SculptGPencil;
+        case OB_MODE_WEIGHT_GPENCIL_LEGACY:
+          return PaintMode::WeightGPencil;
         case OB_MODE_VERTEX_PAINT:
           return PaintMode::Vertex;
         case OB_MODE_WEIGHT_PAINT:

source/blender/blenloader/intern/versioning_400.cc

@@ -2896,6 +2896,16 @@ void blo_do_versions_400(FileData *fd, Library * /*lib*/, Main *bmain)
     }
   }
 
+  if (!MAIN_VERSION_FILE_ATLEAST(bmain, 401, 21)) {
+    LISTBASE_FOREACH (Brush *, brush, &bmain->brushes) {
+      /* The `sculpt_flag` was used to store the `BRUSH_DIR_IN`
+       * With the fix for #115313 this is now just using the `brush->flag`.*/
+      if (brush->gpencil_settings && (brush->gpencil_settings->sculpt_flag & BRUSH_DIR_IN) != 0) {
+        brush->flag |= BRUSH_DIR_IN;
+      }
+    }
+  }
+
   /**
    * Always bump subversion in BKE_blender_version.h when adding versioning
    * code here, and wrap it inside a MAIN_VERSION_FILE_ATLEAST check.

source/blender/editors/gpencil_legacy/gpencil_sculpt_paint.cc

@@ -231,7 +231,7 @@ static bool gpencil_brush_invert_check(tGP_BrushEditData *gso)
 {
   /* The basic setting is the brush's setting (from the panel) */
   bool invert = ((gso->brush->gpencil_settings->sculpt_flag & GP_SCULPT_FLAG_INVERT) != 0) ||
-                (gso->brush->gpencil_settings->sculpt_flag & BRUSH_DIR_IN);
+                (gso->brush->flag & BRUSH_DIR_IN);
   /* During runtime, the user can hold down the Ctrl key to invert the basic behavior */
   if (gso->flag & GP_SCULPT_FLAG_INVERT) {
     invert ^= true;

source/blender/editors/gpencil_legacy/gpencil_weight_paint.cc

@@ -799,7 +799,7 @@ static bool gpencil_weightpaint_brush_init(bContext *C, wmOperator *op)
   }
 
   /* Draw tool: add or subtract weight? */
-  gso->subtract = (gso->brush->gpencil_settings->sculpt_flag & BRUSH_DIR_IN);
+  gso->subtract = (gso->brush->flag & BRUSH_DIR_IN);
 
   /* Setup auto-normalize. */
   gso->auto_normalize = (ts->auto_normalize && gso->vrgroup != -1);
@@ -1524,7 +1524,7 @@ static int gpencil_weight_toggle_direction_invoke(bContext *C,
   Paint *paint = &ts->gp_weightpaint->paint;
 
   /* Toggle Add/Subtract flag. */
-  paint->brush->gpencil_settings->sculpt_flag ^= BRUSH_DIR_IN;
+  paint->brush->flag ^= BRUSH_DIR_IN;
 
   /* Update tool settings. */
   WM_main_add_notifier(NC_BRUSH | NA_EDITED, nullptr);

source/blender/makesrna/intern/rna_brush.cc

@@ -936,6 +936,18 @@ static const EnumPropertyItem *rna_Brush_direction_itemf(bContext *C,
       {0, nullptr, 0, nullptr, nullptr},
   };
 
+  /* gpencil sculpt */
+  static const EnumPropertyItem prop_pinch_items[] = {
+      {0, "ADD", ICON_ADD, "Pinch", "Add effect of brush"},
+      {BRUSH_DIR_IN, "SUBTRACT", ICON_REMOVE, "Inflate", "Subtract effect of brush"},
+      {0, nullptr, 0, nullptr, nullptr},
+  };
+  static const EnumPropertyItem prop_twist_items[] = {
+      {0, "ADD", ICON_ADD, "Counter-Clockwise", "Add effect of brush"},
+      {BRUSH_DIR_IN, "SUBTRACT", ICON_REMOVE, "Clockwise", "Subtract effect of brush"},
+      {0, nullptr, 0, nullptr, nullptr},
+  };
+
   Brush *me = (Brush *)(ptr->data);
 
   switch (mode) {
@@ -1000,6 +1012,25 @@ static const EnumPropertyItem *rna_Brush_direction_itemf(bContext *C,
         default:
           return rna_enum_dummy_DEFAULT_items;
       }
+    case PaintMode::SculptGPencil:
+      switch (me->gpencil_sculpt_tool) {
+        case GPSCULPT_TOOL_THICKNESS:
+        case GPSCULPT_TOOL_STRENGTH:
+          return prop_direction_items;
+        case GPSCULPT_TOOL_TWIST:
+          return prop_twist_items;
+        case GPSCULPT_TOOL_PINCH:
+          return prop_pinch_items;
+        default:
+          return rna_enum_dummy_DEFAULT_items;
+      }
+    case PaintMode::WeightGPencil:
+      switch (me->gpencil_weight_tool) {
+        case GPWEIGHT_TOOL_DRAW:
+          return prop_direction_items;
+        default:
+          return rna_enum_dummy_DEFAULT_items;
+      }
     default:
       return rna_enum_dummy_DEFAULT_items;
   }
@@ -1975,13 +2006,6 @@ static void rna_def_gpencil_options(BlenderRNA *brna)
   RNA_def_property_ui_text(prop, "Outline", "Convert stroke to perimeter");
   RNA_def_property_clear_flag(prop, PROP_ANIMATABLE);
 
-  prop = RNA_def_property(srna, "direction", PROP_ENUM, PROP_NONE);
-  RNA_def_property_enum_bitflag_sdna(prop, nullptr, "sculpt_flag");
-  RNA_def_property_enum_items(prop, prop_direction_items);
-  RNA_def_property_ui_text(prop, "Direction", "");
-  RNA_def_property_clear_flag(prop, PROP_ANIMATABLE);
-  RNA_def_property_update(prop, NC_SCENE | ND_TOOLSETTINGS, nullptr);
-
   prop = RNA_def_property(srna, "use_edit_position", PROP_BOOLEAN, PROP_NONE);
   RNA_def_property_boolean_sdna(
       prop, nullptr, "sculpt_mode_flag", GP_SCULPT_FLAGMODE_APPLY_POSITION);

source/blender/makesrna/intern/rna_sculpt_paint.cc

@@ -1562,14 +1562,6 @@ static void rna_def_gpencil_guides(BlenderRNA *brna)
 
 static void rna_def_gpencil_sculpt(BlenderRNA *brna)
 {
-  static const EnumPropertyItem prop_direction_items[] = {
-      {0, "ADD", ICON_ADD, "Add", "Add effect of brush"},
-      {GP_SCULPT_FLAG_INVERT, "SUBTRACT", ICON_REMOVE, "Subtract", "Subtract effect of brush"},
-      {0, nullptr, 0, nullptr, nullptr},
-  };
-
-  UNUSED_VARS(prop_direction_items);
-
   StructRNA *srna;
   PropertyRNA *prop;