Fix: integer overflows calculating the number of pixels in imbuf

Use size_t for imbuf size calculation to prevent integer overflow.

Back ported from: 6d6c695619b5199d2306eede273b9c50ebd10a28

Pull Request: https://projects.blender.org/blender/blender/pulls/137443

source/blender/blenkernel/intern/image.cc

@@ -2647,10 +2647,9 @@ void BKE_stamp_info_from_imbuf(RenderResult *rr, ImBuf *ibuf)
 
 bool BKE_imbuf_alpha_test(ImBuf *ibuf)
 {
-  int tot;
   if (ibuf->float_buffer.data) {
     const float *buf = ibuf->float_buffer.data;
-    for (tot = ibuf->x * ibuf->y; tot--; buf += 4) {
+    for (size_t tot = size_t(ibuf->x) * size_t(ibuf->y); tot--; buf += 4) {
       if (buf[3] < 1.0f) {
         return true;
       }
@@ -2658,7 +2657,7 @@ bool BKE_imbuf_alpha_test(ImBuf *ibuf)
   }
   else if (ibuf->byte_buffer.data) {
     uchar *buf = ibuf->byte_buffer.data;
-    for (tot = ibuf->x * ibuf->y; tot--; buf += 4) {
+    for (size_t tot = size_t(ibuf->x) * size_t(ibuf->y); tot--; buf += 4) {
       if (buf[3] != 255) {
         return true;
       }

source/blender/editors/sculpt_paint/grease_pencil_trace_util.cc

@@ -90,7 +90,8 @@ ImBuf *bitmap_to_image(const Bitmap &bm)
   const int words_per_scanline = bm.dy;
   const Span<potrace_word> words = {bm.map, num_words};
   MutableSpan<ColorGeometry4b> colors = {
-      reinterpret_cast<ColorGeometry4b *>(ibuf->byte_buffer.data), ibuf->x * ibuf->y};
+      reinterpret_cast<ColorGeometry4b *>(ibuf->byte_buffer.data),
+      int64_t(size_t(ibuf->x) * size_t(ibuf->y))};
   threading::parallel_for(IndexRange(ibuf->y), 4096, [&](const IndexRange range) {
     for (const int y : range) {
       Span<potrace_word> scanline_words = words.slice(

source/blender/imbuf/intern/allocimbuf.cc

@@ -723,7 +723,7 @@ size_t IMB_get_size_in_memory(ImBuf *ibuf)
     channel_size += sizeof(float);
   }
 
-  size += channel_size * ibuf->x * ibuf->y * ibuf->channels;
+  size += channel_size * size_t(ibuf->x) * size_t(ibuf->y) * size_t(ibuf->channels);
 
   if (ibuf->miptot) {
     for (a = 0; a < ibuf->miptot; a++) {

source/blender/imbuf/intern/rectop.cc

@@ -1239,11 +1239,11 @@ void IMB_rectfill_area(
 
 void IMB_rectfill_alpha(ImBuf *ibuf, const float value)
 {
-  int i;
+  size_t i;
 
   if (ibuf->float_buffer.data && (ibuf->channels == 4)) {
     float *fbuf = ibuf->float_buffer.data + 3;
-    for (i = ibuf->x * ibuf->y; i > 0; i--, fbuf += 4) {
+    for (i = size_t(ibuf->x) * size_t(ibuf->y); i > 0; i--, fbuf += 4) {
       *fbuf = value;
     }
   }
@@ -1251,7 +1251,7 @@ void IMB_rectfill_alpha(ImBuf *ibuf, const float value)
   if (ibuf->byte_buffer.data) {
     const uchar cvalue = value * 255;
     uchar *cbuf = ibuf->byte_buffer.data + 3;
-    for (i = ibuf->x * ibuf->y; i > 0; i--, cbuf += 4) {
+    for (i = size_t(ibuf->x) * size_t(ibuf->y); i > 0; i--, cbuf += 4) {
       *cbuf = cvalue;
     }
   }

source/blender/imbuf/intern/util_gpu.cc

@@ -155,8 +155,9 @@ static void *imb_gpu_get_data(const ImBuf *ibuf,
              IMB_colormanagement_space_is_scene_linear(ibuf->byte_buffer.colorspace))
     {
       /* sRGB or scene linear, store as byte texture that the GPU can decode directly. */
-      data_rect = MEM_mallocN(
-          (is_grayscale ? sizeof(float[4]) : sizeof(uchar[4])) * ibuf->x * ibuf->y, __func__);
+      data_rect = MEM_mallocN((is_grayscale ? sizeof(float[4]) : sizeof(uchar[4])) *
+                                  size_t(ibuf->x) * size_t(ibuf->y),
+                              __func__);
       *r_freedata = freedata = true;
 
       if (data_rect == nullptr) {
@@ -224,7 +225,8 @@ static void *imb_gpu_get_data(const ImBuf *ibuf,
     void *src_rect = data_rect;
 
     if (freedata == false) {
-      data_rect = MEM_mallocN((is_float_rect ? sizeof(float) : sizeof(uchar)) * ibuf->x * ibuf->y,
+      data_rect = MEM_mallocN((is_float_rect ? sizeof(float) : sizeof(uchar)) * size_t(ibuf->x) *
+                                  size_t(ibuf->y),
                               __func__);
       *r_freedata = freedata = true;
     }
@@ -233,14 +235,15 @@ static void *imb_gpu_get_data(const ImBuf *ibuf,
       return nullptr;
     }
 
-    int buffer_size = do_rescale ? rescale_size[0] * rescale_size[1] : ibuf->x * ibuf->y;
+    size_t buffer_size = do_rescale ? size_t(rescale_size[0]) * size_t(rescale_size[1]) :
+                                      size_t(ibuf->x) * size_t(ibuf->y);
     if (is_float_rect) {
-      for (uint64_t i = 0; i < buffer_size; i++) {
+      for (size_t i = 0; i < buffer_size; i++) {
         ((float *)data_rect)[i] = ((float *)src_rect)[i * 4];
       }
     }
     else {
-      for (uint64_t i = 0; i < buffer_size; i++) {
+      for (size_t i = 0; i < buffer_size; i++) {
         ((uchar *)data_rect)[i] = ((uchar *)src_rect)[i * 4];
       }
     }

source/blender/makesrna/intern/rna_image.cc

@@ -618,7 +618,7 @@ static int rna_Image_pixels_get_length(const PointerRNA *ptr, int length[RNA_MAX
   ibuf = BKE_image_acquire_ibuf(ima, nullptr, &lock);
 
   if (ibuf) {
-    length[0] = ibuf->x * ibuf->y * ibuf->channels;
+    length[0] = size_t(ibuf->x) * size_t(ibuf->y) * size_t(ibuf->channels);
   }
   else {
     length[0] = 0;
@@ -634,18 +634,17 @@ static void rna_Image_pixels_get(PointerRNA *ptr, float *values)
   Image *ima = (Image *)ptr->owner_id;
   ImBuf *ibuf;
   void *lock;
-  int i, size;
 
   ibuf = BKE_image_acquire_ibuf(ima, nullptr, &lock);
 
   if (ibuf) {
-    size = ibuf->x * ibuf->y * ibuf->channels;
+    const size_t size = size_t(ibuf->x) * size_t(ibuf->y) * size_t(ibuf->channels);
 
     if (ibuf->float_buffer.data) {
       memcpy(values, ibuf->float_buffer.data, sizeof(float) * size);
     }
     else {
-      for (i = 0; i < size; i++) {
+      for (size_t i = 0; i < size; i++) {
         values[i] = ibuf->byte_buffer.data[i] * (1.0f / 255.0f);
       }
     }
@@ -659,18 +658,17 @@ static void rna_Image_pixels_set(PointerRNA *ptr, const float *values)
   Image *ima = (Image *)ptr->owner_id;
   ImBuf *ibuf;
   void *lock;
-  int i, size;
 
   ibuf = BKE_image_acquire_ibuf(ima, nullptr, &lock);
 
   if (ibuf) {
-    size = ibuf->x * ibuf->y * ibuf->channels;
+    const size_t size = size_t(ibuf->x) * size_t(ibuf->y) * size_t(ibuf->channels);
 
     if (ibuf->float_buffer.data) {
       memcpy(ibuf->float_buffer.data, values, sizeof(float) * size);
     }
     else {
-      for (i = 0; i < size; i++) {
+      for (size_t i = 0; i < size; i++) {
         ibuf->byte_buffer.data[i] = unit_float_to_uchar_clamp(values[i]);
       }
     }

source/blender/render/intern/multires_bake.cc

@@ -817,7 +817,7 @@ static void *init_heights_data(MultiresBakeRender *bkr, ImBuf *ibuf)
   BakeImBufuserData *userdata = static_cast<BakeImBufuserData *>(ibuf->userdata);
 
   if (userdata->displacement_buffer == nullptr) {
-    userdata->displacement_buffer = MEM_cnew_array<float>(ibuf->x * ibuf->y,
+    userdata->displacement_buffer = MEM_cnew_array<float>(size_t(ibuf->x) * size_t(ibuf->y),
                                                           "MultiresBake heights");
   }
 
@@ -1420,14 +1420,14 @@ static void bake_ibuf_normalize_displacement(ImBuf *ibuf,
                                              float displacement_min,
                                              float displacement_max)
 {
-  int i;
   const float *current_displacement = displacement;
   const char *current_mask = mask;
   float max_distance;
 
   max_distance = max_ff(fabsf(displacement_min), fabsf(displacement_max));
 
-  for (i = 0; i < ibuf->x * ibuf->y; i++) {
+  const size_t ibuf_pixel_count = size_t(ibuf->x) * size_t(ibuf->y);
+  for (size_t i = 0; i < ibuf_pixel_count; i++) {
     if (*current_mask == FILTER_MASK_USED) {
       float normalized_displacement;