[DIP] Split resize 4D into NCHW and NHWC (#405)

examples/BuddyMobileNetV3/CMakeLists.txt

@@ -71,5 +71,5 @@ SET_TARGET_PROPERTIES(MOBILENETV3 PROPERTIES LINKER_LANGUAGE C)
 add_executable(buddy-mobilenetv3-run buddy-mobilenetv3-main.cpp)
 target_link_directories(buddy-mobilenetv3-run PRIVATE ${LLVM_LIBRARY_DIR})
 
-set(BUDDY_MOBILENETV3_LIBS MOBILENETV3 mlir_c_runner_utils ${PNG_LIBRARIES})
+set(BUDDY_MOBILENETV3_LIBS MOBILENETV3 mlir_c_runner_utils BuddyLibDIP ${PNG_LIBRARIES})
 target_link_libraries(buddy-mobilenetv3-run ${BUDDY_MOBILENETV3_LIBS})

examples/BuddyMobileNetV3/buddy-mobilenetv3-main.cpp

@@ -15,6 +15,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <buddy/Core/Container.h>
+#include <buddy/DIP/DIP.h>
 #include <buddy/DIP/ImgContainer.h>
 #include <chrono>
 #include <cmath>
@@ -27,13 +28,13 @@
 #include <vector>
 
 constexpr size_t ParamsSize = 2554968;
-const std::string ImgName = "dog-224*224.png";
+const std::string ImgName = "dog.png";
 
 // Declare the mobilenet C interface.
 extern "C" void _mlir_ciface_forward(MemRef<float, 2> *output,
                                      MemRef<float, 1> *arg0,
                                      MemRef<long long, 1> *arg1,
-                                     dip::Image<float, 4> *input);
+                                     MemRef<float, 4> *input);
 
 /// Print [Log] label in bold blue format.
 void printLogLabel() { std::cout << "\033[34;1m[Log] \033[0m"; }
@@ -115,7 +116,10 @@ int main() {
   std::string mobilenetDir = getenv("MOBILENETV3_EXAMPLE_PATH");
   std::string imgPath = mobilenetDir + "/images/" + ImgName;
   dip::Image<float, 4> input(imgPath, dip::DIP_RGB, true /* norm */);
-
+  MemRef<float, 4> inputResize = dip::Resize4D_NCHW(
+      &input, dip::INTERPOLATION_TYPE::BILINEAR_INTERPOLATION,
+      {1, 3, 224, 224} /*{image_cols, image_rows}*/);
+
   MemRef<float, 2> output(sizesOutput);
 
   // Load model parameters from the specified file.
@@ -126,7 +130,7 @@ int main() {
   loadParameters(paramsDir, intDir, paramsContainerf32, ParamsContainerInt64);
   // Call the forward function of the model.
   _mlir_ciface_forward(&output, &paramsContainerf32, &ParamsContainerInt64,
-                       &input);
+                       &inputResize);
 
   auto out = output.getData();
   softmax(out, 1000);

examples/DIPDialect/CMakeLists.txt

@@ -26,5 +26,8 @@ target_link_libraries(rotation2D ${DIP_LIBS})
 add_executable(resize2D resize2D.cpp)
 target_link_libraries(resize2D ${DIP_LIBS})
 
-add_executable(resize4D resize4D.cpp)
-target_link_libraries(resize4D ${DIP_LIBS})
+add_executable(resize4D_nhwc resize4D_nhwc.cpp)
+target_link_libraries(resize4D_nhwc ${DIP_LIBS})
+
+add_executable(resize4D_nchw resize4D_nchw.cpp)
+target_link_libraries(resize4D_nchw ${DIP_LIBS})

examples/DIPDialect/resize4D_nchw.cpp

@@ -0,0 +1,58 @@
+//====- resize4D.cpp - Example of buddy-opt tool =============================//
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a 4D resize example with dip.resize_4d operation.
+// The dip.resize_4d operation will be compiled into an object file with the
+// buddy-opt tool.
+// This file will be linked with the object file to generate the executable
+// file.
+//
+//===----------------------------------------------------------------------===//
+#include "buddy/DIP/imgcodecs/loadsave.h"
+#include <buddy/Core/Container.h>
+#include <buddy/DIP/ImgContainer.h>
+#include <buddy/DIP/DIP.h>
+#include <buddy/DIP/ImageContainer.h>
+#include <iostream>
+#include <math.h>
+
+using namespace std;
+
+void testImplementation(int argc, char *argv[]) {
+  // Read as colar image.
+  dip::Image<float, 4> inputBatch(argv[1], dip::DIP_RGB, true);
+
+  // Note : Both values in output image dimensions and scaling ratios must be
+  // positive numbers.
+  MemRef<float, 4> output = dip::Resize4D_NCHW(
+      &inputBatch, dip::INTERPOLATION_TYPE::BILINEAR_INTERPOLATION,
+      {1, 3, 224, 224} /*{image_cols, image_rows}*/);
+
+  // Define Img with the output of Resize4D.
+  intptr_t outSizes[3] = {output.getSizes()[2], output.getSizes()[3],
+                          output.getSizes()[1]};
+
+  Img<float, 3> outputImageResize4D(output.getData(), outSizes);
+
+  // dip::imwrite(argv[2], outputImageResize4D);
+
+  return;
+}
+
+int main(int argc, char *argv[]) {
+  testImplementation(argc, argv);
+  return 0;
+}

examples/DIPDialect/resize4D.cpp → examples/DIPDialect/resize4D_nhwc.cpp

@@ -40,7 +40,7 @@ void testImplementation(int argc, char *argv[]) {
 
   // Note : Both values in output image dimensions and scaling ratios must be
   // positive numbers.
-  MemRef<float, 4> output = dip::Resize4D(
+  MemRef<float, 4> output = dip::Resize4D_NHWC(
       &inputBatch, dip::INTERPOLATION_TYPE::BILINEAR_INTERPOLATION,
       {1, 224, 224, 3} /*{image_cols, image_rows}*/);
 

frontend/Interfaces/buddy/DIP/DIP.h

@@ -23,6 +23,8 @@
 
 #include "buddy/Core/Container.h"
 #include "buddy/DIP/ImageContainer.h"
+#include "buddy/DIP/ImgContainer.h"
+#include <iostream>
 #include <math.h>
 namespace dip {
 // Availale types of boundary extrapolation techniques provided in DIP dialect.
@@ -70,19 +72,27 @@ void _mlir_ciface_resize_2d_nearest_neighbour_interpolation(
     float verticalScalingFactor, MemRef<float, 2> *output);
 
 // Declare the Resize4D C interface.
-void _mlir_ciface_resize_4d_nearest_neighbour_interpolation(
+void _mlir_ciface_resize_4d_nhwc_nearest_neighbour_interpolation(
     Img<float, 4> *input, float horizontalScalingFactor,
     float verticalScalingFactor, MemRef<float, 4> *output);
 
+void _mlir_ciface_resize_4d_nchw_nearest_neighbour_interpolation(
+    dip::Image<float, 4> *input, float horizontalScalingFactor,
+    float verticalScalingFactor, MemRef<float, 4> *output);
+
 void _mlir_ciface_resize_2d_bilinear_interpolation(
     Img<float, 2> *input, float horizontalScalingFactor,
     float verticalScalingFactor, MemRef<float, 2> *output);
 
 // Declare the Resize4D C interface.
-void _mlir_ciface_resize_4d_bilinear_interpolation(
+void _mlir_ciface_resize_4d_nhwc_bilinear_interpolation(
     Img<float, 4> *input, float horizontalScalingFactor,
     float verticalScalingFactor, MemRef<float, 4> *output);
 
+void _mlir_ciface_resize_4d_nchw_bilinear_interpolation(
+    dip::Image<float, 4> *input, float horizontalScalingFactor,
+    float verticalScalingFactor, MemRef<float, 4> *output);
+
 // Declare the Morphology 2D C interface.
 void _mlir_ciface_erosion_2d_constant_padding(
     Img<float, 2> input, MemRef<float, 2> *kernel, MemRef<float, 2> *output,
@@ -213,17 +223,38 @@ inline MemRef<float, 2> Resize2D_Impl(Img<float, 2> *input,
 }
 
 // Helper function for applying 4D resize operation on images.
-inline MemRef<float, 4> Resize4D_Impl(Img<float, 4> *input,
-                                      INTERPOLATION_TYPE type,
-                                      std::vector<float> scalingRatios,
-                                      intptr_t outputSize[4]) {
+inline MemRef<float, 4> Resize4D_NHWC_Impl(Img<float, 4> *input,
+                                           INTERPOLATION_TYPE type,
+                                           std::vector<float> scalingRatios,
+                                           intptr_t outputSize[4]) {
   MemRef<float, 4> output(outputSize);
 
   if (type == INTERPOLATION_TYPE::NEAREST_NEIGHBOUR_INTERPOLATION) {
-    detail::_mlir_ciface_resize_4d_nearest_neighbour_interpolation(
+    detail::_mlir_ciface_resize_4d_nhwc_nearest_neighbour_interpolation(
         input, scalingRatios[0], scalingRatios[1], &output);
   } else if (type == INTERPOLATION_TYPE::BILINEAR_INTERPOLATION) {
-    detail::_mlir_ciface_resize_4d_bilinear_interpolation(
+    detail::_mlir_ciface_resize_4d_nhwc_bilinear_interpolation(
+        input, scalingRatios[0], scalingRatios[1], &output);
+  } else {
+    throw std::invalid_argument(
+        "Please chose a supported type of interpolation "
+        "(Nearest neighbour interpolation or Bilinear interpolation)\n");
+  }
+
+  return output;
+}
+
+inline MemRef<float, 4> Resize4D_NCHW_Impl(dip::Image<float, 4> *input,
+                                           INTERPOLATION_TYPE type,
+                                           std::vector<float> scalingRatios,
+                                           intptr_t outputSize[4]) {
+  MemRef<float, 4> output(outputSize);
+
+  if (type == INTERPOLATION_TYPE::NEAREST_NEIGHBOUR_INTERPOLATION) {
+    detail::_mlir_ciface_resize_4d_nchw_nearest_neighbour_interpolation(
+        input, scalingRatios[0], scalingRatios[1], &output);
+  } else if (type == INTERPOLATION_TYPE::BILINEAR_INTERPOLATION) {
+    detail::_mlir_ciface_resize_4d_nchw_bilinear_interpolation(
         input, scalingRatios[0], scalingRatios[1], &output);
   } else {
     throw std::invalid_argument(
@@ -369,16 +400,32 @@ inline MemRef<float, 2> Resize2D(Img<float, 2> *input, INTERPOLATION_TYPE type,
 }
 
 // User interface for 4D Resize.
-inline MemRef<float, 4> Resize4D(Img<float, 4> *input, INTERPOLATION_TYPE type,
-                                 std::vector<uint> size) {
+inline MemRef<float, 4> Resize4D_NHWC(Img<float, 4> *input,
+                                      INTERPOLATION_TYPE type,
+                                      std::vector<uint> size) {
   if (size.size() != 4) {
     throw std::invalid_argument("Dimension of an image should be 4\n");
   }
   intptr_t outputSize[4] = {size[0], size[1], size[2], size[3]};
-  return detail::Resize4D_Impl(input, type,
-                               {(float)input->getSizes()[1] / (float)size[1],
-                                (float)input->getSizes()[2] / (float)size[2]},
-                               outputSize);
+  return detail::Resize4D_NHWC_Impl(
+      input, type,
+      {(float)input->getSizes()[1] / (float)size[1],
+       (float)input->getSizes()[2] / (float)size[2]},
+      outputSize);
+}
+
+inline MemRef<float, 4> Resize4D_NCHW(dip::Image<float, 4> *input,
+                                      INTERPOLATION_TYPE type,
+                                      std::vector<uint> size) {
+  if (size.size() != 4) {
+    throw std::invalid_argument("Dimension of an image should be 4\n");
+  }
+  intptr_t outputSize[4] = {size[0], size[1], size[2], size[3]};
+  return detail::Resize4D_NCHW_Impl(
+      input, type,
+      {(float)input->getSizes()[2] / (float)size[2],
+       (float)input->getSizes()[3] / (float)size[3]},
+      outputSize);
 }
 
 // User interface for 2D Resize.

frontend/Interfaces/buddy/DIP/ImgContainer.h

@@ -33,7 +33,7 @@ enum ImageModes {
   DIP_RGB = 1,
 };
 
-inline bool isBigEndian() {
+inline bool ifBigEndian() {
   int num = 1;
   char *ptr = (char *)#
   return (*ptr == 0);
@@ -475,7 +475,7 @@ bool Image<T, N>::decodePNG(const std::vector<uint8_t> &fileData) {
     // Convert big or little Endian and convert 16 bits to 8 bits
     if (this->bitDepth == 16)
       png_set_strip_16(png_ptr);
-    else if (!isBigEndian())
+    else if (!ifBigEndian())
       png_set_swap(png_ptr);
 
     // Remove alpha channel

frontend/Interfaces/lib/DIP.mlir

@@ -54,15 +54,27 @@ func.func @resize_2d_bilinear_interpolation(%inputImage : memref<?x?xf32>, %hori
   return
 }
 
-func.func @resize_4d_nearest_neighbour_interpolation(%inputImage : memref<?x?x?x?xf32>, %horizontal_scaling_factor : f32, %vertical_scaling_factor : f32, %outputImage : memref<?x?x?x?xf32>) attributes{llvm.emit_c_interface}
+func.func @resize_4d_nhwc_nearest_neighbour_interpolation(%inputImage : memref<?x?x?x?xf32>, %horizontal_scaling_factor : f32, %vertical_scaling_factor : f32, %outputImage : memref<?x?x?x?xf32>) attributes{llvm.emit_c_interface}
 {
-  dip.resize_4d NEAREST_NEIGHBOUR_INTERPOLATION %inputImage, %horizontal_scaling_factor, %vertical_scaling_factor, %outputImage : memref<?x?x?x?xf32>, f32, f32, memref<?x?x?x?xf32>
+  dip.resize_4d_nhwc NEAREST_NEIGHBOUR_INTERPOLATION %inputImage, %horizontal_scaling_factor, %vertical_scaling_factor, %outputImage : memref<?x?x?x?xf32>, f32, f32, memref<?x?x?x?xf32>
   return
 }
 
-func.func @resize_4d_bilinear_interpolation(%inputImage : memref<?x?x?x?xf32>, %horizontal_scaling_factor : f32, %vertical_scaling_factor : f32, %outputImage : memref<?x?x?x?xf32>) attributes{llvm.emit_c_interface}
+func.func @resize_4d_nhwc_bilinear_interpolation(%inputImage : memref<?x?x?x?xf32>, %horizontal_scaling_factor : f32, %vertical_scaling_factor : f32, %outputImage : memref<?x?x?x?xf32>) attributes{llvm.emit_c_interface}
 {
-  dip.resize_4d BILINEAR_INTERPOLATION %inputImage, %horizontal_scaling_factor, %vertical_scaling_factor, %outputImage : memref<?x?x?x?xf32>, f32, f32, memref<?x?x?x?xf32>
+  dip.resize_4d_nhwc BILINEAR_INTERPOLATION %inputImage, %horizontal_scaling_factor, %vertical_scaling_factor, %outputImage : memref<?x?x?x?xf32>, f32, f32, memref<?x?x?x?xf32>
+  return
+}
+
+func.func @resize_4d_nchw_nearest_neighbour_interpolation(%inputImage : memref<?x?x?x?xf32>, %horizontal_scaling_factor : f32, %vertical_scaling_factor : f32, %outputImage : memref<?x?x?x?xf32>) attributes{llvm.emit_c_interface}
+{
+  dip.resize_4d_nchw NEAREST_NEIGHBOUR_INTERPOLATION %inputImage, %horizontal_scaling_factor, %vertical_scaling_factor, %outputImage : memref<?x?x?x?xf32>, f32, f32, memref<?x?x?x?xf32>
+  return
+}
+
+func.func @resize_4d_nchw_bilinear_interpolation(%inputImage : memref<?x?x?x?xf32>, %horizontal_scaling_factor : f32, %vertical_scaling_factor : f32, %outputImage : memref<?x?x?x?xf32>) attributes{llvm.emit_c_interface}
+{
+  dip.resize_4d_nchw BILINEAR_INTERPOLATION %inputImage, %horizontal_scaling_factor, %vertical_scaling_factor, %outputImage : memref<?x?x?x?xf32>, f32, f32, memref<?x?x?x?xf32>
   return
 }
 

midend/include/Dialect/DIP/DIPOps.td

@@ -210,7 +210,7 @@ def DIP_Resize2DOp : DIP_Op<"resize_2d">
   }];
 }
 
-def DIP_Resize4DOp : DIP_Op<"resize_4d">
+def DIP_Resize4D_NHWCOp : DIP_Op<"resize_4d_nhwc">
 {
   let summary = [{
     This operation intends to provide a utility for resizing images using the DIP dialect.
@@ -228,10 +228,55 @@ def DIP_Resize4DOp : DIP_Op<"resize_4d">
     lowering it every time for each new image (Refer to the example provided in examples
     directory for the DIP dialect).
 
+    The processed image format is (batch, height, weight, channel).
+
+    Syntax :
+
+    ```mlir
+    dip.resize_4d_nhwc INTERPOLATION_TYPE %inputImage, %horizontal_scaling_factor, %vertical_scaling_factor, %outputImage : memref<?x?x?x?xf32>, f32, f32, memref<?x?x?x?xf32>
+    ```
+
+    where ```INTERPOLATION_TYPE``` can be ```NEAREST_NEIGHBOUR_INTERPOLATION``` or
+    ```BILINEAR_INTERPOLATION```.
+  }];
+
+  let arguments = (ins Arg<AnyRankedOrUnrankedMemRef, "inputMemref",
+                           [MemRead]>:$memrefI,
+                       F32 : $horizontal_scaling_factor,
+                       F32 : $vertical_scaling_factor,
+                       Arg<AnyRankedOrUnrankedMemRef, "outputMemref",
+                           [MemRead]>:$memrefO,
+                       DIP_InterpolationAttr:$interpolation_type);
+
+  let assemblyFormat = [{
+     $interpolation_type $memrefI `,` $horizontal_scaling_factor `,` $vertical_scaling_factor `,` $memrefO attr-dict `:` type($memrefI) `,` type($horizontal_scaling_factor) `,` type($vertical_scaling_factor) `,` type($memrefO)
+  }];
+}
+
+def DIP_Resize4D_NCHWOp : DIP_Op<"resize_4d_nchw">
+{
+  let summary = [{
+    This operation intends to provide a utility for resizing images using the DIP dialect.
+    Image resizing has many applications such as data augmentation, dimension adjustment in ML
+    models, etc. and can thus be used in native MLIR pipelines catering to above mentioned
+    use-cases.
+
+    As of now, two different mechanisms for pixel interpolation are provided namely nearest
+    neighbour interpolation and bilinear interpolation. The user can specify the desired type of
+    interpolation via an attribute provided as argument to the operation. The operation also
+    expects scaling ratios (Input image dimension / Output image dimension) for both dimensions
+    of input and output images as arguments.
+
+    The operation is flexible for its use with images of different sizes without necessarily
+    lowering it every time for each new image (Refer to the example provided in examples
+    directory for the DIP dialect).
+
+    The processed image format is (batch, channel, height, weight).
+
     Syntax :
 
     ```mlir
-    dip.resize_4d INTERPOLATION_TYPE %inputImage, %horizontal_scaling_factor, %vertical_scaling_factor, %outputImage : memref<?x?x?x?xf32>, f32, f32, memref<?x?x?x?xf32>
+    dip.resize_4d_nchw INTERPOLATION_TYPE %inputImage, %horizontal_scaling_factor, %vertical_scaling_factor, %outputImage : memref<?x?x?x?xf32>, f32, f32, memref<?x?x?x?xf32>
     ```
 
     where ```INTERPOLATION_TYPE``` can be ```NEAREST_NEIGHBOUR_INTERPOLATION``` or