[DIP] Add resize4D operation into DIP dialect. (#349)

examples/DIPDialect/CMakeLists.txt

@@ -25,3 +25,6 @@ target_link_libraries(rotation2D ${DIP_LIBS})
 
 add_executable(resize2D resize2D.cpp)
 target_link_libraries(resize2D ${DIP_LIBS})
+
+add_executable(resize4D resize4D.cpp)
+target_link_libraries(resize4D ${DIP_LIBS})

examples/DIPDialect/resize4D.cpp

@@ -0,0 +1,61 @@
+//====- resize4D.cpp - Example of buddy-opt tool =============================//
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a 4D resize example with dip.resize_4d operation.
+// The dip.resize_4d operation will be compiled into an object file with the
+// buddy-opt tool.
+// This file will be linked with the object file to generate the executable
+// file.
+//
+//===----------------------------------------------------------------------===//
+#include "buddy/DIP/imgcodecs/loadsave.h"
+#include <buddy/Core/Container.h>
+#include <buddy/DIP/DIP.h>
+#include <buddy/DIP/ImageContainer.h>
+#include <iostream>
+#include <math.h>
+
+using namespace std;
+
+void testImplementation(int argc, char *argv[]) {
+  // Read as colar image.
+  Img<float, 3> input = dip::imread<float, 3>(argv[1], dip::IMGRD_COLOR);
+
+  intptr_t sizes[4] = {1, input.getSizes()[0], input.getSizes()[1],
+                       input.getSizes()[2]};
+  Img<float, 4> inputBatch(input.getData(), sizes);
+
+  // Note : Both values in output image dimensions and scaling ratios must be
+  // positive numbers.
+  MemRef<float, 4> output = dip::Resize4D(
+      &inputBatch, dip::INTERPOLATION_TYPE::BILINEAR_INTERPOLATION,
+      {1, 224, 224, 3} /*{image_cols, image_rows}*/);
+
+  // Define Img with the output of Resize4D.
+  intptr_t outSizes[3] = {output.getSizes()[1], output.getSizes()[2],
+                          output.getSizes()[3]};
+
+  Img<float, 3> outputImageResize4D(output.getData(), outSizes);
+
+  dip::imwrite(argv[2], outputImageResize4D);
+
+  return;
+}
+
+int main(int argc, char *argv[]) {
+  testImplementation(argc, argv);
+  return 0;
+}

frontend/Interfaces/buddy/DIP/DIP.h

@@ -69,10 +69,20 @@ void _mlir_ciface_resize_2d_nearest_neighbour_interpolation(
     Img<float, 2> *input, float horizontalScalingFactor,
     float verticalScalingFactor, MemRef<float, 2> *output);
 
+// Declare the Resize4D C interface.
+void _mlir_ciface_resize_4d_nearest_neighbour_interpolation(
+    Img<float, 4> *input, float horizontalScalingFactor,
+    float verticalScalingFactor, MemRef<float, 4> *output);
+
 void _mlir_ciface_resize_2d_bilinear_interpolation(
     Img<float, 2> *input, float horizontalScalingFactor,
     float verticalScalingFactor, MemRef<float, 2> *output);
 
+// Declare the Resize4D C interface.
+void _mlir_ciface_resize_4d_bilinear_interpolation(
+    Img<float, 4> *input, float horizontalScalingFactor,
+    float verticalScalingFactor, MemRef<float, 4> *output);
+
 // Declare the Morphology 2D C interface.
 void _mlir_ciface_erosion_2d_constant_padding(
     Img<float, 2> input, MemRef<float, 2> *kernel, MemRef<float, 2> *output,
@@ -201,6 +211,28 @@ inline MemRef<float, 2> Resize2D_Impl(Img<float, 2> *input,
 
   return output;
 }
+
+// Helper function for applying 4D resize operation on images.
+inline MemRef<float, 4> Resize4D_Impl(Img<float, 4> *input,
+                                      INTERPOLATION_TYPE type,
+                                      std::vector<float> scalingRatios,
+                                      intptr_t outputSize[4]) {
+  MemRef<float, 4> output(outputSize);
+
+  if (type == INTERPOLATION_TYPE::NEAREST_NEIGHBOUR_INTERPOLATION) {
+    detail::_mlir_ciface_resize_4d_nearest_neighbour_interpolation(
+        input, scalingRatios[0], scalingRatios[1], &output);
+  } else if (type == INTERPOLATION_TYPE::BILINEAR_INTERPOLATION) {
+    detail::_mlir_ciface_resize_4d_bilinear_interpolation(
+        input, scalingRatios[0], scalingRatios[1], &output);
+  } else {
+    throw std::invalid_argument(
+        "Please chose a supported type of interpolation "
+        "(Nearest neighbour interpolation or Bilinear interpolation)\n");
+  }
+
+  return output;
+}
 } // namespace detail
 
 // User interface for 2D Correlation.
@@ -325,25 +357,28 @@ inline MemRef<float, 2> Rotate2D(Img<float, 2> *input, float angle,
 
 // User interface for 2D Resize.
 inline MemRef<float, 2> Resize2D(Img<float, 2> *input, INTERPOLATION_TYPE type,
-                                 std::vector<float> scalingRatios) {
-  if (scalingRatios[0] <= 0 || scalingRatios[1] <= 0) {
-    throw std::invalid_argument(
-        "Please enter positive values of scaling ratios.\n"
-        "Note : scaling ratio = "
-        "output_image_dimension / input_image_dimension\n");
+                                 std::vector<uint> size) {
+  if (size.size() != 2) {
+    throw std::invalid_argument("Dimension of an image should be 2\n");
   }
-  std::reverse(scalingRatios.begin(), scalingRatios.end());
-
-  intptr_t outputSize[2] = {static_cast<unsigned>(std::round(
-                                input->getSizes()[0] * scalingRatios[0])),
-                            static_cast<unsigned>(std::round(
-                                input->getSizes()[1] * scalingRatios[1]))};
-
-  scalingRatios[0] = 1 / scalingRatios[0];
-  scalingRatios[1] = 1 / scalingRatios[1];
+  intptr_t outputSize[2] = {size[0], size[1]};
+  return detail::Resize2D_Impl(input, type,
+                               {(float)input->getSizes()[0] / (float)size[0],
+                                (float)input->getSizes()[1] / (float)size[1]},
+                               outputSize);
+}
 
-  return detail::Resize2D_Impl(
-      input, type, {scalingRatios[1], scalingRatios[0]}, outputSize);
+// User interface for 4D Resize.
+inline MemRef<float, 4> Resize4D(Img<float, 4> *input, INTERPOLATION_TYPE type,
+                                 std::vector<uint> size) {
+  if (size.size() != 4) {
+    throw std::invalid_argument("Dimension of an image should be 4\n");
+  }
+  intptr_t outputSize[4] = {size[0], size[1], size[2], size[3]};
+  return detail::Resize4D_Impl(input, type,
+                               {(float)input->getSizes()[1] / (float)size[1],
+                                (float)input->getSizes()[2] / (float)size[2]},
+                               outputSize);
 }
 
 // User interface for 2D Resize.

frontend/Interfaces/lib/DIP.mlir

@@ -54,6 +54,18 @@ func.func @resize_2d_bilinear_interpolation(%inputImage : memref<?x?xf32>, %hori
   return
 }
 
+func.func @resize_4d_nearest_neighbour_interpolation(%inputImage : memref<?x?x?x?xf32>, %horizontal_scaling_factor : f32, %vertical_scaling_factor : f32, %outputImage : memref<?x?x?x?xf32>) attributes{llvm.emit_c_interface}
+{
+  dip.resize_4d NEAREST_NEIGHBOUR_INTERPOLATION %inputImage, %horizontal_scaling_factor, %vertical_scaling_factor, %outputImage : memref<?x?x?x?xf32>, f32, f32, memref<?x?x?x?xf32>
+  return
+}
+
+func.func @resize_4d_bilinear_interpolation(%inputImage : memref<?x?x?x?xf32>, %horizontal_scaling_factor : f32, %vertical_scaling_factor : f32, %outputImage : memref<?x?x?x?xf32>) attributes{llvm.emit_c_interface}
+{
+  dip.resize_4d BILINEAR_INTERPOLATION %inputImage, %horizontal_scaling_factor, %vertical_scaling_factor, %outputImage : memref<?x?x?x?xf32>, f32, f32, memref<?x?x?x?xf32>
+  return
+}
+
 func.func @erosion_2d_constant_padding(%inputImage : memref<?x?xf32>, %kernel : memref<?x?xf32>, %outputImage : memref<?x?xf32>, %copymemref : memref<?x?xf32>, %centerX : index, %centerY : index, %iterations : index, %constantValue: f32) attributes{llvm.emit_c_interface}
 {
   dip.erosion_2d <CONSTANT_PADDING> %inputImage, %kernel, %outputImage, %copymemref, %centerX, %centerY, %iterations, %constantValue: memref<?x?xf32>, memref<?x?xf32>, memref<?x?xf32>, memref<?x?xf32>, index, index, index, f32

midend/include/Dialect/DIP/DIPOps.td

@@ -210,6 +210,47 @@ def DIP_Resize2DOp : DIP_Op<"resize_2d">
   }];
 }
 
+def DIP_Resize4DOp : DIP_Op<"resize_4d">
+{
+  let summary = [{
+    This operation intends to provide a utility for resizing images using the DIP dialect.
+    Image resizing has many applications such as data augmentation, dimension adjustment in ML
+    models, etc. and can thus be used in native MLIR pipelines catering to above mentioned
+    use-cases.
+
+    As of now, two different mechanisms for pixel interpolation are provided namely nearest
+    neighbour interpolation and bilinear interpolation. The user can specify the desired type of
+    interpolation via an attribute provided as argument to the operation. The operation also
+    expects scaling ratios (Input image dimension / Output image dimension) for both dimensions
+    of input and output images as arguments.
+
+    The operation is flexible for its use with images of different sizes without necessarily
+    lowering it every time for each new image (Refer to the example provided in examples
+    directory for the DIP dialect).
+
+    Syntax :
+
+    ```mlir
+    dip.resize_4d INTERPOLATION_TYPE %inputImage, %horizontal_scaling_factor, %vertical_scaling_factor, %outputImage : memref<?x?x?x?xf32>, f32, f32, memref<?x?x?x?xf32>
+    ```
+
+    where ```INTERPOLATION_TYPE``` can be ```NEAREST_NEIGHBOUR_INTERPOLATION``` or
+    ```BILINEAR_INTERPOLATION```.
+  }];
+
+  let arguments = (ins Arg<AnyRankedOrUnrankedMemRef, "inputMemref",
+                           [MemRead]>:$memrefI,
+                       F32 : $horizontal_scaling_factor,
+                       F32 : $vertical_scaling_factor,
+                       Arg<AnyRankedOrUnrankedMemRef, "outputMemref",
+                           [MemRead]>:$memrefO,
+                       DIP_InterpolationAttr:$interpolation_type);
+
+  let assemblyFormat = [{
+     $interpolation_type $memrefI `,` $horizontal_scaling_factor `,` $vertical_scaling_factor `,` $memrefO attr-dict `:` type($memrefI) `,` type($horizontal_scaling_factor) `,` type($vertical_scaling_factor) `,` type($memrefO)
+  }];
+}
+
 def DIP_Erosion2DOp : DIP_Op<"erosion_2d"> {
   let summary = [{This operation aims to provide utility to perform Erosion on
                       a 2d single channel image.}];

midend/include/Utils/DIPUtils.h

@@ -103,6 +103,15 @@ void fillPixels(OpBuilder &builder, Location loc, Value resXVec, Value resYVec,
                 Value outputColLastElemF32, Value inputRowLastElemF32,
                 Value inputColLastElemF32, Value c0F32);
 
+// Fill appropriate pixel 4D data in its corresponding rotated co-ordinate of
+// output image.
+void fillPixels4D(OpBuilder &builder, Location loc, Value ivs0, Value ivs1,
+                  Value resXVec, Value resYVec, Value xVec, Value yVec,
+                  Value input, Value output, Value c0, Value strideVal,
+                  Value outputRowLastElemF32, Value outputColLastElemF32,
+                  Value inputRowLastElemF32, Value inputColLastElemF32,
+                  Value c0F32);
+
 // Calculate tan(angle / 2) where angle is a function parameter.
 Value customTanVal(OpBuilder &builder, Location loc, Value angleVal);
 
@@ -150,6 +159,15 @@ void fillPixelsBilinearInterpolate(
     Value inputRowLastElemF32, Value inputColLastElemF32, Value c0F32,
     Value c1F32);
 
+// Fills pixels in 4D of bilinear interpolation fashion.
+void fillPixelsBilinearInterpolate4D(
+    OpBuilder &builder, Location loc, Value ivs0, Value ivs1, Value resXVec,
+    Value resYVec, Value xVec_L, Value yVec_L, Value xVec_H, Value yVec_H,
+    Value input, Value output, Value c0, Value strideVal, Value xVecWeight,
+    Value yVecWeight, Value outputRowLastElemF32, Value outputColLastElemF32,
+    Value inputRowLastElemF32, Value inputColLastElemF32, Value c0F32,
+    Value c1F32);
+
 // Helper function for resizing an image using nearest neighbour interpolation
 // mechanism.
 void NearestNeighbourInterpolationResizing(
@@ -161,6 +179,17 @@ void NearestNeighbourInterpolationResizing(
     Value inputRowLastElemF32, Value inputColLastElemF32, VectorType vectorTy32,
     int64_t stride, Value c0, Value c0F32);
 
+// Helper function for resizing 4D an image using nearest neighbour
+// interpolation mechanism.
+void NearestNeighbourInterpolationResizing4D(
+    OpBuilder &builder, Location loc, MLIRContext *ctx,
+    SmallVector<Value, 8> lowerBounds, SmallVector<Value, 8> upperBounds,
+    SmallVector<int64_t, 8> steps, Value strideVal, Value input, Value output,
+    Value horizontalScalingFactorVec, Value verticalScalingFactorVec,
+    Value outputRowLastElemF32, Value outputColLastElemF32,
+    Value inputRowLastElemF32, Value inputColLastElemF32, VectorType vectorTy32,
+    int64_t stride, Value c0, Value c0F32);
+
 // Helper function for resizing an image using bilinear interpolation mechanism.
 void BilinearInterpolationResizing(
     OpBuilder &builder, Location loc, MLIRContext *ctx,
@@ -171,6 +200,17 @@ void BilinearInterpolationResizing(
     Value inputRowLastElemF32, Value inputColLastElemF32, VectorType vectorTy32,
     int64_t stride, Value c0, Value c0F32, Value c1F32);
 
+// Helper function for resizing 4D an image using bilinear interpolation
+// mechanism.
+void BilinearInterpolationResizing4D(
+    OpBuilder &builder, Location loc, MLIRContext *ctx,
+    SmallVector<Value, 8> lowerBounds, SmallVector<Value, 8> upperBounds,
+    SmallVector<int64_t, 8> steps, Value strideVal, Value input, Value output,
+    Value horizontalScalingFactorVec, Value verticalScalingFactorVec,
+    Value outputRowLastElemF32, Value outputColLastElemF32,
+    Value inputRowLastElemF32, Value inputColLastElemF32, VectorType vectorTy32,
+    int64_t stride, Value c0, Value c0F32, Value c1F32);
+
 // Util function for morphological transformations ; compares two vectors and
 // returns a mask
 Value createCompVecMorph(OpBuilder &builder, Location loc, VectorType type,
@@ -191,9 +231,10 @@ void calcAndStorewTailProcessingMorph(
     Value zeroPadding, Value inputCol, VectorType vectorMaskTy, Type elemTy,
     Value kernelValue, Value zeroPaddingElem, DIP_OP op);
 
-// Utility function for traversing an image with support for boundary extrapolation,
-// variable anchor point positioning, and tail processing. It is used to compose more
-// complicated operations on top of it, like 2D Correlation and morphological operations.
+// Utility function for traversing an image with support for boundary
+// extrapolation, variable anchor point positioning, and tail processing. It is
+// used to compose more complicated operations on top of it, like 2D Correlation
+// and morphological operations.
 void traverseImagewBoundaryExtrapolation(
     OpBuilder &rewriter, Location loc, MLIRContext *ctx, Value input,
     Value kernel, Value output, Value centerX, Value centerY,