Add skew function (#36)

Summary:
Add Shearing w.r.t y-axis and x-axis

Pull Request resolved: #36

Reviewed By: jbitton

Differential Revision: D29581632

Pulled By: zpapakipos

fbshipit-source-id: 050df38cd64446b363581daec60b15b1278a2308

Author: tanujdhiman

augly/assets/tests/image/dfdc_expected_output/test_skew.png

@@ -40,6 +40,7 @@
     scale,
     sharpen,
     shuffle_pixels,
+    skew,
     vflip,
 )
 from augly.image.helpers import aug_np_wrapper
@@ -77,6 +78,7 @@
     scale_intensity,
     sharpen_intensity,
     shuffle_pixels_intensity,
+    skew_intensity,
     vflip_intensity,
 )
 from augly.image.transforms import (
@@ -119,6 +121,7 @@
     Scale,
     Sharpen,
     ShufflePixels,
+    Skew,
     VFlip,
 )
 
@@ -165,6 +168,7 @@
     "Scale",
     "Sharpen",
     "ShufflePixels",
+    "Skew",
     "VFlip",
     "apply_lambda",
     "apply_pil_filter",
@@ -200,6 +204,7 @@
     "scale",
     "sharpen",
     "shuffle_pixels",
+    "skew",
     "vflip",
     "apply_lambda_intensity",
     "apply_pil_filter_intensity",
@@ -234,5 +239,6 @@
     "scale_intensity",
     "sharpen_intensity",
     "shuffle_pixels_intensity",
+    "skew_intensity",
     "vflip_intensity",
 ]

augly/image/__init__.py

@@ -15,6 +15,7 @@
 import numpy as np
 from augly import utils
 from augly.image import utils as imutils
+from augly.image.utils.bboxes import spatial_bbox_helper
 from PIL import Image, ImageDraw, ImageEnhance, ImageFilter, ImageFont
 
 
@@ -2368,6 +2369,72 @@ def shuffle_pixels(
     return imutils.ret_and_save_image(aug_image, output_path, src_mode)
 
 
+def skew(
+    image: Union[str, Image.Image],
+    output_path: Optional[str] = None,
+    skew_factor: float = 0.5,
+    axis: int = 0,
+    metadata: Optional[List[Dict[str, Any]]] = None,
+    bboxes: Optional[List[Tuple]] = None,
+    bbox_format: Optional[str] = None,
+) -> Image.Image:
+    """
+    Skews an image with respect to its x or y-axis
+
+    @param image: the path to an image or a variable of type PIL.Image.Image
+        to be augmented
+
+    @param output_path: the path in which the resulting image will be stored.
+        If None, the resulting PIL Image will still be returned
+
+    @param skew_factor: the level of skew to apply to the image; a larger absolute value will
+        result in a more intense skew. Recommended range is between [-2, 2]
+
+    @param axis: the axis along which the image will be skewed; can be set to 0 (x-axis)
+        or 1 (y-axis)
+
+    @param metadata: if set to be a list, metadata about the function execution
+        including its name, the source & dest width, height, etc. will be appended
+        to the inputted list. If set to None, no metadata will be appended or returned
+
+    @param bboxes: a list of bounding boxes can be passed in here if desired. If
+        provided, this list will be modified in place such that each bounding box is
+        transformed according to this function
+
+    @param bbox_format: signifies what bounding box format was used in `bboxes`. Must
+        specify `bbox_format` if `bboxes` is provided. Supported bbox_format values are
+        "pascal_voc", "pascal_voc_norm", "coco", and "yolo"
+
+    @returns: the augmented PIL Image
+    """
+    image = imutils.validate_and_load_image(image)
+    func_kwargs = imutils.get_func_kwargs(metadata, locals())
+    src_mode = image.mode
+
+    w, h = image.size
+
+    if axis == 0:
+        data = (1, skew_factor, -skew_factor * h / 2, 0, 1, 0)
+    elif axis == 1:
+        data = (1, 0, 0, skew_factor, 1, -skew_factor * w / 2)
+    else:
+        raise AssertionError(
+            f"Invalid 'axis' value: Got '{axis}', expected 0 for 'x-axis' or 1 for 'y-axis'"
+        )
+
+    aug_image = image.transform((w, h), Image.AFFINE, data, resample=Image.BILINEAR)
+    imutils.get_metadata(
+        metadata=metadata,
+        function_name="skew",
+        aug_image=aug_image,
+        bboxes_helper_func=spatial_bbox_helper,
+        aug_function=skew,
+        **func_kwargs,
+    )
+
+    return imutils.ret_and_save_image(aug_image, output_path, src_mode)  # pyre-ignore
+
+
 def vflip(
     image: Union[str, Image.Image],
     output_path: Optional[str] = None,

augly/image/functional.py

@@ -308,6 +308,11 @@ def shuffle_pixels_intensity(factor: float, **kwargs) -> float:
     return factor * 100.0
 
 
+def skew_intensity(skew_factor: float, **kwargs) -> float:
+    max_skew_factor = 2.0
+    return min((abs(skew_factor) / max_skew_factor) * 100.0, 100.0)
+
+
 def vflip_intensity(**kwargs) -> float:
     return 100.0
 

augly/image/intensity.py

@@ -2074,6 +2074,55 @@ def apply_transform(
         )
 
 
+class Skew(BaseTransform):
+    def __init__(self, skew_factor: float = 0.5, axis: int = 0, p: float = 1.0):
+        """
+        @param skew_factor: the level of skew to apply to the image; a larger absolute value will
+            result in a more intense skew. Recommended range is between [-2, 2]
+
+        @param axis: the axis along which the image will be skewed; can be set to 0 (x-axis)
+            or 1 (y-axis)
+        """
+        super().__init__(p)
+        self.skew_factor = skew_factor
+        self.axis = axis
+
+    def apply_transform(
+        self,
+        image: Image.Image,
+        metadata: Optional[List[Dict[str, Any]]] = None,
+        bboxes: Optional[List[Tuple]] = None,
+        bbox_format: Optional[str] = None,
+    ) -> Image.Image:
+        """
+        Skews an image with respect to its x or y-axis
+
+        @param image: PIL Image to be augmented
+
+        @param metadata: if set to be a list, metadata about the function execution
+            including its name, the source & dest width, height, etc. will be appended to
+            the inputted list. If set to None, no metadata will be appended or returned
+
+        @param bboxes: a list of bounding boxes can be passed in here if desired. If
+            provided, this list will be modified in place such that each bounding box is
+            transformed according to this function
+
+        @param bbox_format: signifies what bounding box format was used in `bboxes`. Must
+            specify `bbox_format` if `bboxes` is provided. Supported bbox_format values
+            are "pascal_voc", "pascal_voc_norm", "coco", and "yolo"
+
+        @returns: Augmented PIL Image
+        """
+        return F.skew(
+            image,
+            skew_factor=self.skew_factor,
+            axis=self.axis,
+            metadata=metadata,
+            bboxes=bboxes,
+            bbox_format=bbox_format,
+        )
+
+
 class VFlip(BaseTransform):
     def apply_transform(
         self,

augly/image/transforms.py

@@ -132,10 +132,10 @@ def transform_bboxes(
             imbboxes, f"{function_name}_bboxes_helper", lambda bbox, **_: bbox
         )
 
+    func_kwargs = deepcopy(kwargs)
+    func_kwargs.pop("src_bboxes", None)
     transformed_norm_bboxes = [
-        bboxes_helper_func(
-            bbox=bbox, function_name=function_name, src_w=src_w, src_h=src_h, **kwargs
-        )
+        bboxes_helper_func(bbox=bbox, src_w=src_w, src_h=src_h, **func_kwargs)
         for bbox in norm_bboxes
     ]
 
@@ -191,7 +191,7 @@ def get_metadata(
         function_name=function_name,
         image=image,
         aug_image=aug_image,
-        bbox_helper_func=bboxes_helper_func,
+        bboxes_helper_func=bboxes_helper_func,
         **kwargs,
     )
 
@@ -201,6 +201,11 @@ def get_metadata(
     kwargs_types_fixed = dict(
         (k, list(v)) if isinstance(v, tuple) else (k, v) for k, v in kwargs.items()
     )
+    if (
+        bboxes_helper_func is not None
+        and bboxes_helper_func.__name__ == "spatial_bbox_helper"
+    ):
+        kwargs_types_fixed.pop("aug_function", None)
 
     metadata.append(
         {

augly/image/utils/metadata.py

@@ -129,6 +129,9 @@ def test_sharpen(self):
     def test_shuffle_pixels(self):
         self.evaluate_function(imaugs.shuffle_pixels, factor=0.5)
 
+    def test_skew(self):
+        self.evaluate_function(imaugs.skew)
+
     def test_vflip(self):
         self.evaluate_function(imaugs.vflip)
 

augly/tests/image_tests/functional_unit_test.py

@@ -206,6 +206,9 @@ def test_Sharpen(self):
     def test_ShufflePixels(self):
         self.evaluate_class(imaugs.ShufflePixels(factor=0.5), fname="shuffle_pixels")
 
+    def test_Skew(self):
+        self.evaluate_class(imaugs.Skew(), fname="skew")
+
     def test_VFlip(self):
         self.evaluate_class(imaugs.VFlip(), fname="vflip")
 

augly/tests/image_tests/transforms_unit_test.py

@@ -706,6 +706,22 @@
             "src_width": 1920
         }
     ],
+    "skew": [
+        {
+            "bbox_format": "yolo",
+            "dst_bboxes": [[0.5, 0.5, 0.4614583333333333, 0.75]],
+            "dst_height": 1080,
+            "dst_width": 1920,
+            "skew_factor": 0.5,
+            "axis": 0,
+            "intensity": 25.0,
+            "name": "skew",
+            "output_path": null,
+            "src_bboxes": [[0.5, 0.5, 0.25, 0.75]],
+            "src_height": 1080,
+            "src_width": 1920
+        }
+    ],
     "vflip": [
         {
             "bbox_format": "yolo",