vp9: Add check to validate source input

Return invalid_params.

The check is wrapped around a new control:
VP9E_SET_VALIDATE_INPUT_HBD,
and is enabled by default.

Check is done only CONFIG_VP9_HIGHBITDEPTH build
and for bitdepth > 8.

Bug: 488585490
Change-Id: Ic79dff1bd314d4b197087d3c46d315ece8fd355c

Author: marpan-max

test/encode_api_test.cc

@@ -2533,7 +2533,11 @@ TEST(EncodeAPI, Buganizer487259772ScaledRefs) {
   encoder.Encode(/*key_frame=*/false, &rng);
 }
 
-TEST(EncodeAPI, DISABLED_Buganizer488585490CostTableOverflow) {
+#if CONFIG_VP9_HIGHBITDEPTH
+// This test uses source input based on the issue: 488585490,
+// which is invalid for 10 bit. This test checks that the
+// encoder correctly returns VPX_CODEC_INVALID_PARAM.
+TEST(EncodeAPI, Buganizer488585490CostTableOverflow) {
   // Initialize libvpx encoder.
   vpx_codec_iface_t *const iface = vpx_codec_vp9_cx();
   vpx_codec_ctx_t enc;
@@ -2562,10 +2566,12 @@ TEST(EncodeAPI, DISABLED_Buganizer488585490CostTableOverflow) {
   video.Begin();
   ASSERT_EQ(vpx_codec_encode(&enc, video.img(), video.pts(), /*duration=*/66666,
                              /*flags=*/0, VPX_DL_REALTIME),
-            VPX_CODEC_OK);
+            VPX_CODEC_INVALID_PARAM);
 
   ASSERT_EQ(vpx_codec_destroy(&enc), VPX_CODEC_OK);
 }
+#endif
+
 #endif  // CONFIG_VP9_ENCODER
 
 }  // namespace

vp9/vp9_cx_iface.c

@@ -72,6 +72,7 @@ typedef struct vp9_extracfg {
   unsigned int row_mt;
   unsigned int motion_vector_unit_test;
   int delta_q_uv;
+  unsigned int validate_input_hbd;
 } vp9_extracfg;
 
 static struct vp9_extracfg default_extra_cfg = {
@@ -112,6 +113,7 @@ static struct vp9_extracfg default_extra_cfg = {
   0,                     // row_mt
   0,                     // motion_vector_unit_test
   0,                     // delta_q_uv
+  1,                     // validate_input_hbd
 };
 
 struct vpx_codec_alg_priv {
@@ -984,6 +986,12 @@ static vpx_codec_err_t ctrl_set_keyframe_filtering(vpx_codec_alg_priv_t *ctx,
   return update_extra_cfg(ctx, &extra_cfg);
 }
 
+static vpx_codec_err_t ctrl_set_validate_input_hbd(vpx_codec_alg_priv_t *ctx,
+                                                   va_list args) {
+  struct vp9_extracfg extra_cfg = ctx->extra_cfg;
+  extra_cfg.validate_input_hbd = CAST(VP9E_SET_VALIDATE_INPUT_HBD, args);
+  return update_extra_cfg(ctx, &extra_cfg);
+}
 static vpx_codec_err_t ctrl_set_arnr_max_frames(vpx_codec_alg_priv_t *ctx,
                                                 va_list args) {
   struct vp9_extracfg extra_cfg = ctx->extra_cfg;
@@ -1452,6 +1460,36 @@ static vpx_codec_err_t encoder_encode(vpx_codec_alg_priv_t *ctx,
     if (img != NULL) {
       YV12_BUFFER_CONFIG sd;
 
+#if CONFIG_VP9_HIGHBITDEPTH
+      if (ctx->extra_cfg.validate_input_hbd &&
+          (img->fmt & VPX_IMG_FMT_HIGHBITDEPTH) &&
+          ctx->oxcf.input_bit_depth > 8) {
+        const unsigned int h = img->d_h;
+        const unsigned int w = img->d_w;
+        const unsigned int bit_depth = ctx->oxcf.input_bit_depth;
+        const int max_val = 1 << bit_depth;
+        for (int plane = 0; plane < 3; ++plane) {
+          const unsigned short *src =
+              (const unsigned short *)img->planes[plane];
+          const unsigned int stride = img->stride[plane] / 2;
+          const unsigned int ph =
+              (plane == 0) ? h
+                           : (h + img->y_chroma_shift) >> img->y_chroma_shift;
+          const unsigned int pw =
+              (plane == 0) ? w
+                           : (w + img->x_chroma_shift) >> img->x_chroma_shift;
+          for (unsigned int i = 0; i < ph; ++i) {
+            for (unsigned int j = 0; j < pw; ++j) {
+              if (src[j] >= max_val) {
+                return VPX_CODEC_INVALID_PARAM;
+              }
+            }
+            src += stride;
+          }
+        }
+      }
+#endif  // CONFIG_VP9_HIGHBITDEPTH
+
       if (!ctx->pts_offset_initialized) {
         ctx->pts_offset = pts;
         ctx->pts_offset_initialized = 1;
@@ -2158,6 +2196,7 @@ static vpx_codec_ctrl_fn_map_t encoder_ctrl_maps[] = {
   { VP9E_SET_TILE_ROWS, ctrl_set_tile_rows },
   { VP9E_SET_TPL, ctrl_set_tpl_model },
   { VP9E_SET_KEY_FRAME_FILTERING, ctrl_set_keyframe_filtering },
+  { VP9E_SET_VALIDATE_INPUT_HBD, ctrl_set_validate_input_hbd },
   { VP8E_SET_ARNR_MAXFRAMES, ctrl_set_arnr_max_frames },
   { VP8E_SET_ARNR_STRENGTH, ctrl_set_arnr_strength },
   { VP8E_SET_ARNR_TYPE, ctrl_set_arnr_type },

vpx/vp8cx.h

@@ -776,6 +776,16 @@ enum vp8e_enc_control_id {
    * 1. The default value is set to be 0.
    */
   VP9E_SET_KEY_FRAME_FILTERING,
+
+  /*!\brief Codec control function to validate HBD input.
+   *
+   * VP9 allows the encoder to validate the high bitdepth (HBD) input and
+   * ensure that every pixel is within the valid range. To disable/enable,
+   * set this parameter to 0/1. The default value is set to be 1.
+   *
+   * Supported in codecs: VP9
+   */
+  VP9E_SET_VALIDATE_INPUT_HBD,
 };
 
 /*!\brief vpx 1-D scaling mode
@@ -1108,6 +1118,8 @@ VPX_CTRL_USE_TYPE(VP9E_SET_QUANTIZER_ONE_PASS, int)
 #define VPX_CTRL_VP9E_SET_QUANTIZER_ONE_PASS
 VPX_CTRL_USE_TYPE(VP9E_SET_KEY_FRAME_FILTERING, int)
 #define VPX_CTRL_VP9E_SET_KEY_FRAME_FILTERING
+VPX_CTRL_USE_TYPE(VP9E_SET_VALIDATE_INPUT_HBD, int)
+#define VPX_CTRL_VP9E_SET_VALIDATE_INPUT_HBD
 
 /*!\endcond */
 /*! @} - end defgroup vp8_encoder */