Skip to content

Commit f57dc05

Browse files
nosovmiknosovmik
authored
[OV20] Convert NV12 to RGB operation + preprocessing (openvinotoolkit#7508)
* # Conflicts: # docs/template_plugin/tests/functional/op_reference/convert_color_nv12.cpp # inference-engine/tests/functional/plugin/cpu/shared_tests_instances/single_layer_tests/convert_color_nv12.cpp # inference-engine/tests/functional/shared_test_classes/include/shared_test_classes/single_layer/convert_color_nv12.hpp # inference-engine/tests/functional/shared_test_classes/src/single_layer/convert_color_nv12.cpp # ngraph/core/include/openvino/core/preprocess/input_tensor_info.hpp # ngraph/core/include/openvino/core/preprocess/preprocess_steps.hpp # ngraph/core/include/openvino/op/nv12_to_bgr.hpp # ngraph/core/include/openvino/op/nv12_to_rgb.hpp # ngraph/core/src/op/nv12_to_bgr.cpp # ngraph/core/src/op/nv12_to_rgb.cpp # ngraph/core/src/preprocess/pre_post_process.cpp # ngraph/core/src/preprocess/preprocess_steps_impl.hpp # ngraph/test/CMakeLists.txt * Added more test to cover 100% of code Allow convert element type for 'multi-plane' color format * Inherit tensor names for 'convert_color' * Clang * Fix tests * Disable 'int8' preprocessing resize test * Fix review comments * Add more restrictions and tests for planes sub-names * 1) Added check for uniqueness of tensor names generated for nodes Raise error if user's plane sub-name conflicts with some node in a function 2) Added exception safety to preprocess build. Before, when input #2 fail, only one preprocess will be applied to function and it will be corrupted Exception guard will restore function to original state if exception occurs * Fix clang-format
1 parent 659daf6 commit f57dc05

File tree

15 files changed

+1130
-50
lines changed

15 files changed

+1130
-50
lines changed

‎docs/template_plugin/tests/functional/subgraph_reference/preprocess.cpp‎

Lines changed: 176 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ struct RefPreprocessParams {
2424
std::function<std::shared_ptr<ov::Function>()> function;
2525
std::vector<Tensor> inputs;
2626
std::vector<Tensor> expected;
27+
float abs_threshold = 0.01f;
28+
float rel_threshold = 0.01f;
2729
std::string name;
2830
};
2931

@@ -39,6 +41,8 @@ class ReferencePreprocessTest : public testing::TestWithParam<RefPreprocessParam
3941
for (const auto& exp : params.expected) {
4042
refOutData.push_back(exp.data);
4143
}
44+
abs_threshold = params.abs_threshold;
45+
threshold = params.rel_threshold;
4246
}
4347
static std::string getTestCaseName(const testing::TestParamInfo<RefPreprocessParams>& obj) {
4448
const auto& param = obj.param;
@@ -327,6 +331,26 @@ static RefPreprocessParams resize_from_spatial_dims() {
327331
return res;
328332
}
329333

334+
static RefPreprocessParams resize_i8() {
335+
RefPreprocessParams res("resize_i8");
336+
res.function = []() {
337+
auto f = create_simple_function(element::i8, PartialShape{1, 3, 1, 1});
338+
f = PrePostProcessor()
339+
.input(InputInfo()
340+
.tensor(InputTensorInfo()
341+
.set_spatial_dynamic_shape())
342+
.preprocess(PreProcessSteps().resize(ResizeAlgorithm::RESIZE_LINEAR))
343+
.network(InputNetworkInfo().set_layout("NCHW")))
344+
.build(f);
345+
return f;
346+
};
347+
res.inputs.emplace_back(element::i8, Shape{1, 3, 2, 2}, std::vector<int8_t>{0, 0, 0, 0,
348+
1, 1, 1, 1,
349+
2, 2, 2, 2});
350+
res.expected.emplace_back(Shape{1, 3, 1, 1}, element::i8, std::vector<int8_t>{0, 1, 2});
351+
return res;
352+
}
353+
330354
static RefPreprocessParams resize_to_network_width_height() {
331355
RefPreprocessParams res("resize_to_network_width_height");
332356
res.function = []() {
@@ -505,6 +529,152 @@ static RefPreprocessParams resize_and_convert_layout() {
505529
return res;
506530
}
507531

532+
static RefPreprocessParams convert_color_nv12_to_bgr_two_planes() {
533+
RefPreprocessParams res("convert_color_nv12_to_bgr_two_planes");
534+
res.abs_threshold = 2.f; // Allow small color conversion deviations
535+
res.rel_threshold = 1.f; // Ignore relative pixel values comparison (100%)
536+
res.function = []() {
537+
auto f = create_simple_function(element::u8, PartialShape{1, 4, 4, 3});
538+
f = PrePostProcessor()
539+
.input(InputInfo()
540+
.tensor(InputTensorInfo()
541+
.set_color_format(ColorFormat::NV12_TWO_PLANES))
542+
.preprocess(PreProcessSteps()
543+
.convert_color(ColorFormat::BGR)))
544+
.build(f);
545+
return f;
546+
};
547+
548+
// clang-format off
549+
auto input_y = std::vector<uint8_t> {81, 81, 145, 145, // RRGG
550+
81, 81, 145, 145, // RRGG
551+
41, 41, 81, 81, // BBRR
552+
41, 41, 81, 81}; // BBRR
553+
auto input_shape_y = Shape{1, 4, 4, 1};
554+
auto input_uv = std::vector<uint8_t> {240, 90, // R (2x2)
555+
34, 54, // G (2x2)
556+
110, 240, // B (2x2)
557+
240, 90}; // R (2x2)
558+
auto input_shape_uv = Shape{1, 2, 2, 2};
559+
auto exp_out = std::vector<uint8_t> {0, 0, 255, 0, 0, 255, 0, 255, 0, 0, 255, 0,
560+
0, 0, 255, 0, 0, 255, 0, 255, 0, 0, 255, 0,
561+
255, 0, 0, 255, 0, 0, 0, 0, 255, 0, 0, 255,
562+
255, 0, 0, 255, 0, 0, 0, 0, 255, 0, 0, 255};
563+
auto out_shape = Shape{1, 4, 4, 3};
564+
// clang-format on
565+
res.inputs.emplace_back(element::u8, input_shape_y, input_y);
566+
res.inputs.emplace_back(element::u8, input_shape_uv, input_uv);
567+
res.expected.emplace_back(out_shape, element::u8, exp_out);
568+
return res;
569+
}
570+
571+
static RefPreprocessParams convert_color_nv12_single_plane() {
572+
RefPreprocessParams res("convert_color_nv12_single_plane");
573+
res.abs_threshold = 2.f; // Allow small color conversion deviations
574+
res.rel_threshold = 1.f; // Ignore relative pixel values comparison (100%)
575+
res.function = []() {
576+
auto f = create_simple_function(element::f32, PartialShape{1, 4, 4, 3});
577+
f = PrePostProcessor()
578+
.input(InputInfo()
579+
.tensor(InputTensorInfo()
580+
.set_color_format(ColorFormat::NV12_SINGLE_PLANE))
581+
.preprocess(PreProcessSteps()
582+
.convert_color(ColorFormat::RGB)))
583+
.build(f);
584+
return f;
585+
};
586+
587+
// clang-format off
588+
auto input = std::vector<float> { 81, 81, 145, 145, // RRGG
589+
81, 81, 145, 145, // RRGG
590+
41, 41, 81, 81, // BBRR
591+
41, 41, 81, 81, // BBRR
592+
240, 90, 34, 54, 110, 240, 240, 90}; // UV (RGBR)
593+
auto input_shape = Shape{1, 6, 4, 1};
594+
auto exp_out = std::vector<float> {255, 0, 0, 255, 0, 0, 0, 255, 0, 0, 255, 0, // RRGG
595+
255, 0, 0, 255, 0, 0, 0, 255, 0, 0, 255, 0, // RRGG
596+
0, 0, 255, 0, 0, 255, 255, 0, 0, 255, 0, 0, // BBRR
597+
0, 0, 255, 0, 0, 255, 255, 0, 0, 255, 0, 0, // BBRR
598+
};
599+
auto out_shape = Shape{1, 4, 4, 3};
600+
// clang-format on
601+
res.inputs.emplace_back(element::f32, input_shape, input);
602+
res.expected.emplace_back(out_shape, element::f32, exp_out);
603+
return res;
604+
}
605+
606+
static RefPreprocessParams convert_color_nv12_layout_resize() {
607+
RefPreprocessParams res("convert_color_nv12_layout_resize");
608+
res.abs_threshold = 2.f; // Allow small color conversion deviations
609+
res.rel_threshold = 1.f; // Ignore relative pixel values comparison (100%)
610+
res.function = []() {
611+
auto f = create_simple_function(element::f32, PartialShape{1, 3, 2, 2});
612+
f = PrePostProcessor()
613+
.input(InputInfo()
614+
.tensor(InputTensorInfo()
615+
.set_color_format(ColorFormat::NV12_SINGLE_PLANE)
616+
.set_element_type(element::u8)
617+
.set_spatial_dynamic_shape())
618+
.preprocess(PreProcessSteps()
619+
.convert_color(ColorFormat::RGB)
620+
.convert_layout()
621+
.convert_element_type(element::f32)
622+
.resize(ResizeAlgorithm::RESIZE_NEAREST))
623+
.network(InputNetworkInfo().set_layout("NCHW")))
624+
.build(f);
625+
return f;
626+
};
627+
628+
auto result = std::make_shared<HostTensor>();
629+
// clang-format off
630+
auto input = std::vector<uint8_t> {81, 81, 145, 145, // RRGG
631+
81, 81, 145, 145, // RRGG
632+
41, 41, 81, 81, // BBRR
633+
41, 41, 81, 81, // BBRR
634+
240, 90, 34, 54, 110, 240, 240, 90}; // UV (RGBR)
635+
auto input_shape = Shape{1, 6, 4, 1};
636+
auto exp_out = std::vector<float> {255, 0, 0, 255, // R channel
637+
0, 255, 0, 0, // G channel
638+
0, 0, 255, 0}; // B channel
639+
auto out_shape = Shape{1, 2, 2, 3};
640+
// clang-format on
641+
res.inputs.emplace_back(element::u8, input_shape, input);
642+
res.expected.emplace_back(out_shape, element::f32, exp_out);
643+
return res;
644+
}
645+
646+
static RefPreprocessParams element_type_before_convert_color_nv12() {
647+
RefPreprocessParams res("element_type_before_convert_color_nv12");
648+
res.abs_threshold = 2.f; // Allow small color conversion deviations
649+
res.rel_threshold = 1.f; // Ignore relative pixel values comparison (100%)
650+
res.function = []() {
651+
auto f = create_simple_function(element::f32, PartialShape{1, 2, 2, 3});
652+
f = PrePostProcessor()
653+
.input(InputInfo()
654+
.tensor(InputTensorInfo()
655+
.set_element_type(element::u8)
656+
.set_color_format(ColorFormat::NV12_TWO_PLANES))
657+
.preprocess(PreProcessSteps()
658+
.convert_element_type(element::f32)
659+
.convert_color(ColorFormat::RGB))
660+
.network(InputNetworkInfo().set_layout("NHWC")))
661+
.build(f);
662+
return f;
663+
};
664+
665+
// clang-format off
666+
auto input_y = std::vector<uint8_t> {81, 81, 81, 81};
667+
auto input_shape_y = Shape{1, 2, 2, 1};
668+
auto input_uv = std::vector<uint8_t> {240, 90};
669+
auto input_shape_uv = Shape{1, 1, 1, 2};
670+
auto exp_out = std::vector<float> {255, 0, 0, 255, 0, 0, 255, 0, 0, 255, 0, 0};
671+
auto out_shape = Shape{1, 2, 2, 3};
672+
// clang-format on
673+
res.inputs.emplace_back(element::u8, input_shape_y, input_y);
674+
res.inputs.emplace_back(element::u8, input_shape_uv, input_uv);
675+
res.expected.emplace_back(out_shape, element::f32, exp_out);
676+
return res;
677+
}
508678

509679
std::vector<RefPreprocessParams> allPreprocessTests() {
510680
return std::vector<RefPreprocessParams> {
@@ -521,12 +691,17 @@ std::vector<RefPreprocessParams> allPreprocessTests() {
521691
resize_to_network_height(),
522692
resize_to_network_width(),
523693
resize_from_spatial_dims(),
694+
resize_i8(),
524695
resize_to_network_width_height(),
525696
resize_to_specified_width_height(),
526697
resize_lvalues(),
527698
convert_layout_nhwc_to_nchw_lvalue(),
528699
convert_layout_nhwc_to_net_no_tensor_shape(),
529-
resize_and_convert_layout()
700+
resize_and_convert_layout(),
701+
convert_color_nv12_to_bgr_two_planes(),
702+
convert_color_nv12_single_plane(),
703+
convert_color_nv12_layout_resize(),
704+
element_type_before_convert_color_nv12(),
530705
};
531706
}
532707

‎inference-engine/tests/functional/plugin/cpu/shared_tests_instances/skip_tests_config.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,8 @@ std::vector<std::string> disabledTestPatterns() {
111111

112112
// Issue 66685
113113
R"(smoke_PrePostProcess.*resize_linear_nhwc.*)",
114+
// Issue 67214
115+
R"(smoke_PrePostProcess.*resize_and_convert_layout_i8.*)",
114116
};
115117

116118
#define FIX_62820 0

‎inference-engine/tests/functional/shared_test_classes/src/subgraph/preprocess.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ void PrePostProcessTest::SetUp() {
3030
std::tie(func, targetDevice) = GetParam();
3131
function = (std::get<0>(func))();
3232
threshold = std::get<2>(func);
33+
abs_threshold = std::get<2>(func);
3334
}
3435

3536
TEST_P(PrePostProcessTest, CompareWithRefs) {

‎inference-engine/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/preprocess/preprocess_builders.hpp‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -270,6 +270,63 @@ inline std::shared_ptr<Function> resize_and_convert_layout() {
270270
return function;
271271
}
272272

273+
inline std::shared_ptr<Function> resize_and_convert_layout_i8() {
274+
using namespace ov::preprocess;
275+
auto function = create_preprocess_1input(element::i8, PartialShape{1, 30, 20, 3});
276+
function = PrePostProcessor()
277+
.input(InputInfo()
278+
.tensor(InputTensorInfo()
279+
.set_layout("NHWC")
280+
.set_spatial_static_shape(40, 30))
281+
.preprocess(PreProcessSteps()
282+
.convert_layout()
283+
.resize(ResizeAlgorithm::RESIZE_LINEAR))
284+
.network(InputNetworkInfo().set_layout("NCHW")))
285+
.build(function);
286+
return function;
287+
}
288+
289+
inline std::shared_ptr<Function> cvt_color_nv12_to_rgb_single_plane() {
290+
using namespace ov::preprocess;
291+
auto function = create_preprocess_1input(element::f32, PartialShape{1, 20, 20, 3});
292+
function = PrePostProcessor()
293+
.input(InputInfo()
294+
.tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_SINGLE_PLANE))
295+
.preprocess(PreProcessSteps().convert_color(ColorFormat::RGB)))
296+
.build(function);
297+
return function;
298+
}
299+
300+
inline std::shared_ptr<Function> cvt_color_nv12_to_bgr_two_planes() {
301+
using namespace ov::preprocess;
302+
auto function = create_preprocess_1input(element::f32, PartialShape{1, 20, 20, 3});
303+
function = PrePostProcessor()
304+
.input(InputInfo()
305+
.tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_TWO_PLANES))
306+
.preprocess(PreProcessSteps().convert_color(ColorFormat::BGR)))
307+
.build(function);
308+
return function;
309+
}
310+
311+
inline std::shared_ptr<Function> cvt_color_nv12_cvt_layout_resize() {
312+
using namespace ov::preprocess;
313+
auto function = create_preprocess_1input(element::f32, PartialShape{1, 3, 10, 10});
314+
function = PrePostProcessor()
315+
.input(InputInfo()
316+
.tensor(InputTensorInfo()
317+
.set_color_format(ColorFormat::NV12_TWO_PLANES)
318+
.set_element_type(element::u8)
319+
.set_spatial_static_shape(20, 20))
320+
.preprocess(PreProcessSteps()
321+
.convert_color(ColorFormat::RGB)
322+
.convert_layout()
323+
.convert_element_type(element::f32)
324+
.resize(ResizeAlgorithm::RESIZE_LINEAR))
325+
.network(InputNetworkInfo().set_layout("NCHW")))
326+
.build(function);
327+
return function;
328+
}
329+
273330
inline std::vector<preprocess_func> generic_preprocess_functions() {
274331
return std::vector<preprocess_func> {
275332
preprocess_func(mean_only, "mean_only", 0.01f),
@@ -290,6 +347,10 @@ inline std::vector<preprocess_func> generic_preprocess_functions() {
290347
preprocess_func(resize_linear_nhwc, "resize_linear_nhwc", 0.01f),
291348
preprocess_func(resize_cubic, "resize_cubic", 0.01f),
292349
preprocess_func(resize_and_convert_layout, "resize_and_convert_layout", 0.01f),
350+
preprocess_func(resize_and_convert_layout_i8, "resize_and_convert_layout_i8", 0.01f),
351+
preprocess_func(cvt_color_nv12_to_rgb_single_plane, "cvt_color_nv12_to_rgb_single_plane", 2.f),
352+
preprocess_func(cvt_color_nv12_to_bgr_two_planes, "cvt_color_nv12_to_bgr_two_planes", 2.f),
353+
preprocess_func(cvt_color_nv12_cvt_layout_resize, "cvt_color_nv12_cvt_layout_resize", 2.f),
293354
};
294355
}
295356

‎ngraph/core/include/openvino/core/preprocess/color_format.hpp‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
// Copyright (C) 2018-2021 Intel Corporation
2+
// SPDX-License-Identifier: Apache-2.0
3+
//
4+
5+
#pragma once
6+
7+
namespace ov {
8+
namespace preprocess {
9+
10+
/// \brief Color format enumeration for conversion
11+
enum class ColorFormat {
12+
UNDEFINED,
13+
NV12_SINGLE_PLANE, // Image in NV12 format as single tensor
14+
NV12_TWO_PLANES, // Image in NV12 format represented as separate tensors for Y and UV planes
15+
RGB,
16+
BGR
17+
};
18+
19+
} // namespace preprocess
20+
} // namespace ov

‎ngraph/core/include/openvino/core/preprocess/input_tensor_info.hpp‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
#include "openvino/core/core_visibility.hpp"
88
#include "openvino/core/layout.hpp"
9+
#include "openvino/core/preprocess/color_format.hpp"
910
#include "openvino/core/type/element_type.hpp"
1011

1112
namespace ov {
@@ -117,6 +118,44 @@ class OPENVINO_API InputTensorInfo final {
117118
///
118119
/// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner.
119120
InputTensorInfo&& set_spatial_static_shape(size_t height, size_t width) &&;
121+
122+
/// \brief Set color format for user's input tensor.
123+
///
124+
/// In general way, some formats support multi-plane input, e.g. NV12 image can be represented as 2 separate tensors
125+
/// (planes): Y plane and UV plane. set_color_format API also allows to set sub_names for such parameters for
126+
/// convenient usage of plane parameters.
127+
///
128+
/// This version allows chaining for Lvalue objects.
129+
///
130+
/// \param format Color format of input image.
131+
///
132+
/// \param sub_names Optional list of sub-names assigned for each plane (e.g. {"Y", "UV"}). If not specified,
133+
/// sub-names for plane parameters are auto-generated, exact names auto-generation rules depend on specific color
134+
/// format, and client's code shall not rely on these rules. It is not allowed to specify sub-names for single-plane
135+
/// inputs, also is specified, number of sub-names shall match with number of planes.
136+
///
137+
/// \return Reference to 'this' to allow chaining with other calls in a builder-like manner.
138+
InputTensorInfo& set_color_format(const ov::preprocess::ColorFormat& format,
139+
const std::vector<std::string>& sub_names = {}) &;
140+
141+
/// \brief Set color format for user's input tensor.
142+
///
143+
/// In general way, some formats support multi-plane input, e.g. NV12 image can be represented as 2 separate tensors
144+
/// (planes): Y plane and UV plane. set_color_format API also allows to set sub_names for such parameters for
145+
/// convenient usage of plane parameters.
146+
///
147+
/// This version allows chaining for Rvalue objects.
148+
///
149+
/// \param format Color format of input image.
150+
///
151+
/// \param sub_names Optional list of sub-names assigned for each plane (e.g. {"Y", "UV"}). If not specified,
152+
/// sub-names for plane parameters are auto-generated, exact names auto-generation rules depend on specific color
153+
/// format, and client's code shall not rely on these rules. It is not allowed to specify sub-names for single-plane
154+
/// inputs, also is specified, number of sub-names shall match with number of planes.
155+
///
156+
/// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner.
157+
InputTensorInfo&& set_color_format(const ov::preprocess::ColorFormat& format,
158+
const std::vector<std::string>& sub_names = {}) &&;
120159
};
121160

122161
} // namespace preprocess

0 commit comments

Comments
 (0)