Separate executable for paddlepaddle unit tests (openvinotoolkit#6862)

* Separate executable for paddlepaddle unit tests

* Fix CI

* Move PaddlePaddle-specific python requirements to paddlepaddle test folder
Also produce build time warning when paddle test models generation is disabled

* Renamed back PADDLE_TEST_MODELS_DIRNAME to TEST_PADDLE_MODELS_DIRNAME

* Add dependency on CPU plugin (PaddlePaddle fuzzy tests use CPU plugin for inference)

* Fix code style

* Fix review comments #2

* Code style fix

* Add dependency of 'paddlepaddle_test_models' to 'test_model_zoo'

Author: nosovmik

.ci/azure/linux.yml

@@ -85,8 +85,9 @@ jobs:
       python3 -m pip install -r $(REPO_DIR)/inference-engine/ie_bridges/python/wheel/requirements-dev.txt
       # For running Python API tests
       python3 -m pip install -r $(REPO_DIR)/inference-engine/ie_bridges/python/src/requirements-dev.txt
-      # For running nGraph unit tests dependent on Python frameworks
-      python3 -m pip install -r $(REPO_DIR)/ngraph/test/requirements_test.txt
+      # For running PaddlePaddle frontend unit tests
+      python3 -m pip install -r $(REPO_DIR)/ngraph/test/frontend/paddlepaddle/requirements_dev.txt
+      # For running ONNX frontend unit tests
       python3 -m pip install -r $(REPO_DIR)/ngraph/test/requirements_test_onnx.txt
       # For MO unit tests
       python3 -m pip install -r $(REPO_DIR)/model-optimizer/requirements.txt
@@ -171,13 +172,15 @@ jobs:
     displayName: 'Model Optimizer UT'
     continueOnError: false
 
-  - script: |
-      export FE_TEST_MODELS=$(INSTALL_DIR)/tests
-      . $(SETUPVARS) && $(INSTALL_TEST_DIR)/unit-test --gtest_print_time=1 --gtest_filter=-backend_api.config_unsupported:*IE_GPU* --gtest_output=xml:TEST-NGraphUT.xml
+  - script: . $(SETUPVARS) && $(INSTALL_TEST_DIR)/unit-test --gtest_print_time=1 --gtest_filter=-backend_api.config_unsupported:*IE_GPU* --gtest_output=xml:TEST-NGraphUT.xml
     workingDirectory: $(INSTALL_TEST_DIR)
     displayName: 'nGraph UT'
     continueOnError: false
 
+  - script: . $(SETUPVARS) && $(INSTALL_TEST_DIR)/paddlepaddle_tests --gtest_print_time=1 --gtest_filter=-backend_api.config_unsupported:*IE_GPU* --gtest_output=xml:TEST-PaddlePaddle.xml
+    displayName: 'PaddlePaddle Frontend UT'
+    continueOnError: false
+
     # . $(SETUPVARS) && python3 $(WORK_DIR)/gtest-parallel/gtest_parallel.py $(INSTALL_TEST_DIR)/InferenceEngineUnitTests --workers=16 --dump_json_test_results=InferenceEngineUnitTests.json --gtest_filter=*smoke* -- --gtest_print_time=1
   - script: . $(SETUPVARS) && $(INSTALL_TEST_DIR)/InferenceEngineUnitTests --gtest_print_time=1 --gtest_output=xml:TEST-InferenceEngineUnitTests.xml
     displayName: 'IE UT old'

.ci/azure/windows.yml

@@ -86,8 +86,9 @@ jobs:
       python -m pip install --upgrade pip
       rem For running Python API tests
       python -m pip install -r $(REPO_DIR)\inference-engine\ie_bridges\python\src\requirements-dev.txt
-      rem For running nGraph unit tests dependent on Python frameworks
-      python -m pip install -r $(REPO_DIR)\ngraph\test\requirements_test.txt
+      rem For running PaddlePaddle frontend unit tests
+      python -m pip install -r $(REPO_DIR)\ngraph\test\frontend\paddlepaddle\requirements_dev.txt
+      rem For running ONNX frontend unit tests
       python -m pip install -r $(REPO_DIR)\ngraph\test\requirements_test_onnx.txt
       rem For MO unit tests
       python -m pip install -r $(REPO_DIR)\model-optimizer\requirements.txt
@@ -143,13 +144,15 @@ jobs:
     displayName: 'Clean build dir'
     continueOnError: false
 
-  - script: |
-      set FE_TEST_MODELS=$(INSTALL_DIR)\tests
-      call $(SETUPVARS) && $(INSTALL_TEST_DIR)\unit-test --gtest_print_time=1 --gtest_filter=-backend_api.config_unsupported:*IE_GPU* --gtest_output=xml:TEST-NGraphUT.xml
+  - script: call $(SETUPVARS) && $(INSTALL_TEST_DIR)\unit-test --gtest_print_time=1 --gtest_filter=-backend_api.config_unsupported:*IE_GPU* --gtest_output=xml:TEST-NGraphUT.xml
     workingDirectory: $(INSTALL_TEST_DIR)
     displayName: 'nGraph UT'
     continueOnError: false
 
+  - script: call $(SETUPVARS) && $(INSTALL_TEST_DIR)\paddlepaddle_tests --gtest_print_time=1 --gtest_filter=-backend_api.config_unsupported:*IE_GPU* --gtest_output=xml:TEST-PaddlePaddle.xml
+    displayName: 'PaddlePaddle Frontend UT'
+    continueOnError: false
+
   - script: |
       set PATH=$(IB_DIR);%PATH%
       call $(SETUPVARS) && "$(IB_TESTCONSOLE)" $(INSTALL_TEST_DIR)\InferenceEngineUnitTests.exe --gtest_output=xml:TEST-InferenceEngineUnitTests-IB.xml

cmake/test_model_zoo.cmake

@@ -120,6 +120,10 @@ if(ENABLE_TESTS)
         add_dependencies(test_model_zoo test_pip_prerequsites)
     endif()
 
+    if (NGRAPH_PDPD_FRONTEND_ENABLE)
+        add_dependencies(test_model_zoo paddlepaddle_test_models)
+    endif()
+
     install(DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/test_model_zoo"
             DESTINATION tests COMPONENT tests EXCLUDE_FROM_ALL)
 

ngraph/test/CMakeLists.txt

@@ -558,31 +558,10 @@ if (NGRAPH_ONNX_IMPORT_ENABLE)
 endif()
 
 # SOURCE FOR FRONTEND TESTING
-
 file(GLOB FRONTEND_TESTS_SRC ${CMAKE_CURRENT_SOURCE_DIR}/frontend/frontend_manager.cpp)
 set(SRC ${FRONTEND_TESTS_SRC} ${SRC})
 
-file(GLOB FRONTEND_SHARED_TESTS_SRC ${CMAKE_CURRENT_SOURCE_DIR}/frontend/shared/src/*.cpp)
-file(GLOB FRONTEND_SHARED_TESTS_HDR ${CMAKE_CURRENT_SOURCE_DIR}/frontend/shared/include/*.hpp)
-set(SRC ${FRONTEND_SHARED_TESTS_SRC} ${SRC})
-
-add_definitions("-DTEST_MODEL_BUILD_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\"")
-
-# ---- PaddlePaddle FrontEnd testing ------
-if (NGRAPH_PDPD_FRONTEND_ENABLE)
-    ie_check_pip_package(paddlepaddle WARNING)
-
-    if(paddlepaddle_FOUND)
-        file(GLOB FRONTEND_PDPD_TESTS_SRC ${CMAKE_CURRENT_SOURCE_DIR}/frontend/paddlepaddle/*.cpp)
-        set(SRC ${FRONTEND_PDPD_TESTS_SRC} ${SRC})
-        set(PADDLE_TEST_MODELS_DIRNAME paddle_test_models)
-        set(TEST_PADDLE_MODELS ${CMAKE_CURRENT_BINARY_DIR}/${PADDLE_TEST_MODELS_DIRNAME}/)
-        add_definitions("-DTEST_PADDLE_MODELS_DIRNAME=\"${PADDLE_TEST_MODELS_DIRNAME}/\"")
-    endif()
-endif()
-# ---- End PaddlePaddle FrontEnd testing ------
-
-add_clang_format_target(unit-test_clang FOR_SOURCES ${SRC} ${MULTI_TEST_SRC} ${FRONTEND_SHARED_TESTS_HDR})
+add_clang_format_target(unit-test_clang FOR_SOURCES ${SRC} ${MULTI_TEST_SRC})
 
 foreach(BACKEND_NAME ${ACTIVE_BACKEND_LIST})
     string(TOLOWER ${BACKEND_NAME} BACKEND_DIR)
@@ -603,7 +582,6 @@ add_executable(unit-test ${SRC})
 
 target_include_directories(unit-test PRIVATE ".")
 target_include_directories(unit-test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/runtime)
-target_include_directories(unit-test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/frontend/shared/include)
 
 add_definitions("-DCURDIR=\"${CMAKE_CURRENT_SOURCE_DIR}\"")
 add_definitions("-DJSON_INCLUDES=\"${JSON_INCLUDE_DIR}\"")
@@ -618,7 +596,8 @@ target_link_libraries(unit-test PRIVATE ngraph_test_util
                                         ie_backend
                                         interpreter_backend
                                         Threads::Threads
-                                        openvino::conditional_compilation)
+                                        openvino::conditional_compilation
+                                        frontend_manager)
 
 # Protobuf-lite does not support parsing files from prototxt format
 # Since most of the onnx models are stored in this format it have to be disabled
@@ -655,39 +634,7 @@ install(TARGETS unit-test
         COMPONENT tests
         EXCLUDE_FROM_ALL)
 
-############ FRONTEND ############
-target_include_directories(unit-test PRIVATE ${FRONTEND_INCLUDE_PATH} frontend/shared/include)
-target_link_libraries(unit-test PRIVATE frontend_manager cnpy commonTestUtils)
-
 add_subdirectory(frontend)
-### END FRONTEND ###
-
-#PaddlePaddle - test models generator
-if (NGRAPH_PDPD_FRONTEND_ENABLE AND paddlepaddle_FOUND)
-    file(GLOB_RECURSE PDPD_GEN_SCRIPTS ${CMAKE_CURRENT_SOURCE_DIR}/files/paddlepaddle/gen_scripts/generate_*.py)
-    set(OUT_FILES "")
-    foreach(GEN_SCRIPT ${PDPD_GEN_SCRIPTS})
-        get_filename_component(FILE_WE ${GEN_SCRIPT} NAME_WE)
-        set(OUT_DONE_FILE ${TEST_PADDLE_MODELS}/${FILE_WE}_done.txt)
-        set(OUT_FILES ${OUT_DONE_FILE} ${OUT_FILES})
-        add_custom_command(OUTPUT ${OUT_DONE_FILE}
-                COMMAND ${PYTHON_EXECUTABLE}
-                        ${CMAKE_CURRENT_SOURCE_DIR}/files/paddlepaddle/gen_wrapper.py
-                        ${GEN_SCRIPT}
-                        ${TEST_PADDLE_MODELS}
-                        ${OUT_DONE_FILE}
-                DEPENDS ${GEN_SCRIPT} ${CMAKE_CURRENT_SOURCE_DIR}/files/paddlepaddle/gen_wrapper.py
-                )
-    endforeach()
-    add_custom_target(paddlepaddle_test_models DEPENDS ${OUT_FILES})
-    add_dependencies(unit-test paddlepaddle_test_models)
-    add_dependencies(unit-test paddlepaddle_ngraph_frontend)
-
-    install(DIRECTORY ${TEST_PADDLE_MODELS}
-            DESTINATION tests/${PADDLE_TEST_MODELS_DIRNAME}
-            COMPONENT tests
-            EXCLUDE_FROM_ALL)
-endif()
 
 # process models
 add_dependencies(unit-test test_model_zoo)

ngraph/test/frontend/CMakeLists.txt

@@ -1,6 +1,11 @@
 # Copyright (C) 2018-2021 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 #
+add_subdirectory(shared)
+
+if (NGRAPH_PDPD_FRONTEND_ENABLE)
+    add_subdirectory(paddlepaddle)
+endif()
 
 set(SRC ${CMAKE_CURRENT_SOURCE_DIR}/mock_frontend.cpp)
 add_library(mock1_ngraph_frontend SHARED ${SRC})

ngraph/test/frontend/paddlepaddle/CMakeLists.txt

@@ -0,0 +1,70 @@
+# Copyright (C) 2018-2021 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set(TARGET_NAME "paddlepaddle_tests")
+
+file(GLOB_RECURSE SRC ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp)
+
+add_executable(${TARGET_NAME} ${SRC})
+
+target_link_libraries(${TARGET_NAME} PRIVATE frontend_shared_test_classes)
+
+add_clang_format_target(${TARGET_NAME}_clang FOR_TARGETS ${TARGET_NAME})
+
+install(TARGETS ${TARGET_NAME}
+        RUNTIME DESTINATION tests
+        COMPONENT tests
+        EXCLUDE_FROM_ALL)
+
+# Test model generating
+ie_check_pip_package(paddlepaddle WARNING)
+
+set(TEST_PADDLE_MODELS_DIRNAME test_model_zoo/paddle_test_models)
+target_compile_definitions(${TARGET_NAME} PRIVATE -D TEST_PADDLE_MODELS_DIRNAME=\"${TEST_PADDLE_MODELS_DIRNAME}/\")
+
+# If 'paddlepaddle' is not found, code will still be compiled
+# but models will not be generated and tests will fail
+# This is done this way for 'code style' and check cases - cmake shall pass, but CI machine doesn't need to have
+# 'paddlepaddle' installed to check code style
+if (paddlepaddle_FOUND)
+    set(TEST_PADDLE_MODELS ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TEST_PADDLE_MODELS_DIRNAME}/)
+
+    file(GLOB_RECURSE PADDLE_GEN_SCRIPTS ${CMAKE_CURRENT_SOURCE_DIR}/test_models/gen_scripts/generate_*.py)
+    file(GLOB_RECURSE PADDLE_ALL_SCRIPTS ${CMAKE_CURRENT_SOURCE_DIR}/*.py)
+    set(OUT_FILES "")
+    foreach(GEN_SCRIPT ${PADDLE_GEN_SCRIPTS})
+        get_filename_component(FILE_WE ${GEN_SCRIPT} NAME_WE)
+        set(OUT_DONE_FILE ${TEST_PADDLE_MODELS}/${FILE_WE}_done.txt)
+        set(OUT_FILES ${OUT_DONE_FILE} ${OUT_FILES})
+        add_custom_command(OUTPUT ${OUT_DONE_FILE}
+                COMMAND ${PYTHON_EXECUTABLE}
+                ${CMAKE_CURRENT_SOURCE_DIR}/test_models/gen_wrapper.py
+                ${GEN_SCRIPT}
+                ${TEST_PADDLE_MODELS}
+                ${OUT_DONE_FILE}
+                DEPENDS ${PADDLE_ALL_SCRIPTS}
+                )
+    endforeach()
+    add_custom_target(paddlepaddle_test_models DEPENDS ${OUT_FILES})
+
+    install(DIRECTORY ${TEST_PADDLE_MODELS}
+            DESTINATION tests/${TEST_PADDLE_MODELS_DIRNAME}
+            COMPONENT tests
+            EXCLUDE_FROM_ALL)
+else()
+    # Produce warning message at build time as well
+    add_custom_command(OUTPUT unable_build_paddle_models.txt
+            COMMAND ${CMAKE_COMMAND}
+            -E cmake_echo_color --red "Warning: Unable to generate PaddlePaddle test models. Running '${TARGET_NAME}' will likely fail"
+            )
+    add_custom_target(paddlepaddle_test_models DEPENDS unable_build_paddle_models.txt)
+endif()
+
+add_dependencies(${TARGET_NAME} paddlepaddle_test_models)
+add_dependencies(${TARGET_NAME} paddlepaddle_ngraph_frontend)
+
+# Fuzzy tests for PaddlePaddle use IE_CPU engine
+if (ENABLE_MKL_DNN)
+    add_dependencies(${TARGET_NAME} MKLDNNPlugin)
+endif()

ngraph/test/frontend/paddlepaddle/incorrect_cut_model.cpp

@@ -21,8 +21,7 @@ TEST(FrontEndIncorrectCutModelTest, test_incorrect_cut)
     ASSERT_NO_THROW(frontEnd = fem.load_by_framework(PADDLE_FE));
     ASSERT_NE(frontEnd, nullptr);
     auto model_filename = FrontEndTestUtils::make_model_path(
-        std::string(TEST_PADDLE_MODELS_DIRNAME) +
-        std::string("2in_2out/2in_2out.pdmodel"));
+        std::string(TEST_PADDLE_MODELS_DIRNAME) + std::string("2in_2out/2in_2out.pdmodel"));
     ASSERT_NO_THROW(inputModel = frontEnd->load(model_filename));
     ASSERT_NE(inputModel, nullptr);
 

ngraph/test/frontend/paddlepaddle/main.cpp

@@ -0,0 +1,10 @@
+// Copyright (C) 2018-2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "utils.hpp"
+
+int main(int argc, char** argv)
+{
+    return FrontEndTestUtils::run_tests(argc, argv);
+}

ngraph/test/frontend/paddlepaddle/paddle_utils.hpp

@@ -7,3 +7,4 @@
 #include <string>
 
 static const std::string PADDLE_FE = "paddle";
+