diff --git a/tools/inference_engine/include/inference_engine_tensor.hpp b/tools/inference_engine/include/inference_engine_tensor.hpp index 5a41bc6..2923912 100644 --- a/tools/inference_engine/include/inference_engine_tensor.hpp +++ b/tools/inference_engine/include/inference_engine_tensor.hpp @@ -6,20 +6,26 @@ namespace inference_engine { + enum class Layout + { + NCHW, + NHWC, + UNKNOWN + }; + struct Tensor { inference_engine_data_type_t data_type = INFERENCE_ENGINE_DATA_TYPE_UNKNOWN; - std::vector dims; - std::vector strides; + std::vector dims; // specified in NCHW layout as DirectML + std::vector strides; // specified in NCHW layout as DirectML Tensor() = default; - Tensor(inference_engine_data_type_t dt, std::vector&& dimensions) + Tensor(inference_engine_data_type_t dt, std::vector&& dimensions, Layout layout = Layout::NCHW) : data_type(dt) , dims(std::move(dimensions)) { - // ToDo: Decide if should we allow for default strides? - // if Yes: NCHW strides calculation should be default strides.resize(dims.size()); + calculate_strides(layout); } Tensor(const inference_engine_tensor_t& tensor_desc) : data_type(tensor_desc.data_type) @@ -31,6 +37,11 @@ namespace inference_engine } } + bool operator==(const Tensor& other) const + { + return data_type == other.data_type && dims == other.dims && strides == other.strides; + } + operator inference_engine_tensor_t() const { inference_engine_tensor_t ret{}; @@ -61,5 +72,35 @@ namespace inference_engine } return 1; } + + protected: + void calculate_strides(Layout layout) + { + if (layout == Layout::NCHW && dims.size() > 1) + { + strides[dims.size() - 1] = 1; + for (size_t i = dims.size() - 2; i < dims.size(); --i) + strides[i] = strides[i + 1] * dims[i + 1]; + } + else if (layout == Layout::NHWC && dims.size() == 4) + { + // dims are also given in NCHW order, as DML + strides[0] = dims[1] * dims[2] * dims[3]; // n stride = H * W * C + strides[1] = 1; // c stride = 1 + strides[2] = dims[3] * dims[1]; // h stride = W * C + strides[3] = dims[1]; // w stride = C + } + else + { + assert(!"unsupported layout"); + } + } }; + + struct IdToTensor + { + std::size_t id; + Tensor tensor; + }; + } // namespace inference_engine \ No newline at end of file diff --git a/tools/inference_engine/src/impl/model.cpp b/tools/inference_engine/src/impl/model.cpp index 0ace78e..b4e392f 100644 --- a/tools/inference_engine/src/impl/model.cpp +++ b/tools/inference_engine/src/impl/model.cpp @@ -14,9 +14,9 @@ namespace inference_engine { namespace { - std::vector build_output_mapping(const std::vector>& nodes) + std::vector build_output_mapping(const std::vector>& nodes) { - std::vector ret{}; + std::vector ret{}; for (const auto& n : nodes) { if (n->get_outputs().empty()) @@ -34,7 +34,7 @@ namespace inference_engine nodes_.reserve(1024); } - std::vector> DAG::compile(std::span input_mappings) + std::vector> DAG::compile(std::span input_mappings) { create_adjacency_list(); @@ -62,7 +62,7 @@ namespace inference_engine // maybe we can move it to a separate function? // set input tensor if this is port (important: we have topological sorted, so we assume here that all inputs are traversed first)! - auto it = std::find_if(std::begin(input_mappings), std::end(input_mappings), [&](const TensorMapping& im) + auto it = std::find_if(std::begin(input_mappings), std::end(input_mappings), [&](const IdToTensor& im) { return im.id == ret[i]->get_id(); }); @@ -172,7 +172,7 @@ void ExecutableModel::set_resource(inference_engine_node_id_t id, GpuResource::P } } -const std::vector& ExecutableModel::get_outputs() const +const std::vector& ExecutableModel::get_outputs() const { return output_mappings_; } @@ -268,7 +268,7 @@ class FusionVisitor : public GpuVisitor } }; -inference_engine::ExecutableModel ModelDescriptor::compile(GpuContext& ctx, GpuStream& stream, std::span input_mappings) +inference_engine::ExecutableModel ModelDescriptor::compile(GpuContext& ctx, GpuStream& stream, std::span input_mappings) { //ToDo: we need some data structure to represent graph (random order of example features below) // 1) Sorting graph diff --git a/tools/inference_engine/src/impl/model.h b/tools/inference_engine/src/impl/model.h index 41a0e31..f362113 100644 --- a/tools/inference_engine/src/impl/model.h +++ b/tools/inference_engine/src/impl/model.h @@ -1,8 +1,7 @@ #pragma once #include "gpu_context.h" -#include "inference_engine_tensor.h" #include "inference_engine_operators.h" -#include "tensor.h" +#include "inference_engine_tensor.hpp" #include "node.h" #include @@ -49,7 +48,7 @@ class DAG } public: - std::vector> compile(std::span input_mappings); + std::vector> compile(std::span input_mappings); private: @@ -73,11 +72,11 @@ struct ExecutableModel void execute(GpuStream& stream); void set_resource(inference_engine_node_id_t id, GpuResource::Ptr rsc); - const std::vector& get_outputs() const; + const std::vector& get_outputs() const; private: const std::vector> nodes_; - const std::vector output_mappings_; + const std::vector output_mappings_; }; class ModelDescriptor @@ -92,7 +91,7 @@ class ModelDescriptor return dag_.add_node(desc, name); } - ExecutableModel compile(GpuContext& ctx, GpuStream& stream, std::span input_mappings); + ExecutableModel compile(GpuContext& ctx, GpuStream& stream, std::span input_mappings); private: DAG dag_; diff --git a/tools/inference_engine/src/impl/node.h b/tools/inference_engine/src/impl/node.h index 2e3058c..6df9363 100644 --- a/tools/inference_engine/src/impl/node.h +++ b/tools/inference_engine/src/impl/node.h @@ -1,7 +1,7 @@ #pragma once #include "inference_engine_operators.h" -#include "tensor.h" +#include "inference_engine_tensor.hpp" #include "gpu_context.h" #include #include diff --git a/tools/inference_engine/src/impl/tensor.h b/tools/inference_engine/src/impl/tensor.h deleted file mode 100644 index 7392756..0000000 --- a/tools/inference_engine/src/impl/tensor.h +++ /dev/null @@ -1,68 +0,0 @@ -#pragma once -#include - -#include -#include - -namespace inference_engine -{ - struct Tensor - { - inference_engine_data_type_t data_type = INFERENCE_ENGINE_DATA_TYPE_UNKNOWN; - std::vector dims; - std::vector strides; - - Tensor() = default; - Tensor(const inference_engine_tensor_t& tensor_desc) - : data_type(tensor_desc.data_type) - { - for (int i = 0; i < INFERENCE_ENGINE_MAX_TENSOR_DIMS && tensor_desc.dims[i] != 0; ++i) - { - dims.push_back(tensor_desc.dims[i]); - strides.push_back(tensor_desc.strides[i]); - } - } - - bool operator==(const Tensor& other) const - { - return data_type == other.data_type && dims == other.dims && strides == other.strides; - } - - operator inference_engine_tensor_t() const - { - inference_engine_tensor_t ret{}; - ret.data_type = data_type; - for (auto i = 0; i < dims.size(); i++) - { - ret.dims[i] = dims[i]; - ret.strides[i] = strides[i]; - } - return ret; - } - - std::size_t bytes_width() const - { - std::size_t size = 1; - for (const auto& d : dims) - { - size *= d; - } - switch (data_type) - { - case inference_engine_data_type_t::INFERENCE_ENGINE_DATA_TYPE_FP32: - return size * sizeof(float); - case inference_engine_data_type_t::INFERENCE_ENGINE_DATA_TYPE_FP16: - return size * sizeof(std::uint16_t); - default: - assert(!"unsupported"); - } - return 1; - } - }; - - struct TensorMapping - { - std::size_t id; - Tensor tensor; - }; -} // namespace inference_engine \ No newline at end of file diff --git a/tools/inference_engine/src/inference_engine_model.cpp b/tools/inference_engine/src/inference_engine_model.cpp index e796926..f7ca201 100644 --- a/tools/inference_engine/src/inference_engine_model.cpp +++ b/tools/inference_engine/src/inference_engine_model.cpp @@ -39,7 +39,7 @@ INFERENCE_ENGINE_API inference_engine_model_t inferenceEngineCompileModelDescrip std::cout << "Wrong param input_mapping_list is nullptr or input_mapping_size is 0 " << std::endl; return nullptr; } - std::vector im{}; + std::vector im{}; for (auto i = 0; i < input_mapping_size; i++) { im.push_back({ input_mapping_list[i].id, inference_engine::Tensor(input_mapping_list[i].tensor) });