gcpukernel.hpp 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534
  1. // This file is part of OpenCV project.
  2. // It is subject to the license terms in the LICENSE file found in the top-level directory
  3. // of this distribution and at http://opencv.org/license.html.
  4. //
  5. // Copyright (C) 2018-2020 Intel Corporation
  6. #ifndef OPENCV_GAPI_GCPUKERNEL_HPP
  7. #define OPENCV_GAPI_GCPUKERNEL_HPP
  8. #include <functional>
  9. #include <unordered_map>
  10. #include <utility>
  11. #include <vector>
  12. #include <opencv2/core/mat.hpp>
  13. #include <opencv2/gapi/gcommon.hpp>
  14. #include <opencv2/gapi/gkernel.hpp>
  15. #include <opencv2/gapi/garg.hpp>
  16. #include <opencv2/gapi/gmetaarg.hpp>
  17. #include <opencv2/gapi/util/compiler_hints.hpp> //suppress_unused_warning
  18. #include <opencv2/gapi/util/util.hpp>
  19. // FIXME: namespace scheme for backends?
  20. namespace cv {
  21. namespace gimpl
  22. {
  23. // Forward-declare an internal class
  24. class GCPUExecutable;
  25. } // namespace gimpl
  26. namespace gapi
  27. {
  28. /**
  29. * @brief This namespace contains G-API CPU backend functions,
  30. * structures, and symbols.
  31. */
  32. namespace cpu
  33. {
  34. /**
  35. * \addtogroup gapi_std_backends
  36. * @{
  37. *
  38. * @brief G-API backends available in this OpenCV version
  39. *
  40. * G-API backends play a corner stone role in G-API execution
  41. * stack. Every backend is hardware-oriented and thus can run its
  42. * kernels efficiently on the target platform.
  43. *
  44. * Backends are usually "black boxes" for G-API users -- on the API
  45. * side, all backends are represented as different objects of the
  46. * same class cv::gapi::GBackend.
  47. * User can manipulate with backends by specifying which kernels to use.
  48. *
  49. * @sa @ref gapi_hld
  50. */
  51. /**
  52. * @brief Get a reference to CPU (OpenCV) backend.
  53. *
  54. * This is the default backend in G-API at the moment, providing
  55. * broader functional coverage but losing some graph model
  56. * advantages. Provided mostly for reference and prototyping
  57. * purposes.
  58. *
  59. * @sa gapi_std_backends
  60. */
  61. GAPI_EXPORTS cv::gapi::GBackend backend();
  62. /** @} */
  63. class GOCVFunctor;
  64. //! @cond IGNORED
  65. template<typename K, typename Callable>
  66. GOCVFunctor ocv_kernel(const Callable& c);
  67. template<typename K, typename Callable>
  68. GOCVFunctor ocv_kernel(Callable& c);
  69. //! @endcond
  70. } // namespace cpu
  71. } // namespace gapi
  72. // Represents arguments which are passed to a wrapped CPU function
  73. // FIXME: put into detail?
  74. class GAPI_EXPORTS GCPUContext
  75. {
  76. public:
  77. // Generic accessor API
  78. template<typename T>
  79. const T& inArg(int input) { return m_args.at(input).get<T>(); }
  80. // Syntax sugar
  81. const cv::Mat& inMat(int input);
  82. cv::Mat& outMatR(int output); // FIXME: Avoid cv::Mat m = ctx.outMatR()
  83. const cv::Scalar& inVal(int input);
  84. cv::Scalar& outValR(int output); // FIXME: Avoid cv::Scalar s = ctx.outValR()
  85. cv::MediaFrame& outFrame(int output);
  86. template<typename T> std::vector<T>& outVecR(int output) // FIXME: the same issue
  87. {
  88. return outVecRef(output).wref<T>();
  89. }
  90. template<typename T> T& outOpaqueR(int output) // FIXME: the same issue
  91. {
  92. return outOpaqueRef(output).wref<T>();
  93. }
  94. GArg state()
  95. {
  96. return m_state;
  97. }
  98. protected:
  99. detail::VectorRef& outVecRef(int output);
  100. detail::OpaqueRef& outOpaqueRef(int output);
  101. std::vector<GArg> m_args;
  102. GArg m_state;
  103. //FIXME: avoid conversion of arguments from internal representation to OpenCV one on each call
  104. //to OCV kernel. (This can be achieved by a two single time conversions in GCPUExecutable::run,
  105. //once on enter for input and output arguments, and once before return for output arguments only
  106. std::unordered_map<std::size_t, GRunArgP> m_results;
  107. friend class gimpl::GCPUExecutable;
  108. };
  109. class GAPI_EXPORTS GCPUKernel
  110. {
  111. public:
  112. // This function is a kernel's execution entry point (does the processing work)
  113. using RunF = std::function<void(GCPUContext &)>;
  114. // This function is a stateful kernel's setup routine (configures state)
  115. using SetupF = std::function<void(const GMetaArgs &, const GArgs &,
  116. GArg &, const GCompileArgs &)>;
  117. GCPUKernel();
  118. GCPUKernel(const RunF& runF, const SetupF& setupF = nullptr);
  119. RunF m_runF = nullptr;
  120. SetupF m_setupF = nullptr;
  121. bool m_isStateful = false;
  122. };
  123. // FIXME: This is an ugly ad-hoc implementation. TODO: refactor
  124. namespace detail
  125. {
  126. template<class T> struct get_in;
  127. template<> struct get_in<cv::GMat>
  128. {
  129. static cv::Mat get(GCPUContext &ctx, int idx) { return ctx.inMat(idx); }
  130. };
  131. template<> struct get_in<cv::GMatP>
  132. {
  133. static cv::Mat get(GCPUContext &ctx, int idx) { return get_in<cv::GMat>::get(ctx, idx); }
  134. };
  135. template<> struct get_in<cv::GFrame>
  136. {
  137. static cv::MediaFrame get(GCPUContext &ctx, int idx) { return ctx.inArg<cv::MediaFrame>(idx); }
  138. };
  139. template<> struct get_in<cv::GScalar>
  140. {
  141. static cv::Scalar get(GCPUContext &ctx, int idx) { return ctx.inVal(idx); }
  142. };
  143. template<typename U> struct get_in<cv::GArray<U> >
  144. {
  145. static const std::vector<U>& get(GCPUContext &ctx, int idx) { return ctx.inArg<VectorRef>(idx).rref<U>(); }
  146. };
  147. template<typename U> struct get_in<cv::GOpaque<U> >
  148. {
  149. static const U& get(GCPUContext &ctx, int idx) { return ctx.inArg<OpaqueRef>(idx).rref<U>(); }
  150. };
  151. //FIXME(dm): GArray<Mat>/GArray<GMat> conversion should be done more gracefully in the system
  152. template<> struct get_in<cv::GArray<cv::GMat> >: public get_in<cv::GArray<cv::Mat> >
  153. {
  154. };
  155. //FIXME(dm): GArray<Scalar>/GArray<GScalar> conversion should be done more gracefully in the system
  156. template<> struct get_in<cv::GArray<cv::GScalar> >: public get_in<cv::GArray<cv::Scalar> >
  157. {
  158. };
  159. // FIXME(dm): GArray<vector<U>>/GArray<GArray<U>> conversion should be done more gracefully in the system
  160. template<typename U> struct get_in<cv::GArray<cv::GArray<U>> >: public get_in<cv::GArray<std::vector<U>> >
  161. {
  162. };
  163. //FIXME(dm): GOpaque<Mat>/GOpaque<GMat> conversion should be done more gracefully in the system
  164. template<> struct get_in<cv::GOpaque<cv::GMat> >: public get_in<cv::GOpaque<cv::Mat> >
  165. {
  166. };
  167. //FIXME(dm): GOpaque<Scalar>/GOpaque<GScalar> conversion should be done more gracefully in the system
  168. template<> struct get_in<cv::GOpaque<cv::GScalar> >: public get_in<cv::GOpaque<cv::Mat> >
  169. {
  170. };
  171. template<class T> struct get_in
  172. {
  173. static T get(GCPUContext &ctx, int idx) { return ctx.inArg<T>(idx); }
  174. };
  175. struct tracked_cv_mat{
  176. tracked_cv_mat(cv::Mat& m) : r{m}, original_data{m.data} {}
  177. cv::Mat r;
  178. uchar* original_data;
  179. operator cv::Mat& (){ return r;}
  180. void validate() const{
  181. if (r.data != original_data)
  182. {
  183. util::throw_error
  184. (std::logic_error
  185. ("OpenCV kernel output parameter was reallocated. \n"
  186. "Incorrect meta data was provided ?"));
  187. }
  188. }
  189. };
  190. template<typename... Outputs>
  191. void postprocess(Outputs&... outs)
  192. {
  193. struct
  194. {
  195. void operator()(tracked_cv_mat* bm) { bm->validate(); }
  196. void operator()(...) { }
  197. } validate;
  198. //dummy array to unfold parameter pack
  199. int dummy[] = { 0, (validate(&outs), 0)... };
  200. cv::util::suppress_unused_warning(dummy);
  201. }
  202. template<class T> struct get_out;
  203. template<> struct get_out<cv::GMat>
  204. {
  205. static tracked_cv_mat get(GCPUContext &ctx, int idx)
  206. {
  207. auto& r = ctx.outMatR(idx);
  208. return {r};
  209. }
  210. };
  211. template<> struct get_out<cv::GMatP>
  212. {
  213. static tracked_cv_mat get(GCPUContext &ctx, int idx)
  214. {
  215. return get_out<cv::GMat>::get(ctx, idx);
  216. }
  217. };
  218. template<> struct get_out<cv::GScalar>
  219. {
  220. static cv::Scalar& get(GCPUContext &ctx, int idx)
  221. {
  222. return ctx.outValR(idx);
  223. }
  224. };
  225. template<> struct get_out<cv::GFrame>
  226. {
  227. static cv::MediaFrame& get(GCPUContext &ctx, int idx)
  228. {
  229. return ctx.outFrame(idx);
  230. }
  231. };
  232. template<typename U> struct get_out<cv::GArray<U>>
  233. {
  234. static std::vector<U>& get(GCPUContext &ctx, int idx)
  235. {
  236. return ctx.outVecR<U>(idx);
  237. }
  238. };
  239. //FIXME(dm): GArray<Mat>/GArray<GMat> conversion should be done more gracefully in the system
  240. template<> struct get_out<cv::GArray<cv::GMat> >: public get_out<cv::GArray<cv::Mat> >
  241. {
  242. };
  243. // FIXME(dm): GArray<vector<U>>/GArray<GArray<U>> conversion should be done more gracefully in the system
  244. template<typename U> struct get_out<cv::GArray<cv::GArray<U>> >: public get_out<cv::GArray<std::vector<U>> >
  245. {
  246. };
  247. template<typename U> struct get_out<cv::GOpaque<U>>
  248. {
  249. static U& get(GCPUContext &ctx, int idx)
  250. {
  251. return ctx.outOpaqueR<U>(idx);
  252. }
  253. };
  254. template<typename, typename>
  255. struct OCVSetupHelper;
  256. template<typename Impl, typename... Ins>
  257. struct OCVSetupHelper<Impl, std::tuple<Ins...>>
  258. {
  259. // Using 'auto' return type and 'decltype' specifier in both 'setup_impl' versions
  260. // to check existence of required 'Impl::setup' functions.
  261. // While 'decltype' specifier accepts expression we pass expression with 'comma-operator'
  262. // where first operand of comma-operator is call attempt to desired 'Impl::setup' and
  263. // the second operand is 'void()' expression.
  264. //
  265. // SFINAE for 'Impl::setup' which accepts compile arguments.
  266. template<int... IIs>
  267. static auto setup_impl(const GMetaArgs &metaArgs, const GArgs &args,
  268. GArg &state, const GCompileArgs &compileArgs,
  269. detail::Seq<IIs...>) ->
  270. decltype(Impl::setup(detail::get_in_meta<Ins>(metaArgs, args, IIs)...,
  271. std::declval<typename std::add_lvalue_reference<
  272. std::shared_ptr<typename Impl::State>
  273. >::type
  274. >(),
  275. compileArgs)
  276. , void())
  277. {
  278. // TODO: unique_ptr <-> shared_ptr conversion ?
  279. // To check: Conversion is possible only if the state which should be passed to
  280. // 'setup' user callback isn't required to have previous value
  281. std::shared_ptr<typename Impl::State> stPtr;
  282. Impl::setup(detail::get_in_meta<Ins>(metaArgs, args, IIs)..., stPtr, compileArgs);
  283. state = GArg(stPtr);
  284. }
  285. // SFINAE for 'Impl::setup' which doesn't accept compile arguments.
  286. template<int... IIs>
  287. static auto setup_impl(const GMetaArgs &metaArgs, const GArgs &args,
  288. GArg &state, const GCompileArgs &/* compileArgs */,
  289. detail::Seq<IIs...>) ->
  290. decltype(Impl::setup(detail::get_in_meta<Ins>(metaArgs, args, IIs)...,
  291. std::declval<typename std::add_lvalue_reference<
  292. std::shared_ptr<typename Impl::State>
  293. >::type
  294. >()
  295. )
  296. , void())
  297. {
  298. // The same comment as in 'setup' above.
  299. std::shared_ptr<typename Impl::State> stPtr;
  300. Impl::setup(detail::get_in_meta<Ins>(metaArgs, args, IIs)..., stPtr);
  301. state = GArg(stPtr);
  302. }
  303. static void setup(const GMetaArgs &metaArgs, const GArgs &args,
  304. GArg& state, const GCompileArgs &compileArgs)
  305. {
  306. setup_impl(metaArgs, args, state, compileArgs,
  307. typename detail::MkSeq<sizeof...(Ins)>::type());
  308. }
  309. };
  310. // OCVCallHelper is a helper class to call stateless OCV kernels and OCV kernel functors.
  311. template<typename, typename, typename>
  312. struct OCVCallHelper;
  313. // FIXME: probably can be simplified with std::apply or analogue.
  314. template<typename Impl, typename... Ins, typename... Outs>
  315. struct OCVCallHelper<Impl, std::tuple<Ins...>, std::tuple<Outs...>>
  316. {
  317. template<typename... Inputs>
  318. struct call_and_postprocess
  319. {
  320. template<typename... Outputs>
  321. static void call(Inputs&&... ins, Outputs&&... outs)
  322. {
  323. //not using a std::forward on outs is deliberate in order to
  324. //cause compilation error, by trying to bind rvalue references to lvalue references
  325. Impl::run(std::forward<Inputs>(ins)..., outs...);
  326. postprocess(outs...);
  327. }
  328. template<typename... Outputs>
  329. static void call(Impl& impl, Inputs&&... ins, Outputs&&... outs)
  330. {
  331. impl(std::forward<Inputs>(ins)..., outs...);
  332. }
  333. };
  334. template<int... IIs, int... OIs>
  335. static void call_impl(GCPUContext &ctx, detail::Seq<IIs...>, detail::Seq<OIs...>)
  336. {
  337. //Make sure that OpenCV kernels do not reallocate memory for output parameters
  338. //by comparing it's state (data ptr) before and after the call.
  339. //This is done by converting each output Mat into tracked_cv_mat object, and binding
  340. //them to parameters of ad-hoc function
  341. call_and_postprocess<decltype(get_in<Ins>::get(ctx, IIs))...>
  342. ::call(get_in<Ins>::get(ctx, IIs)..., get_out<Outs>::get(ctx, OIs)...);
  343. }
  344. template<int... IIs, int... OIs>
  345. static void call_impl(cv::GCPUContext &ctx, Impl& impl,
  346. detail::Seq<IIs...>, detail::Seq<OIs...>)
  347. {
  348. call_and_postprocess<decltype(get_in<Ins>::get(ctx, IIs))...>
  349. ::call(impl, get_in<Ins>::get(ctx, IIs)..., get_out<Outs>::get(ctx, OIs)...);
  350. }
  351. static void call(GCPUContext &ctx)
  352. {
  353. call_impl(ctx,
  354. typename detail::MkSeq<sizeof...(Ins)>::type(),
  355. typename detail::MkSeq<sizeof...(Outs)>::type());
  356. }
  357. // NB: Same as call but calling the object
  358. // This necessary for kernel implementations that have a state
  359. // and are represented as an object
  360. static void callFunctor(cv::GCPUContext &ctx, Impl& impl)
  361. {
  362. call_impl(ctx, impl,
  363. typename detail::MkSeq<sizeof...(Ins)>::type(),
  364. typename detail::MkSeq<sizeof...(Outs)>::type());
  365. }
  366. };
  367. // OCVStCallHelper is a helper class to call stateful OCV kernels.
  368. template<typename, typename, typename>
  369. struct OCVStCallHelper;
  370. template<typename Impl, typename... Ins, typename... Outs>
  371. struct OCVStCallHelper<Impl, std::tuple<Ins...>, std::tuple<Outs...>> :
  372. OCVCallHelper<Impl, std::tuple<Ins...>, std::tuple<Outs...>>
  373. {
  374. template<typename... Inputs>
  375. struct call_and_postprocess
  376. {
  377. template<typename... Outputs>
  378. static void call(typename Impl::State& st, Inputs&&... ins, Outputs&&... outs)
  379. {
  380. Impl::run(std::forward<Inputs>(ins)..., outs..., st);
  381. postprocess(outs...);
  382. }
  383. };
  384. template<int... IIs, int... OIs>
  385. static void call_impl(GCPUContext &ctx, detail::Seq<IIs...>, detail::Seq<OIs...>)
  386. {
  387. auto& st = *ctx.state().get<std::shared_ptr<typename Impl::State>>();
  388. call_and_postprocess<decltype(get_in<Ins>::get(ctx, IIs))...>
  389. ::call(st, get_in<Ins>::get(ctx, IIs)..., get_out<Outs>::get(ctx, OIs)...);
  390. }
  391. static void call(GCPUContext &ctx)
  392. {
  393. call_impl(ctx,
  394. typename detail::MkSeq<sizeof...(Ins)>::type(),
  395. typename detail::MkSeq<sizeof...(Outs)>::type());
  396. }
  397. };
  398. } // namespace detail
  399. template<class Impl, class K>
  400. class GCPUKernelImpl: public cv::detail::KernelTag
  401. {
  402. using CallHelper = cv::detail::OCVCallHelper<Impl, typename K::InArgs, typename K::OutArgs>;
  403. public:
  404. using API = K;
  405. static cv::gapi::GBackend backend() { return cv::gapi::cpu::backend(); }
  406. static cv::GCPUKernel kernel() { return GCPUKernel(&CallHelper::call); }
  407. };
  408. template<class Impl, class K, class S>
  409. class GCPUStKernelImpl: public cv::detail::KernelTag
  410. {
  411. using StSetupHelper = detail::OCVSetupHelper<Impl, typename K::InArgs>;
  412. using StCallHelper = detail::OCVStCallHelper<Impl, typename K::InArgs, typename K::OutArgs>;
  413. public:
  414. using API = K;
  415. using State = S;
  416. static cv::gapi::GBackend backend() { return cv::gapi::cpu::backend(); }
  417. static cv::GCPUKernel kernel() { return GCPUKernel(&StCallHelper::call,
  418. &StSetupHelper::setup); }
  419. };
  420. #define GAPI_OCV_KERNEL(Name, API) struct Name: public cv::GCPUKernelImpl<Name, API>
  421. // TODO: Reuse Anatoliy's logic for support of types with commas in macro.
  422. // Retrieve the common part from Anatoliy's logic to the separate place.
  423. #define GAPI_OCV_KERNEL_ST(Name, API, State) \
  424. struct Name: public cv::GCPUStKernelImpl<Name, API, State> \
  425. /// @private
  426. class gapi::cpu::GOCVFunctor : public gapi::GFunctor
  427. {
  428. public:
  429. using Impl = std::function<void(GCPUContext &)>;
  430. using Meta = cv::GKernel::M;
  431. GOCVFunctor(const char* id, const Meta &meta, const Impl& impl)
  432. : gapi::GFunctor(id), impl_{GCPUKernel(impl), meta}
  433. {
  434. }
  435. GKernelImpl impl() const override { return impl_; }
  436. gapi::GBackend backend() const override { return gapi::cpu::backend(); }
  437. private:
  438. GKernelImpl impl_;
  439. };
  440. //! @cond IGNORED
  441. template<typename K, typename Callable>
  442. gapi::cpu::GOCVFunctor gapi::cpu::ocv_kernel(Callable& c)
  443. {
  444. using P = cv::detail::OCVCallHelper<Callable, typename K::InArgs, typename K::OutArgs>;
  445. return GOCVFunctor{ K::id()
  446. , &K::getOutMeta
  447. , std::bind(&P::callFunctor, std::placeholders::_1, std::ref(c))
  448. };
  449. }
  450. template<typename K, typename Callable>
  451. gapi::cpu::GOCVFunctor gapi::cpu::ocv_kernel(const Callable& c)
  452. {
  453. using P = cv::detail::OCVCallHelper<Callable, typename K::InArgs, typename K::OutArgs>;
  454. return GOCVFunctor{ K::id()
  455. , &K::getOutMeta
  456. , std::bind(&P::callFunctor, std::placeholders::_1, c)
  457. };
  458. }
  459. //! @endcond
  460. } // namespace cv
  461. #endif // OPENCV_GAPI_GCPUKERNEL_HPP