tracking_legacy.hpp 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539
  1. /*M///////////////////////////////////////////////////////////////////////////////////////
  2. //
  3. // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
  4. //
  5. // By downloading, copying, installing or using the software you agree to this license.
  6. // If you do not agree to this license, do not download, install,
  7. // copy or use the software.
  8. //
  9. //
  10. // License Agreement
  11. // For Open Source Computer Vision Library
  12. //
  13. // Copyright (C) 2013, OpenCV Foundation, all rights reserved.
  14. // Third party copyrights are property of their respective owners.
  15. //
  16. // Redistribution and use in source and binary forms, with or without modification,
  17. // are permitted provided that the following conditions are met:
  18. //
  19. // * Redistribution's of source code must retain the above copyright notice,
  20. // this list of conditions and the following disclaimer.
  21. //
  22. // * Redistribution's in binary form must reproduce the above copyright notice,
  23. // this list of conditions and the following disclaimer in the documentation
  24. // and/or other materials provided with the distribution.
  25. //
  26. // * The name of the copyright holders may not be used to endorse or promote products
  27. // derived from this software without specific prior written permission.
  28. //
  29. // This software is provided by the copyright holders and contributors "as is" and
  30. // any express or implied warranties, including, but not limited to, the implied
  31. // warranties of merchantability and fitness for a particular purpose are disclaimed.
  32. // In no event shall the Intel Corporation or contributors be liable for any direct,
  33. // indirect, incidental, special, exemplary, or consequential damages
  34. // (including, but not limited to, procurement of substitute goods or services;
  35. // loss of use, data, or profits; or business interruption) however caused
  36. // and on any theory of liability, whether in contract, strict liability,
  37. // or tort (including negligence or otherwise) arising in any way out of
  38. // the use of this software, even if advised of the possibility of such damage.
  39. //
  40. //M*/
  41. #ifndef OPENCV_TRACKING_LEGACY_HPP
  42. #define OPENCV_TRACKING_LEGACY_HPP
  43. /*
  44. * Partially based on:
  45. * ====================================================================================================================
  46. * - [AAM] S. Salti, A. Cavallaro, L. Di Stefano, Adaptive Appearance Modeling for Video Tracking: Survey and Evaluation
  47. * - [AMVOT] X. Li, W. Hu, C. Shen, Z. Zhang, A. Dick, A. van den Hengel, A Survey of Appearance Models in Visual Object Tracking
  48. *
  49. * This Tracking API has been designed with PlantUML. If you modify this API please change UML files under modules/tracking/doc/uml
  50. *
  51. */
  52. #include "tracking_internals.hpp"
  53. namespace cv {
  54. namespace legacy {
  55. #ifndef CV_DOXYGEN
  56. inline namespace tracking {
  57. #endif
  58. using namespace cv::detail::tracking;
  59. /** @addtogroup tracking_legacy
  60. @{
  61. */
  62. /************************************ Tracker Base Class ************************************/
  63. /** @brief Base abstract class for the long-term tracker:
  64. */
  65. class CV_EXPORTS_W Tracker : public virtual Algorithm
  66. {
  67. public:
  68. Tracker();
  69. virtual ~Tracker() CV_OVERRIDE;
  70. /** @brief Initialize the tracker with a known bounding box that surrounded the target
  71. @param image The initial frame
  72. @param boundingBox The initial bounding box
  73. @return True if initialization went succesfully, false otherwise
  74. */
  75. CV_WRAP bool init( InputArray image, const Rect2d& boundingBox );
  76. /** @brief Update the tracker, find the new most likely bounding box for the target
  77. @param image The current frame
  78. @param boundingBox The bounding box that represent the new target location, if true was returned, not
  79. modified otherwise
  80. @return True means that target was located and false means that tracker cannot locate target in
  81. current frame. Note, that latter *does not* imply that tracker has failed, maybe target is indeed
  82. missing from the frame (say, out of sight)
  83. */
  84. CV_WRAP bool update( InputArray image, CV_OUT Rect2d& boundingBox );
  85. virtual void read( const FileNode& fn ) CV_OVERRIDE = 0;
  86. virtual void write( FileStorage& fs ) const CV_OVERRIDE = 0;
  87. protected:
  88. virtual bool initImpl( const Mat& image, const Rect2d& boundingBox ) = 0;
  89. virtual bool updateImpl( const Mat& image, Rect2d& boundingBox ) = 0;
  90. bool isInit;
  91. Ptr<TrackerContribFeatureSet> featureSet;
  92. Ptr<TrackerContribSampler> sampler;
  93. Ptr<TrackerModel> model;
  94. };
  95. /************************************ Specific Tracker Classes ************************************/
  96. /** @brief The MIL algorithm trains a classifier in an online manner to separate the object from the
  97. background.
  98. Multiple Instance Learning avoids the drift problem for a robust tracking. The implementation is
  99. based on @cite MIL .
  100. Original code can be found here <http://vision.ucsd.edu/~bbabenko/project_miltrack.shtml>
  101. */
  102. class CV_EXPORTS_W TrackerMIL : public cv::legacy::Tracker
  103. {
  104. public:
  105. struct CV_EXPORTS Params : cv::TrackerMIL::Params
  106. {
  107. void read( const FileNode& fn );
  108. void write( FileStorage& fs ) const;
  109. };
  110. /** @brief Constructor
  111. @param parameters MIL parameters TrackerMIL::Params
  112. */
  113. static Ptr<legacy::TrackerMIL> create(const TrackerMIL::Params &parameters);
  114. CV_WRAP static Ptr<legacy::TrackerMIL> create();
  115. virtual ~TrackerMIL() CV_OVERRIDE {}
  116. };
  117. /** @brief the Boosting tracker
  118. This is a real-time object tracking based on a novel on-line version of the AdaBoost algorithm.
  119. The classifier uses the surrounding background as negative examples in update step to avoid the
  120. drifting problem. The implementation is based on @cite OLB .
  121. */
  122. class CV_EXPORTS_W TrackerBoosting : public cv::legacy::Tracker
  123. {
  124. public:
  125. struct CV_EXPORTS Params
  126. {
  127. Params();
  128. int numClassifiers; //!<the number of classifiers to use in a OnlineBoosting algorithm
  129. float samplerOverlap; //!<search region parameters to use in a OnlineBoosting algorithm
  130. float samplerSearchFactor; //!< search region parameters to use in a OnlineBoosting algorithm
  131. int iterationInit; //!<the initial iterations
  132. int featureSetNumFeatures; //!< # features
  133. /**
  134. * \brief Read parameters from a file
  135. */
  136. void read( const FileNode& fn );
  137. /**
  138. * \brief Write parameters to a file
  139. */
  140. void write( FileStorage& fs ) const;
  141. };
  142. /** @brief Constructor
  143. @param parameters BOOSTING parameters TrackerBoosting::Params
  144. */
  145. static Ptr<legacy::TrackerBoosting> create(const TrackerBoosting::Params &parameters);
  146. CV_WRAP static Ptr<legacy::TrackerBoosting> create();
  147. virtual ~TrackerBoosting() CV_OVERRIDE {}
  148. };
  149. /** @brief the Median Flow tracker
  150. Implementation of a paper @cite MedianFlow .
  151. The tracker is suitable for very smooth and predictable movements when object is visible throughout
  152. the whole sequence. It's quite and accurate for this type of problems (in particular, it was shown
  153. by authors to outperform MIL). During the implementation period the code at
  154. <http://www.aonsquared.co.uk/node/5>, the courtesy of the author Arthur Amarra, was used for the
  155. reference purpose.
  156. */
  157. class CV_EXPORTS_W TrackerMedianFlow : public cv::legacy::Tracker
  158. {
  159. public:
  160. struct CV_EXPORTS Params
  161. {
  162. Params(); //!<default constructor
  163. //!<note that the default values of parameters are recommended for most of use cases
  164. int pointsInGrid; //!<square root of number of keypoints used; increase it to trade
  165. //!<accurateness for speed
  166. cv::Size winSize; //!<window size parameter for Lucas-Kanade optical flow
  167. int maxLevel; //!<maximal pyramid level number for Lucas-Kanade optical flow
  168. TermCriteria termCriteria; //!<termination criteria for Lucas-Kanade optical flow
  169. cv::Size winSizeNCC; //!<window size around a point for normalized cross-correlation check
  170. double maxMedianLengthOfDisplacementDifference; //!<criterion for loosing the tracked object
  171. void read( const FileNode& /*fn*/ );
  172. void write( FileStorage& /*fs*/ ) const;
  173. };
  174. /** @brief Constructor
  175. @param parameters Median Flow parameters TrackerMedianFlow::Params
  176. */
  177. static Ptr<legacy::TrackerMedianFlow> create(const TrackerMedianFlow::Params &parameters);
  178. CV_WRAP static Ptr<legacy::TrackerMedianFlow> create();
  179. virtual ~TrackerMedianFlow() CV_OVERRIDE {}
  180. };
  181. /** @brief the TLD (Tracking, learning and detection) tracker
  182. TLD is a novel tracking framework that explicitly decomposes the long-term tracking task into
  183. tracking, learning and detection.
  184. The tracker follows the object from frame to frame. The detector localizes all appearances that
  185. have been observed so far and corrects the tracker if necessary. The learning estimates detector's
  186. errors and updates it to avoid these errors in the future. The implementation is based on @cite TLD .
  187. The Median Flow algorithm (see cv::TrackerMedianFlow) was chosen as a tracking component in this
  188. implementation, following authors. The tracker is supposed to be able to handle rapid motions, partial
  189. occlusions, object absence etc.
  190. */
  191. class CV_EXPORTS_W TrackerTLD : public cv::legacy::Tracker
  192. {
  193. public:
  194. struct CV_EXPORTS Params
  195. {
  196. Params();
  197. void read( const FileNode& /*fn*/ );
  198. void write( FileStorage& /*fs*/ ) const;
  199. };
  200. /** @brief Constructor
  201. @param parameters TLD parameters TrackerTLD::Params
  202. */
  203. static Ptr<legacy::TrackerTLD> create(const TrackerTLD::Params &parameters);
  204. CV_WRAP static Ptr<legacy::TrackerTLD> create();
  205. virtual ~TrackerTLD() CV_OVERRIDE {}
  206. };
  207. /** @brief the KCF (Kernelized Correlation Filter) tracker
  208. * KCF is a novel tracking framework that utilizes properties of circulant matrix to enhance the processing speed.
  209. * This tracking method is an implementation of @cite KCF_ECCV which is extended to KCF with color-names features (@cite KCF_CN).
  210. * The original paper of KCF is available at <http://www.robots.ox.ac.uk/~joao/publications/henriques_tpami2015.pdf>
  211. * as well as the matlab implementation. For more information about KCF with color-names features, please refer to
  212. * <http://www.cvl.isy.liu.se/research/objrec/visualtracking/colvistrack/index.html>.
  213. */
  214. class CV_EXPORTS_W TrackerKCF : public cv::legacy::Tracker
  215. {
  216. public:
  217. /**
  218. * \brief Feature type to be used in the tracking grayscale, colornames, compressed color-names
  219. * The modes available now:
  220. - "GRAY" -- Use grayscale values as the feature
  221. - "CN" -- Color-names feature
  222. */
  223. typedef enum cv::tracking::TrackerKCF::MODE MODE;
  224. struct CV_EXPORTS Params : cv::tracking::TrackerKCF::Params
  225. {
  226. void read(const FileNode& /*fn*/);
  227. void write(FileStorage& /*fs*/) const;
  228. };
  229. virtual void setFeatureExtractor(void(*)(const Mat, const Rect, Mat&), bool pca_func = false) = 0;
  230. /** @brief Constructor
  231. @param parameters KCF parameters TrackerKCF::Params
  232. */
  233. static Ptr<legacy::TrackerKCF> create(const TrackerKCF::Params &parameters);
  234. CV_WRAP static Ptr<legacy::TrackerKCF> create();
  235. virtual ~TrackerKCF() CV_OVERRIDE {}
  236. };
  237. #if 0 // legacy variant is not available
  238. /** @brief the GOTURN (Generic Object Tracking Using Regression Networks) tracker
  239. * GOTURN (@cite GOTURN) is kind of trackers based on Convolutional Neural Networks (CNN). While taking all advantages of CNN trackers,
  240. * GOTURN is much faster due to offline training without online fine-tuning nature.
  241. * GOTURN tracker addresses the problem of single target tracking: given a bounding box label of an object in the first frame of the video,
  242. * we track that object through the rest of the video. NOTE: Current method of GOTURN does not handle occlusions; however, it is fairly
  243. * robust to viewpoint changes, lighting changes, and deformations.
  244. * Inputs of GOTURN are two RGB patches representing Target and Search patches resized to 227x227.
  245. * Outputs of GOTURN are predicted bounding box coordinates, relative to Search patch coordinate system, in format X1,Y1,X2,Y2.
  246. * Original paper is here: <http://davheld.github.io/GOTURN/GOTURN.pdf>
  247. * As long as original authors implementation: <https://github.com/davheld/GOTURN#train-the-tracker>
  248. * Implementation of training algorithm is placed in separately here due to 3d-party dependencies:
  249. * <https://github.com/Auron-X/GOTURN_Training_Toolkit>
  250. * GOTURN architecture goturn.prototxt and trained model goturn.caffemodel are accessible on opencv_extra GitHub repository.
  251. */
  252. class CV_EXPORTS_W TrackerGOTURN : public cv::legacy::Tracker
  253. {
  254. public:
  255. struct CV_EXPORTS Params
  256. {
  257. Params();
  258. void read(const FileNode& /*fn*/);
  259. void write(FileStorage& /*fs*/) const;
  260. String modelTxt;
  261. String modelBin;
  262. };
  263. /** @brief Constructor
  264. @param parameters GOTURN parameters TrackerGOTURN::Params
  265. */
  266. static Ptr<legacy::TrackerGOTURN> create(const TrackerGOTURN::Params &parameters);
  267. CV_WRAP static Ptr<legacy::TrackerGOTURN> create();
  268. virtual ~TrackerGOTURN() CV_OVERRIDE {}
  269. };
  270. #endif
  271. /** @brief the MOSSE (Minimum Output Sum of Squared %Error) tracker
  272. The implementation is based on @cite MOSSE Visual Object Tracking using Adaptive Correlation Filters
  273. @note this tracker works with grayscale images, if passed bgr ones, they will get converted internally.
  274. */
  275. class CV_EXPORTS_W TrackerMOSSE : public cv::legacy::Tracker
  276. {
  277. public:
  278. /** @brief Constructor
  279. */
  280. CV_WRAP static Ptr<legacy::TrackerMOSSE> create();
  281. virtual ~TrackerMOSSE() CV_OVERRIDE {}
  282. };
  283. /************************************ MultiTracker Class ---By Laksono Kurnianggoro---) ************************************/
  284. /** @brief This class is used to track multiple objects using the specified tracker algorithm.
  285. * The %MultiTracker is naive implementation of multiple object tracking.
  286. * It process the tracked objects independently without any optimization accross the tracked objects.
  287. */
  288. class CV_EXPORTS_W MultiTracker : public Algorithm
  289. {
  290. public:
  291. /**
  292. * \brief Constructor.
  293. */
  294. CV_WRAP MultiTracker();
  295. /**
  296. * \brief Destructor
  297. */
  298. ~MultiTracker() CV_OVERRIDE;
  299. /**
  300. * \brief Add a new object to be tracked.
  301. *
  302. * @param newTracker tracking algorithm to be used
  303. * @param image input image
  304. * @param boundingBox a rectangle represents ROI of the tracked object
  305. */
  306. CV_WRAP bool add(Ptr<cv::legacy::Tracker> newTracker, InputArray image, const Rect2d& boundingBox);
  307. /**
  308. * \brief Add a set of objects to be tracked.
  309. * @param newTrackers list of tracking algorithms to be used
  310. * @param image input image
  311. * @param boundingBox list of the tracked objects
  312. */
  313. bool add(std::vector<Ptr<legacy::Tracker> > newTrackers, InputArray image, std::vector<Rect2d> boundingBox);
  314. /**
  315. * \brief Update the current tracking status.
  316. * The result will be saved in the internal storage.
  317. * @param image input image
  318. */
  319. bool update(InputArray image);
  320. /**
  321. * \brief Update the current tracking status.
  322. * @param image input image
  323. * @param boundingBox the tracking result, represent a list of ROIs of the tracked objects.
  324. */
  325. CV_WRAP bool update(InputArray image, CV_OUT std::vector<Rect2d> & boundingBox);
  326. /**
  327. * \brief Returns a reference to a storage for the tracked objects, each object corresponds to one tracker algorithm
  328. */
  329. CV_WRAP const std::vector<Rect2d>& getObjects() const;
  330. /**
  331. * \brief Returns a pointer to a new instance of MultiTracker
  332. */
  333. CV_WRAP static Ptr<MultiTracker> create();
  334. protected:
  335. //!< storage for the tracker algorithms.
  336. std::vector< Ptr<Tracker> > trackerList;
  337. //!< storage for the tracked objects, each object corresponds to one tracker algorithm.
  338. std::vector<Rect2d> objects;
  339. };
  340. /************************************ Multi-Tracker Classes ---By Tyan Vladimir---************************************/
  341. /** @brief Base abstract class for the long-term Multi Object Trackers:
  342. @sa Tracker, MultiTrackerTLD
  343. */
  344. class CV_EXPORTS MultiTracker_Alt
  345. {
  346. public:
  347. /** @brief Constructor for Multitracker
  348. */
  349. MultiTracker_Alt()
  350. {
  351. targetNum = 0;
  352. }
  353. /** @brief Add a new target to a tracking-list and initialize the tracker with a known bounding box that surrounded the target
  354. @param image The initial frame
  355. @param boundingBox The initial bounding box of target
  356. @param tracker_algorithm Multi-tracker algorithm
  357. @return True if new target initialization went succesfully, false otherwise
  358. */
  359. bool addTarget(InputArray image, const Rect2d& boundingBox, Ptr<legacy::Tracker> tracker_algorithm);
  360. /** @brief Update all trackers from the tracking-list, find a new most likely bounding boxes for the targets
  361. @param image The current frame
  362. @return True means that all targets were located and false means that tracker couldn't locate one of the targets in
  363. current frame. Note, that latter *does not* imply that tracker has failed, maybe target is indeed
  364. missing from the frame (say, out of sight)
  365. */
  366. bool update(InputArray image);
  367. /** @brief Current number of targets in tracking-list
  368. */
  369. int targetNum;
  370. /** @brief Trackers list for Multi-Object-Tracker
  371. */
  372. std::vector <Ptr<Tracker> > trackers;
  373. /** @brief Bounding Boxes list for Multi-Object-Tracker
  374. */
  375. std::vector <Rect2d> boundingBoxes;
  376. /** @brief List of randomly generated colors for bounding boxes display
  377. */
  378. std::vector<Scalar> colors;
  379. };
  380. /** @brief Multi Object %Tracker for TLD.
  381. TLD is a novel tracking framework that explicitly decomposes
  382. the long-term tracking task into tracking, learning and detection.
  383. The tracker follows the object from frame to frame. The detector localizes all appearances that
  384. have been observed so far and corrects the tracker if necessary. The learning estimates detector's
  385. errors and updates it to avoid these errors in the future. The implementation is based on @cite TLD .
  386. The Median Flow algorithm (see cv::TrackerMedianFlow) was chosen as a tracking component in this
  387. implementation, following authors. The tracker is supposed to be able to handle rapid motions, partial
  388. occlusions, object absence etc.
  389. @sa Tracker, MultiTracker, TrackerTLD
  390. */
  391. class CV_EXPORTS MultiTrackerTLD : public MultiTracker_Alt
  392. {
  393. public:
  394. /** @brief Update all trackers from the tracking-list, find a new most likely bounding boxes for the targets by
  395. optimized update method using some techniques to speedup calculations specifically for MO TLD. The only limitation
  396. is that all target bounding boxes should have approximately same aspect ratios. Speed boost is around 20%
  397. @param image The current frame.
  398. @return True means that all targets were located and false means that tracker couldn't locate one of the targets in
  399. current frame. Note, that latter *does not* imply that tracker has failed, maybe target is indeed
  400. missing from the frame (say, out of sight)
  401. */
  402. bool update_opt(InputArray image);
  403. };
  404. /*********************************** CSRT ************************************/
  405. /** @brief the CSRT tracker
  406. The implementation is based on @cite Lukezic_IJCV2018 Discriminative Correlation Filter with Channel and Spatial Reliability
  407. */
  408. class CV_EXPORTS_W TrackerCSRT : public cv::legacy::Tracker
  409. {
  410. public:
  411. struct CV_EXPORTS Params : cv::tracking::TrackerCSRT::Params
  412. {
  413. /**
  414. * \brief Read parameters from a file
  415. */
  416. void read(const FileNode& /*fn*/);
  417. /**
  418. * \brief Write parameters to a file
  419. */
  420. void write(cv::FileStorage& fs) const;
  421. };
  422. /** @brief Constructor
  423. @param parameters CSRT parameters TrackerCSRT::Params
  424. */
  425. static Ptr<legacy::TrackerCSRT> create(const TrackerCSRT::Params &parameters);
  426. CV_WRAP static Ptr<legacy::TrackerCSRT> create();
  427. CV_WRAP virtual void setInitialMask(InputArray mask) = 0;
  428. virtual ~TrackerCSRT() CV_OVERRIDE {}
  429. };
  430. CV_EXPORTS_W Ptr<cv::Tracker> upgradeTrackingAPI(const Ptr<legacy::Tracker>& legacy_tracker);
  431. //! @}
  432. #ifndef CV_DOXYGEN
  433. } // namespace
  434. #endif
  435. }} // namespace
  436. #endif // OPENCV_TRACKING_LEGACY_HPP