Fleshed out the spec for the new track association stuff.

dlib/svm/cross_validate_track_association_trainer.h

@@ -41,6 +41,7 @@ namespace dlib
                 const std::vector<std::pair<detection_type,detection_id_type> >& dets = samples[j];
                 for (unsigned long k = 0; k < assignments.size(); ++k)
                 {
+                    // If the detection is associated to tracks[assignments[k]]
                     if (assignments[k] != -1)
                     {
                         tracks[assignments[k]].update_track(dets[k].first);

dlib/svm/cross_validate_track_association_trainer_abstract.h

@@ -4,6 +4,7 @@
 #ifdef DLIB_CROSS_VALIDATE_TRACK_ASSOCIATION_TrAINER_ABSTRACT_H__
 
 #include "structural_track_association_trainer_abstract.h"
+#include "svm_abstract.h"
 
 namespace dlib
 {
@@ -20,6 +21,16 @@ namespace dlib
         const std::vector<std::vector<std::vector<std::pair<detection_type,detection_id_type> > > >& samples
     );
     /*!
+        requires
+            - is_track_association_problem(samples)
+            - track_association_function == an instantiation of the dlib::track_association_function
+              template or an object with a compatible interface.
+        ensures
+            - Tests assoc against the given samples and returns the fraction of detections
+              which were correctly associated to their tracks.  That is, if assoc produces
+              perfect tracks when used then this function returns a value of 1.  Similarly,
+              if 5% of the detections were associated to the incorrect track then the
+              return value is 0.05.
     !*/
 
 // ----------------------------------------------------------------------------------------
@@ -34,6 +45,20 @@ namespace dlib
         const std::vector<std::vector<std::vector<std::pair<detection_type,detection_id_type> > > >& samples,
         const long folds
     );
+    /*!
+        requires
+            - is_track_association_problem(samples)
+            - 1 < folds <= samples.size()
+            - trainer_type == dlib::structural_track_association_trainer or an object with
+              a compatible interface.
+        ensures
+            - Performs k-fold cross validation by using the given trainer to solve the
+              given track association learning problem for the given number of folds.  Each
+              fold is tested using the output of the trainer and the fraction of
+              mis-associated detections is returned (i.e. this function returns the same
+              measure of track association quality as test_track_association_function()).
+            - The number of folds used is given by the folds argument.
+    !*/
 
 // ----------------------------------------------------------------------------------------
 

dlib/svm/structural_track_association_trainer.h

@@ -49,8 +49,7 @@ namespace dlib
         typedef detection_id_type_ detection_id_type;
         typedef std::pair<detection_type, detection_id_type> labeled_detection;
         typedef std::vector<labeled_detection> detections_at_single_time_step;
-        // This type logically represents an entire track history
-        typedef std::vector<detections_at_single_time_step> sample_type;
+        typedef std::vector<detections_at_single_time_step> track_history;
 
         typedef track_association_function<detection_type> trained_function_type;
 
@@ -161,7 +160,7 @@ namespace dlib
         }
 
         const track_association_function<detection_type> train (  
-            const std::vector<sample_type>& samples
+            const std::vector<track_history>& samples
         ) const
         {
             // make sure requires clause is not broken
@@ -197,10 +196,10 @@ namespace dlib
         }
 
         const track_association_function<detection_type> train (  
-            const sample_type& sample
+            const track_history& sample
         ) const
         {
-            std::vector<sample_type> samples;
+            std::vector<track_history> samples;
             samples.push_back(sample);
             return train(samples);
         }
@@ -208,7 +207,7 @@ namespace dlib
     private:
 
         static unsigned long find_num_dims (
-            const std::vector<sample_type>& samples
+            const std::vector<track_history>& samples
         )
         {
             // find a detection_type object so we can call get_similarity_features() and
@@ -326,7 +325,7 @@ namespace dlib
         {
             C = 100;
             verbose = false;
-            eps = 0.1;
+            eps = 0.01;
             num_threads = 2;
             max_cache_size = 5;
             learn_nonnegative_weights = false;

dlib/svm/structural_track_association_trainer_abstract.h

@@ -17,97 +17,221 @@ namespace dlib
         >
     class structural_track_association_trainer
     {
+        /*!
+            REQUIREMENTS ON detection_type_
+                It must be an object that implements an interface compatible with the
+                example_detection defined in dlib/svm/track_association_function_abstract.h.  
+
+            REQUIREMENTS ON detection_id_type_
+                This can be any type that can be used as the key in a std::map.  e.g.
+                unsigned long, std::string, etc.
+            
+            WHAT THIS OBJECT REPRESENTS
+                This object is a tool for learning to solve a track association problem.  That
+                is, it takes in a set of training data and outputs a track_association_function
+                you can use to do detection to track association.  The training data takes the
+                form of a set or sets of "track histories".  Each track history is a
+                std::vector where each element contains all the detections from a single time
+                step.  Moreover, each detection is labeled with a detection_id_type_ that
+                uniquely identifies which object (e.g. person or whatever) the detection really
+                corresponds to.  That is, the detection_id_type_ values indicate the correct
+                detection to track associations.  The goal of this object is then to produce a
+                track_association_function that can perform a correct detection to track
+                association at each time step.
+
+        !*/
+
     public:
         typedef detection_type_ detection_type;
         typedef typename detection_type::track_type track_type;
         typedef detection_id_type_ detection_id_type;
         typedef std::pair<detection_type, detection_id_type> labeled_detection;
         typedef std::vector<labeled_detection> detections_at_single_time_step;
-        // This type logically represents an entire track history
-        typedef std::vector<detections_at_single_time_step> sample_type;
+        typedef std::vector<detections_at_single_time_step> track_history;
 
         typedef track_association_function<detection_type> trained_function_type;
 
         structural_track_association_trainer (
         );  
         /*!
-            C = 100;
-            verbose = false;
-            eps = 0.1;
-            num_threads = 2;
-            max_cache_size = 5;
-            learn_nonnegative_weights = false;
+            ensures
+                - #get_c() == 100
+                - this object isn't verbose
+                - #get_epsilon() == 0.01
+                - #get_num_threads() == 2
+                - #get_max_cache_size() == 5
+                - #learns_nonnegative_weights() == false
         !*/
 
         void set_num_threads (
             unsigned long num
         );
+        /*!
+            ensures
+                - #get_num_threads() == num
+        !*/
 
         unsigned long get_num_threads (
         ) const;
+        /*!
+            ensures
+                - returns the number of threads used during training.  You should 
+                  usually set this equal to the number of processing cores on your
+                  machine.
+        !*/
 
         void set_epsilon (
             double eps
         );
+        /*!
+            requires
+                - eps > 0
+            ensures
+                - #get_epsilon() == eps
+        !*/
 
         double get_epsilon (
         ) const; 
+        /*!
+            ensures
+                - returns the error epsilon that determines when training should stop.
+                  Smaller values may result in a more accurate solution but take longer to
+                  train.  You can think of this epsilon value as saying "solve the
+                  optimization problem until the average number of association mistakes per
+                  time step is within epsilon of its optimal value".
+        !*/
 
         void set_max_cache_size (
             unsigned long max_size
         );
+        /*!
+            ensures
+                - #get_max_cache_size() == max_size
+        !*/
 
         unsigned long get_max_cache_size (
         ) const;
+        /*!
+            ensures
+                - During training, this object basically runs the track_association_function on 
+                  each training sample, over and over.  To speed this up, it is possible to 
+                  cache the results of these invocations.  This function returns the number 
+                  of cache elements per training sample kept in the cache.  Note that a value 
+                  of 0 means caching is not used at all.  
+        !*/
 
         void be_verbose (
         );
+        /*!
+            ensures
+                - This object will print status messages to standard out so that a user can
+                  observe the progress of the algorithm.
+        !*/
 
         void be_quiet (
         );
+        /*!
+            ensures
+                - this object will not print anything to standard out
+        !*/
 
         void set_oca (
             const oca& item
         );
+        /*!
+            ensures
+                - #get_oca() == item 
+        !*/
 
         const oca get_oca (
         ) const;
+        /*!
+            ensures
+                - Internally this object treats track association learning as a structural
+                  SVM problem.  This routine returns a copy of the optimizer used to solve
+                  the structural SVM problem.  
+        !*/
 
         void set_c (
             double C
         );
+        /*!
+            ensures
+                - returns the SVM regularization parameter.  It is the parameter that
+                  determines the trade-off between trying to fit the training data (i.e.
+                  minimize the loss) or allowing more errors but hopefully improving the
+                  generalization of the resulting track_association_function.  Larger
+                  values encourage exact fitting while smaller values of C may encourage
+                  better generalization. 
+        !*/
 
         double get_c (
         ) const;
+        /*!
+            requires
+                - C > 0
+            ensures
+                - #get_c() = C
+        !*/
 
         bool learns_nonnegative_weights (
         ) const; 
+        /*!
+            ensures
+                - Ultimately, the output of training is a parameter vector that defines the
+                  behavior of the track_association_function.  If
+                  learns_nonnegative_weights() == true then the resulting learned parameter
+                  vector will always have non-negative entries.
+        !*/
        
         void set_learns_nonnegative_weights (
             bool value
         );
+        /*!
+            ensures
+                - #learns_nonnegative_weights() == value
+        !*/
 
         const track_association_function<detection_type> train (  
-            const std::vector<sample_type>& samples
+            const track_history& sample
         ) const;
         /*!
             requires
                 - is_track_association_problem(samples) == true
             ensures
-                - 
+                - This function attempts to learn to do track association from the given
+                  training data.
+                - returns a function F such that:
+                    - Executing F(tracks, detections) will try to correctly associate the
+                      contents of detections to the contents of tracks and perform track
+                      updating and creation.
+                    - if (learns_nonnegative_weights() == true) then
+                        - min(F.get_assignment_function().get_weights()) >= 0
         !*/
 
         const track_association_function<detection_type> train (  
-            const sample_type& sample
+            const std::vector<track_history>& samples
         ) const;
         /*!
             requires
                 - is_track_association_problem(samples) == true
             ensures
-                - 
+                - This function attempts to learn to do track association from the given
+                  training data.  In this case, we take a set of track histories as
+                  training data instead of just one track history as with the above train()
+                  method.
+                - returns a function F such that:
+                    - Executing F(tracks, detections) will try to correctly associate the
+                      contents of detections to the contents of tracks and perform track
+                      updating and creation.
+                    - if (learns_nonnegative_weights() == true) then
+                        - min(F.get_assignment_function().get_weights()) >= 0
         !*/
+
     };
 
+// ----------------------------------------------------------------------------------------
+
 }
 
 #endif // DLIB_STRUCTURAL_TRACK_ASSOCIATION_TRAnER_ABSTRACT_H__

dlib/svm/track_association_function_abstract.h

@@ -15,12 +15,16 @@ namespace dlib
     {
         /*!
             WHAT THIS OBJECT REPRESENTS
-
+                This object defines the interface a detection must implement if it is to be
+                used with the track_association_function defined at the bottom of this
+                file.  In this case, the interface is very simple.  A detection object is
+                only required to define the track_type typedef and it must also be possible
+                to store detection objects in a std::vector.
         !*/
 
     public:
         // Each detection object should be designed to work with a specific track object.
-        // This typedef lets you determine which track type is meant for use with this
+        // This typedef lets us determine which track type is meant for use with this
         // detection object.
         typedef struct example_track track_type;
 
@@ -32,11 +36,14 @@ namespace dlib
     {
         /*!
             WHAT THIS OBJECT REPRESENTS
+                This object defines the interface a track must implement if it is to be
+                used with the track_association_function defined at the bottom of this
+                file.   
         !*/
 
     public:
-        // This type should be a dlib::matrix capable of storing column vectors
-        // or an unsorted sparse vector type as defined in dlib/svm/sparse_vector_abstract.h.
+        // This type should be a dlib::matrix capable of storing column vectors or an
+        // unsorted sparse vector type as defined in dlib/svm/sparse_vector_abstract.h.
         typedef matrix_or_sparse_vector_type feature_vector_type;
 
         example_track(
@@ -75,6 +82,8 @@ namespace dlib
         !*/
     };
 
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
 // ----------------------------------------------------------------------------------------
 
     template <
@@ -83,12 +92,18 @@ namespace dlib
     class feature_extractor_track_association
     {
         /*!
+            REQUIREMENTS ON detection_type
+                It must be an object that implements an interface compatible with the
+                example_detection discussed above.  This also means that detection_type::track_type 
+                must be an object that implements an interface compatible with example_track 
+                defined above.
+
             WHAT THIS OBJECT REPRESENTS 
                 This object is an adapter that converts from the detection/track style
                 interface defined above to the feature extraction interface required by the
                 association rule learning tools in dlib.  Specifically, it converts the
                 detection/track interface into a form usable by the assignment_function and
-                its trainer structural_assignment_trainer.
+                its trainer object structural_assignment_trainer.
         !*/
 
     public:
@@ -115,10 +130,26 @@ namespace dlib
         !*/
     };
 
-    void serialize (const feature_extractor_track_association& item, std::ostream& out);
-    void deserialize (feature_extractor_track_association& item, std::istream& in);
+    template <
+        typename detection_type
+        > 
+    void serialize (
+        const feature_extractor_track_association<detection_type>& item, 
+        std::ostream& out
+    );
     /*!
-        Provides serialization and deserialization support.
+        Provides serialization support.
+    !*/
+
+    template <
+        typename detection_type
+        > 
+    void deserialize (
+        feature_extractor_track_association<detection_type>& item,
+        std::istream& in
+    );
+    /*!
+        Provides deserialization support.
     !*/
 
 // ----------------------------------------------------------------------------------------
@@ -129,7 +160,30 @@ namespace dlib
     class track_association_function
     {
         /*!
+            REQUIREMENTS ON detection_type
+                It must be an object that implements an interface compatible with the
+                example_detection discussed above.  This also means that detection_type::track_type 
+                must be an object that implements an interface compatible with example_track 
+                defined above.
+
             WHAT THIS OBJECT REPRESENTS
+                This object is a tool that helps you implement an object tracker.  So for
+                example, if you wanted to track people moving around in a video then this
+                object can help.  In particular, imagine you have a tool for detecting the
+                positions of each person in an image.  Then you can run this person
+                detector on the video and at each time step, i.e. at each frame, you get a
+                set of person detections.  However, that by itself doesn't tell you how
+                many people there are in the video and where they are moving to and from.
+                To get that information you need to figure out which detections match each
+                other from frame to frame.  This is where the track_association_function
+                comes in.  It performs the detection to track association.  It will also do
+                some of the track management tasks like creating a new track when a
+                detection doesn't match any of the existing tracks.
+
+                Internally, this object is implemented using the assignment_function object.  
+                In fact, it's really just a thin wrapper around assignment_function and
+                exists just to provide a more convenient interface to users doing detection
+                to track association.   
         !*/
     public:
 
@@ -145,7 +199,7 @@ namespace dlib
         !*/
 
         track_association_function (
-            const association_function_type& assoc_
+            const association_function_type& assoc
         ); 
         /*!
             ensures
@@ -181,10 +235,26 @@ namespace dlib
         !*/
     };
 
-    void serialize (const track_association_function& item, std::ostream& out);
-    void deserialize (track_association_function& item, std::istream& in);
+    template <
+        typename detection_type
+        > 
+    void serialize (
+        const track_association_function<detection_type>& item,
+        std::ostream& out
+    );
+    /*!
+        Provides serialization support.
+    !*/
+
+    template <
+        typename detection_type
+        > 
+    void deserialize (
+        track_association_function<detection_type>& item, 
+        std::istream& in
+    );
     /*!
-        Provides serialization and deserialization support.
+        Provides deserialization support.
     !*/
 
 // ----------------------------------------------------------------------------------------