Moved the reduced set stuff to its own file. Also added a prototype

version of another reduced set algorithm.

--HG--
extra : convert_revision : svn%3Afdd8eb12-d10e-0410-9acb-85c331704f74/trunk%402419

dlib/svm.h

@@ -10,6 +10,7 @@
 #include "svm/feature_ranking.h"
 #include "svm/rbf_network.h"
 #include "svm/linearly_independent_subset_finder.h"
+#include "svm/reduced.h"
 
 #endif // DLIB_SVm_HEADER
 

dlib/svm/reduced_abstract.h

+// Copyright (C) 2008  Davis E. King (davisking@users.sourceforge.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#undef DLIB_REDUCED_TRAINERs_ABSTRACT_
+#ifdef DLIB_REDUCED_TRAINERs_ABSTRACT_
+
+#include "../matrix.h"
+#include "../algs.h"
+#include "function_abstract.h"
+#include "kernel_abstract.h"
+#include "../optimization.h"
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename trainer_type 
+        >
+    class reduced_decision_function_trainer
+    {
+        /*!
+            WHAT THIS OBJECT REPRESENTS
+                This object represents an implementation of a reduced set algorithm
+                for support vector decision functions.  This object acts as a post
+                processor for anything that creates decision_function objects.  It
+                wraps another trainer object and performs this reduced set post 
+                processing with the goal of representing the original decision 
+                function in a form that involves fewer support vectors.
+        !*/
+
+    public:
+        typedef typename trainer_type::kernel_type kernel_type;
+        typedef typename trainer_type::scalar_type scalar_type;
+        typedef typename trainer_type::sample_type sample_type;
+        typedef typename trainer_type::mem_manager_type mem_manager_type;
+        typedef typename trainer_type::trained_function_type trained_function_type;
+
+        explicit reduced_decision_function_trainer (
+            const trainer_type& trainer,
+            const unsigned long num_sv 
+        );
+        /*!
+            requires
+                - num_sv > 0
+                - trainer_type == some kind of trainer object (e.g. svm_nu_trainer)
+            ensures
+                - returns a trainer object that applies post processing to the decision_function
+                  objects created by the given trainer object with the goal of creating
+                  decision_function objects with fewer support vectors.
+                - The reduced decision functions that are output will have at most
+                  num_sv support vectors.
+        !*/
+
+        template <
+            typename in_sample_vector_type,
+            typename in_scalar_vector_type
+            >
+        const decision_function<kernel_type> train (
+            const in_sample_vector_type& x,
+            const in_scalar_vector_type& y
+        ) const;
+        /*!
+            ensures
+                - trains a decision_function using the trainer that was supplied to
+                  this object's constructor and then finds a reduced representation
+                  for it and returns the reduced version.  
+            throws
+                - std::bad_alloc
+                - any exceptions thrown by the trainer_type object
+        !*/
+
+    }; 
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename trainer_type
+        >
+    const reduced_decision_function_trainer<trainer_type> reduced (
+        const trainer_type& trainer,
+        const unsigned long num_sv
+    ) { return reduced_decision_function_trainer<trainer_type>(trainer, num_sv); }
+    /*!
+        requires
+            - num_sv > 0
+            - trainer_type == some kind of trainer object that creates decision_function
+              objects (e.g. svm_nu_trainer)
+        ensures
+            - returns a reduced_decision_function_trainer object that has been
+              instantiated with the given arguments.
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_REDUCED_TRAINERs_ABSTRACT_
+

dlib/svm/svm.h

@@ -15,8 +15,6 @@
 #include "function.h"
 #include "kernel.h"
 #include "../enable_if.h"
-#include "kcentroid.h"
-#include "linearly_independent_subset_finder.h"
 
 namespace dlib
 {
@@ -1424,146 +1422,6 @@ namespace dlib
 // ----------------------------------------------------------------------------------------
 // ----------------------------------------------------------------------------------------
 // ----------------------------------------------------------------------------------------
-
-    template <
-        typename trainer_type 
-        >
-    class reduced_decision_function_trainer
-    {
-    public:
-        typedef typename trainer_type::kernel_type kernel_type;
-        typedef typename trainer_type::scalar_type scalar_type;
-        typedef typename trainer_type::sample_type sample_type;
-        typedef typename trainer_type::mem_manager_type mem_manager_type;
-        typedef typename trainer_type::trained_function_type trained_function_type;
-
-        explicit reduced_decision_function_trainer (
-            const trainer_type& trainer_,
-            const unsigned long num_sv_ 
-        ) :
-            trainer(trainer_),
-            num_sv(num_sv_)
-        {
-        }
-
-        template <
-            typename in_sample_vector_type,
-            typename in_scalar_vector_type
-            >
-        const decision_function<kernel_type> train (
-            const in_sample_vector_type& x,
-            const in_scalar_vector_type& y
-        ) const
-        {
-            return do_train(vector_to_matrix(x), vector_to_matrix(y));
-        }
-
-    private:
-
-    // ------------------------------------------------------------------------------------
-
-        template <
-            typename in_sample_vector_type,
-            typename in_scalar_vector_type
-            >
-        const decision_function<kernel_type> do_train (
-            const in_sample_vector_type& x,
-            const in_scalar_vector_type& y
-        ) const
-        {
-            // make sure requires clause is not broken
-            DLIB_ASSERT(is_binary_classification_problem(x,y) == true,
-                "\tdecision_function reduced_decision_function_trainer::train(x,y)"
-                << "\n\t invalid inputs were given to this function"
-                << "\n\t x.nr(): " << x.nr() 
-                << "\n\t y.nr(): " << y.nr() 
-                << "\n\t x.nc(): " << x.nc() 
-                << "\n\t y.nc(): " << y.nc() 
-                << "\n\t is_binary_classification_problem(x,y): " << ((is_binary_classification_problem(x,y))? "true":"false")
-                );
-
-            // get the decision function object we are going to try and approximate
-            const decision_function<kernel_type> dec_funct = trainer.train(x,y);
-            
-            // now find a linearly independent subset of the training points of num_sv points.
-            linearly_independent_subset_finder<kernel_type> lisf(trainer.get_kernel(), num_sv);
-            for (long i = 0; i < x.nr(); ++i)
-            {
-                lisf.add(x(i));
-            }
-
-            // make num be the number of points in the lisf object.  Just do this so we don't have
-            // to write out lisf.dictionary_size() all over the place.
-            const long num = lisf.dictionary_size();
-
-            // The next few blocks of code just find the best weights with which to approximate 
-            // the dec_funct object with the smaller set of vectors in the lisf dictionary.  This
-            // is really just a simple application of some linear algebra.  For the details 
-            // see page 554 of Learning with kernels by Scholkopf and Smola where they talk 
-            // about "Optimal Expansion Coefficients."
-            matrix<scalar_type, 0, 0, mem_manager_type> K_inv(num, num); 
-            matrix<scalar_type, 0, 0, mem_manager_type> K(num, dec_funct.alpha.size()); 
-
-            const kernel_type kernel(trainer.get_kernel());
-
-            for (long r = 0; r < K_inv.nr(); ++r)
-            {
-                for (long c = 0; c < K_inv.nc(); ++c)
-                {
-                    K_inv(r,c) = kernel(lisf[r], lisf[c]);
-                }
-            }
-            K_inv = pinv(K_inv);
-
-
-            for (long r = 0; r < K.nr(); ++r)
-            {
-                for (long c = 0; c < K.nc(); ++c)
-                {
-                    K(r,c) = kernel(lisf[r], dec_funct.support_vectors(c));
-                }
-            }
-
-
-            // Now we compute the approximate decision function.  Note that the weights come out
-            // of the expression K_inv*K*dec_funct.alpha.
-            decision_function<kernel_type> new_df(K_inv*K*dec_funct.alpha, 
-                                                  0,
-                                                  kernel, 
-                                                  lisf.get_dictionary());
-
-            // now we have to figure out what the new bias should be.  It might be a little
-            // different since we just messed with all the weights and vectors.
-            double bias = 0;
-            for (long i = 0; i < x.nr(); ++i)
-            {
-                bias += new_df(x(i)) - dec_funct(x(i));
-            }
-            
-            new_df.b = bias/x.nr();
-
-            return new_df;
-        }
-
-    // ------------------------------------------------------------------------------------
-
-        const trainer_type& trainer;
-        const unsigned long num_sv; 
-
-
-    }; // end of class reduced_decision_function_trainer
-
-    template <typename trainer_type>
-    const reduced_decision_function_trainer<trainer_type> reduced (
-        const trainer_type& trainer,
-        const unsigned long num_sv
-    )
-    {
-        return reduced_decision_function_trainer<trainer_type>(trainer, num_sv);
-    }
-
-// ----------------------------------------------------------------------------------------
-
 }
 
 #endif // DLIB_SVm_

dlib/svm/svm_abstract.h

@@ -296,86 +296,6 @@ namespace dlib
             - std::bad_alloc
     !*/
 
-// ----------------------------------------------------------------------------------------
-
-    template <
-        typename trainer_type 
-        >
-    class reduced_decision_function_trainer
-    {
-        /*!
-            WHAT THIS OBJECT REPRESENTS
-                This object represents an implementation of a reduced set algorithm
-                for support vector decision functions.  This object acts as a post
-                processor for anything that creates decision_function objects.  It
-                wraps another trainer object and performs this reduced set post 
-                processing with the goal of representing the original decision 
-                function in a form that involves fewer support vectors.
-        !*/
-
-    public:
-        typedef typename trainer_type::kernel_type kernel_type;
-        typedef typename trainer_type::scalar_type scalar_type;
-        typedef typename trainer_type::sample_type sample_type;
-        typedef typename trainer_type::mem_manager_type mem_manager_type;
-        typedef typename trainer_type::trained_function_type trained_function_type;
-
-        explicit reduced_decision_function_trainer (
-            const trainer_type& trainer_,
-            const scalar_type tolerance_ = 0.001
-        );
-        /*!
-            requires
-                - tolerance >= 0
-                - trainer_type == some kind of trainer object (e.g. svm_nu_trainer)
-            ensures
-                - returns a trainer object that applies post processing to the decision_function
-                  objects created by the given trainer object with the goal of creating
-                  decision_function objects with fewer support vectors.
-                - tolerance == a parameter that controls how accurate the post processing
-                  is in preserving the original decision_function.  Larger values
-                  result in a decision_function with fewer support vectors but may
-                  decrease the accuracy of the resulting decision_function.
-        !*/
-
-        template <
-            typename in_sample_vector_type,
-            typename in_scalar_vector_type
-            >
-        const decision_function<kernel_type> train (
-            const in_sample_vector_type& x,
-            const in_scalar_vector_type& y
-        ) const;
-        /*!
-            ensures
-                - trains a decision_function using the trainer that was supplied to
-                  this object's constructor and then finds a reduced representation
-                  for it and returns the reduced version.  
-            throws
-                - std::bad_alloc
-                - any exceptions thrown by the trainer_type object
-        !*/
-
-    }; 
-
-    template <
-        typename trainer_type
-        >
-    const reduced_decision_function_trainer<trainer_type> reduced (
-        const trainer_type& trainer,
-        const typename trainer_type::scalar_type& tolerance = 0.001
-    ) { return reduced_decision_function_trainer<trainer_type>(trainer, tolerance); }
-    /*!
-        requires
-            - tolerance >= 0
-            - trainer_type == some kind of trainer object that creates decision_function
-              objects (e.g. svm_nu_trainer)
-        ensures
-            - returns a reduced_decision_function_trainer object that has been
-              instantiated with the given arguments.
-    !*/
-
-
 // ----------------------------------------------------------------------------------------
 // ----------------------------------------------------------------------------------------
 //                                  Miscellaneous functions