Added a bunch of new multiclass classification tools.

--HG--
extra : convert_revision : svn%3Afdd8eb12-d10e-0410-9acb-85c331704f74/trunk%404047

dlib/svm.h

@@ -28,6 +28,12 @@
 #include "svm/svm_one_class_trainer.h"
 #include "svm/svr_trainer.h"
 
+#include "svm/one_vs_one_decision_function.h"
+#include "svm/one_vs_one_trainer.h"
+#include "svm/multiclass_tools.h"
+#include "svm/cross_validate_multiclass_trainer.h"
+
+
 #endif // DLIB_SVm_HEADER
 
 

dlib/svm/cross_validate_multiclass_trainer.h

+// Copyright (C) 2010  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_CROSS_VALIDATE_MULTICLASS_TRaINER_H__
+#define DLIB_CROSS_VALIDATE_MULTICLASS_TRaINER_H__
+
+#include <vector>
+#include "../matrix.h"
+#include "one_vs_one_trainer.h"
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename dec_funct_type,
+        typename sample_type,
+        typename label_type
+        >
+    const matrix<typename dec_funct_type::scalar_type, 0, 0, typename dec_funct_type::mem_manager_type> 
+    test_multiclass_decision_function (
+        const dec_funct_type& dec_funct,
+        const std::vector<sample_type>& x_test,
+        const std::vector<label_type>& y_test
+    )
+    {
+        typedef typename dec_funct_type::scalar_type scalar_type;
+        typedef typename dec_funct_type::mem_manager_type mem_manager_type;
+
+        // make sure requires clause is not broken
+        DLIB_ASSERT( is_learning_problem(x_test,y_test) == true,
+                    "\tmatrix test_multiclass_decision_function()"
+                    << "\n\t invalid inputs were given to this function"
+                    << "\n\t is_learning_problem(x_test,y_test): " 
+                    << is_learning_problem(x_test,y_test));
+
+
+        const std::vector<label_type> all_labels = dec_funct.get_labels();
+
+        // make a lookup table that maps from labels to their index in all_labels
+        std::map<label_type,unsigned long> label_to_int;
+        for (unsigned long i = 0; i < all_labels.size(); ++i)
+            label_to_int[all_labels[i]] = i;
+
+        matrix<typename dec_funct_type::scalar_type, 0, 0, typename dec_funct_type::mem_manager_type> res;
+        res.set_size(all_labels.size(), all_labels.size());
+
+        res = 0;
+
+        typename std::map<label_type,unsigned long>::const_iterator iter;
+
+        // now test this trained object 
+        for (unsigned long i = 0; i < x_test.size(); ++i)
+        {
+            iter = label_to_int.find(y_test[i]);
+            // ignore samples with labels that the decision function doesn't know about.
+            if (iter == label_to_int.end())
+                continue;
+
+            const unsigned long truth = iter->second;
+            const unsigned long pred  = label_to_int[dec_funct(x_test[i])];
+
+            res(truth,pred) += 1;
+        }
+
+        return res;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename trainer_type,
+        typename sample_type,
+        typename label_type 
+        >
+    const matrix<typename trainer_type::scalar_type, 0, 0, typename trainer_type::mem_manager_type> 
+    cross_validate_multiclass_trainer (
+        const trainer_type& trainer,
+        const std::vector<sample_type>& x,
+        const std::vector<label_type>& y,
+        const long folds
+    )
+    {
+        typedef typename trainer_type::scalar_type scalar_type;
+        typedef typename trainer_type::mem_manager_type mem_manager_type;
+
+        // make sure requires clause is not broken
+        DLIB_ASSERT(is_learning_problem(x,y) == true &&
+                    1 < folds && folds <= static_cast<long>(x.size()),
+            "\tmatrix cross_validate_multiclass_trainer()"
+            << "\n\t invalid inputs were given to this function"
+            << "\n\t x.size(): " << x.size() 
+            << "\n\t folds:  " << folds 
+            << "\n\t is_learning_problem(x,y): " << is_learning_problem(x,y)
+            );
+
+        const std::vector<label_type> all_labels = select_all_distinct_labels(y);
+
+        // count the number of times each label shows up 
+        std::map<label_type,long> label_counts;
+        for (unsigned long i = 0; i < y.size(); ++i)
+            label_counts[y[i]] += 1;
+
+
+        // figure out how many samples from each class will be in the test and train splits 
+        std::map<label_type,long> num_in_test, num_in_train;
+        for (typename std::map<label_type,long>::iterator i = label_counts.begin(); i != label_counts.end(); ++i)
+        {
+            const long in_test = i->second/folds;
+            num_in_test[i->first] = in_test; 
+            num_in_train[i->first] = i->second - in_test;
+        }
+
+
+
+        std::vector<sample_type> x_test, x_train;
+        std::vector<label_type> y_test, y_train;
+
+        matrix<scalar_type, 0, 0, mem_manager_type> res;
+
+        std::map<label_type,long> next_test_idx;
+        for (unsigned long i = 0; i < all_labels.size(); ++i)
+            next_test_idx[all_labels[i]] = 0;
+
+        label_type label;
+
+        for (long i = 0; i < folds; ++i)
+        {
+            x_test.clear();
+            y_test.clear();
+            x_train.clear();
+            y_train.clear();
+
+            // load up the test samples
+            for (unsigned long j = 0; j < all_labels.size(); ++j)
+            {
+                label = all_labels[j];
+                long next = next_test_idx[label];
+
+                long cur = 0;
+                const long num_needed = num_in_test[label];
+                while (cur < num_needed)
+                {
+                    if (y[next] == label)
+                    {
+                        x_test.push_back(x[next]);
+                        y_test.push_back(label);
+                        ++cur;
+                    }
+                    next = (next + 1)%x.size();
+                }
+
+                next_test_idx[label] = next;
+            }
+
+            // load up the training samples
+            for (unsigned long j = 0; j < all_labels.size(); ++j)
+            {
+                label = all_labels[j];
+                long next = next_test_idx[label];
+
+                long cur = 0;
+                const long num_needed = num_in_train[label];
+                while (cur < num_needed)
+                {
+                    if (y[next] == label)
+                    {
+                        x_train.push_back(x[next]);
+                        y_train.push_back(label);
+                        ++cur;
+                    }
+                    next = (next + 1)%x.size();
+                }
+            }
+
+
+            try
+            {
+                // do the training and testing
+                res += test_multiclass_decision_function(trainer.train(x_train,y_train),x_test,y_test);
+            }
+            catch (invalid_nu_error&)
+            {
+                // just ignore cases which result in an invalid nu
+            }
+
+        } // for (long i = 0; i < folds; ++i)
+
+        return res;
+    }
+
+}
+
+// ----------------------------------------------------------------------------------------
+
+#endif // DLIB_CROSS_VALIDATE_MULTICLASS_TRaINER_H__
+

dlib/svm/cross_validate_multiclass_trainer_abstract.h

+// Copyright (C) 2010  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#undef DLIB_CROSS_VALIDATE_MULTICLASS_TRaINER_ABSTRACT_H__
+#ifdef DLIB_CROSS_VALIDATE_MULTICLASS_TRaINER_ABSTRACT_H__
+
+#include <vector>
+#include "../matrix.h"
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename dec_funct_type,
+        typename sample_type,
+        typename label_type
+        >
+    const matrix<typename dec_funct_type::scalar_type, 0, 0, typename dec_funct_type::mem_manager_type> 
+    test_multiclass_decision_function (
+        const dec_funct_type& dec_funct,
+        const std::vector<sample_type>& x_test,
+        const std::vector<label_type>& y_test
+    );
+    /*!
+        requires
+            - is_learning_problem(x_test, y_test)
+            - dec_funct_type == some kind of multiclass decision function object 
+              (e.g. one_vs_one_decision_function)
+        ensures
+            - Tests dec_funct against the given samples in x_test and labels in y_test
+              and returns a confusion matrix summarizing the results.
+            - let L = dec_funct.get_labels().  Then the confusion matrix C returned 
+              by this function has the following properties.
+                - C.nr() == C.nc() == L.size()
+                - C(r,c) == the number of times a sample with label L(r) was predicted
+                  to have a label of L(c)
+            - Any samples with a y_test value not in L are ignored.  That is, samples
+              with labels the decision function hasn't ever seen before are ignored.
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename trainer_type,
+        typename sample_type,
+        typename label_type 
+        >
+    const matrix<typename trainer_type::scalar_type, 0, 0, typename trainer_type::mem_manager_type> 
+    cross_validate_multiclass_trainer (
+        const trainer_type& trainer,
+        const std::vector<sample_type>& x,
+        const std::vector<label_type>& y,
+        const long folds
+    );
+    /*!
+        requires
+            - is_learning_problem(x,y)
+            - 1 < folds <= x.size()
+            - trainer_type == some kind of multiclass classification trainer object (e.g. one_vs_one_trainer)
+        ensures
+            - performs k-fold cross validation by using the given trainer to solve the
+              given multiclass classification problem for the given number of folds.
+              Each fold is tested using the output of the trainer and the confusion
+              matrix from all folds is summed and returned.
+            - The total confusion matrix is computed by running test_binary_decision_function()
+              on each fold and summing its output.
+            - The number of folds used is given by the folds argument.
+            - let L = select_all_distinct_labels(y).  Then the confusion matrix C returned 
+              by this function has the following properties.
+                - C.nr() == C.nc() == L.size()
+                - C(r,c) == the number of times a sample with label L(r) was predicted
+                  to have a label of L(c)
+    !*/
+
+}
+
+// ----------------------------------------------------------------------------------------
+
+#endif // DLIB_CROSS_VALIDATE_MULTICLASS_TRaINER_ABSTRACT_H__
+
+

dlib/svm/multiclass_tools.h

+// Copyright (C) 2010  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_MULTICLASS_TOoLS_H__
+#define DLIB_MULTICLASS_TOoLS_H__
+
+#include "multiclass_tools_abstract.h"
+
+#include <vector>
+#include <set>
+#include "../unordered_pair.h"
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+
+    template <typename label_type>
+    std::vector<label_type> select_all_distinct_labels (
+        const std::vector<label_type>& labels
+    )
+    {
+        std::set<label_type> temp;
+        temp.insert(labels.begin(), labels.end());
+        return std::vector<label_type>(temp.begin(), temp.end());
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <typename label_type, typename U>
+    std::vector<unordered_pair<label_type> > find_missing_pairs (
+        const std::map<unordered_pair<label_type>,U>& bdfs 
+    )
+    {
+        typedef std::map<unordered_pair<label_type>,U> map_type;
+
+        // find all the labels
+        std::set<label_type> temp;
+        for (typename map_type::const_iterator i = bdfs.begin(); i != bdfs.end(); ++i)
+        {
+            temp.insert(i->first.first);
+            temp.insert(i->first.second);
+        }
+
+        std::vector<unordered_pair<label_type> > missing_pairs;
+
+        // now make sure all label pairs are present
+        typename std::set<label_type>::const_iterator i, j;
+        for (i = temp.begin(); i != temp.end(); ++i)
+        {
+            for (j = i, ++j; j != temp.end(); ++j)
+            {
+                const unordered_pair<label_type> p(*i, *j);
+
+                if (bdfs.count(p) == 0)
+                    missing_pairs.push_back(p);
+            }
+        }
+
+        return missing_pairs;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_MULTICLASS_TOoLS_H__
+
+

dlib/svm/multiclass_tools_abstract.h

+// Copyright (C) 2010  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#undef DLIB_MULTICLASS_TOoLS_ABSTRACT_H__
+#ifdef DLIB_MULTICLASS_TOoLS_ABSTRACT_H__
+
+#include <vector>
+#include <map>
+#include "../unordered_pair.h"
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+
+    template <typename label_type>
+    std::vector<label_type> select_all_distinct_labels (
+        const std::vector<label_type>& labels
+    );
+    /*!
+        ensures
+            - Determines all distinct values present in labels and stores them
+              into a sorted vector and returns it.  They are sorted in ascending 
+              order.
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    template <typename label_type, typename U>
+    std::vector<unordered_pair<label_type> > find_missing_pairs (
+        const std::map<unordered_pair<label_type>,U>& binary_decision_functions 
+    );
+    /*!
+        ensures
+            - Let L denote the set of all label_type values present in binary_decision_functions.
+            - This function finds all the label pairs with both elements distinct and in L but
+              not also in binary_decision_functions.  All these missing pairs are stored
+              in a sorted vector and returned.  They are sorted in ascending order.
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_MULTICLASS_TOoLS_ABSTRACT_H__
+

dlib/svm/one_vs_one_decision_function_abstract.h

+// Copyright (C) 2010  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#undef DLIB_ONE_VS_ONE_DECISION_FUnCTION_ABSTRACT_H__
+#ifdef DLIB_ONE_VS_ONE_DECISION_FUnCTION_ABSTRACT_H__
+
+
+#include "../serialize.h"
+#include <map>
+#include "../any/any_decision_function_abstract.h"
+#include "../unordered_pair.h"
+#include "one_vs_one_trainer_abstract.h"
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+
+    struct null_df
+    {
+        /*!
+            WHAT THIS OBJECT REPRESENTS
+                This is a type used to represent an unused field in the 
+                list of template arguments of the one_vs_one_decision_function
+                defined below.  As such, this object doesn't actually do anything.
+        !*/
+        template <typename T>
+        double operator() ( const T&) const { return 0; }
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename one_vs_one_trainer,
+        typename DF1 = null_df, typename DF2 = null_df, typename DF3 = null_df,
+        typename DF4 = null_df, typename DF5 = null_df, typename DF6 = null_df,
+        typename DF7 = null_df, typename DF8 = null_df, typename DF9 = null_df,
+        typename DF10 = null_df
+        >
+    class one_vs_one_decision_function
+    {
+        /*!
+            REQUIREMENTS ON one_vs_one_trainer
+                This should be an instantiation of the one_vs_one_trainer template.  
+                It is used to infer which types are used for various things, such as 
+                representing labels.
+
+            REQUIREMENTS ON DF*
+                These types can either be left at their default values or set
+                to any kind of decision function object capable of being
+                stored in an any_decision_function<sample_type,scalar_type>
+                object.  These types should also be serializable.
+
+            WHAT THIS OBJECT REPRESENTS
+                This object represents a multiclass classifier built out
+                of a set of binary classifiers.  Each binary classifier
+                is used to vote for the correct multiclass label using a 
+                one vs. one strategy.  Therefore, if you have N classes then
+                there will be N*(N-1)/2 binary classifiers inside this object.
+
+                Note that the DF* template arguments are only used if you want
+                to serialize and deserialize one_vs_one_decision_function objects. 
+                Specifically, all the types of binary decision function contained
+                within a one_vs_one_decision_function must be listed in the
+                template arguments if serialization and deserialization is to
+                be used.
+        !*/
+    public:
+
+        typedef typename one_vs_one_trainer::label_type label_type;
+        typedef typename one_vs_one_trainer::sample_type sample_type;
+        typedef typename one_vs_one_trainer::scalar_type scalar_type;
+        typedef typename one_vs_one_trainer::mem_manager_type mem_manager_type;
+
+        typedef std::map<unordered_pair<label_type>, any_decision_function<sample_type, scalar_type> > binary_function_table;
+
+        one_vs_one_decision_function(
+        );
+        /*!
+            ensures
+                - #number_of_classes() == 0
+                - #get_binary_decision_functions().size() == 0
+                - #get_labels().size() == 0
+        !*/
+
+        explicit one_vs_one_decision_function(
+            const binary_function_table& decision_functions
+        ); 
+        /*!
+            requires
+                - find_missing_pairs(decision_functions).size() == 0
+                  (i.e. all pairs of labels have an associated decision function)
+            ensures
+                - #get_binary_decision_functions() == decision_functions
+                - #get_labels() == a list of all the labels which appear in the
+                  given set of decision functions
+                - #number_of_classes() == #get_labels().size() 
+        !*/
+
+        template <
+            typename df1, typename df2, typename df3, typename df4, typename df5,
+            typename df6, typename df7, typename df8, typename df9, typename df10
+            >
+        one_vs_one_decision_function (
+            const one_vs_one_decision_function<one_vs_one_trainer, 
+                                               df1, df2, df3, df4, df5,
+                                               df6, df7, df8, df9, df10>& item
+        ); 
+        /*!
+            ensures
+                - #*this will be a copy of item
+                - #number_of_classes() == item.number_of_classes()
+                - #get_labels() == item.get_labels()
+                - #get_binary_decision_functions() == item.get_binary_decision_functions()
+        !*/
+
+        const binary_function_table& get_binary_decision_functions (
+        ) const;
+        /*!
+            ensures
+                - returns the table of binary decision functions used by this
+                  object.  The correspondence between binary decision functions
+                  and multiclass labels is the following:
+                    - for each element i of get_binary_decision_functions()
+                        - i->first == the label pair associated with binary decision
+                          function i->second.  
+                        - if (decision function i->second outputs a value > 0) then
+                            - i->second is indicating that a test sample should
+                              receive a label of i->first.first
+                        - else
+                            - i->second is indicating that a test sample should
+                              receive a label of i->first.second
+        !*/
+
+        const std::vector<label_type> get_labels (
+        ) const;
+        /*!
+            ensures
+                - returns a vector containing all the labels which can be
+                  predicted by this object.
+        !*/
+
+        unsigned long number_of_classes (
+        ) const;
+        /*!
+            ensures
+                - returns get_labels().size()
+                  (i.e. returns the number of different labels/classes predicted by
+                  this object)
+        !*/
+
+        label_type operator() (
+            const sample_type& sample
+        ) const
+        /*!
+            requires
+                - number_of_classes() != 0
+            ensures
+                - evaluates all the decision functions in get_binary_decision_functions()
+                  and returns the label which received the most votes.
+        !*/
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename T,
+        typename DF1, typename DF2, typename DF3,
+        typename DF4, typename DF5, typename DF6,
+        typename DF7, typename DF8, typename DF9,
+        typename DF10 
+        >
+    void serialize(
+        const one_vs_one_decision_function<T,DF1,DF2,DF3,DF4,DF5,DF6,DF7,DF8,DF9,DF10>& item, 
+        std::ostream& out
+    );
+    /*!
+        ensures
+            - writes the given item to the output stream out.
+        throws
+            - serialization_error.  
+              This is thrown if there is a problem writing to the ostream or if item 
+              contains a type of decision function not listed among the DF* template 
+              arguments.
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename T,
+        typename DF1, typename DF2, typename DF3,
+        typename DF4, typename DF5, typename DF6,
+        typename DF7, typename DF8, typename DF9,
+        typename DF10 
+        >
+    void deserialize(
+        one_vs_one_decision_function<T,DF1,DF2,DF3,DF4,DF5,DF6,DF7,DF8,DF9,DF10>& item, 
+        std::istream& in 
+    );
+    /*!
+        ensures
+            - deserializes a one_vs_one_decision_function from in and stores it in item.
+        throws
+            - serialization_error.  
+              This is thrown if there is a problem reading from the istream or if the
+              serialized data contains decision functions not listed among the DF*
+              template arguments.
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_ONE_VS_ONE_DECISION_FUnCTION_ABSTRACT_H__
+
+
+

dlib/svm/one_vs_one_trainer.h

+// Copyright (C) 2010  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_ONE_VS_ONE_TRAiNER_H__
+#define DLIB_ONE_VS_ONE_TRAiNER_H__
+
+#include "one_vs_one_trainer_abstract.h"
+
+#include "one_vs_one_decision_function.h"
+#include <vector>
+
+#include "../unordered_pair.h"
+#include "multiclass_tools.h"
+
+#include <sstream>
+#include <iostream>
+
+#include "../any.h"
+#include <map>
+#include <set>
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename any_trainer,
+        typename label_type_ = double
+        >
+    class one_vs_one_trainer
+    {
+    public:
+        typedef label_type_ label_type;
+
+        typedef typename any_trainer::sample_type sample_type;
+        typedef typename any_trainer::scalar_type scalar_type;
+        typedef typename any_trainer::mem_manager_type mem_manager_type;
+
+        typedef one_vs_one_decision_function<one_vs_one_trainer> trained_function_type;
+
+        one_vs_one_trainer (
+        ) : 
+            verbose(false)
+        {}
+
+        void set_trainer (
+            const any_trainer& trainer
+        )
+        /*!
+            ensures
+                - sets the trainer used for all pairs of training
+        !*/
+        {
+            default_trainer = trainer;
+            trainers.clear();
+        }
+
+        void set_trainer (
+            const any_trainer& trainer,
+            const label_type& l1,
+            const label_type& l2
+        )
+        /*!
+            requires
+                - l1 != l2
+            ensures
+                - sets the trainer used for just the l1 l2 class pair
+        !*/
+        {
+            trainers[make_unordered_pair(l1,l2)] = trainer;
+        }
+
+        void be_verbose (
+        )
+        {
+            verbose = true;
+        }
+
+        void be_quiet (
+        )
+        {
+            verbose = false;
+        }
+
+        struct invalid_label : public dlib::error 
+        { 
+            invalid_label(const std::string& msg, const label_type& l1_, const label_type& l2_
+                ) : dlib::error(msg), l1(l1_), l2(l2_) {};
+
+            label_type l1, l2;
+        };
+
+        trained_function_type train (
+            const std::vector<sample_type>& all_samples,
+            const std::vector<label_type>& all_labels
+        ) const
+        {
+            const std::vector<label_type> distinct_labels = select_all_distinct_labels(all_labels);
+
+            std::vector<sample_type> samples;
+            std::vector<scalar_type> labels;
+
+
+            typename trained_function_type::binary_function_table dfs;
+
+            for (unsigned long i = 0; i < distinct_labels.size(); ++i)
+            {
+                for (unsigned long j = i+1; j < distinct_labels.size(); ++j)
+                {
+                    samples.clear();
+                    labels.clear();
+
+                    const unordered_pair<label_type> p(distinct_labels[i], distinct_labels[j]);
+
+                    // pick out the samples corresponding to these two classes
+                    for (unsigned long k = 0; k < all_samples.size(); ++k)
+                    {
+                        if (all_labels[k] == p.first)
+                        {
+                            samples.push_back(all_samples[k]);
+                            labels.push_back(+1);
+                        }
+                        else if (all_labels[k] == p.second)
+                        {
+                            samples.push_back(all_samples[k]);
+                            labels.push_back(-1);
+                        }
+                    }
+
+
+                    if (verbose)
+                    {
+                        std::cout << "Training classifier for " << p.first << " vs. " << p.second << std::endl;
+                    }
+
+                    // now train a binary classifier using the samples we selected
+                    const typename binary_function_table::const_iterator itr = trainers.find(p);
+
+                    if (itr != trainers.end())
+                    {
+                        dfs[p] = itr->second.train(samples, labels);
+                    }
+                    else if (default_trainer.is_empty() == false)
+                    {
+                        dfs[p] = default_trainer.train(samples, labels);
+                    }
+                    else
+                    {
+                        std::ostringstream sout;
+                        sout << "In one_vs_one_trainer, no trainer registered for the (" << p.first << ", " << p.second << ") label pair.";
+                        throw invalid_label(sout.str(), p.first, p.second);
+                    }
+                }
+            }
+
+            return trained_function_type(dfs);
+        }
+
+    private:
+
+        any_trainer default_trainer;
+
+        typedef std::map<unordered_pair<label_type>, any_trainer> binary_function_table;
+        binary_function_table trainers;
+
+        bool verbose;
+
+    };
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_ONE_VS_ONE_TRAiNER_H__
+

dlib/svm/one_vs_one_trainer_abstract.h

+// Copyright (C) 2010  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#undef DLIB_ONE_VS_ONE_TRAiNER_ABSTRACT_H__
+#ifdef DLIB_ONE_VS_ONE_TRAiNER_ABSTRACT_H__
+
+
+#include "one_vs_one_decision_function_abstract.h"
+#include <vector>
+
+#include "../any/any_trainer_abstract.h"
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename any_trainer,
+        typename label_type_ = double
+        >
+    class one_vs_one_trainer
+    {
+        /*!
+            REQUIREMENTS ON any_trainer
+                must be an instantiation of the dlib::any_trainer template.   
+
+            REQUIREMENTS ON label_type_
+                label_type_ must be default constructable, copyable, and comparable using
+                operator < and ==.  It must also be possible to write it to an std::ostream
+                using operator<<.
+
+            WHAT THIS OBJECT REPRESENTS
+                This object is a tool for turning a bunch of binary classifiers
+                into a multiclass classifier.  It does this by training the binary
+                classifiers in a one vs. one fashion.  That is, if you have N possible
+                classes then it trains N*(N-1)/2 binary classifiers which are then used
+                to vote on the identity of a test sample.
+
+                This object works with any kind of binary classification trainer object
+                capable of being assigned to an any_trainer object.  (e.g. the svm_nu_trainer) 
+        !*/
+
+    public:
+
+
+        typedef label_type_ label_type;
+
+        typedef typename any_trainer::sample_type sample_type;
+        typedef typename any_trainer::scalar_type scalar_type;
+        typedef typename any_trainer::mem_manager_type mem_manager_type;
+
+        typedef one_vs_one_decision_function<one_vs_one_trainer> trained_function_type;
+
+        one_vs_one_trainer (
+        );
+        /*!
+            ensures
+                - this object is properly initialized
+                - this object will not be verbose unless be_verbose() is called
+                - no binary trainers are associated with *this.  I.e. you have to
+                  call set_trainer() before calling train()
+        !*/
+
+        void set_trainer (
+            const any_trainer& trainer
+        );
+        /*!
+            ensures
+                - sets the trainer used for all pairs of training.  Any previous 
+                  calls to set_trainer() are overridden by this function.  Even the
+                  more specific set_trainer(trainer, l1, l2) form. 
+        !*/
+
+        void set_trainer (
+            const any_trainer& trainer,
+            const label_type& l1,
+            const label_type& l2
+        );
+        /*!
+            requires
+                - l1 != l2
+            ensures
+                - Sets the trainer object used to create binary classifiers to
+                  distinguish l1 labeled samples from l2 labeled samples.
+        !*/
+
+        struct invalid_label : public dlib::error 
+        { 
+            /*!
+                This is the exception thrown by the train() function below.
+            !*/
+            label_type l1, l2;
+        };
+
+        void be_verbose (
+        );
+        /*!
+            ensures
+                - This object will print status messages to standard out so that a 
+                  user can observe the progress of the algorithm.
+        !*/
+
+        void be_quiet (
+        );
+        /*!
+            ensures
+                - this object will not print anything to standard out
+        !*/
+
+        trained_function_type train (
+            const std::vector<sample_type>& all_samples,
+            const std::vector<label_type>& all_labels
+        ) const;
+        /*!
+            requires
+                - is_learning_problem(all_samples, all_labels)
+            ensures
+                - trains a bunch of binary classifiers in a one vs one fashion to solve the given 
+                  multiclass classification problem.  
+                - returns a one_vs_one_decision_function F with the following properties:
+                    - F contains all the learned binary classifiers and can be used to predict
+                      the labels of new samples.
+                    - if (new_x is a sample predicted to have a label of L) then
+                        - F(new_x) == L
+                    - F.get_labels() == select_all_distinct_labels(all_labels)
+                    - F.number_of_classes() == select_all_distinct_labels(all_labels).size()
+            throws
+                - invalid_label
+                  This exception is thrown if there are labels in all_labels which don't have
+                  any corresponding trainer object.  This will never happen if set_trainer(trainer)
+                  has been called.  However, if only the set_trainer(trainer,l1,l2) form has been
+                  used then this exception is thrown if not all necessary label pairs have been
+                  given a trainer.
+
+                  invalid_label::l1 and invalid_label::l2 will contain the label pair which is
+                  missing a trainer object.  Additionally, the exception will contain an
+                  informative error message available via invalid_label::what().
+        !*/
+
+    };
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_ONE_VS_ONE_TRAiNER_ABSTRACT_H__
+
+