Skip to content

D
dlib
Commit 3c7c8fee authored by Davis King's avatar Davis King
Browse files
Options

Refactored a bunch of the svm training code into a much cleaner form. 

--HG--
extra : convert_revision : svn%3Afdd8eb12-d10e-0410-9acb-85c331704f74/trunk%402380
parent b51cc5c2
Hide whitespace changes
Inline Side-by-side
Showing with 983 additions and 682 deletions
dlib/svm/svm.h
View file @ 3c7c8fee
...@@ -19,12 +19,21 @@ ...@@ -19,12 +19,21 @@
namespace dlib namespace dlib
{ {
// ----------------------------------------------------------------------------------------
class invalid_svm_nu_error : public dlib::error
{
public:
invalid_svm_nu_error(const std::string& msg, double nu_) : dlib::error(msg), nu(nu_) {};
const double nu;
};
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
template < template <
typename T typename T
> >
typename T::type maximum_nu ( typename T::type maximum_nu_impl (
const T& y const T& y
) )
{ {
...@@ -63,10 +72,67 @@ namespace dlib ...@@ -63,10 +72,67 @@ namespace dlib
return static_cast<scalar_type>(2.0*(scalar_type)std::min(pos_count,neg_count)/(scalar_type)y.nr()); return static_cast<scalar_type>(2.0*(scalar_type)std::min(pos_count,neg_count)/(scalar_type)y.nr());
} }
template <
typename T
>
typename T::type maximum_nu (
const T& y
)
{
return maximum_nu_impl(vector_to_matrix(y));
}
template <
typename T
>
typename T::value_type maximum_nu (
const T& y
)
{
return maximum_nu_impl(vector_to_matrix(y));
}
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
template < template <
typename K typename T,
typename U
>
bool is_binary_classification_problem_impl (
const T& x,
const U& x_labels
)
{
if (x.nc() != 1 || x_labels.nc() != 1) return false;
if (x.nr() != x_labels.nr()) return false;
if (x.nr() <= 1) return false;
for (long r = 0; r < x_labels.nr(); ++r)
{
if (x_labels(r) != -1 && x_labels(r) != 1)
return false;
}
return true;
}
template <
typename T,
typename U
>
bool is_binary_classification_problem (
const T& x,
const U& x_labels
)
{
return is_binary_classification_problem_impl(vector_to_matrix(x), vector_to_matrix(x_labels));
}
// ----------------------------------------------------------------------------------------
template <
typename K,
typename sample_vector_type,
typename scalar_vector_type
> >
class kernel_matrix_cache class kernel_matrix_cache
{ {
...@@ -74,8 +140,8 @@ namespace dlib ...@@ -74,8 +140,8 @@ namespace dlib
typedef typename K::sample_type sample_type; typedef typename K::sample_type sample_type;
typedef typename K::mem_manager_type mem_manager_type; typedef typename K::mem_manager_type mem_manager_type;
const matrix<sample_type,0,1,mem_manager_type>& x; const sample_vector_type& x;
const matrix<scalar_type,0,1,mem_manager_type>& y; const scalar_vector_type& y;
K kernel_function; K kernel_function;
mutable matrix<scalar_type,0,0,mem_manager_type> cache; mutable matrix<scalar_type,0,0,mem_manager_type> cache;
...@@ -104,8 +170,8 @@ namespace dlib ...@@ -104,8 +170,8 @@ namespace dlib
public: public:
kernel_matrix_cache ( kernel_matrix_cache (
const matrix<sample_type,0,1,mem_manager_type>& x_, const sample_vector_type& x_,
const matrix<scalar_type,0,1,mem_manager_type>& y_, const scalar_vector_type& y_,
K kernel_function_, K kernel_function_,
long max_size_megabytes long max_size_megabytes
) : x(x_), y(y_), kernel_function(kernel_function_) ) : x(x_), y(y_), kernel_function(kernel_function_)
...@@ -180,530 +246,37 @@ namespace dlib ...@@ -180,530 +246,37 @@ namespace dlib
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
template < template <
typename scalar_type, typename trainer_type,
typename scalar_vector_type typename in_sample_vector_type,
> typename in_scalar_vector_type
inline void set_initial_alpha (
const scalar_vector_type& y,
const scalar_type nu,
scalar_vector_type& alpha
)
{
set_all_elements(alpha,0);
const scalar_type l = y.nr();
scalar_type temp = nu*l/2;
long num = (long)std::floor(temp);
long num_total = (long)std::ceil(temp);
int count = 0;
for (int i = 0; i < alpha.nr(); ++i)
{
if (y(i) == 1)
{
if (count < num)
{
++count;
alpha(i) = 1;
}
else
{
if (temp > num)
{
++count;
alpha(i) = temp - std::floor(temp);
}
break;
}
}
}
if (count != num_total)
{
std::ostringstream sout;
sout << "invalid nu of " << nu << ". Must be between 0 and " << (scalar_type)count/y.nr();
throw error(sout.str());
}
count = 0;
for (int i = 0; i < alpha.nr(); ++i)
{
if (y(i) == -1)
{
if (count < num)
{
++count;
alpha(i) = 1;
}
else
{
if (temp > num)
{
++count;
alpha(i) = temp - std::floor(temp);
}
break;
}
}
}
if (count != num_total)
{
std::ostringstream sout;
sout << "invalid nu of " << nu << ". Must be between 0 and " << (scalar_type)count/y.nr();
throw error(sout.str());
}
}
// ----------------------------------------------------------------------------------------
template <
typename K,
typename scalar_vector_type,
typename scalar_type
>
inline bool find_working_group (
const scalar_vector_type& y,
const scalar_vector_type& alpha,
const kernel_matrix_cache<K>& Q,
const scalar_vector_type& df,
const scalar_type tau,
const scalar_type eps,
long& i_out,
long& j_out
)
{
using namespace std;
long ip = -1;
long jp = -1;
long in = -1;
long jn = -1;
scalar_type ip_val = -numeric_limits<scalar_type>::infinity();
scalar_type jp_val = numeric_limits<scalar_type>::infinity();
scalar_type in_val = -numeric_limits<scalar_type>::infinity();
scalar_type jn_val = numeric_limits<scalar_type>::infinity();
// loop over the alphas and find the maximum ip and in indices.
for (long i = 0; i < alpha.nr(); ++i)
{
if (y(i) == 1)
{
if (alpha(i) < 1.0)
{
if (-df(i) > ip_val)
{
ip_val = -df(i);
ip = i;
}
}
}
else
{
if (alpha(i) > 0.0)
{
if (df(i) > in_val)
{
in_val = df(i);
in = i;
}
}
}
}
scalar_type Mp = numeric_limits<scalar_type>::infinity();
scalar_type Mn = numeric_limits<scalar_type>::infinity();
scalar_type bp = -numeric_limits<scalar_type>::infinity();
scalar_type bn = -numeric_limits<scalar_type>::infinity();
// now we need to find the minimum jp and jn indices
for (long j = 0; j < alpha.nr(); ++j)
{
if (y(j) == 1)
{
if (alpha(j) > 0.0)
{
scalar_type b = ip_val + df(j);
if (-df(j) < Mp)
Mp = -df(j);
if (b > 0 && (Q.is_cached(j) || b > bp || jp == -1 ))
{
bp = b;
scalar_type a = Q(ip,ip) + Q(j,j) - 2*Q(j,ip);
if (a <= 0)
a = tau;
scalar_type temp = -b*b/a;
if (temp < jp_val)
{
jp_val = temp;
jp = j;
}
}
}
}
else
{
if (alpha(j) < 1.0)
{
scalar_type b = in_val - df(j);
if (df(j) < Mn)
Mn = df(j);
if (b > 0 && (Q.is_cached(j) || b > bn || jn == -1 ))
{
bn = b;
scalar_type a = Q(in,in) + Q(j,j) - 2*Q(j,in);
if (a <= 0)
a = tau;
scalar_type temp = -b*b/a;
if (temp < jn_val)
{
jn_val = temp;
jn = j;
}
}
}
}
}
// if we are at the optimal point then return false so the caller knows
// to stop optimizing
if (std::max(ip_val - Mp, in_val - Mn) < eps)
return false;
if (jp_val < jn_val)
{
i_out = ip;
j_out = jp;
}
else
{
i_out = in;
j_out = jn;
}
if (j_out >= 0 && i_out >= 0)
return true;
else
return false;
}
// ----------------------------------------------------------------------------------------
template <
typename scalar_vector_type,
typename scalar_type
> >
void calculate_rho_and_b( const matrix<typename trainer_type::scalar_type, 1, 2, typename trainer_type::mem_manager_type>
const scalar_vector_type& y, cross_validate_trainer_impl (
const scalar_vector_type& alpha, const trainer_type& trainer,
const scalar_vector_type& df, const in_sample_vector_type& x,
scalar_type& rho, const in_scalar_vector_type& y,
scalar_type& b const long folds
) )
{ {
using namespace std; typedef typename trainer_type::scalar_type scalar_type;
long num_p_free = 0; typedef typename trainer_type::sample_type sample_type;
long num_n_free = 0; typedef typename trainer_type::mem_manager_type mem_manager_type;
scalar_type sum_p_free = 0; typedef matrix<sample_type,0,1,mem_manager_type> sample_vector_type;
scalar_type sum_n_free = 0; typedef matrix<scalar_type,0,1,mem_manager_type> scalar_vector_type;
scalar_type upper_bound_p = -numeric_limits<scalar_type>::infinity();
scalar_type upper_bound_n = -numeric_limits<scalar_type>::infinity();
scalar_type lower_bound_p = numeric_limits<scalar_type>::infinity();
scalar_type lower_bound_n = numeric_limits<scalar_type>::infinity();
for(long i = 0; i < alpha.nr(); ++i)
{
if(y(i) == 1)
{
if(alpha(i) == 1)
{
if (df(i) > upper_bound_p)
upper_bound_p = df(i);
}
else if(alpha(i) == 0)
{
if (df(i) < lower_bound_p)
lower_bound_p = df(i);
}
else
{
++num_p_free;
sum_p_free += df(i);
}
}
else
{
if(alpha(i) == 1)
{
if (df(i) > upper_bound_n)
upper_bound_n = df(i);
}
else if(alpha(i) == 0)
{
if (df(i) < lower_bound_n)
lower_bound_n = df(i);
}
else
{
++num_n_free;
sum_n_free += df(i);
}
}
}
scalar_type r1,r2;
if(num_p_free > 0)
r1 = sum_p_free/num_p_free;
else
r1 = (upper_bound_p+lower_bound_p)/2;
if(num_n_free > 0)
r2 = sum_n_free/num_n_free;
else
r2 = (upper_bound_n+lower_bound_n)/2;
rho = (r1+r2)/2;
b = (r1-r2)/2/rho;
}
// ----------------------------------------------------------------------------------------
template <
typename K,
typename scalar_vector_type,
typename scalar_type
>
inline void optimize_working_pair (
const scalar_vector_type& y,
scalar_vector_type& alpha,
const kernel_matrix_cache<K>& Q,
const scalar_vector_type& df,
const scalar_type tau,
const long i,
const long j
)
{
scalar_type quad_coef = Q(i,i)+Q(j,j)-2*Q(j,i);
if (quad_coef <= 0)
quad_coef = tau;
scalar_type delta = (df(i)-df(j))/quad_coef;
scalar_type sum = alpha(i) + alpha(j);
alpha(i) -= delta;
alpha(j) += delta;
if(sum > 1)
{
if(alpha(i) > 1)
{
alpha(i) = 1;
alpha(j) = sum - 1;
}
else if(alpha(j) > 1)
{
alpha(j) = 1;
alpha(i) = sum - 1;
}
}
else
{
if(alpha(j) < 0)
{
alpha(j) = 0;
alpha(i) = sum;
}
else if(alpha(i) < 0)
{
alpha(i) = 0;
alpha(j) = sum;
}
}
}
// ----------------------------------------------------------------------------------------
template <
typename K
>
const decision_function<K> svm_nu_train (
const typename decision_function<K>::sample_vector_type& x,
const typename decision_function<K>::scalar_vector_type& y,
const K& kernel_function,
const typename K::scalar_type nu,
const long cache_size = 200,
const typename K::scalar_type eps = 0.001
)
{
typedef typename K::scalar_type scalar_type;
typedef typename decision_function<K>::sample_vector_type sample_vector_type;
typedef typename decision_function<K>::scalar_vector_type scalar_vector_type;
// make sure requires clause is not broken // make sure requires clause is not broken
#ifdef ENABLE_ASSERTS DLIB_ASSERT(is_binary_classification_problem(x,y) == true &&
for (long r = 0; r < y.nr(); ++r) 1 < folds && folds <= x.nr(),
{ "\tmatrix cross_validate_trainer()"
DLIB_ASSERT(y(r) == -1.0 || y(r) == 1.0, << "\n\t invalid inputs were given to this function"
"\tdecision_function svm_nu_train()" << "\n\t x.nr(): " << x.nr()
<< "\n\tinvalid inputs were given to this function" << "\n\t y.nr(): " << y.nr()
<< "\n\tr: " << r << "\n\t x.nc(): " << x.nc()
<< "\n\ty(r): " << y(r) << "\n\t y.nc(): " << y.nc()
<< "\n\t folds: " << folds
<< "\n\t is_binary_classification_problem(x,y): " << ((is_binary_classification_problem(x,y))? "true":"false")
); );
}
#endif
DLIB_ASSERT(x.nr() > 1 && y.nr() == x.nr(),
"\tdecision_function svm_nu_train()"
<< "\n\tinvalid inputs were given to this function"
<< "\n\tx.nr(): " << x.nr()
<< "\n\ty.nr(): " << y.nr()
<< "\n\tx.nc(): " << x.nc()
<< "\n\ty.nc(): " << y.nc()
);
DLIB_ASSERT(eps > 0 &&
cache_size > 0 &&
0 < nu && nu < maximum_nu(y),
"\tdecision_function svm_nu_train()"
<< "\n\tinvalid inputs were given to this function"
<< "\n\teps: " << eps
<< "\n\tcache_size: " << cache_size
<< "\n\tnu: " << nu
<< "\n\tmaximum_nu(y): " << maximum_nu(y)
);
const scalar_type tau = 1e-12;
scalar_vector_type df; // delta f(alpha)
scalar_vector_type alpha;
kernel_matrix_cache<K> Q(x,y,kernel_function,cache_size);
alpha.set_size(x.nr());
df.set_size(x.nr());
// now initialize alpha
set_initial_alpha(y, nu, alpha);
// initialize df. Compute df = Q*alpha
for (long r = 0; r < df.nr(); ++r)
{
df(r) = 0;
for (long c = 0; c < alpha.nr(); ++c)
{
if (alpha(c) != 0)
df(r) += Q(c,r)*alpha(c);
}
}
// now perform the actual optimization of alpha
long i, j;
while (find_working_group(y,alpha,Q,df,tau,eps,i,j))
{
const scalar_type old_alpha_i = alpha(i);
const scalar_type old_alpha_j = alpha(j);
optimize_working_pair(y,alpha,Q,df,tau,i,j);
// update the df vector now that we have modified alpha(i) and alpha(j)
scalar_type delta_alpha_i = alpha(i) - old_alpha_i;
scalar_type delta_alpha_j = alpha(j) - old_alpha_j;
for(long k = 0; k < df.nr(); ++k)
df(k) += Q(k,i)*delta_alpha_i + Q(k,j)*delta_alpha_j;
}
scalar_type rho, b;
calculate_rho_and_b(y,alpha,df,rho,b);
alpha = pointwise_multiply(alpha,y)/rho;
// count the number of support vectors
long sv_count = 0;
for (long i = 0; i < alpha.nr(); ++i)
{
if (alpha(i) != 0)
++sv_count;
}
scalar_vector_type sv_alpha;
sample_vector_type support_vectors;
// size these column vectors so that they have an entry for each support vector
sv_alpha.set_size(sv_count);
support_vectors.set_size(sv_count);
// load the support vectors and their alpha values into these new column matrices
long idx = 0;
for (long i = 0; i < alpha.nr(); ++i)
{
if (alpha(i) != 0)
{
sv_alpha(idx) = alpha(i);
support_vectors(idx) = x(i);
++idx;
}
}
// now return the decision function
return decision_function<K> (sv_alpha, b, kernel_function, support_vectors);
}
// ----------------------------------------------------------------------------------------
template <
typename K
>
const matrix<typename K::scalar_type, 1, 2, typename K::mem_manager_type> svm_nu_cross_validate (
const typename decision_function<K>::sample_vector_type& x,
const typename decision_function<K>::scalar_vector_type& y,
const K& kernel_function,
const typename K::scalar_type nu,
const long folds,
const long cache_size = 200,
const typename K::scalar_type eps = 0.001
)
{
typedef typename K::scalar_type scalar_type;
typedef typename decision_function<K>::sample_vector_type sample_vector_type;
typedef typename decision_function<K>::scalar_vector_type scalar_vector_type;
// make sure requires clause is not broken
#ifdef ENABLE_ASSERTS
for (long r = 0; r < y.nr(); ++r)
{
DLIB_ASSERT(y(r) == -1.0 || y(r) == 1.0,
"\tdecision_function svm_nu_cross_validate()"
<< "\n\tinvalid inputs were given to this function"
<< "\n\tr: " << r
<< "\n\ty(r): " << y(r)
);
}
#endif
DLIB_ASSERT(x.nr() > 1 && y.nr() == x.nr(),
"\tdecision_function svm_nu_cross_validate()"
<< "\n\tinvalid inputs were given to this function"
<< "\n\tx.nr(): " << x.nr()
<< "\n\ty.nr(): " << y.nr()
<< "\n\tx.nc(): " << x.nc()
<< "\n\ty.nc(): " << y.nc()
);
DLIB_ASSERT(eps > 0 &&
folds > 1 && folds <= x.nr() &&
cache_size > 0 &&
0 < nu && nu < maximum_nu(y),
"\tdecision_function svm_nu_cross_validate()"
<< "\n\tinvalid inputs were given to this function"
<< "\n\teps: " << eps
<< "\n\tfolds: " << folds
<< "\n\tcache_size: " << cache_size
<< "\n\tnu: " << nu
<< "\n\tmaximum_nu(y): " << maximum_nu(y)
);
// count the number of positive and negative examples // count the number of positive and negative examples
long num_pos = 0; long num_pos = 0;
...@@ -725,9 +298,9 @@ namespace dlib ...@@ -725,9 +298,9 @@ namespace dlib
long num_pos_correct = 0; long num_pos_correct = 0;
long num_neg_correct = 0; long num_neg_correct = 0;
decision_function<K> d; typename trainer_type::trained_function_type d;
typename decision_function<K>::sample_vector_type x_test, x_train; sample_vector_type x_test, x_train;
typename decision_function<K>::scalar_vector_type y_test, y_train; scalar_vector_type y_test, y_train;
x_test.set_size (num_pos_test_samples + num_neg_test_samples); x_test.set_size (num_pos_test_samples + num_neg_test_samples);
y_test.set_size (num_pos_test_samples + num_neg_test_samples); y_test.set_size (num_pos_test_samples + num_neg_test_samples);
x_train.set_size(num_pos_train_samples + num_neg_train_samples); x_train.set_size(num_pos_train_samples + num_neg_train_samples);
...@@ -795,7 +368,7 @@ namespace dlib ...@@ -795,7 +368,7 @@ namespace dlib
} }
// do the training // do the training
d = svm_nu_train (x_train,y_train,kernel_function,nu,cache_size,eps); d = trainer.train(x_train,y_train);
// now test this fold // now test this fold
for (long i = 0; i < x_test.nr(); ++i) for (long i = 0; i < x_test.nr(); ++i)
...@@ -813,37 +386,56 @@ namespace dlib ...@@ -813,37 +386,56 @@ namespace dlib
} }
else else
{ {
throw dlib::error("invalid input labels to the svm_nu_cross_validate() function"); throw dlib::error("invalid input labels to the cross_validate_trainer() function");
} }
} }
} // for (long i = 0; i < folds; ++i) } // for (long i = 0; i < folds; ++i)
matrix<typename K::scalar_type, 1, 2, typename K::mem_manager_type> res; matrix<scalar_type, 1, 2, mem_manager_type> res;
res(0) = (scalar_type)num_pos_correct/(scalar_type)(num_pos_test_samples*folds); res(0) = (scalar_type)num_pos_correct/(scalar_type)(num_pos_test_samples*folds);
res(1) = (scalar_type)num_neg_correct/(scalar_type)(num_neg_test_samples*folds); res(1) = (scalar_type)num_neg_correct/(scalar_type)(num_neg_test_samples*folds);
return res; return res;
} }
template <
typename trainer_type,
typename in_sample_vector_type,
typename in_scalar_vector_type
>
const matrix<typename trainer_type::scalar_type, 1, 2, typename trainer_type::mem_manager_type>
cross_validate_trainer (
const trainer_type& trainer,
const in_sample_vector_type& x,
const in_scalar_vector_type& y,
const long folds
)
{
return cross_validate_trainer_impl(trainer,
vector_to_matrix(x),
vector_to_matrix(y),
folds);
}
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
template < template <
typename K typename trainer_type,
typename in_sample_vector_type,
typename in_scalar_vector_type
> >
const probabilistic_decision_function<K> svm_nu_train_prob ( const probabilistic_decision_function<typename trainer_type::kernel_type> train_probabilistic_decision_function_impl (
const typename decision_function<K>::sample_vector_type& x, const trainer_type& trainer,
const typename decision_function<K>::scalar_vector_type& y, const in_sample_vector_type& x,
const K& kernel_function, const in_scalar_vector_type& y,
const typename K::scalar_type nu, const long folds
const long folds,
const long cache_size = 200,
const typename K::scalar_type eps = 0.001
) )
{ {
typedef typename K::scalar_type scalar_type; typedef typename trainer_type::sample_type sample_type;
typedef typename K::mem_manager_type mem_manager_type; typedef typename trainer_type::scalar_type scalar_type;
typedef typename decision_function<K>::sample_vector_type sample_vector_type; typedef typename trainer_type::mem_manager_type mem_manager_type;
typedef typename decision_function<K>::scalar_vector_type scalar_vector_type; typedef typename trainer_type::kernel_type K;
/* /*
This function fits a sigmoid function to the output of the This function fits a sigmoid function to the output of the
...@@ -856,39 +448,16 @@ namespace dlib ...@@ -856,39 +448,16 @@ namespace dlib
*/ */
// make sure requires clause is not broken // make sure requires clause is not broken
#ifdef ENABLE_ASSERTS DLIB_ASSERT(is_binary_classification_problem(x,y) == true &&
for (long r = 0; r < y.nr(); ++r) 1 < folds && folds <= x.nr(),
{ "\tprobabilistic_decision_function train_probabilistic_decision_function()"
DLIB_ASSERT(y(r) == -1.0 || y(r) == 1.0, << "\n\t invalid inputs were given to this function"
"\tdecision_function svm_nu_train()" << "\n\t x.nr(): " << x.nr()
<< "\n\tinvalid inputs were given to this function" << "\n\t y.nr(): " << y.nr()
<< "\n\tr: " << r << "\n\t x.nc(): " << x.nc()
<< "\n\ty(r): " << y(r) << "\n\t y.nc(): " << y.nc()
); << "\n\t folds: " << folds
<< "\n\t is_binary_classification_problem(x,y): " << ((is_binary_classification_problem(x,y))? "true":"false")
}
#endif
DLIB_ASSERT(x.nr() > 1 && y.nr() == x.nr(),
"\tdecision_function svm_nu_train()"
<< "\n\tinvalid inputs were given to this function"
<< "\n\tx.nr(): " << x.nr()
<< "\n\ty.nr(): " << y.nr()
<< "\n\tx.nc(): " << x.nc()
<< "\n\ty.nc(): " << y.nc()
);
DLIB_ASSERT(eps > 0 &&
folds > 1 && folds <= x.nr() &&
cache_size > 0 &&
0 < nu && nu < maximum_nu(y),
"\tdecision_function svm_nu_train()"
<< "\n\tinvalid inputs were given to this function"
<< "\n\teps: " << eps
<< "\n\tfolds: " << folds
<< "\n\tcache_size: " << cache_size
<< "\n\tnu: " << nu
<< "\n\tmaximum_nu(y): " << maximum_nu(y)
); );
// count the number of positive and negative examples // count the number of positive and negative examples
...@@ -986,7 +555,7 @@ namespace dlib ...@@ -986,7 +555,7 @@ namespace dlib
} }
// do the training // do the training
d = svm_nu_train (x_train,y_train,kernel_function,nu,cache_size,eps); d = trainer.train (x_train,y_train);
// now test this fold // now test this fold
for (long i = 0; i < x_test.nr(); ++i) for (long i = 0; i < x_test.nr(); ++i)
...@@ -1003,7 +572,7 @@ namespace dlib ...@@ -1003,7 +572,7 @@ namespace dlib
} }
else else
{ {
throw dlib::error("invalid input labels to the svm_nu_train_prob() function"); throw dlib::error("invalid input labels to the train_probabilistic_decision_function() function");
} }
} }
...@@ -1113,9 +682,25 @@ namespace dlib ...@@ -1113,9 +682,25 @@ namespace dlib
break; break;
} }
return probabilistic_decision_function<K>( return probabilistic_decision_function<K>( A, B, trainer.train(x,y) );
A, B, }
svm_nu_train (x,y,kernel_function,nu,cache_size,eps) );
template <
typename trainer_type,
typename in_sample_vector_type,
typename in_scalar_vector_type
>
const probabilistic_decision_function<typename trainer_type::kernel_type> train_probabilistic_decision_function (
const trainer_type& trainer,
const in_sample_vector_type& x,
const in_scalar_vector_type& y,
const long folds
)
{
return train_probabilistic_decision_function_impl(trainer,
vector_to_matrix(x),
vector_to_matrix(y),
folds);
} }
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
...@@ -1232,6 +817,576 @@ namespace dlib ...@@ -1232,6 +817,576 @@ namespace dlib
} }
} }
// ----------------------------------------------------------------------------------------
// ----------------------------------------------------------------------------------------
// ----------------------------------------------------------------------------------------
template <
typename K
>
class svm_nu_trainer
{
public:
typedef K kernel_type;
typedef typename kernel_type::scalar_type scalar_type;
typedef typename kernel_type::sample_type sample_type;
typedef typename kernel_type::mem_manager_type mem_manager_type;
typedef decision_function<kernel_type> trained_function_type;
svm_nu_trainer (
) :
nu(0.1),
cache_size(200),
eps(0.001)
{
}
svm_nu_trainer (
const kernel_type& kernel_,
const scalar_type& nu_
) :
kernel_function(kernel_),
nu(nu_),
cache_size(200),
eps(0.001)
{
}
void set_cache_size (
long cache_size_
)
{
cache_size = cache_size_;
}
const long get_cache_size (
) const
{
return cache_size;
}
void set_epsilon (
scalar_type eps_
)
{
eps = eps_;
}
const scalar_type get_epsilon (
) const
{
return eps;
}
void set_kernel (
const kernel_type& k
)
{
kernel_function = k;
}
const kernel_type& get_kernel (
) const
{
return kernel_function;
}
void set_nu (
scalar_type nu_
)
{
nu = nu_;
}
const scalar_type get_nu (
) const
{
return nu;
}
template <
typename in_sample_vector_type,
typename in_scalar_vector_type
>
const decision_function<kernel_type> train (
const in_sample_vector_type& x,
const in_scalar_vector_type& y
) const
{
return do_train(vector_to_matrix(x), vector_to_matrix(y));
}
void swap (
svm_nu_trainer& item
)
{
exchange(kernel_function, item.kernel_function);
exchange(nu, item.nu);
exchange(cache_size, item.cache_size);
exchange(eps, item.eps);
}
private:
// ------------------------------------------------------------------------------------
template <
typename in_sample_vector_type,
typename in_scalar_vector_type
>
const decision_function<kernel_type> do_train (
const in_sample_vector_type& x,
const in_scalar_vector_type& y
) const
{
typedef typename K::scalar_type scalar_type;
typedef typename decision_function<K>::sample_vector_type sample_vector_type;
typedef typename decision_function<K>::scalar_vector_type scalar_vector_type;
// make sure requires clause is not broken
DLIB_ASSERT(is_binary_classification_problem(x,y) == true,
"\tdecision_function svm_nu_trainer::train(x,y)"
<< "\n\t invalid inputs were given to this function"
<< "\n\t x.nr(): " << x.nr()
<< "\n\t y.nr(): " << y.nr()
<< "\n\t x.nc(): " << x.nc()
<< "\n\t y.nc(): " << y.nc()
<< "\n\t is_binary_classification_problem(x,y): " << ((is_binary_classification_problem(x,y))? "true":"false")
);
const scalar_type tau = 1e-12;
scalar_vector_type df; // delta f(alpha)
scalar_vector_type alpha;
kernel_matrix_cache<K, in_sample_vector_type, in_scalar_vector_type> Q(x,y,kernel_function,cache_size);
alpha.set_size(x.nr());
df.set_size(x.nr());
// now initialize alpha
set_initial_alpha(y, nu, alpha);
// initialize df. Compute df = Q*alpha
for (long r = 0; r < df.nr(); ++r)
{
df(r) = 0;
for (long c = 0; c < alpha.nr(); ++c)
{
if (alpha(c) != 0)
df(r) += Q(c,r)*alpha(c);
}
}
// now perform the actual optimization of alpha
long i, j;
while (find_working_group(y,alpha,Q,df,tau,eps,i,j))
{
const scalar_type old_alpha_i = alpha(i);
const scalar_type old_alpha_j = alpha(j);
optimize_working_pair(y,alpha,Q,df,tau,i,j);
// update the df vector now that we have modified alpha(i) and alpha(j)
scalar_type delta_alpha_i = alpha(i) - old_alpha_i;
scalar_type delta_alpha_j = alpha(j) - old_alpha_j;
for(long k = 0; k < df.nr(); ++k)
df(k) += Q(k,i)*delta_alpha_i + Q(k,j)*delta_alpha_j;
}
scalar_type rho, b;
calculate_rho_and_b(y,alpha,df,rho,b);
alpha = pointwise_multiply(alpha,y)/rho;
// count the number of support vectors
long sv_count = 0;
for (long i = 0; i < alpha.nr(); ++i)
{
if (alpha(i) != 0)
++sv_count;
}
scalar_vector_type sv_alpha;
sample_vector_type support_vectors;
// size these column vectors so that they have an entry for each support vector
sv_alpha.set_size(sv_count);
support_vectors.set_size(sv_count);
// load the support vectors and their alpha values into these new column matrices
long idx = 0;
for (long i = 0; i < alpha.nr(); ++i)
{
if (alpha(i) != 0)
{
sv_alpha(idx) = alpha(i);
support_vectors(idx) = x(i);
++idx;
}
}
// now return the decision function
return decision_function<K> (sv_alpha, b, kernel_function, support_vectors);
}
// ------------------------------------------------------------------------------------
template <
typename scalar_type,
typename scalar_vector_type,
typename scalar_vector_type2
>
inline void set_initial_alpha (
const scalar_vector_type& y,
const scalar_type nu,
scalar_vector_type2& alpha
) const
{
set_all_elements(alpha,0);
const scalar_type l = y.nr();
scalar_type temp = nu*l/2;
long num = (long)std::floor(temp);
long num_total = (long)std::ceil(temp);
int count = 0;
for (int i = 0; i < alpha.nr(); ++i)
{
if (y(i) == 1)
{
if (count < num)
{
++count;
alpha(i) = 1;
}
else
{
if (temp > num)
{
++count;
alpha(i) = temp - std::floor(temp);
}
break;
}
}
}
if (count != num_total)
{
std::ostringstream sout;
sout << "invalid nu of " << nu << ". Must be between 0 and " << (scalar_type)count/y.nr();
throw invalid_svm_nu_error(sout.str(),nu);
}
count = 0;
for (int i = 0; i < alpha.nr(); ++i)
{
if (y(i) == -1)
{
if (count < num)
{
++count;
alpha(i) = 1;
}
else
{
if (temp > num)
{
++count;
alpha(i) = temp - std::floor(temp);
}
break;
}
}
}
if (count != num_total)
{
std::ostringstream sout;
sout << "invalid nu of " << nu << ". Must be between 0 and " << (scalar_type)count/y.nr();
throw invalid_svm_nu_error(sout.str(),nu);
}
}
// ------------------------------------------------------------------------------------
template <
typename sample_vector_type,
typename scalar_vector_type,
typename scalar_vector_type2,
typename scalar_type
>
inline bool find_working_group (
const scalar_vector_type2& y,
const scalar_vector_type& alpha,
const kernel_matrix_cache<K,sample_vector_type, scalar_vector_type2>& Q,
const scalar_vector_type& df,
const scalar_type tau,
const scalar_type eps,
long& i_out,
long& j_out
) const
{
using namespace std;
long ip = -1;
long jp = -1;
long in = -1;
long jn = -1;
scalar_type ip_val = -numeric_limits<scalar_type>::infinity();
scalar_type jp_val = numeric_limits<scalar_type>::infinity();
scalar_type in_val = -numeric_limits<scalar_type>::infinity();
scalar_type jn_val = numeric_limits<scalar_type>::infinity();
// loop over the alphas and find the maximum ip and in indices.
for (long i = 0; i < alpha.nr(); ++i)
{
if (y(i) == 1)
{
if (alpha(i) < 1.0)
{
if (-df(i) > ip_val)
{
ip_val = -df(i);
ip = i;
}
}
}
else
{
if (alpha(i) > 0.0)
{
if (df(i) > in_val)
{
in_val = df(i);
in = i;
}
}
}
}
scalar_type Mp = numeric_limits<scalar_type>::infinity();
scalar_type Mn = numeric_limits<scalar_type>::infinity();
scalar_type bp = -numeric_limits<scalar_type>::infinity();
scalar_type bn = -numeric_limits<scalar_type>::infinity();
// now we need to find the minimum jp and jn indices
for (long j = 0; j < alpha.nr(); ++j)
{
if (y(j) == 1)
{
if (alpha(j) > 0.0)
{
scalar_type b = ip_val + df(j);
if (-df(j) < Mp)
Mp = -df(j);
if (b > 0 && (Q.is_cached(j) || b > bp || jp == -1 ))
{
bp = b;
scalar_type a = Q(ip,ip) + Q(j,j) - 2*Q(j,ip);
if (a <= 0)
a = tau;
scalar_type temp = -b*b/a;
if (temp < jp_val)
{
jp_val = temp;
jp = j;
}
}
}
}
else
{
if (alpha(j) < 1.0)
{
scalar_type b = in_val - df(j);
if (df(j) < Mn)
Mn = df(j);
if (b > 0 && (Q.is_cached(j) || b > bn || jn == -1 ))
{
bn = b;
scalar_type a = Q(in,in) + Q(j,j) - 2*Q(j,in);
if (a <= 0)
a = tau;
scalar_type temp = -b*b/a;
if (temp < jn_val)
{
jn_val = temp;
jn = j;
}
}
}
}
}
// if we are at the optimal point then return false so the caller knows
// to stop optimizing
if (std::max(ip_val - Mp, in_val - Mn) < eps)
return false;
if (jp_val < jn_val)
{
i_out = ip;
j_out = jp;
}
else
{
i_out = in;
j_out = jn;
}
if (j_out >= 0 && i_out >= 0)
return true;
else
return false;
}
// ------------------------------------------------------------------------------------
template <
typename scalar_vector_type,
typename scalar_vector_type2,
typename scalar_type
>
void calculate_rho_and_b(
const scalar_vector_type2& y,
const scalar_vector_type& alpha,
const scalar_vector_type& df,
scalar_type& rho,
scalar_type& b
) const
{
using namespace std;
long num_p_free = 0;
long num_n_free = 0;
scalar_type sum_p_free = 0;
scalar_type sum_n_free = 0;
scalar_type upper_bound_p = -numeric_limits<scalar_type>::infinity();
scalar_type upper_bound_n = -numeric_limits<scalar_type>::infinity();
scalar_type lower_bound_p = numeric_limits<scalar_type>::infinity();
scalar_type lower_bound_n = numeric_limits<scalar_type>::infinity();
for(long i = 0; i < alpha.nr(); ++i)
{
if(y(i) == 1)
{
if(alpha(i) == 1)
{
if (df(i) > upper_bound_p)
upper_bound_p = df(i);
}
else if(alpha(i) == 0)
{
if (df(i) < lower_bound_p)
lower_bound_p = df(i);
}
else
{
++num_p_free;
sum_p_free += df(i);
}
}
else
{
if(alpha(i) == 1)
{
if (df(i) > upper_bound_n)
upper_bound_n = df(i);
}
else if(alpha(i) == 0)
{
if (df(i) < lower_bound_n)
lower_bound_n = df(i);
}
else
{
++num_n_free;
sum_n_free += df(i);
}
}
}
scalar_type r1,r2;
if(num_p_free > 0)
r1 = sum_p_free/num_p_free;
else
r1 = (upper_bound_p+lower_bound_p)/2;
if(num_n_free > 0)
r2 = sum_n_free/num_n_free;
else
r2 = (upper_bound_n+lower_bound_n)/2;
rho = (r1+r2)/2;
b = (r1-r2)/2/rho;
}
// ------------------------------------------------------------------------------------
template <
typename sample_vector_type,
typename scalar_vector_type,
typename scalar_vector_type2,
typename scalar_type
>
inline void optimize_working_pair (
const scalar_vector_type2& y,
scalar_vector_type& alpha,
const kernel_matrix_cache<K, sample_vector_type, scalar_vector_type2>& Q,
const scalar_vector_type& df,
const scalar_type tau,
const long i,
const long j
) const
{
scalar_type quad_coef = Q(i,i)+Q(j,j)-2*Q(j,i);
if (quad_coef <= 0)
quad_coef = tau;
scalar_type delta = (df(i)-df(j))/quad_coef;
scalar_type sum = alpha(i) + alpha(j);
alpha(i) -= delta;
alpha(j) += delta;
if(sum > 1)
{
if(alpha(i) > 1)
{
alpha(i) = 1;
alpha(j) = sum - 1;
}
else if(alpha(j) > 1)
{
alpha(j) = 1;
alpha(i) = sum - 1;
}
}
else
{
if(alpha(j) < 0)
{
alpha(j) = 0;
alpha(i) = sum;
}
else if(alpha(i) < 0)
{
alpha(i) = 0;
alpha(j) = sum;
}
}
}
// ------------------------------------------------------------------------------------
kernel_type kernel_function;
scalar_type nu;
long cache_size;
scalar_type eps;
}; // end of class svm_nu_trainer
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
} }
......
dlib/svm/svm_abstract.h
View file @ 3c7c8fee
...@@ -17,8 +17,24 @@ namespace dlib ...@@ -17,8 +17,24 @@ namespace dlib
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
// Functions that perform SVM training
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
class invalid_svm_nu_error : public dlib::error
{
/*!
WHAT THIS OBJECT REPRESENTS
This object is an exception class used to indicate that a
value of nu used for svm training is incompatible with a
particular data set.
this->nu will be set to the invalid value of nu used.
!*/
public:
invalid_svm_nu_error(const std::string& msg, double nu_) : dlib::error(msg), nu(nu_) {};
const double nu;
};
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
template < template <
...@@ -29,102 +45,242 @@ namespace dlib ...@@ -29,102 +45,242 @@ namespace dlib
); );
/*! /*!
requires requires
- T == a matrix object - T == a matrix object or an object convertible to a matrix via
vector_to_matrix()
- y.nc() == 1 - y.nc() == 1
- y.nr() > 1 - y.nr() > 1
- for all valid i: - for all valid i:
- y(i) == -1 or +1 - y(i) == -1 or +1
ensures ensures
- returns the maximum valid nu that can be used with svm_nu_train(). - returns the maximum valid nu that can be used with the svm_nu_trainer and
the training set labels from the given y vector.
(i.e. 2.0*min(number of +1 examples in y, number of -1 examples in y)/y.nr()) (i.e. 2.0*min(number of +1 examples in y, number of -1 examples in y)/y.nr())
!*/ !*/
template < // ----------------------------------------------------------------------------------------
typename K
bool template <
typename T,
typename U
> >
const decision_function<K> svm_nu_train ( bool is_binary_classification_problem (
const typename decision_function<K>::sample_vector_type& x, const T& x,
const typename decision_function<K>::scalar_vector_type& y, const U& x_labels
const K& kernel_function,
const typename K::scalar_type nu,
const long cache_size = 200,
const typename K::scalar_type eps = 0.001
); );
/*! /*!
requires requires
- eps > 0 - T == a matrix or something convertible to a matrix via vector_to_matrix()
- x.nc() == 1 (i.e. x is a column vector) - U == a matrix or something convertible to a matrix via vector_to_matrix()
- y.nc() == 1 (i.e. y is a column vector)
- x.nr() == y.nr()
- x.nr() > 1
- cache_size > 0
- for all valid i:
- y(i) == -1 or +1
- y(i) is the class that should be assigned to training example x(i)
- 0 < nu < maximum_nu(y)
- kernel_function == a kernel function object type as defined at the
top of dlib/svm/kernel_abstract.h
ensures ensures
- trains a nu support vector classifier given the training samples in x and - returns true if all of the following are true and false otherwise:
labels in y. Training is done when the error is less than eps. - x.nc() == 1 (i.e. x is a column vector)
- caches approximately at most cache_size megabytes of the kernel matrix. - x_labels.nc() == 1 (i.e. x_labels is a column vector)
(bigger values of this may make training go faster but doesn't affect the - x.nr() == x_labels.nr()
result. However, too big a value will cause you to run out of memory.) - x.nr() > 1
- returns a decision function F with the following properties: - for all valid i:
- if (new_x is a sample predicted have +1 label) then - x_labels(i) == -1 or +1
- F(new_x) >= 0
- else
- F(new_x) < 0
!*/ !*/
/* // ----------------------------------------------------------------------------------------
The implementation of the nu-svm training algorithm used by this library is based // ----------------------------------------------------------------------------------------
on the following excellent papers: // ----------------------------------------------------------------------------------------
- Chang and Lin, Training {nu}-Support Vector Classifiers: Theory and Algorithms
- Chih-Chung Chang and Chih-Jen Lin, LIBSVM : a library for support vector template <
machines, 2001. Software available at http://www.csie.ntu.edu.tw/~cjlin/libsvm typename K
*/ >
class svm_nu_trainer
{
/*!
REQUIREMENTS ON K
is a kernel function object as defined in dlib/svm/kernel_abstract.h
WHAT THIS OBJECT REPRESENTS
This object implements a trainer for a nu support vector machine for
solving binary classification problems.
The implementation of the nu-svm training algorithm used by this object is based
on the following excellent papers:
- Chang and Lin, Training {nu}-Support Vector Classifiers: Theory and Algorithms
- Chih-Chung Chang and Chih-Jen Lin, LIBSVM : a library for support vector
machines, 2001. Software available at http://www.csie.ntu.edu.tw/~cjlin/libsvm
!*/
public:
typedef K kernel_type;
typedef typename kernel_type::scalar_type scalar_type;
typedef typename kernel_type::sample_type sample_type;
typedef typename kernel_type::mem_manager_type mem_manager_type;
typedef decision_function<kernel_type> trained_function_type;
svm_nu_trainer (
);
/*!
ensures
- This object is properly initialized and ready to be used
to train a support vector machine.
- #get_kernel() == kernel_type()
- #get_nu() == 0.1
- #get_cache_size() == 200
- #get_epsilon() == 0.001
!*/
svm_nu_trainer (
const kernel_type& kernel,
const scalar_type& nu
);
/*!
requires
- 0 < nu <= 1
ensures
- This object is properly initialized and ready to be used
to train a support vector machine.
- #get_kernel() == kernel
- #get_nu() == nu
- #get_cache_size() == 200
- #get_epsilon() == 0.001
!*/
void set_cache_size (
long cache_size
);
/*!
requires
- cache_size > 0
ensures
- #get_cache_size() == cache_size
!*/
const long get_cache_size (
) const;
/*!
ensures
- returns the number of megabytes of cache this object will use
when it performs training via the this->train() function.
(bigger values of this may make training go faster but doesn't affect
the result. However, too big a value will cause you to run out of
memory obviously.)
!*/
void set_epsilon (
scalar_type eps
);
/*!
requires
- eps > 0
ensures
- #get_epsilon() == eps
!*/
const scalar_type get_epsilon (
) const;
/*!
ensures
- returns the error epsilon that determines when training should stop.
Generally a good value for this is 0.001. Smaller values may result
in a more accurate solution but take longer to execute.
!*/
void set_kernel (
const kernel_type& k
);
/*!
ensures
- #get_kernel() == k
!*/
const kernel_type& get_kernel (
) const;
/*!
ensures
- returns a copy of the kernel function in use by this object
!*/
void set_nu (
scalar_type nu
);
/*!
requires
- 0 < nu <= 1
ensures
- #get_nu() == nu
!*/
const scalar_type get_nu (
) const;
/*!
ensures
- returns the nu svm parameter. This is a value between 0 and
1. It is the parameter that determines the trade off between
trying to fit the training data exactly or allowing more errors
but hopefully improving the generalization ability of the
resulting classifier. For more information you should consult
the papers referenced above.
!*/
template <
typename in_sample_vector_type,
typename in_scalar_vector_type
>
const decision_function<kernel_type> train (
const in_sample_vector_type& x,
const in_scalar_vector_type& y
) const;
/*!
requires
- is_binary_classification_problem(x,y) == true
ensures
- trains a nu support vector classifier given the training samples in x and
labels in y. Training is done when the error is less than get_epsilon().
- returns a decision function F with the following properties:
- if (new_x is a sample predicted have +1 label) then
- F(new_x) >= 0
- else
- F(new_x) < 0
throws
- invalid_svm_nu_error
This exception is thrown if get_nu() > maximum_nu(y)
- std::bad_alloc
!*/
void swap (
svm_nu_trainer& item
);
/*!
ensures
- swaps *this and item
!*/
};
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
template < template <
typename K typename trainer_type,
typename in_sample_vector_type,
typename in_scalar_vector_type
> >
const probabilistic_decision_function<K> svm_nu_train_prob ( const probabilistic_decision_function<typename trainer_type::kernel_type>
const typename decision_function<K>::sample_vector_type& x, train_probabilistic_decision_function (
const typename decision_function<K>::scalar_vector_type& y, const trainer_type& trainer,
const K& kernel_function, const in_sample_vector_type& x,
const typename K::scalar_type nu, const in_scalar_vector_type& y,
const long folds, const long folds
const long cache_size = 200, )
const typename K::scalar_type eps = 0.001
);
/*! /*!
requires requires
- eps > 0
- 1 < folds <= x.nr() - 1 < folds <= x.nr()
- x.nc() == 1 (i.e. x is a column vector) - is_binary_classification_problem(x,y) == true
- y.nc() == 1 (i.e. y is a column vector) - trainer_type == some kind of trainer object (e.g. svm_nu_trainer)
- x.nr() == y.nr()
- x.nr() > 1
- cache_size > 0
- for all valid i:
- y(i) == -1 or +1
- y(i) is the class that should be assigned to training example x(i)
- 0 < nu < maximum_nu(y)
- kernel_function == a kernel function object type as defined at the
top of dlib/svm/kernel_abstract.h
ensures ensures
- trains a nu support vector classifier given the training samples in x and - trains a nu support vector classifier given the training samples in x and
labels in y. Training is done when the error is less than eps. labels in y.
- caches approximately at most cache_size megabytes of the kernel matrix. - returns a probabilistic_decision_function that represents the trained svm.
(bigger values of this may make training go faster but doesn't affect the
result. However, too big a value will cause you to run out of memory.)
- returns a probabilistic_decision_function that represents the trained
svm.
- The parameters of the probability model are estimated by performing k-fold - The parameters of the probability model are estimated by performing k-fold
cross validation. cross validation.
- The number of folds used is given by the folds argument. - The number of folds used is given by the folds argument.
throws
- any exceptions thrown by trainer.train()
- std::bad_alloc
!*/ !*/
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
...@@ -134,46 +290,36 @@ namespace dlib ...@@ -134,46 +290,36 @@ namespace dlib
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
template < template <
typename K typename trainer_type,
typename in_sample_vector_type,
typename in_scalar_vector_type
> >
const matrix<typename K::scalar_type, 1, 2, typename K::mem_manager_type> svm_nu_cross_validate ( const matrix<typename trainer_type::scalar_type, 1, 2, typename trainer_type::mem_manager_type>
const typename decision_function<K>::sample_vector_type& x, cross_validate_trainer (
const typename decision_function<K>::scalar_vector_type& y, const trainer_type& trainer,
const K& kernel_function, const in_sample_vector_type& x,
const typename K::scalar_type nu, const in_scalar_vector_type& y,
const long folds, const long folds
const long cache_size = 200,
const typename K::scalar_type eps = 0.001
); );
/*! /*!
requires requires
- eps > 0 - is_binary_classification_problem(x,y) == true
- 1 < folds <= x.nr() - 1 < folds <= x.nr()
- x.nc() == 1 (i.e. x is a column vector) - trainer_type == some kind of trainer object (e.g. svm_nu_trainer)
- y.nc() == 1 (i.e. y is a column vector)
- x.nr() == y.nr()
- x.nr() > 1
- cache_size > 0
- for all valid i:
- y(i) == -1 or +1
- y(i) is the class that should be assigned to training example x(i)
- 0 < nu < maximum_nu(y)
- kernel_function == a kernel function object type as defined at the
top of dlib/svm/kernel_abstract.h
ensures ensures
- performs k-fold cross validation by training a nu-svm using the svm_nu_train() - performs k-fold cross validation by using the given trainer to solve the
function. Each fold is tested using the learned decision_function and the given binary classification problem for the given number of folds.
average accuracy from all folds is returned. The accuracy is returned in Each fold is tested using the output of the trainer and the average
a column vector, let us call it R. Both quantities in R are numbers between classification accuracy from all folds is returned.
0 and 1 which represent the fraction of examples correctly classified. R(0) - The accuracy is returned in a column vector, let us call it R. Both
is the fraction of +1 examples correctly classified and R(1) is the fraction quantities in R are numbers between 0 and 1 which represent the fraction
of -1 examples correctly classified. of examples correctly classified. R(0) is the fraction of +1 examples
correctly classified and R(1) is the fraction of -1 examples correctly
classified.
- The number of folds used is given by the folds argument. - The number of folds used is given by the folds argument.
- in each fold: trains a nu support vector classifier given the training samples throws
in x and labels in y. Training is done when the error is less than eps. - any exceptions thrown by trainer.train()
- caches approximately at most cache_size megabytes of the kernel matrix. - std::bad_alloc
(bigger values of this may make training go faster but doesn't affect the
result. However, too big a value will cause you to run out of memory.)
!*/ !*/
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment