Skip to content

D
dlib
Commit 60d25f3e authored by Davis King's avatar Davis King
Browse files
Options

Improved the interface for the oca optimizer. Now the user has a lot more 

control of when to stop the algorithm.

--HG--
extra : convert_revision : svn%3Afdd8eb12-d10e-0410-9acb-85c331704f74/trunk%403502
parent 3d118c63
Hide whitespace changes
Inline Side-by-side
Showing with 64 additions and 119 deletions
dlib/optimization/optimization_oca.h
View file @ 60d25f3e
...@@ -21,16 +21,17 @@ namespace dlib ...@@ -21,16 +21,17 @@ namespace dlib
virtual ~oca_problem() {} virtual ~oca_problem() {}
virtual void optimization_status (
scalar_type ,
scalar_type ,
unsigned long
) const {}
virtual bool r_has_lower_bound ( virtual bool r_has_lower_bound (
scalar_type& scalar_type&
) const { return false; } ) const { return false; }
virtual bool optimization_status (
scalar_type ,
scalar_type ,
unsigned long,
unsigned long
) const = 0;
virtual scalar_type get_c ( virtual scalar_type get_c (
) const = 0; ) const = 0;
...@@ -53,51 +54,12 @@ namespace dlib ...@@ -53,51 +54,12 @@ namespace dlib
oca () oca ()
{ {
eps = 0.001; sub_eps = 1e-2;
max_iter = 1000000; sub_max_iter = 200000;
sub_eps = 1e-5;
sub_max_iter = 20000;
inactive_thresh = 15;
}
void set_epsilon (
double eps_
)
{
// make sure requires clause is not broken
DLIB_ASSERT(eps_ > 0,
"\t void oca::set_epsilon"
<< "\n\t epsilon must be greater than 0"
<< "\n\t eps_: " << eps_
<< "\n\t this: " << this
);
eps = eps_; inactive_thresh = 10;
} }
double get_epsilon (
) const { return eps; }
void set_max_iterations (
unsigned long max_iter_
)
{
// make sure requires clause is not broken
DLIB_ASSERT(max_iter_ > 0,
"\t void oca::set_max_iterations"
<< "\n\t max iterations must be greater than 0"
<< "\n\t max_iter_: " << max_iter_
<< "\n\t this: " << this
);
max_iter = max_iter_;
}
unsigned long get_max_iterations (
) const { return max_iter; }
void set_subproblem_epsilon ( void set_subproblem_epsilon (
double eps_ double eps_
) { sub_eps = eps_; } ) { sub_eps = eps_; }
...@@ -195,8 +157,10 @@ namespace dlib ...@@ -195,8 +157,10 @@ namespace dlib
matrix<scalar_type,0,0,mem_manager_type, layout_type> K; matrix<scalar_type,0,0,mem_manager_type, layout_type> K;
for (unsigned long iter = 0; iter < max_iter; ++iter) unsigned long counter = 0;
while (true)
{ {
++counter;
// add the next cutting plane // add the next cutting plane
scalar_type cur_risk; scalar_type cur_risk;
...@@ -214,10 +178,6 @@ namespace dlib ...@@ -214,10 +178,6 @@ namespace dlib
w = w_cur; w = w_cur;
} }
// check stopping condition and stop if we can
if (best_obj - cp_obj <= eps)
break;
// compute kernel matrix for all the planes // compute kernel matrix for all the planes
K.set_size(planes.size(), planes.size()); K.set_size(planes.size(), planes.size());
...@@ -236,8 +196,13 @@ namespace dlib ...@@ -236,8 +196,13 @@ namespace dlib
alpha = uniform_matrix<scalar_type>(planes.size(),1, C/planes.size()); alpha = uniform_matrix<scalar_type>(planes.size(),1, C/planes.size());
// solve the cutting plane subproblem for the next w_cur // solve the cutting plane subproblem for the next w_cur. We solve it to an
solve_qp_using_smo(K, vector_to_matrix(bs), alpha, static_cast<scalar_type>(sub_eps), sub_max_iter); // accuracy that is related to how big the error gap is
scalar_type eps = std::min<scalar_type>(sub_eps, 0.1*(best_obj-cp_obj)) ;
// just a sanaty check
if (eps < 1e-7)
eps = 1e-7;
solve_qp_using_smo(K, vector_to_matrix(bs), alpha, eps, sub_max_iter);
// construct the w_cur that minimized the subproblem. // construct the w_cur that minimized the subproblem.
w_cur = 0; w_cur = 0;
...@@ -260,12 +225,9 @@ namespace dlib ...@@ -260,12 +225,9 @@ namespace dlib
// plane subproblem. // plane subproblem.
cp_obj = -0.5*trans(w_cur)*w_cur + trans(alpha)*vector_to_matrix(bs); cp_obj = -0.5*trans(w_cur)*w_cur + trans(alpha)*vector_to_matrix(bs);
// check stopping condition and stop if we can
if (best_obj - cp_obj <= eps)
break;
// report current status // report current status
problem.optimization_status(best_obj, best_obj - cp_obj, planes.size()); if (problem.optimization_status(best_obj, best_obj - cp_obj, planes.size(), counter))
break;
// If it has been a while since a cutting plane was an active constraint then // If it has been a while since a cutting plane was an active constraint then
// we should throw it away. // we should throw it away.
...@@ -281,18 +243,13 @@ namespace dlib ...@@ -281,18 +243,13 @@ namespace dlib
} }
// report current status
problem.optimization_status(best_obj, best_obj - cp_obj, planes.size());
return best_obj; return best_obj;
} }
private: private:
double eps;
double sub_eps; double sub_eps;
unsigned long max_iter;
unsigned long sub_max_iter; unsigned long sub_max_iter;
unsigned long inactive_thresh; unsigned long inactive_thresh;
......
dlib/optimization/optimization_oca_abstract.h
View file @ 60d25f3e
...@@ -22,6 +22,10 @@ namespace dlib ...@@ -22,6 +22,10 @@ namespace dlib
Minimize: f(w) == 0.5*dot(w,w) + C*R(w) Minimize: f(w) == 0.5*dot(w,w) + C*R(w)
Where R(w) is a convex function and C > 0 Where R(w) is a convex function and C > 0
Note that the stopping condition must be provided by the user
in the form of the optimization_status() function.
!*/ !*/
public: public:
...@@ -30,16 +34,6 @@ namespace dlib ...@@ -30,16 +34,6 @@ namespace dlib
virtual ~oca_problem() {} virtual ~oca_problem() {}
virtual void optimization_status (
scalar_type current_objective_value,
scalar_type current_error_gap,
unsigned long num_cutting_planes
) const {}
/*!
This function is called by the OCA optimizer each iteration. It
exists to allow the user to monitor the progress of the optimization.
!*/
virtual bool r_has_lower_bound ( virtual bool r_has_lower_bound (
scalar_type& lower_bound scalar_type& lower_bound
) const { return false; } ) const { return false; }
...@@ -52,6 +46,27 @@ namespace dlib ...@@ -52,6 +46,27 @@ namespace dlib
- returns false - returns false
!*/ !*/
virtual bool optimization_status (
scalar_type current_objective_value,
scalar_type current_error_gap,
unsigned long num_cutting_planes,
unsigned long num_iterations
) const = 0;
/*!
requires
- This function is called by the OCA optimizer each iteration.
- current_objective_value == the current value of the objective function f(w)
- current_error_gap == the bound on how much lower the objective function
can drop before we reach the optimal point
- num_cutting_planes == the number of cutting planes the algorithm is currently
using
- num_iterations == A count of the total number of iterations that have executed
since we started running the optimization.
ensures
- If it is appropriate to terminate the optimization then this function returns true
and false otherwise.
!*/
virtual scalar_type get_c ( virtual scalar_type get_c (
) const = 0; ) const = 0;
/*! /*!
...@@ -93,11 +108,9 @@ namespace dlib ...@@ -93,11 +108,9 @@ namespace dlib
{ {
/*! /*!
INITIAL VALUE INITIAL VALUE
- get_epsilon() == 0.001 - get_subproblem_epsilon() == 1e-2
- get_max_iterations() == 1000000 - get_subproblem_max_iterations() == 200000
- get_subproblem_epsilon() == 1e-5 - get_inactive_plane_threshold() == 10
- get_subproblem_max_iterations() == 20000
- get_inactive_plane_threshold() == 15
WHAT THIS OBJECT REPRESENTS WHAT THIS OBJECT REPRESENTS
This object is a tool for solving the optimization problem defined above This object is a tool for solving the optimization problem defined above
...@@ -139,46 +152,11 @@ namespace dlib ...@@ -139,46 +152,11 @@ namespace dlib
- problem.get_num_dimensions() > 0 - problem.get_num_dimensions() > 0
ensures ensures
- solves the given oca problem and stores the solution in #w - solves the given oca problem and stores the solution in #w
- The optimization algorithm runs until problem.optimization_status()
indicates it is time to stop.
- returns the objective value at the solution #w - returns the objective value at the solution #w
!*/ !*/
void set_epsilon (
double eps_
);
/*!
requires
- eps > 0
ensures
- #get_epsilon() == eps
!*/
double get_epsilon (
) const;
/*!
ensures
- returns the error epsilon that determines when training should stop.
Smaller values may result in a more accurate solution but may cause
the algorithm to take longer to execute.
!*/
void set_max_iterations (
unsigned long max_iter
);
/*!
requires
- max_iter > 0
ensures
- #get_max_iterations() == max_iter
!*/
unsigned long get_max_iterations (
) const;
/*!
ensures
- returns the maximum number of iterations this object will perform
while attempting to solve an oca_problem.
!*/
void set_subproblem_epsilon ( void set_subproblem_epsilon (
double eps double eps
); );
......
dlib/svm/svm_c_linear_trainer.h
View file @ 60d25f3e
...@@ -66,10 +66,11 @@ namespace dlib ...@@ -66,10 +66,11 @@ namespace dlib
return num_dimensions_in_samples(samples) + 1; return num_dimensions_in_samples(samples) + 1;
} }
virtual void optimization_status ( virtual bool optimization_status (
scalar_type current_objective_value, scalar_type current_objective_value,
scalar_type current_error_gap, scalar_type current_error_gap,
unsigned long num_cutting_planes unsigned long num_cutting_planes,
unsigned long num_iterations
) const ) const
{ {
if (be_verbose) if (be_verbose)
...@@ -78,8 +79,17 @@ namespace dlib ...@@ -78,8 +79,17 @@ namespace dlib
cout << "svm objective: " << current_objective_value << endl; cout << "svm objective: " << current_objective_value << endl;
cout << "gap: " << current_error_gap << endl; cout << "gap: " << current_error_gap << endl;
cout << "num planes: " << num_cutting_planes << endl; cout << "num planes: " << num_cutting_planes << endl;
cout << "iter: " << num_iterations << endl;
cout << endl; cout << endl;
} }
if (current_error_gap/current_objective_value < 0.001)
return true;
if (num_iterations > 10000)
return true;
return false;
} }
virtual bool r_has_lower_bound ( virtual bool r_has_lower_bound (
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment