Skip to content

D
dlib
Commit 60d25f3e authored by Davis King's avatar Davis King
Browse files
Options

Improved the interface for the oca optimizer. Now the user has a lot more 

control of when to stop the algorithm.

--HG--
extra : convert_revision : svn%3Afdd8eb12-d10e-0410-9acb-85c331704f74/trunk%403502
parent 3d118c63
Hide whitespace changes
Inline Side-by-side
Showing with 64 additions and 119 deletions
dlib/optimization/optimization_oca.h
View file @ 60d25f3e
......@@ -21,16 +21,17 @@ namespace dlib
virtual ~oca_problem() {}
virtual void optimization_status (
scalar_type ,
scalar_type ,
unsigned long
) const {}
virtual bool r_has_lower_bound (
scalar_type&
) const { return false; }
virtual bool optimization_status (
scalar_type ,
scalar_type ,
unsigned long,
unsigned long
) const = 0;
virtual scalar_type get_c (
) const = 0;
......@@ -53,51 +54,12 @@ namespace dlib
oca ()
{
eps = 0.001;
max_iter = 1000000;
sub_eps = 1e-5;
sub_max_iter = 20000;
inactive_thresh = 15;
}
void set_epsilon (
double eps_
)
{
// make sure requires clause is not broken
DLIB_ASSERT(eps_ > 0,
"\t void oca::set_epsilon"
<< "\n\t epsilon must be greater than 0"
<< "\n\t eps_: " << eps_
<< "\n\t this: " << this
);
sub_eps = 1e-2;
sub_max_iter = 200000;
eps = eps_;
inactive_thresh = 10;
}
double get_epsilon (
) const { return eps; }
void set_max_iterations (
unsigned long max_iter_
)
{
// make sure requires clause is not broken
DLIB_ASSERT(max_iter_ > 0,
"\t void oca::set_max_iterations"
<< "\n\t max iterations must be greater than 0"
<< "\n\t max_iter_: " << max_iter_
<< "\n\t this: " << this
);
max_iter = max_iter_;
}
unsigned long get_max_iterations (
) const { return max_iter; }
void set_subproblem_epsilon (
double eps_
) { sub_eps = eps_; }
......@@ -195,8 +157,10 @@ namespace dlib
matrix<scalar_type,0,0,mem_manager_type, layout_type> K;
for (unsigned long iter = 0; iter < max_iter; ++iter)
unsigned long counter = 0;
while (true)
{
++counter;
// add the next cutting plane
scalar_type cur_risk;
......@@ -214,10 +178,6 @@ namespace dlib
w = w_cur;
}
// check stopping condition and stop if we can
if (best_obj - cp_obj <= eps)
break;
// compute kernel matrix for all the planes
K.set_size(planes.size(), planes.size());
......@@ -236,8 +196,13 @@ namespace dlib
alpha = uniform_matrix<scalar_type>(planes.size(),1, C/planes.size());
// solve the cutting plane subproblem for the next w_cur
solve_qp_using_smo(K, vector_to_matrix(bs), alpha, static_cast<scalar_type>(sub_eps), sub_max_iter);
// solve the cutting plane subproblem for the next w_cur. We solve it to an
// accuracy that is related to how big the error gap is
scalar_type eps = std::min<scalar_type>(sub_eps, 0.1*(best_obj-cp_obj)) ;
// just a sanaty check
if (eps < 1e-7)
eps = 1e-7;
solve_qp_using_smo(K, vector_to_matrix(bs), alpha, eps, sub_max_iter);
// construct the w_cur that minimized the subproblem.
w_cur = 0;
......@@ -260,12 +225,9 @@ namespace dlib
// plane subproblem.
cp_obj = -0.5*trans(w_cur)*w_cur + trans(alpha)*vector_to_matrix(bs);
// check stopping condition and stop if we can
if (best_obj - cp_obj <= eps)
break;
// report current status
problem.optimization_status(best_obj, best_obj - cp_obj, planes.size());
if (problem.optimization_status(best_obj, best_obj - cp_obj, planes.size(), counter))
break;
// If it has been a while since a cutting plane was an active constraint then
// we should throw it away.
......@@ -281,18 +243,13 @@ namespace dlib
}
// report current status
problem.optimization_status(best_obj, best_obj - cp_obj, planes.size());
return best_obj;
}
private:
double eps;
double sub_eps;
unsigned long max_iter;
unsigned long sub_max_iter;
unsigned long inactive_thresh;
......
dlib/optimization/optimization_oca_abstract.h
View file @ 60d25f3e
......@@ -22,6 +22,10 @@ namespace dlib
Minimize: f(w) == 0.5*dot(w,w) + C*R(w)
Where R(w) is a convex function and C > 0
Note that the stopping condition must be provided by the user
in the form of the optimization_status() function.
!*/
public:
......@@ -30,16 +34,6 @@ namespace dlib
virtual ~oca_problem() {}
virtual void optimization_status (
scalar_type current_objective_value,
scalar_type current_error_gap,
unsigned long num_cutting_planes
) const {}
/*!
This function is called by the OCA optimizer each iteration. It
exists to allow the user to monitor the progress of the optimization.
!*/
virtual bool r_has_lower_bound (
scalar_type& lower_bound
) const { return false; }
......@@ -52,6 +46,27 @@ namespace dlib
- returns false
!*/
virtual bool optimization_status (
scalar_type current_objective_value,
scalar_type current_error_gap,
unsigned long num_cutting_planes,
unsigned long num_iterations
) const = 0;
/*!
requires
- This function is called by the OCA optimizer each iteration.
- current_objective_value == the current value of the objective function f(w)
- current_error_gap == the bound on how much lower the objective function
can drop before we reach the optimal point
- num_cutting_planes == the number of cutting planes the algorithm is currently
using
- num_iterations == A count of the total number of iterations that have executed
since we started running the optimization.
ensures
- If it is appropriate to terminate the optimization then this function returns true
and false otherwise.
!*/
virtual scalar_type get_c (
) const = 0;
/*!
......@@ -93,11 +108,9 @@ namespace dlib
{
/*!
INITIAL VALUE
- get_epsilon() == 0.001
- get_max_iterations() == 1000000
- get_subproblem_epsilon() == 1e-5
- get_subproblem_max_iterations() == 20000
- get_inactive_plane_threshold() == 15
- get_subproblem_epsilon() == 1e-2
- get_subproblem_max_iterations() == 200000
- get_inactive_plane_threshold() == 10
WHAT THIS OBJECT REPRESENTS
This object is a tool for solving the optimization problem defined above
......@@ -139,46 +152,11 @@ namespace dlib
- problem.get_num_dimensions() > 0
ensures
- solves the given oca problem and stores the solution in #w
- The optimization algorithm runs until problem.optimization_status()
indicates it is time to stop.
- returns the objective value at the solution #w
!*/
void set_epsilon (
double eps_
);
/*!
requires
- eps > 0
ensures
- #get_epsilon() == eps
!*/
double get_epsilon (
) const;
/*!
ensures
- returns the error epsilon that determines when training should stop.
Smaller values may result in a more accurate solution but may cause
the algorithm to take longer to execute.
!*/
void set_max_iterations (
unsigned long max_iter
);
/*!
requires
- max_iter > 0
ensures
- #get_max_iterations() == max_iter
!*/
unsigned long get_max_iterations (
) const;
/*!
ensures
- returns the maximum number of iterations this object will perform
while attempting to solve an oca_problem.
!*/
void set_subproblem_epsilon (
double eps
);
......
dlib/svm/svm_c_linear_trainer.h
View file @ 60d25f3e
......@@ -66,10 +66,11 @@ namespace dlib
return num_dimensions_in_samples(samples) + 1;
}
virtual void optimization_status (
virtual bool optimization_status (
scalar_type current_objective_value,
scalar_type current_error_gap,
unsigned long num_cutting_planes
unsigned long num_cutting_planes,
unsigned long num_iterations
) const
{
if (be_verbose)
......@@ -78,8 +79,17 @@ namespace dlib
cout << "svm objective: " << current_objective_value << endl;
cout << "gap: " << current_error_gap << endl;
cout << "num planes: " << num_cutting_planes << endl;
cout << "iter: " << num_iterations << endl;
cout << endl;
}
if (current_error_gap/current_objective_value < 0.001)
return true;
if (num_iterations > 10000)
return true;
return false;
}
virtual bool r_has_lower_bound (
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment