Changed the oca optimizer so that it warm starts the QP subproblem

rather than resolving it from scratch during each iteration. This
improves the speed and stability of the algorithm.

--HG--
extra : convert_revision : svn%3Afdd8eb12-d10e-0410-9acb-85c331704f74/trunk%404155

dlib/optimization/optimization_oca.h

@@ -55,9 +55,9 @@ namespace dlib
         oca () 
         {
             sub_eps = 1e-2;
-            sub_max_iter = 200000;
+            sub_max_iter = 50000;
 
-            inactive_thresh = 10;
+            inactive_thresh = 20;
         }
 
         void set_subproblem_epsilon (
@@ -154,6 +154,7 @@ namespace dlib
                 // what it is.
                 bs.push_back(R_lower_bound);
                 planes.push_back(zeros_matrix<scalar_type>(w.size(),1));
+                alpha = uniform_matrix<scalar_type>(1,1, C);
                 miss_count.push_back(0);
 
                 K.set_size(1,1);
@@ -173,6 +174,13 @@ namespace dlib
                 bs.push_back(cur_risk - dot(w_cur,planes.back()));
                 miss_count.push_back(0);
 
+                // If alpha is empty then initialize it (we must always have sum(alpha) == C).  
+                // But otherwise, just append a zero.
+                if (alpha.size() == 0)
+                    alpha = uniform_matrix<scalar_type>(1,1, C);
+                else
+                    alpha = join_cols(alpha,zeros_matrix<scalar_type>(1,1));
+
                 // Check the objective value at w_cur and see if it is better than
                 // the best seen so far.
                 const scalar_type cur_obj = 0.5*trans(w_cur)*w_cur + C*cur_risk;
@@ -199,7 +207,6 @@ namespace dlib
                     ++rr;
                 }
 
-                alpha = uniform_matrix<scalar_type>(planes.size(),1, C/planes.size());
 
                 // solve the cutting plane subproblem for the next w_cur.   We solve it to an
                 // accuracy that is related to how big the error gap is
@@ -207,6 +214,8 @@ namespace dlib
                 // just a sanity check
                 if (eps < 1e-16)
                     eps = 1e-16;
+                // Note that we warm start this optimization by using the alpha from the last
+                // iteration as the starting point.
                 solve_qp_using_smo(K, vector_to_matrix(bs), alpha, eps, sub_max_iter); 
 
                 // construct the w_cur that minimized the subproblem.
@@ -245,6 +254,7 @@ namespace dlib
                     bs.erase(bs.begin()+idx);
                     miss_count.erase(miss_count.begin()+idx);
                     K = removerc(K, idx, idx);
+                    alpha = remove_row(alpha,idx);
                 }
 
             }

dlib/optimization/optimization_oca_abstract.h

@@ -110,8 +110,8 @@ namespace dlib
         /*!
             INITIAL VALUE
                 - get_subproblem_epsilon() == 1e-2
-                - get_subproblem_max_iterations() == 200000
-                - get_inactive_plane_threshold() == 10
+                - get_subproblem_max_iterations() == 50000
+                - get_inactive_plane_threshold() == 20
 
             WHAT THIS OBJECT REPRESENTS
                 This object is a tool for solving the optimization problem defined above