Add cosine_delay learning rate policy

Summary:
This adds a cosine schedule, based on:
 - "Bag of Freebies for Training Object Detection Neural Networks" https://arxiv.org/pdf/1902.04103.pdf
  - "SGDR: Stochastic Gradient Descent with Warm Restarts" https://arxiv.org/pdf/1608.03983.pdf

Reviewed By: ashwinb

Differential Revision: D14593944

fbshipit-source-id: f0f7eedee784fa4d0aebd092d718526bbf627d81

detectron/core/config.py

@@ -581,6 +581,8 @@ __C.SOLVER.LR_POLICY = 'step'
 #   SOLVER.STEPS = [0, 60000, 80000]
 #   SOLVER.LRS = [0.02, 0.002, 0.0002]
 #   lr = LRS[current_step]
+# 'cosine_decay'
+#   lr = SOLVER.BASE_LR * (cos(PI * cur_iter / SOLVER.MAX_ITER) * 0.5 + 0.5)
 
 # Hyperparameter used by the specified policy
 # For 'step', the current LR is multiplied by SOLVER.GAMMA at each step

detectron/utils/lr_policy.py

@@ -91,6 +91,14 @@ def lr_func_step(cur_iter):
         cfg.SOLVER.GAMMA ** (cur_iter // cfg.SOLVER.STEP_SIZE))
 
 
+def lr_func_cosine_decay(cur_iter):
+    """For cfg.SOLVER.LR_POLICY = 'cosine_decay'
+    """
+    iter_frac = float(cur_iter) / cfg.SOLVER.MAX_ITER
+    cos_frac = 0.5 * (np.cos(np.pi * iter_frac) + 1)
+    return cfg.SOLVER.BASE_LR * cos_frac
+
+
 # ---------------------------------------------------------------------------- #
 # Helpers
 # ---------------------------------------------------------------------------- #