Use getters instead of exposing dataset catalog dict

Reviewed By: rbgirshick

Differential Revision: D8116895

fbshipit-source-id: 6d8b5be01189db16ec92c8b5b0010db810777170

detectron/core/config.py

@@ -75,7 +75,7 @@ __C.TRAIN = AttrDict()
 __C.TRAIN.WEIGHTS = b''
 
 # Datasets to train on
-# Available dataset list: datasets.dataset_catalog.DATASETS.keys()
+# Available dataset list: detectron.datasets.dataset_catalog.datasets()
 # If multiple datasets are listed, the model is trained on their union
 __C.TRAIN.DATASETS = ()
 
@@ -216,7 +216,7 @@ __C.TEST = AttrDict()
 __C.TEST.WEIGHTS = b''
 
 # Datasets to test on
-# Available dataset list: datasets.dataset_catalog.DATASETS.keys()
+# Available dataset list: detectron.datasets.dataset_catalog.datasets()
 # If multiple datasets are listed, testing is performed on each one sequentially
 __C.TEST.DATASETS = ()
 

detectron/datasets/cityscapes_json_dataset_evaluator.py

@@ -28,8 +28,7 @@ import uuid
 import pycocotools.mask as mask_util
 
 from detectron.core.config import cfg
-from detectron.datasets.dataset_catalog import DATASETS
-from detectron.datasets.dataset_catalog import RAW_DIR
+from detectron.datasets.dataset_catalog import get_raw_dir
 
 logger = logging.getLogger(__name__)
 
@@ -55,7 +54,7 @@ def evaluate_masks(
     if not os.path.exists(results_dir):
         os.mkdir(results_dir)
 
-    os.environ['CITYSCAPES_DATASET'] = DATASETS[json_dataset.name][RAW_DIR]
+    os.environ['CITYSCAPES_DATASET'] = get_raw_dir(json_dataset.name)
     os.environ['CITYSCAPES_RESULTS'] = output_dir
 
     # Load the Cityscapes eval script *after* setting the required env vars,

detectron/datasets/dataset_catalog.py

@@ -27,163 +27,198 @@ import os
 _DATA_DIR = os.path.join(os.path.dirname(__file__), 'data')
 
 # Required dataset entry keys
-IM_DIR = 'image_directory'
-ANN_FN = 'annotation_file'
+_IM_DIR = 'image_directory'
+_ANN_FN = 'annotation_file'
 
 # Optional dataset entry keys
-IM_PREFIX = 'image_prefix'
-DEVKIT_DIR = 'devkit_directory'
-RAW_DIR = 'raw_dir'
+_IM_PREFIX = 'image_prefix'
+_DEVKIT_DIR = 'devkit_directory'
+_RAW_DIR = 'raw_dir'
 
 # Available datasets
-DATASETS = {
+_DATASETS = {
     'cityscapes_fine_instanceonly_seg_train': {
-        IM_DIR:
+        _IM_DIR:
             _DATA_DIR + '/cityscapes/images',
-        ANN_FN:
+        _ANN_FN:
             _DATA_DIR + '/cityscapes/annotations/instancesonly_gtFine_train.json',
-        RAW_DIR:
+        _RAW_DIR:
             _DATA_DIR + '/cityscapes/raw'
     },
     'cityscapes_fine_instanceonly_seg_val': {
-        IM_DIR:
+        _IM_DIR:
             _DATA_DIR + '/cityscapes/images',
         # use filtered validation as there is an issue converting contours
-        ANN_FN:
+        _ANN_FN:
             _DATA_DIR + '/cityscapes/annotations/instancesonly_filtered_gtFine_val.json',
-        RAW_DIR:
+        _RAW_DIR:
             _DATA_DIR + '/cityscapes/raw'
     },
     'cityscapes_fine_instanceonly_seg_test': {
-        IM_DIR:
+        _IM_DIR:
             _DATA_DIR + '/cityscapes/images',
-        ANN_FN:
+        _ANN_FN:
             _DATA_DIR + '/cityscapes/annotations/instancesonly_gtFine_test.json',
-        RAW_DIR:
+        _RAW_DIR:
             _DATA_DIR + '/cityscapes/raw'
     },
     'coco_2014_train': {
-        IM_DIR:
+        _IM_DIR:
             _DATA_DIR + '/coco/coco_train2014',
-        ANN_FN:
+        _ANN_FN:
             _DATA_DIR + '/coco/annotations/instances_train2014.json'
     },
     'coco_2014_val': {
-        IM_DIR:
+        _IM_DIR:
             _DATA_DIR + '/coco/coco_val2014',
-        ANN_FN:
+        _ANN_FN:
             _DATA_DIR + '/coco/annotations/instances_val2014.json'
     },
     'coco_2014_minival': {
-        IM_DIR:
+        _IM_DIR:
             _DATA_DIR + '/coco/coco_val2014',
-        ANN_FN:
+        _ANN_FN:
             _DATA_DIR + '/coco/annotations/instances_minival2014.json'
     },
     'coco_2014_valminusminival': {
-        IM_DIR:
+        _IM_DIR:
             _DATA_DIR + '/coco/coco_val2014',
-        ANN_FN:
+        _ANN_FN:
             _DATA_DIR + '/coco/annotations/instances_valminusminival2014.json'
     },
     'coco_2015_test': {
-        IM_DIR:
+        _IM_DIR:
             _DATA_DIR + '/coco/coco_test2015',
-        ANN_FN:
+        _ANN_FN:
             _DATA_DIR + '/coco/annotations/image_info_test2015.json'
     },
     'coco_2015_test-dev': {
-        IM_DIR:
+        _IM_DIR:
             _DATA_DIR + '/coco/coco_test2015',
-        ANN_FN:
+        _ANN_FN:
             _DATA_DIR + '/coco/annotations/image_info_test-dev2015.json'
     },
     'coco_2017_test': {  # 2017 test uses 2015 test images
-        IM_DIR:
+        _IM_DIR:
             _DATA_DIR + '/coco/coco_test2015',
-        ANN_FN:
+        _ANN_FN:
             _DATA_DIR + '/coco/annotations/image_info_test2017.json',
-        IM_PREFIX:
+        _IM_PREFIX:
             'COCO_test2015_'
     },
     'coco_2017_test-dev': {  # 2017 test-dev uses 2015 test images
-        IM_DIR:
+        _IM_DIR:
             _DATA_DIR + '/coco/coco_test2015',
-        ANN_FN:
+        _ANN_FN:
             _DATA_DIR + '/coco/annotations/image_info_test-dev2017.json',
-        IM_PREFIX:
+        _IM_PREFIX:
             'COCO_test2015_'
     },
     'coco_stuff_train': {
-        IM_DIR:
+        _IM_DIR:
             _DATA_DIR + '/coco/coco_train2014',
-        ANN_FN:
+        _ANN_FN:
             _DATA_DIR + '/coco/annotations/coco_stuff_train.json'
     },
     'coco_stuff_val': {
-        IM_DIR:
+        _IM_DIR:
             _DATA_DIR + '/coco/coco_val2014',
-        ANN_FN:
+        _ANN_FN:
             _DATA_DIR + '/coco/annotations/coco_stuff_val.json'
     },
     'keypoints_coco_2014_train': {
-        IM_DIR:
+        _IM_DIR:
             _DATA_DIR + '/coco/coco_train2014',
-        ANN_FN:
+        _ANN_FN:
             _DATA_DIR + '/coco/annotations/person_keypoints_train2014.json'
     },
     'keypoints_coco_2014_val': {
-        IM_DIR:
+        _IM_DIR:
             _DATA_DIR + '/coco/coco_val2014',
-        ANN_FN:
+        _ANN_FN:
             _DATA_DIR + '/coco/annotations/person_keypoints_val2014.json'
     },
     'keypoints_coco_2014_minival': {
-        IM_DIR:
+        _IM_DIR:
             _DATA_DIR + '/coco/coco_val2014',
-        ANN_FN:
+        _ANN_FN:
             _DATA_DIR + '/coco/annotations/person_keypoints_minival2014.json'
     },
     'keypoints_coco_2014_valminusminival': {
-        IM_DIR:
+        _IM_DIR:
             _DATA_DIR + '/coco/coco_val2014',
-        ANN_FN:
+        _ANN_FN:
             _DATA_DIR + '/coco/annotations/person_keypoints_valminusminival2014.json'
     },
     'keypoints_coco_2015_test': {
-        IM_DIR:
+        _IM_DIR:
             _DATA_DIR + '/coco/coco_test2015',
-        ANN_FN:
+        _ANN_FN:
             _DATA_DIR + '/coco/annotations/image_info_test2015.json'
     },
     'keypoints_coco_2015_test-dev': {
-        IM_DIR:
+        _IM_DIR:
             _DATA_DIR + '/coco/coco_test2015',
-        ANN_FN:
+        _ANN_FN:
             _DATA_DIR + '/coco/annotations/image_info_test-dev2015.json'
     },
     'voc_2007_trainval': {
-        IM_DIR:
+        _IM_DIR:
             _DATA_DIR + '/VOC2007/JPEGImages',
-        ANN_FN:
+        _ANN_FN:
             _DATA_DIR + '/VOC2007/annotations/voc_2007_trainval.json',
-        DEVKIT_DIR:
+        _DEVKIT_DIR:
             _DATA_DIR + '/VOC2007/VOCdevkit2007'
     },
     'voc_2007_test': {
-        IM_DIR:
+        _IM_DIR:
             _DATA_DIR + '/VOC2007/JPEGImages',
-        ANN_FN:
+        _ANN_FN:
             _DATA_DIR + '/VOC2007/annotations/voc_2007_test.json',
-        DEVKIT_DIR:
+        _DEVKIT_DIR:
             _DATA_DIR + '/VOC2007/VOCdevkit2007'
     },
     'voc_2012_trainval': {
-        IM_DIR:
+        _IM_DIR:
             _DATA_DIR + '/VOC2012/JPEGImages',
-        ANN_FN:
+        _ANN_FN:
             _DATA_DIR + '/VOC2012/annotations/voc_2012_trainval.json',
-        DEVKIT_DIR:
+        _DEVKIT_DIR:
             _DATA_DIR + '/VOC2012/VOCdevkit2012'
     }
 }
+
+
+def datasets():
+    """Retrieve the list of available dataset names."""
+    return _DATASETS.keys()
+
+
+def contains(name):
+    """Determine if the dataset is in the catalog."""
+    return name in _DATASETS.keys()
+
+
+def get_im_dir(name):
+    """Retrieve the image directory for the dataset."""
+    return _DATASETS[name][_IM_DIR]
+
+
+def get_ann_fn(name):
+    """Retrieve the annotation file for the dataset."""
+    return _DATASETS[name][_ANN_FN]
+
+
+def get_im_prefix(name):
+    """Retrieve the image prefix for the dataset."""
+    return _DATASETS[name][_IM_PREFIX] if _IM_PREFIX in _DATASETS[name] else ''
+
+
+def get_devkit_dir(name):
+    """Retrieve the devkit dir for the dataset."""
+    return _DATASETS[name][_DEVKIT_DIR]
+
+
+def get_raw_dir(name):
+    """Retrieve the raw dir for the dataset."""
+    return _DATASETS[name][_RAW_DIR]

detectron/datasets/json_dataset.py

@@ -40,11 +40,8 @@ from pycocotools import mask as COCOmask
 from pycocotools.coco import COCO
 
 from detectron.core.config import cfg
-from detectron.datasets.dataset_catalog import ANN_FN
-from detectron.datasets.dataset_catalog import DATASETS
-from detectron.datasets.dataset_catalog import IM_DIR
-from detectron.datasets.dataset_catalog import IM_PREFIX
 from detectron.utils.timer import Timer
+import detectron.datasets.dataset_catalog as dataset_catalog
 import detectron.utils.boxes as box_utils
 
 logger = logging.getLogger(__name__)
@@ -54,19 +51,17 @@ class JsonDataset(object):
     """A class representing a COCO json dataset."""
 
     def __init__(self, name):
-        assert name in DATASETS.keys(), \
+        assert dataset_catalog.contains(name), \
             'Unknown dataset name: {}'.format(name)
-        assert os.path.exists(DATASETS[name][IM_DIR]), \
-            'Image directory \'{}\' not found'.format(DATASETS[name][IM_DIR])
-        assert os.path.exists(DATASETS[name][ANN_FN]), \
-            'Annotation file \'{}\' not found'.format(DATASETS[name][ANN_FN])
+        assert os.path.exists(dataset_catalog.get_im_dir(name)), \
+            'Im dir \'{}\' not found'.format(dataset_catalog.get_im_dir(name))
+        assert os.path.exists(dataset_catalog.get_ann_fn(name)), \
+            'Ann fn \'{}\' not found'.format(dataset_catalog.get_ann_fn(name))
         logger.debug('Creating: {}'.format(name))
         self.name = name
-        self.image_directory = DATASETS[name][IM_DIR]
-        self.image_prefix = (
-            '' if IM_PREFIX not in DATASETS[name] else DATASETS[name][IM_PREFIX]
-        )
-        self.COCO = COCO(DATASETS[name][ANN_FN])
+        self.image_directory = dataset_catalog.get_im_dir(name)
+        self.image_prefix = dataset_catalog.get_im_prefix(name)
+        self.COCO = COCO(dataset_catalog.get_ann_fn(name))
         self.debug_timer = Timer()
         # Set up dataset classes
         category_ids = self.COCO.getCatIds()

detectron/datasets/voc_dataset_evaluator.py

@@ -27,8 +27,7 @@ import shutil
 import uuid
 
 from detectron.core.config import cfg
-from detectron.datasets.dataset_catalog import DATASETS
-from detectron.datasets.dataset_catalog import DEVKIT_DIR
+from detectron.datasets.dataset_catalog import get_devkit_dir
 from detectron.datasets.voc_eval import voc_eval
 from detectron.utils.io import save_object
 
@@ -164,7 +163,7 @@ def _do_matlab_eval(json_dataset, salt, output_dir='output'):
 def voc_info(json_dataset):
     year = json_dataset.name[4:8]
     image_set = json_dataset.name[9:]
-    devkit_path = DATASETS[json_dataset.name][DEVKIT_DIR]
+    devkit_path = get_devkit_dir(json_dataset.name)
     assert os.path.exists(devkit_path), \
         'Devkit directory {} not found'.format(devkit_path)
     anno_path = os.path.join(

tools/generate_testdev_from_test.py

@@ -31,8 +31,7 @@ import json
 import os
 import sys
 
-from detectron.datasets.dataset_catalog import ANN_FN
-from detectron.datasets.dataset_catalog import DATASETS
+from detectron.datasets.dataset_catalog import get_ann_fn
 from detectron.utils.timer import Timer
 
 
@@ -59,14 +58,14 @@ def convert(json_file, output_dir):
         dt = json.load(fid)
     print('done!')
 
-    test_image_info = DATASETS['coco_2017_test'][ANN_FN]
+    test_image_info = get_ann_fn('coco_2017_test')
     with open(test_image_info, 'r') as fid:
         info_test = json.load(fid)
     image_test = info_test['images']
     image_test_id = [i['id'] for i in image_test]
     print('{} has {} images'.format(test_image_info, len(image_test_id)))
 
-    test_dev_image_info = DATASETS['coco_2017_test-dev'][ANN_FN]
+    test_dev_image_info = get_ann_fn('coco_2017_test-dev')
     with open(test_dev_image_info, 'r') as fid:
         info_testdev = json.load(fid)
     image_testdev = info_testdev['images']