[C API] Additional API coverage: IndexLSH, I/O, meta-indexes (#425)

* [c_api] leverage index downcast

- make index downcast declaration+definition possible with macros
- use macros on existing index types

* [c_api] Add IndexLSH to C API

* [c_api] Add Index IO interface

- new header index_io_c.h for access to the index_io.h API
- function`faiss_write_index` and overloads for dumping index to a file
- function`faiss_read_index` and overloads for reading index from a file
- function`faiss_clone_index` for cloning an index
- update C example to save index to a file

* [c_api] Add IndexIDMap and IndexShards API

c_api/IndexFlat_c.cpp

@@ -24,6 +24,9 @@ using faiss::IndexFlatL2BaseShift;
 using faiss::IndexRefineFlat;
 using faiss::IndexFlat1D;
 
+DEFINE_DESTRUCTOR(IndexFlat)
+DEFINE_INDEX_DOWNCAST(IndexFlat)
+
 int faiss_IndexFlat_new(FaissIndexFlat** p_index) {
     try {
         *p_index = reinterpret_cast<FaissIndexFlat*>(new IndexFlat());
@@ -39,8 +42,6 @@ int faiss_IndexFlat_new_with(FaissIndexFlat** p_index, idx_t d, FaissMetricType
     } CATCH_AND_HANDLE
 }
 
-DEFINE_DESTRUCTOR(IndexFlat)
-
 void faiss_IndexFlat_xb(FaissIndexFlat* index, float** p_xb, size_t* p_size) {
     auto& xb = reinterpret_cast<IndexFlat*>(index)->xb;
     *p_xb = xb.data();
@@ -49,11 +50,6 @@ void faiss_IndexFlat_xb(FaissIndexFlat* index, float** p_xb, size_t* p_size) {
     }
 }
 
-FaissIndexFlat* faiss_IndexFlat_cast(FaissIndex* index) {
-    return reinterpret_cast<FaissIndexFlat*>(
-        dynamic_cast<IndexFlat*>(reinterpret_cast<Index*>(index)));
-}
-
 int faiss_IndexFlat_compute_distance_subset(
     FaissIndex* index,
     idx_t n,

c_api/IndexFlat_c.h

@@ -44,7 +44,7 @@ void faiss_IndexFlat_xb(FaissIndexFlat* index, float** p_xb, size_t* p_size);
  * @param index opaque pointer to index object
  * @return the same pointer if the index is a flat index, NULL otherwise
  */
-FaissIndexFlat* faiss_IndexFlat_cast(FaissIndex* index);
+FAISS_DECLARE_INDEX_DOWNCAST(IndexFlat)
 
 FAISS_DECLARE_DESTRUCTOR(IndexFlat)
 

c_api/IndexIVFFlat_c.cpp

@@ -20,6 +20,7 @@ using faiss::IndexIVFFlat;
 using faiss::MetricType;
 
 DEFINE_DESTRUCTOR(IndexIVFFlat)
+DEFINE_INDEX_DOWNCAST(IndexIVFFlat)
 
 int faiss_IndexIVFFlat_new(FaissIndexIVFFlat** p_index) {
     try {

c_api/IndexIVFFlat_c.h

@@ -26,6 +26,7 @@ extern "C" {
  */
 FAISS_DECLARE_CLASS(IndexIVFFlat)
 FAISS_DECLARE_DESTRUCTOR(IndexIVFFlat)
+FAISS_DECLARE_INDEX_DOWNCAST(IndexIVFFlat)
 
 int faiss_IndexIVFFlat_new(FaissIndexIVFFlat** p_index);
 

c_api/IndexIVF_c.cpp

@@ -19,6 +19,7 @@ using faiss::IndexIVF;
 using faiss::IndexIVFStats;
 
 DEFINE_DESTRUCTOR(IndexIVF)
+DEFINE_INDEX_DOWNCAST(IndexIVF)
 
 /// number of possible key values
 DEFINE_GETTER(IndexIVF, size_t, nlist)

c_api/IndexIVF_c.h

@@ -39,6 +39,7 @@ extern "C" {
  */
 FAISS_DECLARE_CLASS_INHERITED(IndexIVF, Index)
 FAISS_DECLARE_DESTRUCTOR(IndexIVF)
+FAISS_DECLARE_INDEX_DOWNCAST(IndexIVF)
 
 /// number of possible key values
 FAISS_DECLARE_GETTER(IndexIVF, size_t, nlist)

c_api/IndexLSH_c.cpp

+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD+Patents license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c++ -*-
+
+#include "IndexLSH_c.h"
+#include "IndexLSH.h"
+#include "macros_impl.h"
+
+using faiss::Index;
+using faiss::IndexLSH;
+
+DEFINE_DESTRUCTOR(IndexLSH)
+DEFINE_INDEX_DOWNCAST(IndexLSH)
+
+DEFINE_GETTER(IndexLSH, int, nbits)
+DEFINE_GETTER(IndexLSH, int, bytes_per_vec)
+DEFINE_GETTER_PERMISSIVE(IndexLSH, int, rotate_data)
+DEFINE_GETTER_PERMISSIVE(IndexLSH, int, train_thresholds)
+
+int faiss_IndexLSH_new(FaissIndexLSH** p_index, idx_t d, int nbits) {
+    try {
+        *p_index = reinterpret_cast<FaissIndexLSH*>(new IndexLSH(d, nbits));
+    } CATCH_AND_HANDLE
+}
+
+int faiss_IndexLSH_new_with_options(FaissIndexLSH** p_index, idx_t d, int nbits, int rotate_data, int train_thresholds) {
+    try {
+        *p_index = reinterpret_cast<FaissIndexLSH*>(
+            new IndexLSH(d, nbits, static_cast<bool>(rotate_data), static_cast<bool>(train_thresholds)));
+    } CATCH_AND_HANDLE
+}

c_api/IndexLSH_c.h

+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD+Patents license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c++ -*-
+
+#ifndef INDEX_LSH_C_H
+#define INDEX_LSH_C_H
+
+#include "faiss_c.h"
+#include "Index_c.h"
+#include "Clustering_c.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** The sign of each vector component is put in a binary signature */
+FAISS_DECLARE_CLASS_INHERITED(IndexLSH, Index)
+FAISS_DECLARE_DESTRUCTOR(IndexLSH)
+FAISS_DECLARE_INDEX_DOWNCAST(IndexLSH)
+
+FAISS_DECLARE_GETTER(IndexLSH, int, nbits)
+FAISS_DECLARE_GETTER(IndexLSH, int, bytes_per_vec)
+FAISS_DECLARE_GETTER(IndexLSH, int, rotate_data)
+FAISS_DECLARE_GETTER(IndexLSH, int, train_thresholds)
+
+int faiss_IndexLSH_new(FaissIndexLSH** p_index, idx_t d, int nbits);
+
+int faiss_IndexLSH_new_with_options(FaissIndexLSH** p_index, idx_t d, int nbits, int rotate_data, int train_thresholds);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

c_api/Makefile

@@ -13,8 +13,8 @@ DEBUGFLAG=-DNDEBUG # no debugging
 
 LIBNAME=libfaiss
 CLIBNAME=libfaiss_c
-LIBCOBJ=error_impl.o Index_c.o IndexFlat_c.o Clustering_c.o \
-	AuxIndexStructures_c.o AutoTune_c.o IndexIVF_c.o IndexIVFFlat_c.o
+LIBCOBJ=error_impl.o Index_c.o IndexFlat_c.o Clustering_c.o AuxIndexStructures_c.o \
+	AutoTune_c.o IndexIVF_c.o IndexIVFFlat_c.o IndexLSH_c.o index_io_c.o MetaIndexes_c.o
 
 # Build shared object file by default
 all: $(CLIBNAME).$(SHAREDEXT)
@@ -40,6 +40,9 @@ clean:
 error_impl.o: CFLAGS += -I.. $(DEBUGFLAG)
 error_impl.o: error_impl.cpp error_c.h error_impl.h macros_impl.h
 
+index_io_c.o: CFLAGS += -I.. $(DEBUGFLAG)
+index_io_c.o: index_io_c.cpp error_impl.cpp ../index_io.h macros_impl.h
+
 Index_c.o: CFLAGS += -I.. $(DEBUGFLAG)
 Index_c.o: Index_c.cpp Index_c.h ../Index.h macros_impl.h
 
@@ -52,6 +55,9 @@ IndexIVF_c.o: IndexIVF_c.cpp IndexIVF_c.h ../IndexIVF.h macros_impl.h
 IndexIVFFlat_c.o: CFLAGS += -I.. $(DEBUGFLAG)
 IndexIVFFlat_c.o: IndexIVFFlat_c.cpp IndexIVFFlat_c.h ../IndexIVFFlat.h macros_impl.h
 
+IndexLSH_c.o: CFLAGS += -I.. $(DEBUGFLAG)
+IndexLSH_c.o: IndexLSH_c.cpp IndexLSH_c.h ../IndexLSH.h macros_impl.h
+
 Clustering_c.o: CFLAGS += -I.. $(DEBUGFLAG)
 Clustering_c.o: Clustering_c.cpp Clustering_c.h ../Clustering.h macros_impl.h
 
@@ -60,3 +66,6 @@ AutoTune_c.o: AutoTune_c.cpp AutoTune_c.h ../AutoTune.h macros_impl.h
 
 AuxIndexStructures_c.o: CFLAGS += -I.. $(DEBUGFLAG)
 AuxIndexStructures_c.o: AuxIndexStructures_c.cpp AuxIndexStructures_c.h ../AuxIndexStructures.h macros_impl.h
+
+MetaIndexes_c.o: CFLAGS += -I.. $(DEBUGFLAG)
+MetaIndexes_c.o: MetaIndexes_c.cpp MetaIndexes_c.h ../MetaIndexes.h macros_impl.h

c_api/MetaIndexes_c.cpp

+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD+Patents license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c++ -*-
+
+#include "MetaIndexes_c.h"
+#include "MetaIndexes.h"
+#include "macros_impl.h"
+
+using faiss::Index;
+using faiss::IndexIDMap;
+using faiss::IndexIDMap2;
+using faiss::IndexShards;
+
+DEFINE_GETTER(IndexIDMap, int, own_fields)
+DEFINE_SETTER(IndexIDMap, int, own_fields)
+
+int faiss_IndexIDMap_new(FaissIndexIDMap** p_index, FaissIndex* index) {
+    try {
+        auto out = new IndexIDMap(reinterpret_cast<Index*>(index));
+        *p_index = reinterpret_cast<FaissIndexIDMap*>(out);
+    } CATCH_AND_HANDLE
+}
+
+void faiss_IndexIDMap_id_map(FaissIndexIDMap* index, long** p_id_map, size_t* p_size) {
+    auto idx = reinterpret_cast<IndexIDMap*>(index);
+    if (p_id_map)
+        *p_id_map = idx->id_map.data();
+    if (p_size)
+        *p_size = idx->id_map.size();
+}
+
+int faiss_IndexIDMap2_new(FaissIndexIDMap2** p_index, FaissIndex* index) {
+    try {
+        auto out = new IndexIDMap2(reinterpret_cast<Index*>(index));
+        *p_index = reinterpret_cast<FaissIndexIDMap2*>(out);
+    } CATCH_AND_HANDLE
+}
+
+int faiss_IndexIDMap2_construct_rev_map(FaissIndexIDMap2* index) {
+    try {
+        reinterpret_cast<IndexIDMap2*>(index)->construct_rev_map();
+    } CATCH_AND_HANDLE
+}
+
+DEFINE_GETTER(IndexShards, int, own_fields)
+DEFINE_SETTER(IndexShards, int, own_fields)
+
+DEFINE_GETTER(IndexShards, int, threaded)
+DEFINE_SETTER(IndexShards, int, threaded)
+
+DEFINE_GETTER(IndexShards, int, successive_ids)
+DEFINE_SETTER(IndexShards, int, successive_ids)
+
+int faiss_IndexShards_new(FaissIndexShards** p_index, idx_t d) {
+    try {
+        auto out = new IndexShards(d);
+        *p_index = reinterpret_cast<FaissIndexShards*>(out);
+    } CATCH_AND_HANDLE
+}
+
+int faiss_IndexShards_new_with_options(FaissIndexShards** p_index, idx_t d, int threaded, int successive_ids) {
+    try {
+        auto out = new IndexShards(d, static_cast<bool>(threaded), static_cast<bool>(successive_ids));
+        *p_index = reinterpret_cast<FaissIndexShards*>(out);
+    } CATCH_AND_HANDLE
+}
+
+/** get a pointer to the index' shards (the `shard_indexes` field). The
+ * outputs of this function become invalid after any operation that can modify the index.
+ * 
+ * @param index   opaque pointer to index object
+ * @param p_shard_indexes    output, the pointer to the beginning of `shard_indexes`.
+ * @param p_size  output, the current length of `shard_indexes`.
+ */
+void faiss_IndexShards_shard_indexes(FaissIndexShards* index, FaissIndex** p_shard_indexes, size_t* p_size) {
+    auto idx = reinterpret_cast<IndexShards*>(index);
+    if (p_shard_indexes)
+        *p_shard_indexes = reinterpret_cast<FaissIndex*>(idx->shard_indexes.data());
+    if (p_size)
+        *p_size = idx->shard_indexes.size();
+}
+
+int faiss_IndexShards_add_shard(FaissIndexShards* index, FaissIndex* shard) {
+    try {
+        reinterpret_cast<IndexShards*>(index)->add_shard(
+            reinterpret_cast<Index*>(shard));
+    } CATCH_AND_HANDLE
+}
+
+int faiss_IndexShards_sync_with_shard_indexes(FaissIndexShards* index) {
+    try {
+        reinterpret_cast<IndexShards*>(index)->sync_with_shard_indexes();
+    } CATCH_AND_HANDLE
+}
+
+FaissIndex* faiss_IndexShards_at(FaissIndexShards* index, int i) {
+    auto shard = reinterpret_cast<IndexShards*>(index)->at(i);
+    return reinterpret_cast<FaissIndex*>(shard);
+}

c_api/MetaIndexes_c.h

+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD+Patents license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c++ -*-
+
+#ifndef METAINDEXES_C_H
+#define METAINDEXES_C_H
+
+#include "faiss_c.h"
+#include "Index_c.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** Index that translates search results to ids */
+FAISS_DECLARE_CLASS_INHERITED(IndexIDMap, Index)
+
+FAISS_DECLARE_GETTER_SETTER(IndexIDMap, int, own_fields)
+
+int faiss_IndexIDMap_new(FaissIndexIDMap** p_index, FaissIndex* index);
+
+/** get a pointer to the index map's internal ID vector (the `id_map` field). The
+ * outputs of this function become invalid after any operation that can modify the index.
+ * 
+ * @param index   opaque pointer to index object
+ * @param p_id_map    output, the pointer to the beginning of `id_map`.
+ * @param p_size  output, the current length of `id_map`.
+ */
+void faiss_IndexIDMap_id_map(FaissIndexIDMap* index, long** p_id_map, size_t* p_size);
+
+/** same as IndexIDMap but also provides an efficient reconstruction
+    implementation via a 2-way index */
+FAISS_DECLARE_CLASS_INHERITED(IndexIDMap2, IndexIDMap)
+
+int faiss_IndexIDMap2_new(FaissIndexIDMap2** p_index, FaissIndex* index);
+
+/// make the rev_map from scratch
+int faiss_IndexIDMap2_construct_rev_map(FaissIndexIDMap2* index);
+
+/** Index that concatenates the results from several sub-indexes
+ */
+FAISS_DECLARE_CLASS_INHERITED(IndexShards, Index)
+
+FAISS_DECLARE_GETTER_SETTER(IndexShards, int, own_fields)
+FAISS_DECLARE_GETTER_SETTER(IndexShards, int, threaded)
+FAISS_DECLARE_GETTER_SETTER(IndexShards, int, successive_ids)
+
+int faiss_IndexShards_new(FaissIndexShards** p_index, idx_t d);
+
+int faiss_IndexShards_new_with_options(FaissIndexShards** p_index, idx_t d, int threaded, int successive_ids);
+
+/** get a pointer to the index' shards (the `shard_indexes` field). The
+ * outputs of this function become invalid after any operation that can modify the index.
+ * 
+ * @param index   opaque pointer to index object
+ * @param p_shard_indexes    output, the pointer to the beginning of `shard_indexes`.
+ * @param p_size  output, the current length of `shard_indexes`.
+ */
+void faiss_IndexShards_shard_indexes(FaissIndexShards* index, FaissIndex** p_shard_indexes, size_t* p_size);
+
+int faiss_IndexShards_add_shard(FaissIndexShards* index, FaissIndex* shard);
+
+/// update metric_type and ntotal
+int faiss_IndexShards_sync_with_shard_indexes(FaissIndexShards* index);
+
+FaissIndex* faiss_IndexShards_at(FaissIndexShards* index, int i);
+
+#ifdef __cplusplus
+}
+#endif
+#endif

c_api/example_c.c

@@ -14,6 +14,7 @@
 #include <time.h>
 
 #include "error_c.h"
+#include "index_io_c.h"
 #include "Index_c.h"
 #include "IndexFlat_c.h"
 #include "AutoTune_c.h"
@@ -85,6 +86,9 @@ int main() {
         free(D);
     }
 
+    printf("Saving index to disk...\n");
+    FAISS_TRY(faiss_write_index_fname(index, "example.index"));
+
     printf("Freeing index...\n");
     faiss_Index_free(index);
     printf("Done.\n");

c_api/faiss_c.h

@@ -25,6 +25,12 @@ typedef long idx_t;    ///< all indices are this type
 #define FAISS_DECLARE_CLASS_INHERITED(clazz, parent) \
     typedef struct Faiss ## parent ## _H Faiss ## clazz;
 
+/// Declare a dynamic downcast operation from a base `FaissIndex*` pointer
+/// type to a more specific index type. The function returns the same pointer
+/// if the downcast is valid, and `NULL` otherwise.
+#define FAISS_DECLARE_INDEX_DOWNCAST(clazz) \
+    Faiss ## clazz * faiss_ ## clazz ## _cast (FaissIndex*);
+
 /// Declare a getter for the field `name` in class `clazz`,
 /// of return type `ty`
 #define FAISS_DECLARE_GETTER(clazz, ty, name) \
@@ -33,7 +39,7 @@ typedef long idx_t;    ///< all indices are this type
 /// Declare a setter for the field `name` in class `clazz`,
 /// in which the user provides a value of type `ty`
 #define FAISS_DECLARE_SETTER(clazz, ty, name) \
-    void faiss_ ## clazz ## _set_ ## name (Faiss ## clazz *, ty); \
+    void faiss_ ## clazz ## _set_ ## name (Faiss ## clazz *, ty);
 
 /// Declare a getter and setter for the field `name` in class `clazz`.
 #define FAISS_DECLARE_GETTER_SETTER(clazz, ty, name) \

c_api/index_io_c.cpp

+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD+Patents license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+//  Copyright 2004-present Facebook. All Rights Reserved
+// -*- c++ -*-
+// I/O code for indexes
+
+#include "index_io_c.h"
+#include "index_io.h"
+#include "macros_impl.h"
+
+using faiss::Index;
+
+int faiss_write_index(const FaissIndex *idx, FILE *f) {
+    try {
+        faiss::write_index(reinterpret_cast<const Index*>(idx), f);
+    } CATCH_AND_HANDLE
+}
+
+int faiss_write_index_fname(const FaissIndex *idx, const char *fname) {
+    try {
+        faiss::write_index(reinterpret_cast<const Index*>(idx), fname);
+    } CATCH_AND_HANDLE
+}
+
+int faiss_read_index(FILE *f, int io_flags, FaissIndex **p_out) {
+    try {
+        auto out = faiss::read_index(f, io_flags);
+        *p_out = reinterpret_cast<FaissIndex*>(out);
+    } CATCH_AND_HANDLE
+}
+
+int faiss_read_index_fname(const char *fname, int io_flags, FaissIndex **p_out) {
+    try {
+        auto out = faiss::read_index(fname, io_flags);
+        *p_out = reinterpret_cast<FaissIndex*>(out);
+    } CATCH_AND_HANDLE
+}
+
+int faiss_clone_index (const FaissIndex *idx, FaissIndex **p_out) {
+    try {
+        auto out = faiss::clone_index(reinterpret_cast<const Index*>(idx));
+        *p_out = reinterpret_cast<FaissIndex*>(out);
+    } CATCH_AND_HANDLE
+}

c_api/index_io_c.h

+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD+Patents license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+//  Copyright 2004-present Facebook. All Rights Reserved
+// -*- c++ -*-
+// I/O code for indexes
+
+
+#ifndef FAISS_INDEX_IO_C_H
+#define FAISS_INDEX_IO_C_H
+
+#include <stdio.h>
+#include "faiss_c.h"
+#include "Index_c.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** Write index to a file.
+ * This is equivalent to `faiss::write_index` when a file descriptor is provided.
+ */
+int faiss_write_index(const FaissIndex *idx, FILE *f);
+
+/** Write index to a file.
+ * This is equivalent to `faiss::write_index` when a file path is provided.
+ */
+int faiss_write_index_fname(const FaissIndex *idx, const char *fname);
+
+#define FAISS_IO_FLAG_MMAP 1
+#define FAISS_IO_FLAG_READ_ONLY 2
+
+/** Read index from a file.
+ * This is equivalent to `faiss:read_index` when a file descriptor is given.
+ */
+int faiss_read_index(FILE *f, int io_flags, FaissIndex **p_out);
+
+/** Read index from a file.
+ * This is equivalent to `faiss:read_index` when a file path is given.
+ */
+int faiss_read_index_fname(const char *fname, int io_flags, FaissIndex **p_out);
+
+/* cloning functions */
+
+/** Clone an index. This is equivalent to `faiss::clone_index` */
+int faiss_clone_index (const FaissIndex *, FaissIndex ** p_out);
+
+#ifdef __cplusplus
+}
+#endif
+#endif

c_api/macros_impl.h

@@ -101,4 +101,11 @@
         delete reinterpret_cast<faiss::clazz *>(obj);      \
     }
 
+#define DEFINE_INDEX_DOWNCAST(clazz)                                \
+    Faiss ## clazz * faiss_ ## clazz ## _cast (FaissIndex* index) { \
+        return reinterpret_cast<Faiss ## clazz *>(                  \
+            dynamic_cast< faiss::clazz *>(                          \
+                reinterpret_cast<faiss::Index*>(index)));           \
+    }
+
 #endif