Faiss

- Main Page
- Namespaces
- Classes
- Files

- File List

All Classes Namespaces Functions Variables Typedefs Enumerations Enumerator Friends
/data/users/matthijs/github_faiss/faiss/MetaIndexes.h
1 /**
2 * Copyright (c) 2015-present, Facebook, Inc.
3 * All rights reserved.
4 *
5 * This source code is licensed under the BSD+Patents license found in the
6 * LICENSE file in the root directory of this source tree.
7 */
8
9 // Copyright 2004-present Facebook. All Rights Reserved.
10 // -*- c++ -*-
11
12 #ifndef META_INDEXES_H
13 #define META_INDEXES_H
14
15
16 #include <vector>
17 #include <unordered_map>
18
19
20 #include "Index.h"
21
22
23 namespace faiss {
24
25 /** Index that translates search results to ids */
26 struct IndexIDMap : Index {
27 Index * index; ///! the sub-index
28 bool own_fields; ///! whether pointers are deleted in destructo
29 std::vector<long> id_map;
30
31 explicit IndexIDMap (Index *index);
32
33 /// Same as add_core, but stores xids instead of sequential ids
34 /// @param xids if non-null, ids to store for the vectors (size n)
35 void add_with_ids(idx_t n, const float* x, const long* xids) override;
36
37 /// this will fail. Use add_with_ids
38 void add(idx_t n, const float* x) override;
39
40 void search(
41 idx_t n,
42 const float* x,
43 idx_t k,
44 float* distances,
45 idx_t* labels) const override;
46
47 void train(idx_t n, const float* x) override;
48
49 void reset() override;
50
51 /// remove ids adapted to IndexFlat
52 long remove_ids(const IDSelector& sel) override;
53
54 void range_search (idx_t n, const float *x, float radius,
55 RangeSearchResult *result) const override;
56
57 ~IndexIDMap() override;
58 IndexIDMap () {own_fields=false; index=nullptr; }
59 };
60
61 /** same as IndexIDMap but also provides an efficient reconstruction
62 implementation via a 2-way index */
63 struct IndexIDMap2 : IndexIDMap {
64
65 std::unordered_map<idx_t, idx_t> rev_map;
66
67 explicit IndexIDMap2 (Index *index);
68
69 /// make the rev_map from scratch
70 void construct_rev_map ();
71
72 void add_with_ids(idx_t n, const float* x, const long* xids) override;
73
74 long remove_ids(const IDSelector& sel) override;
75
76 void reconstruct (idx_t key, float * recons) const override;
77
78 ~IndexIDMap2() override {}
79 IndexIDMap2 () {}
80 };
81
82
83 /** Index that concatenates the results from several sub-indexes
84 *
85 */
86 struct IndexShards : Index {
87
88 std::vector<Index*> shard_indexes;
89 bool own_fields; /// should the sub-indexes be deleted along with this?
90 bool threaded;
91 bool successive_ids;
92
93 /**
94 * @param threaded do we use one thread per sub_index or do
95 * queries sequentially?
96 * @param successive_ids should we shift the returned ids by
97 * the size of each sub-index or return them
98 * as they are?
99 */
100 explicit IndexShards (idx_t d, bool threaded = false,
101 bool successive_ids = true);
102
103 void add_shard (Index *);
104
105 // update metric_type and ntotal. Call if you changes something in
106 // the shard indexes.
107 void sync_with_shard_indexes ();
108
109 Index *at(int i) {return shard_indexes[i]; }
110
111 /// supported only for sub-indices that implement add_with_ids
112 void add(idx_t n, const float* x) override;
113
114 /**
115 * Cases (successive_ids, xids):
116 * - true, non-NULL ERROR: it makes no sense to pass in ids and
117 * request them to be shifted
118 * - true, NULL OK, but should be called only once (calls add()
119 * on sub-indexes).
120 * - false, non-NULL OK: will call add_with_ids with passed in xids
121 * distributed evenly over shards
122 * - false, NULL OK: will call add_with_ids on each sub-index,
123 * starting at ntotal
124 */
125 void add_with_ids(idx_t n, const float* x, const long* xids) override;
126
127 void search(
128 idx_t n,
129 const float* x,
130 idx_t k,
131 float* distances,
132 idx_t* labels) const override;
133
134 void train(idx_t n, const float* x) override;
135
136 void reset() override;
137
138 ~IndexShards() override;
139 };
140
141 /** splits input vectors in segments and assigns each segment to a sub-index
142 * used to distribute a MultiIndexQuantizer
143 */
144
145 struct IndexSplitVectors: Index {
146 bool own_fields;
147 bool threaded;
148 std::vector<Index*> sub_indexes;
149 idx_t sum_d; /// sum of dimensions seen so far
150
151 explicit IndexSplitVectors (idx_t d, bool threaded = false);
152
153 void add_sub_index (Index *);
154 void sync_with_sub_indexes ();
155
156 void add(idx_t n, const float* x) override;
157
158 void search(
159 idx_t n,
160 const float* x,
161 idx_t k,
162 float* distances,
163 idx_t* labels) const override;
164
165 void train(idx_t n, const float* x) override;
166
167 void reset() override;
168
169 ~IndexSplitVectors() override;
170 };
171
172
173
174 }
175
176
177 #endif
faiss::IndexShards::train
void train(idx_t n, const float *x) override
Definition: MetaIndexes.cpp:397
faiss::IndexIDMap2
Definition: MetaIndexes.h:63
faiss::IndexShards::IndexShards
IndexShards(idx_t d, bool threaded=false, bool successive_ids=true)
Definition: MetaIndexes.cpp:367
faiss::IndexIDMap::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: MetaIndexes.cpp:69
faiss::IndexShards::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
Definition: MetaIndexes.cpp:426
faiss::IDSelector
Definition: AuxIndexStructures.h:52
faiss::IndexShards::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: MetaIndexes.cpp:487
faiss::IndexIDMap2::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
Definition: MetaIndexes.cpp:142
faiss::IndexIDMap
Definition: MetaIndexes.h:26
faiss::IndexSplitVectors::reset
void reset() override
removes all elements from the database.
Definition: MetaIndexes.cpp:716
faiss::IndexIDMap::add
void add(idx_t n, const float *x) override
this will fail. Use add_with_ids
Definition: MetaIndexes.cpp:40
faiss::IndexSplitVectors
Definition: MetaIndexes.h:145
faiss::Index::d
int d
vector dimension
Definition: Index.h:64
faiss::IndexIDMap::id_map
std::vector< long > id_map
! whether pointers are deleted in destructo
Definition: MetaIndexes.h:29
faiss::IndexIDMap::range_search
void range_search(idx_t n, const float *x, float radius, RangeSearchResult *result) const override
Definition: MetaIndexes.cpp:80
faiss::IndexSplitVectors::train
void train(idx_t n, const float *x) override
Definition: MetaIndexes.cpp:712
faiss::IndexShards::add
void add(idx_t n, const float *x) override
supported only for sub-indices that implement add_with_ids
Definition: MetaIndexes.cpp:420
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
faiss::IndexIDMap2::construct_rev_map
void construct_rev_map()
make the rev_map from scratch
Definition: MetaIndexes.cpp:151
faiss::IndexSplitVectors::add
void add(idx_t n, const float *x) override
Definition: MetaIndexes.cpp:611
faiss::IndexIDMap2::remove_ids
long remove_ids(const IDSelector &sel) override
remove ids adapted to IndexFlat
Definition: MetaIndexes.cpp:160
faiss::IndexShards::threaded
bool threaded
should the sub-indexes be deleted along with this?
Definition: MetaIndexes.h:90
faiss::IndexShards::reset
void reset() override
removes all elements from the database.
Definition: MetaIndexes.cpp:479
faiss::RangeSearchResult
Definition: AuxIndexStructures.h:29
faiss::IndexIDMap::reset
void reset() override
removes all elements from the database.
Definition: MetaIndexes.cpp:53
faiss::IndexSplitVectors::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: MetaIndexes.cpp:655
faiss::IndexIDMap::remove_ids
long remove_ids(const IDSelector &sel) override
remove ids adapted to IndexFlat
Definition: MetaIndexes.cpp:106
faiss::IndexIDMap2::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: MetaIndexes.cpp:168
faiss::IndexIDMap::train
void train(idx_t n, const float *x) override
Definition: MetaIndexes.cpp:47
faiss::Index
Definition: Index.h:60
faiss::IndexShards
Definition: MetaIndexes.h:86
faiss::IndexIDMap::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
Definition: MetaIndexes.cpp:60
faiss::IndexSplitVectors::IndexSplitVectors
IndexSplitVectors(idx_t d, bool threaded=false)
sum of dimensions seen so far
Definition: MetaIndexes.cpp:581
faiss::IndexIDMap::own_fields
bool own_fields
! the sub-index
Definition: MetaIndexes.h:28