Faiss

- Main Page
- Namespaces
- Classes
- Files

- File List

All Classes Namespaces Functions Variables Typedefs Enumerations Enumerator Friends
/data/users/matthijs/github_faiss/faiss/MetaIndexes.h
1 /**
2 * Copyright (c) 2015-present, Facebook, Inc.
3 * All rights reserved.
4 *
5 * This source code is licensed under the CC-by-NC license found in the
6 * LICENSE file in the root directory of this source tree.
7 */
8
9 // Copyright 2004-present Facebook. All Rights Reserved.
10 // -*- c++ -*-
11
12 #ifndef META_INDEXES_H
13 #define META_INDEXES_H
14
15
16 #include <vector>
17
18
19 #include "Index.h"
20
21
22 namespace faiss {
23
24 /** Index that translates search results to ids */
25 struct IndexIDMap : Index {
26 Index * index; ///! the sub-index
27 bool own_fields; ///! whether pointers are deleted in destructo
28 std::vector<long> id_map;
29
30 explicit IndexIDMap (Index *index);
31
32 /// Same as add_core, but stores xids instead of sequential ids
33 /// @param xids if non-null, ids to store for the vectors (size n)
34 void add_with_ids(idx_t n, const float* x, const long* xids) override;
35
36 /// this will fail. Use add_with_ids
37 void add(idx_t n, const float* x) override;
38
39 void search(
40 idx_t n,
41 const float* x,
42 idx_t k,
43 float* distances,
44 idx_t* labels) const override;
45
46 void train(idx_t n, const float* x) override;
47
48 void reset() override;
49
50 /// remove ids adapted to IndexFlat
51 long remove_ids(const IDSelector& sel) override;
52
53 ~IndexIDMap() override;
54 IndexIDMap () {own_fields=false; index=nullptr; }
55 };
56
57 /** Index that concatenates the results from several sub-indexes
58 *
59 */
60 struct IndexShards : Index {
61
62 std::vector<Index*> shard_indexes;
63 bool own_fields; /// should the sub-indexes be deleted along with this?
64 bool threaded;
65 bool successive_ids;
66
67 /**
68 * @param threaded do we use one thread per sub_index or do
69 * queries sequentially?
70 * @param successive_ids should we shift the returned ids by
71 * the size of each sub-index or return them
72 * as they are?
73 */
74 explicit IndexShards (idx_t d, bool threaded = false,
75 bool successive_ids = true);
76
77 void add_shard (Index *);
78
79 // update metric_type and ntotal. Call if you changes something in
80 // the shard indexes.
81 void sync_with_shard_indexes ();
82
83 Index *at(int i) {return shard_indexes[i]; }
84
85 /// supported only for sub-indices that implement add_with_ids
86 void add(idx_t n, const float* x) override;
87
88 void add_with_ids(idx_t n, const float* x, const long* xids) override;
89
90 void search(
91 idx_t n,
92 const float* x,
93 idx_t k,
94 float* distances,
95 idx_t* labels) const override;
96
97 void train(idx_t n, const float* x) override;
98
99 void reset() override;
100
101 ~IndexShards() override;
102 };
103
104 /** splits input vectors in segments and assigns each segment to a sub-index
105 * used to distribute a MultiIndexQuantizer
106 */
107
108 struct IndexSplitVectors: Index {
109 bool own_fields;
110 bool threaded;
111 std::vector<Index*> sub_indexes;
112 idx_t sum_d; /// sum of dimensions seen so far
113
114 explicit IndexSplitVectors (idx_t d, bool threaded = false);
115
116 void add_sub_index (Index *);
117 void sync_with_sub_indexes ();
118
119 void add(idx_t n, const float* x) override;
120
121 void search(
122 idx_t n,
123 const float* x,
124 idx_t k,
125 float* distances,
126 idx_t* labels) const override;
127
128 void train(idx_t n, const float* x) override;
129
130 void reset() override;
131
132 ~IndexSplitVectors() override;
133 };
134
135
136
137 }
138
139
140 #endif
faiss::IndexShards::train
void train(idx_t n, const float *x) override
Definition: MetaIndexes.cpp:343
faiss::IndexShards::IndexShards
IndexShards(idx_t d, bool threaded=false, bool successive_ids=true)
Definition: MetaIndexes.cpp:313
faiss::IndexIDMap::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: MetaIndexes.cpp:69
faiss::IndexShards::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
Definition: MetaIndexes.cpp:383
faiss::IDSelector
Definition: AuxIndexStructures.h:57
faiss::IndexShards::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: MetaIndexes.cpp:444
faiss::IndexIDMap
Definition: MetaIndexes.h:25
faiss::IndexSplitVectors::reset
void reset() override
removes all elements from the database.
Definition: MetaIndexes.cpp:676
faiss::IndexIDMap::add
void add(idx_t n, const float *x) override
this will fail. Use add_with_ids
Definition: MetaIndexes.cpp:40
faiss::IndexSplitVectors
Definition: MetaIndexes.h:108
faiss::Index::d
int d
vector dimension
Definition: Index.h:64
faiss::IndexIDMap::id_map
std::vector< long > id_map
! whether pointers are deleted in destructo
Definition: MetaIndexes.h:28
faiss::IndexSplitVectors::train
void train(idx_t n, const float *x) override
Definition: MetaIndexes.cpp:671
faiss::IndexShards::add
void add(idx_t n, const float *x) override
supported only for sub-indices that implement add_with_ids
Definition: MetaIndexes.cpp:366
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
faiss::IndexSplitVectors::add
void add(idx_t n, const float *x) override
Definition: MetaIndexes.cpp:568
faiss::IndexShards::threaded
bool threaded
should the sub-indexes be deleted along with this?
Definition: MetaIndexes.h:64
faiss::IndexShards::reset
void reset() override
removes all elements from the database.
Definition: MetaIndexes.cpp:436
faiss::IndexIDMap::reset
void reset() override
removes all elements from the database.
Definition: MetaIndexes.cpp:53
faiss::IndexSplitVectors::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: MetaIndexes.cpp:613
faiss::IndexIDMap::remove_ids
long remove_ids(const IDSelector &sel) override
remove ids adapted to IndexFlat
Definition: MetaIndexes.cpp:95
faiss::IndexIDMap::train
void train(idx_t n, const float *x) override
Definition: MetaIndexes.cpp:47
faiss::Index
Definition: Index.h:60
faiss::IndexShards
Definition: MetaIndexes.h:60
faiss::IndexIDMap::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
Definition: MetaIndexes.cpp:60
faiss::IndexSplitVectors::IndexSplitVectors
IndexSplitVectors(idx_t d, bool threaded=false)
sum of dimensions seen so far
Definition: MetaIndexes.cpp:538
faiss::IndexIDMap::own_fields
bool own_fields
! the sub-index
Definition: MetaIndexes.h:27