Faiss

- Main Page
- Namespaces
- Classes
- Files

- File List

All Classes Namespaces Functions Variables Typedefs Enumerations Enumerator Friends
/data/users/matthijs/github_faiss/faiss/IndexIVFScalarQuantizer.h
1 /**
2 * Copyright (c) 2015-present, Facebook, Inc.
3 * All rights reserved.
4 *
5 * This source code is licensed under the CC-by-NC license found in the
6 * LICENSE file in the root directory of this source tree.
7 */
8
9 #ifndef FAISS_INDEX_IVF_SCALAR_QUANTIZER_H
10 #define FAISS_INDEX_IVF_SCALAR_QUANTIZER_H
11
12 #include <stdint.h>
13
14
15 #include <vector>
16
17
18 #include "IndexIVF.h"
19
20
21 namespace faiss {
22
23 /** An IVF implementation where the components of the residuals are
24 * encoded with a scalar uniform quantizer. All distance computations
25 * are asymmetric, so the encoded vectors are decoded and approximate
26 * distances are computed.
27 *
28 * The uniform quantizer has a range [vmin, vmax]. The range can be
29 * the same for all dimensions (uniform) or specific per dimension
30 * (default).
31 */
32
33
34 struct ScalarQuantizer {
35
36 enum QuantizerType {
37 QT_8bit, ///< 8 bits per component
38 QT_4bit, ///< 4 bits per component
39 QT_8bit_uniform, ///< same, shared range for all dimensions
40 QT_4bit_uniform,
41 };
42
43 QuantizerType qtype;
44
45 /** The uniform encoder can estimate the range of representable
46 * values of the unform encoder using different statistics. Here
47 * rs = rangestat_arg */
48
49 // rangestat_arg.
50 enum RangeStat {
51 RS_minmax, ///< [min - rs*(max-min), max + rs*(max-min)]
52 RS_meanstd, ///< [mean - std * rs, mean + std * rs]
53 RS_quantiles, ///< [Q(rs), Q(1-rs)]
54 RS_optim, ///< alternate optimization of reconstruction error
55 };
56
57 RangeStat rangestat;
58 float rangestat_arg;
59
60 /// dimension of input vectors
61 size_t d;
62
63 /// bytes per vector
64 size_t code_size;
65
66 /// trained values (including the range)
67 std::vector<float> trained;
68
69 ScalarQuantizer (size_t d, QuantizerType qtype);
70 ScalarQuantizer ();
71
72 void train (size_t n, const float *x);
73
74
75 /// same as compute_code for several vectors
76 void compute_codes (const float * x,
77 uint8_t * codes,
78 size_t n) const ;
79
80 /// decode a vector from a given code (or n vectors if third argument)
81 void decode (const uint8_t *code, float *x, size_t n) const;
82
83 };
84
85
86 struct IndexIVFScalarQuantizer:IndexIVF {
87 ScalarQuantizer sq;
88
89 size_t code_size;
90
91 /// inverted list codes.
92 std::vector<std::vector<uint8_t> > codes;
93
94 IndexIVFScalarQuantizer(Index *quantizer, size_t d, size_t nlist,
95 ScalarQuantizer::QuantizerType qtype,
96 MetricType metric = METRIC_L2);
97
98 IndexIVFScalarQuantizer();
99
100 void train_residual(idx_t n, const float* x) override;
101
102 void add_with_ids(idx_t n, const float* x, const long* xids) override;
103
104 void search(
105 idx_t n,
106 const float* x,
107 idx_t k,
108 float* distances,
109 idx_t* labels) const override;
110
111 void merge_from_residuals(IndexIVF& other) override;
112 };
113
114
115 }
116
117
118 #endif
faiss::IndexIVF
Definition: IndexIVF.h:45
faiss::ScalarQuantizer::code_size
size_t code_size
bytes per vector
Definition: IndexIVFScalarQuantizer.h:64
faiss::ScalarQuantizer::QuantizerType
QuantizerType
Definition: IndexIVFScalarQuantizer.h:36
faiss::ScalarQuantizer::trained
std::vector< float > trained
trained values (including the range)
Definition: IndexIVFScalarQuantizer.h:67
faiss::IndexIVFScalarQuantizer::train_residual
void train_residual(idx_t n, const float *x) override
Definition: IndexIVFScalarQuantizer.cpp:714
faiss::IndexIVFScalarQuantizer::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: IndexIVFScalarQuantizer.cpp:849
faiss::IndexIVFScalarQuantizer::merge_from_residuals
void merge_from_residuals(IndexIVF &other) override
Definition: IndexIVFScalarQuantizer.cpp:882
faiss::ScalarQuantizer
Definition: IndexIVFScalarQuantizer.h:34
faiss::ScalarQuantizer::RS_optim
alternate optimization of reconstruction error
Definition: IndexIVFScalarQuantizer.h:54
faiss::IndexIVFScalarQuantizer::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
Definition: IndexIVFScalarQuantizer.cpp:733
faiss::Index::d
int d
vector dimension
Definition: Index.h:64
faiss::IndexIVF::quantizer
Index * quantizer
quantizer that maps vectors to inverted lists
Definition: IndexIVF.h:49
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
faiss::ScalarQuantizer::QT_8bit_uniform
same, shared range for all dimensions
Definition: IndexIVFScalarQuantizer.h:39
faiss::ScalarQuantizer::RangeStat
RangeStat
Definition: IndexIVFScalarQuantizer.h:50
faiss::IndexIVFScalarQuantizer
Definition: IndexIVFScalarQuantizer.h:86
faiss::ScalarQuantizer::QT_8bit
8 bits per component
Definition: IndexIVFScalarQuantizer.h:37
faiss::ScalarQuantizer::RS_quantiles
[Q(rs), Q(1-rs)]
Definition: IndexIVFScalarQuantizer.h:53
faiss::ScalarQuantizer::decode
void decode(const uint8_t *code, float *x, size_t n) const
decode a vector from a given code (or n vectors if third argument)
Definition: IndexIVFScalarQuantizer.cpp:685
faiss::ScalarQuantizer::compute_codes
void compute_codes(const float *x, uint8_t *codes, size_t n) const
same as compute_code for several vectors
Definition: IndexIVFScalarQuantizer.cpp:674
faiss::ScalarQuantizer::QT_4bit
4 bits per component
Definition: IndexIVFScalarQuantizer.h:38
faiss::ScalarQuantizer::RS_minmax
[min - rs*(max-min), max + rs*(max-min)]
Definition: IndexIVFScalarQuantizer.h:51
faiss::IndexIVF::nlist
size_t nlist
number of possible key values
Definition: IndexIVF.h:46
faiss::Index
Definition: Index.h:60
faiss::IndexIVFScalarQuantizer::codes
std::vector< std::vector< uint8_t > > codes
inverted list codes.
Definition: IndexIVFScalarQuantizer.h:92
faiss::ScalarQuantizer::RS_meanstd
[mean - std * rs, mean + std * rs]
Definition: IndexIVFScalarQuantizer.h:52
faiss::ScalarQuantizer::d
size_t d
dimension of input vectors
Definition: IndexIVFScalarQuantizer.h:61
faiss::MetricType
MetricType
Some algorithms support both an inner product vetsion and a L2 search version.
Definition: Index.h:43