Faiss

- Main Page
- Namespaces
- Classes
- Files

- File List

All Classes Namespaces Functions Variables Typedefs Enumerations Enumerator Friends
/data/users/hoss/faiss/IndexScalarQuantizer.h
1 /**
2 * Copyright (c) Facebook, Inc. and its affiliates.
3 *
4 * This source code is licensed under the MIT license found in the
5 * LICENSE file in the root directory of this source tree.
6 */
7
8 // -*- c++ -*-
9
10 #ifndef FAISS_INDEX_SCALAR_QUANTIZER_H
11 #define FAISS_INDEX_SCALAR_QUANTIZER_H
12
13 #include <stdint.h>
14
15
16 #include <vector>
17
18
19 #include "IndexIVF.h"
20
21
22 namespace faiss {
23
24 /**
25 * The uniform quantizer has a range [vmin, vmax]. The range can be
26 * the same for all dimensions (uniform) or specific per dimension
27 * (default).
28 */
29
30 struct SQDistanceComputer;
31
32 struct ScalarQuantizer {
33
34 enum QuantizerType {
35 QT_8bit, ///< 8 bits per component
36 QT_4bit, ///< 4 bits per component
37 QT_8bit_uniform, ///< same, shared range for all dimensions
38 QT_4bit_uniform,
39 QT_fp16,
40 QT_8bit_direct, /// fast indexing of uint8s
41 QT_6bit, ///< 6 bits per component
42 };
43
44 QuantizerType qtype;
45
46 /** The uniform encoder can estimate the range of representable
47 * values of the unform encoder using different statistics. Here
48 * rs = rangestat_arg */
49
50 // rangestat_arg.
51 enum RangeStat {
52 RS_minmax, ///< [min - rs*(max-min), max + rs*(max-min)]
53 RS_meanstd, ///< [mean - std * rs, mean + std * rs]
54 RS_quantiles, ///< [Q(rs), Q(1-rs)]
55 RS_optim, ///< alternate optimization of reconstruction error
56 };
57
58 RangeStat rangestat;
59 float rangestat_arg;
60
61 /// dimension of input vectors
62 size_t d;
63
64 /// bytes per vector
65 size_t code_size;
66
67 /// trained values (including the range)
68 std::vector<float> trained;
69
70 ScalarQuantizer (size_t d, QuantizerType qtype);
71 ScalarQuantizer ();
72
73 void train (size_t n, const float *x);
74
75
76 /// same as compute_code for several vectors
77 void compute_codes (const float * x,
78 uint8_t * codes,
79 size_t n) const ;
80
81 /// decode a vector from a given code (or n vectors if third argument)
82 void decode (const uint8_t *code, float *x, size_t n) const;
83
84
85 SQDistanceComputer *get_distance_computer (MetricType metric = METRIC_L2)
86 const;
87
88 };
89
90 struct DistanceComputer;
91
92 struct IndexScalarQuantizer: Index {
93 /// Used to encode the vectors
94 ScalarQuantizer sq;
95
96 /// Codes. Size ntotal * pq.code_size
97 std::vector<uint8_t> codes;
98
99 size_t code_size;
100
101 /** Constructor.
102 *
103 * @param d dimensionality of the input vectors
104 * @param M number of subquantizers
105 * @param nbits number of bit per subvector index
106 */
107 IndexScalarQuantizer (int d,
108 ScalarQuantizer::QuantizerType qtype,
109 MetricType metric = METRIC_L2);
110
111 IndexScalarQuantizer ();
112
113 void train(idx_t n, const float* x) override;
114
115 void add(idx_t n, const float* x) override;
116
117 void search(
118 idx_t n,
119 const float* x,
120 idx_t k,
121 float* distances,
122 idx_t* labels) const override;
123
124 void reset() override;
125
126 void reconstruct_n(idx_t i0, idx_t ni, float* recons) const override;
127
128 void reconstruct(idx_t key, float* recons) const override;
129
130 DistanceComputer *get_distance_computer () const;
131
132 };
133
134
135 /** An IVF implementation where the components of the residuals are
136 * encoded with a scalar uniform quantizer. All distance computations
137 * are asymmetric, so the encoded vectors are decoded and approximate
138 * distances are computed.
139 */
140
141 struct IndexIVFScalarQuantizer: IndexIVF {
142 ScalarQuantizer sq;
143 bool by_residual;
144
145 IndexIVFScalarQuantizer(Index *quantizer, size_t d, size_t nlist,
146 ScalarQuantizer::QuantizerType qtype,
147 MetricType metric = METRIC_L2);
148
149 IndexIVFScalarQuantizer();
150
151 void train_residual(idx_t n, const float* x) override;
152
153 void encode_vectors(idx_t n, const float* x,
154 const idx_t *list_nos,
155 uint8_t * codes) const override;
156
157 void add_with_ids(idx_t n, const float* x, const long* xids) override;
158
159 InvertedListScanner *get_InvertedListScanner (bool store_pairs)
160 const override;
161
162
163 void reconstruct_from_offset (long list_no, long offset,
164 float* recons) const override;
165
166 };
167
168
169 }
170
171
172 #endif
faiss::IndexIVF
Definition: IndexIVF.h:90
faiss::IndexIVFScalarQuantizer::encode_vectors
void encode_vectors(idx_t n, const float *x, const idx_t *list_nos, uint8_t *codes) const override
Definition: IndexScalarQuantizer.cpp:1764
faiss::ScalarQuantizer::code_size
size_t code_size
bytes per vector
Definition: IndexScalarQuantizer.h:65
faiss::ScalarQuantizer::QuantizerType
QuantizerType
Definition: IndexScalarQuantizer.h:34
faiss::IndexScalarQuantizer::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: IndexScalarQuantizer.cpp:1640
faiss::SQDistanceComputer
Definition: IndexScalarQuantizer.cpp:47
faiss::IndexIVFScalarQuantizer::train_residual
void train_residual(idx_t n, const float *x) override
Definition: IndexScalarQuantizer.cpp:1735
faiss::DistanceComputer
Definition: AuxIndexStructures.h:231
faiss::ScalarQuantizer::RS_optim
alternate optimization of reconstruction error
Definition: IndexScalarQuantizer.h:55
faiss::IndexScalarQuantizer
Definition: IndexScalarQuantizer.h:92
faiss::ScalarQuantizer::QT_8bit_uniform
same, shared range for all dimensions
Definition: IndexScalarQuantizer.h:37
faiss::IndexIVFScalarQuantizer::reconstruct_from_offset
void reconstruct_from_offset(long list_no, long offset, float *recons) const override
Definition: IndexScalarQuantizer.cpp:1850
faiss::IndexScalarQuantizer::add
void add(idx_t n, const float *x) override
Definition: IndexScalarQuantizer.cpp:1631
faiss::ScalarQuantizer::QT_4bit
4 bits per component
Definition: IndexScalarQuantizer.h:36
faiss::IndexScalarQuantizer::reset
void reset() override
removes all elements from the database.
Definition: IndexScalarQuantizer.cpp:1690
faiss::ScalarQuantizer
Definition: IndexScalarQuantizer.h:32
faiss::IndexIVFScalarQuantizer::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
default implementation that calls encode_vectors
Definition: IndexScalarQuantizer.cpp:1796
faiss::Index::d
int d
vector dimension
Definition: Index.h:66
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
faiss::IndexScalarQuantizer::codes
std::vector< uint8_t > codes
Codes. Size ntotal * pq.code_size.
Definition: IndexScalarQuantizer.h:97
faiss::IndexScalarQuantizer::sq
ScalarQuantizer sq
Used to encode the vectors.
Definition: IndexScalarQuantizer.h:94
faiss::ScalarQuantizer::RangeStat
RangeStat
Definition: IndexScalarQuantizer.h:51
faiss::IndexIVFScalarQuantizer
Definition: IndexScalarQuantizer.h:141
faiss::ScalarQuantizer::RS_quantiles
[Q(rs), Q(1-rs)]
Definition: IndexScalarQuantizer.h:54
faiss::ScalarQuantizer::RS_meanstd
[mean - std * rs, mean + std * rs]
Definition: IndexScalarQuantizer.h:53
faiss::ScalarQuantizer::QT_6bit
fast indexing of uint8s
Definition: IndexScalarQuantizer.h:41
faiss::InvertedListScanner
Definition: IndexIVF.h:272
faiss::ScalarQuantizer::decode
void decode(const uint8_t *code, float *x, size_t n) const
decode a vector from a given code (or n vectors if third argument)
Definition: IndexScalarQuantizer.cpp:1290
faiss::IndexIVFScalarQuantizer::get_InvertedListScanner
InvertedListScanner * get_InvertedListScanner(bool store_pairs) const override
get a scanner for this index (store_pairs means ignore labels)
Definition: IndexScalarQuantizer.cpp:1843
faiss::ScalarQuantizer::compute_codes
void compute_codes(const float *x, uint8_t *codes, size_t n) const
same as compute_code for several vectors
Definition: IndexScalarQuantizer.cpp:1278
faiss::ScalarQuantizer::QT_8bit
8 bits per component
Definition: IndexScalarQuantizer.h:35
faiss::Index
Definition: Index.h:61
faiss::IndexScalarQuantizer::reconstruct_n
void reconstruct_n(idx_t i0, idx_t ni, float *recons) const override
Definition: IndexScalarQuantizer.cpp:1696
faiss::IndexScalarQuantizer::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: IndexScalarQuantizer.cpp:1706
faiss::ScalarQuantizer::RS_minmax
[min - rs*(max-min), max + rs*(max-min)]
Definition: IndexScalarQuantizer.h:52
faiss::ScalarQuantizer::trained
std::vector< float > trained
trained values (including the range)
Definition: IndexScalarQuantizer.h:68
faiss::Level1Quantizer::quantizer
Index * quantizer
quantizer that maps vectors to inverted lists
Definition: IndexIVF.h:32
faiss::IndexScalarQuantizer::train
void train(idx_t n, const float *x) override
Definition: IndexScalarQuantizer.cpp:1625
faiss::ScalarQuantizer::d
size_t d
dimension of input vectors
Definition: IndexScalarQuantizer.h:62
faiss::Level1Quantizer::nlist
size_t nlist
number of possible key values
Definition: IndexIVF.h:33
faiss::MetricType
MetricType
Some algorithms support both an inner product version and a L2 search version.
Definition: Index.h:44