Faiss

- Main Page
- Namespaces
- Classes
- Files

- File List

All Classes Namespaces Functions Variables Typedefs Enumerations Enumerator Friends
/tmp/faiss/MetaIndexes.h
1 /**
2 * Copyright (c) 2015-present, Facebook, Inc.
3 * All rights reserved.
4 *
5 * This source code is licensed under the BSD+Patents license found in the
6 * LICENSE file in the root directory of this source tree.
7 */
8
9 // -*- c++ -*-
10
11 #ifndef META_INDEXES_H
12 #define META_INDEXES_H
13
14
15 #include <vector>
16 #include <unordered_map>
17
18
19 #include "Index.h"
20
21
22 namespace faiss {
23
24 /** Index that translates search results to ids */
25 struct IndexIDMap : Index {
26 Index * index; ///! the sub-index
27 bool own_fields; ///! whether pointers are deleted in destructo
28 std::vector<long> id_map;
29
30 explicit IndexIDMap (Index *index);
31
32 /// Same as add_core, but stores xids instead of sequential ids
33 /// @param xids if non-null, ids to store for the vectors (size n)
34 void add_with_ids(idx_t n, const float* x, const long* xids) override;
35
36 /// this will fail. Use add_with_ids
37 void add(idx_t n, const float* x) override;
38
39 void search(
40 idx_t n,
41 const float* x,
42 idx_t k,
43 float* distances,
44 idx_t* labels) const override;
45
46 void train(idx_t n, const float* x) override;
47
48 void reset() override;
49
50 /// remove ids adapted to IndexFlat
51 long remove_ids(const IDSelector& sel) override;
52
53 void range_search (idx_t n, const float *x, float radius,
54 RangeSearchResult *result) const override;
55
56 ~IndexIDMap() override;
57 IndexIDMap () {own_fields=false; index=nullptr; }
58 };
59
60 /** same as IndexIDMap but also provides an efficient reconstruction
61 implementation via a 2-way index */
62 struct IndexIDMap2 : IndexIDMap {
63
64 std::unordered_map<idx_t, idx_t> rev_map;
65
66 explicit IndexIDMap2 (Index *index);
67
68 /// make the rev_map from scratch
69 void construct_rev_map ();
70
71 void add_with_ids(idx_t n, const float* x, const long* xids) override;
72
73 long remove_ids(const IDSelector& sel) override;
74
75 void reconstruct (idx_t key, float * recons) const override;
76
77 ~IndexIDMap2() override {}
78 IndexIDMap2 () {}
79 };
80
81
82 /** Index that concatenates the results from several sub-indexes
83 *
84 */
85 struct IndexShards : Index {
86
87 std::vector<Index*> shard_indexes;
88 bool own_fields; /// should the sub-indexes be deleted along with this?
89 bool threaded;
90 bool successive_ids;
91
92 /**
93 * @param threaded do we use one thread per sub_index or do
94 * queries sequentially?
95 * @param successive_ids should we shift the returned ids by
96 * the size of each sub-index or return them
97 * as they are?
98 */
99 explicit IndexShards (idx_t d, bool threaded = false,
100 bool successive_ids = true);
101
102 void add_shard (Index *);
103
104 // update metric_type and ntotal. Call if you changes something in
105 // the shard indexes.
106 void sync_with_shard_indexes ();
107
108 Index *at(int i) {return shard_indexes[i]; }
109
110 /// supported only for sub-indices that implement add_with_ids
111 void add(idx_t n, const float* x) override;
112
113 /**
114 * Cases (successive_ids, xids):
115 * - true, non-NULL ERROR: it makes no sense to pass in ids and
116 * request them to be shifted
117 * - true, NULL OK, but should be called only once (calls add()
118 * on sub-indexes).
119 * - false, non-NULL OK: will call add_with_ids with passed in xids
120 * distributed evenly over shards
121 * - false, NULL OK: will call add_with_ids on each sub-index,
122 * starting at ntotal
123 */
124 void add_with_ids(idx_t n, const float* x, const long* xids) override;
125
126 void search(
127 idx_t n,
128 const float* x,
129 idx_t k,
130 float* distances,
131 idx_t* labels) const override;
132
133 void train(idx_t n, const float* x) override;
134
135 void reset() override;
136
137 ~IndexShards() override;
138 };
139
140 /** splits input vectors in segments and assigns each segment to a sub-index
141 * used to distribute a MultiIndexQuantizer
142 */
143
144 struct IndexSplitVectors: Index {
145 bool own_fields;
146 bool threaded;
147 std::vector<Index*> sub_indexes;
148 idx_t sum_d; /// sum of dimensions seen so far
149
150 explicit IndexSplitVectors (idx_t d, bool threaded = false);
151
152 void add_sub_index (Index *);
153 void sync_with_sub_indexes ();
154
155 void add(idx_t n, const float* x) override;
156
157 void search(
158 idx_t n,
159 const float* x,
160 idx_t k,
161 float* distances,
162 idx_t* labels) const override;
163
164 void train(idx_t n, const float* x) override;
165
166 void reset() override;
167
168 ~IndexSplitVectors() override;
169 };
170
171
172 } // namespace faiss
173
174
175 #endif
faiss::IndexShards::train
void train(idx_t n, const float *x) override
Definition: MetaIndexes.cpp:399
faiss::IndexIDMap2
Definition: MetaIndexes.h:62
faiss::IndexShards::IndexShards
IndexShards(idx_t d, bool threaded=false, bool successive_ids=true)
Definition: MetaIndexes.cpp:369
faiss::IndexIDMap::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: MetaIndexes.cpp:69
faiss::IndexShards::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
Definition: MetaIndexes.cpp:428
faiss::IDSelector
Definition: AuxIndexStructures.h:54
faiss::IndexShards::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: MetaIndexes.cpp:489
faiss::IndexIDMap2::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
Definition: MetaIndexes.cpp:144
faiss::IndexIDMap
Definition: MetaIndexes.h:25
faiss::IndexSplitVectors::reset
void reset() override
removes all elements from the database.
Definition: MetaIndexes.cpp:718
faiss::IndexIDMap::add
void add(idx_t n, const float *x) override
this will fail. Use add_with_ids
Definition: MetaIndexes.cpp:39
faiss::IndexSplitVectors
Definition: MetaIndexes.h:144
faiss::Index::d
int d
vector dimension
Definition: Index.h:66
faiss::IndexIDMap::id_map
std::vector< long > id_map
! whether pointers are deleted in destructo
Definition: MetaIndexes.h:28
faiss::IndexIDMap::range_search
void range_search(idx_t n, const float *x, float radius, RangeSearchResult *result) const override
Definition: MetaIndexes.cpp:81
faiss::IndexSplitVectors::train
void train(idx_t n, const float *x) override
Definition: MetaIndexes.cpp:714
faiss::IndexShards::add
void add(idx_t n, const float *x) override
supported only for sub-indices that implement add_with_ids
Definition: MetaIndexes.cpp:422
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:64
faiss::IndexIDMap2::construct_rev_map
void construct_rev_map()
make the rev_map from scratch
Definition: MetaIndexes.cpp:153
faiss::IndexSplitVectors::add
void add(idx_t n, const float *x) override
Definition: MetaIndexes.cpp:613
faiss::IndexIDMap2::remove_ids
long remove_ids(const IDSelector &sel) override
remove ids adapted to IndexFlat
Definition: MetaIndexes.cpp:162
faiss::IndexShards::threaded
bool threaded
should the sub-indexes be deleted along with this?
Definition: MetaIndexes.h:89
faiss::IndexShards::reset
void reset() override
removes all elements from the database.
Definition: MetaIndexes.cpp:481
faiss::RangeSearchResult
Definition: AuxIndexStructures.h:31
faiss::IndexIDMap::reset
void reset() override
removes all elements from the database.
Definition: MetaIndexes.cpp:52
faiss::IndexSplitVectors::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: MetaIndexes.cpp:657
faiss::IndexIDMap::remove_ids
long remove_ids(const IDSelector &sel) override
remove ids adapted to IndexFlat
Definition: MetaIndexes.cpp:108
faiss::IndexIDMap2::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: MetaIndexes.cpp:170
faiss::IndexIDMap::train
void train(idx_t n, const float *x) override
Definition: MetaIndexes.cpp:46
faiss::Index
Definition: Index.h:62
faiss::IndexShards
Definition: MetaIndexes.h:85
faiss::IndexIDMap::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
Definition: MetaIndexes.cpp:60
faiss::IndexSplitVectors::IndexSplitVectors
IndexSplitVectors(idx_t d, bool threaded=false)
sum of dimensions seen so far
Definition: MetaIndexes.cpp:583
faiss::IndexIDMap::own_fields
bool own_fields
! the sub-index
Definition: MetaIndexes.h:27