Faiss

- Main Page
- Namespaces
- Classes
- Files

- File List

All Classes Namespaces Functions Variables Typedefs Enumerations Enumerator Friends
/data/users/hoss/faiss/IndexShards.h
1 /**
2 * Copyright (c) Facebook, Inc. and its affiliates.
3 *
4 * This source code is licensed under the MIT license found in the
5 * LICENSE file in the root directory of this source tree.
6 */
7
8 #pragma once
9
10 #include "Index.h"
11 #include "IndexBinary.h"
12 #include "ThreadedIndex.h"
13
14 namespace faiss {
15
16 /**
17 * Index that concatenates the results from several sub-indexes
18 */
19 template <typename IndexT>
20 struct IndexShardsTemplate : public ThreadedIndex<IndexT> {
21 using idx_t = typename IndexT::idx_t;
22 using component_t = typename IndexT::component_t;
23 using distance_t = typename IndexT::distance_t;
24
25 /**
26 * The dimension that all sub-indices must share will be the dimension of the
27 * first sub-index added
28 *
29 * @param threaded do we use one thread per sub_index or do
30 * queries sequentially?
31 * @param successive_ids should we shift the returned ids by
32 * the size of each sub-index or return them
33 * as they are?
34 */
35 explicit IndexShardsTemplate(bool threaded = false,
36 bool successive_ids = true);
37
38 /**
39 * @param threaded do we use one thread per sub_index or do
40 * queries sequentially?
41 * @param successive_ids should we shift the returned ids by
42 * the size of each sub-index or return them
43 * as they are?
44 */
45 explicit IndexShardsTemplate(idx_t d,
46 bool threaded = false,
47 bool successive_ids = true);
48
49 /// int version due to the implicit bool conversion ambiguity of int as
50 /// dimension
51 explicit IndexShardsTemplate(int d,
52 bool threaded = false,
53 bool successive_ids = true);
54
55 /// Alias for addIndex()
56 void add_shard(IndexT* index) { this->addIndex(index); }
57
58 /// Alias for removeIndex()
59 void remove_shard(IndexT* index) { this->removeIndex(index); }
60
61 /// supported only for sub-indices that implement add_with_ids
62 void add(idx_t n, const component_t* x) override;
63
64 /**
65 * Cases (successive_ids, xids):
66 * - true, non-NULL ERROR: it makes no sense to pass in ids and
67 * request them to be shifted
68 * - true, NULL OK, but should be called only once (calls add()
69 * on sub-indexes).
70 * - false, non-NULL OK: will call add_with_ids with passed in xids
71 * distributed evenly over shards
72 * - false, NULL OK: will call add_with_ids on each sub-index,
73 * starting at ntotal
74 */
75 void add_with_ids(idx_t n, const component_t* x, const idx_t* xids) override;
76
77 void search(idx_t n, const component_t* x, idx_t k,
78 distance_t* distances, idx_t* labels) const override;
79
80 void train(idx_t n, const component_t* x) override;
81
82 // update metric_type and ntotal. Call if you changes something in
83 // the shard indexes.
84 void sync_with_shard_indexes();
85
86 bool successive_ids;
87
88 protected:
89 /// Called just after an index is added
90 void onAfterAddIndex(IndexT* index) override;
91
92 /// Called just after an index is removed
93 void onAfterRemoveIndex(IndexT* index) override;
94 };
95
96 using IndexShards = IndexShardsTemplate<Index>;
97 using IndexBinaryShards = IndexShardsTemplate<IndexBinary>;
98
99
100 } // namespace faiss
faiss::ThreadedIndex::removeIndex
void removeIndex(IndexT *index)
Definition: ThreadedIndex-inl.h:88
faiss::ThreadedIndex
Definition: ThreadedIndex.h:21
faiss::IndexShardsTemplate::add_with_ids
void add_with_ids(idx_t n, const component_t *x, const idx_t *xids) override
Definition: IndexShards.cpp:198
faiss::ThreadedIndex::addIndex
void addIndex(IndexT *index)
Definition: ThreadedIndex-inl.h:49
faiss::IndexShardsTemplate::add_shard
void add_shard(IndexT *index)
Alias for addIndex()
Definition: IndexShards.h:56
faiss::IndexShardsTemplate::IndexShardsTemplate
IndexShardsTemplate(bool threaded=false, bool successive_ids=true)
Definition: IndexShards.cpp:126
faiss::IndexShardsTemplate
Definition: IndexShards.h:20
faiss::IndexShardsTemplate::add
void add(idx_t n, const component_t *x) override
supported only for sub-indices that implement add_with_ids
Definition: IndexShards.cpp:191
faiss::IndexShardsTemplate::onAfterAddIndex
void onAfterAddIndex(IndexT *index) override
Called just after an index is added.
Definition: IndexShards.cpp:134
faiss::IndexShardsTemplate::remove_shard
void remove_shard(IndexT *index)
Alias for removeIndex()
Definition: IndexShards.h:59
faiss::IndexShardsTemplate::onAfterRemoveIndex
void onAfterRemoveIndex(IndexT *index) override
Called just after an index is removed.
Definition: IndexShards.cpp:140