Faiss

- Main Page
- Namespaces
- Classes
- Files

- File List

All Classes Namespaces Functions Variables Typedefs Enumerations Enumerator Friends
/data/users/hoss/faiss/IVFlib.h
1 /**
2 * Copyright (c) Facebook, Inc. and its affiliates.
3 *
4 * This source code is licensed under the MIT license found in the
5 * LICENSE file in the root directory of this source tree.
6 */
7
8 // -*- c++ -*-
9
10 #ifndef FAISS_IVFLIB_H
11 #define FAISS_IVFLIB_H
12
13 /** Since IVF (inverted file) indexes are of so much use for
14 * large-scale use cases, we group a few functions related to them in
15 * this small library. Most functions work both on IndexIVFs and
16 * IndexIVFs embedded within an IndexPreTransform.
17 */
18
19 #include <vector>
20 #include "IndexIVF.h"
21
22 namespace faiss { namespace ivflib {
23
24
25 /** check if two indexes have the same parameters and are trained in
26 * the same way, otherwise throw. */
27 void check_compatible_for_merge (const Index * index1,
28 const Index * index2);
29
30 /** get an IndexIVF from an index. The index may be an IndexIVF or
31 * some wrapper class that encloses an IndexIVF
32 *
33 * throws an exception if this is not the case.
34 */
35 const IndexIVF * extract_index_ivf (const Index * index);
36 IndexIVF * extract_index_ivf (Index * index);
37
38 /** Merge index1 into index0. Works on IndexIVF's and IndexIVF's
39 * embedded in a IndexPreTransform. On output, the index1 is empty.
40 *
41 * @param shift_ids: translate the ids from index1 to index0->prev_ntotal
42 */
43 void merge_into(Index *index0, Index *index1, bool shift_ids);
44
45 typedef Index::idx_t idx_t;
46
47 /* Returns the cluster the embeddings belong to.
48 *
49 * @param index Index, which should be an IVF index
50 * (otherwise there are no clusters)
51 * @param embeddings object descriptors for which the centroids should be found,
52 * size num_objects * d
53 * @param centroid_ids
54 * cluster id each object belongs to, size num_objects
55 */
56 void search_centroid(Index *index,
57 const float* x, int n,
58 idx_t* centroid_ids);
59
60 /* Returns the cluster the embeddings belong to.
61 *
62 * @param index Index, which should be an IVF index
63 * (otherwise there are no clusters)
64 * @param query_centroid_ids
65 * centroid ids corresponding to the query vectors (size n)
66 * @param result_centroid_ids
67 * centroid ids corresponding to the results (size n * k)
68 * other arguments are the same as the standard search function
69 */
70 void search_and_return_centroids(Index *index,
71 size_t n,
72 const float* xin,
73 long k,
74 float *distances,
75 idx_t* labels,
76 idx_t* query_centroid_ids,
77 idx_t* result_centroid_ids);
78
79
80 /** A set of IndexIVFs concatenated together in a FIFO fashion.
81 * at each "step", the oldest index slice is removed and a new index is added.
82 */
83 struct SlidingIndexWindow {
84 /// common index that contains the sliding window
85 Index * index;
86
87 /// InvertedLists of index
88 ArrayInvertedLists *ils;
89
90 /// number of slices currently in index
91 int n_slice;
92
93 /// same as index->nlist
94 size_t nlist;
95
96 /// cumulative list sizes at each slice
97 std::vector<std::vector<size_t> > sizes;
98
99 /// index should be initially empty and trained
100 SlidingIndexWindow (Index *index);
101
102 /** Add one index to the current index and remove the oldest one.
103 *
104 * @param sub_index slice to swap in (can be NULL)
105 * @param remove_oldest if true, remove the oldest slices */
106 void step(const Index *sub_index, bool remove_oldest);
107
108 };
109
110
111 /// Get a subset of inverted lists [i0, i1)
112 ArrayInvertedLists * get_invlist_range (const Index *index,
113 long i0, long i1);
114
115 /// Set a subset of inverted lists
116 void set_invlist_range (Index *index, long i0, long i1,
117 ArrayInvertedLists * src);
118
119
120 // search an IndexIVF, possibly embedded in an IndexPreTransform
121 // with given parameters
122 void search_with_parameters (const Index *index,
123 idx_t n, const float *x, idx_t k,
124 float *distances, idx_t *labels,
125 IVFSearchParameters *params);
126
127 } } // namespace faiss::ivflib
128
129 #endif
faiss::ArrayInvertedLists
simple (default) implementation as an array of inverted lists
Definition: InvertedLists.h:182
faiss::ivflib::SlidingIndexWindow::nlist
size_t nlist
same as index-&gt;nlist
Definition: IVFlib.h:94
faiss::IVFSearchParameters
Definition: IndexIVF.h:60
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
faiss::ivflib::SlidingIndexWindow::ils
ArrayInvertedLists * ils
InvertedLists of index.
Definition: IVFlib.h:88
faiss::ivflib::SlidingIndexWindow::n_slice
int n_slice
number of slices currently in index
Definition: IVFlib.h:91
faiss::ivflib::SlidingIndexWindow
Definition: IVFlib.h:83
faiss::ivflib::SlidingIndexWindow::sizes
std::vector< std::vector< size_t > > sizes
cumulative list sizes at each slice
Definition: IVFlib.h:97
faiss::ivflib::SlidingIndexWindow::SlidingIndexWindow
SlidingIndexWindow(Index *index)
index should be initially empty and trained
Definition: IVFlib.cpp:159
faiss::ivflib::SlidingIndexWindow::step
void step(const Index *sub_index, bool remove_oldest)
Definition: IVFlib.cpp:190
faiss::Index
Definition: Index.h:61
faiss::ivflib::SlidingIndexWindow::index
Index * index
common index that contains the sliding window
Definition: IVFlib.h:85