Faiss

- Main Page
- Namespaces
- Classes
- Files

- File List

All Classes Namespaces Functions Variables Typedefs Enumerations Enumerator Friends
/tmp/faiss/IVFlib.h
1 /**
2 * Copyright (c) 2015-present, Facebook, Inc.
3 * All rights reserved.
4 *
5 * This source code is licensed under the BSD+Patents license found in the
6 * LICENSE file in the root directory of this source tree.
7 */
8
9 // -*- c++ -*-
10
11 #ifndef FAISS_IVFLIB_H
12 #define FAISS_IVFLIB_H
13
14 /** Since IVF (inverted file) indexes are of so much use for
15 * large-scale use cases, we group a few functions related to them in
16 * this small library. Most functions work both on IndexIVFs and
17 * IndexIVFs embedded within an IndexPreTransform.
18 */
19
20 #include <vector>
21 #include "IndexIVF.h"
22
23 namespace faiss { namespace ivflib {
24
25
26 /** check if two indexes have the same parameters and are trained in
27 * the same way, otherwise throw. */
28 void check_compatible_for_merge (const Index * index1,
29 const Index * index2);
30
31 /** get an IndexIVF from an index. The index may be an IndexIVF or
32 * some wrapper class that encloses an IndexIVF
33 *
34 * throws an exception if this is not the case.
35 */
36 const IndexIVF * extract_index_ivf (const Index * index);
37 IndexIVF * extract_index_ivf (Index * index);
38
39 /** Merge index1 into index0. Works on IndexIVF's and IndexIVF's
40 * embedded in a IndexPreTransform. On output, the index1 is empty.
41 *
42 * @param shift_ids: translate the ids from index1 to index0->prev_ntotal
43 */
44 void merge_into(Index *index0, Index *index1, bool shift_ids);
45
46 typedef Index::idx_t idx_t;
47
48 /* Returns the cluster the embeddings belong to.
49 *
50 * @param index Index, which should be an IVF index
51 * (otherwise there are no clusters)
52 * @param embeddings object descriptors for which the centroids should be found,
53 * size num_objects * d
54 * @param centroid_ids
55 * cluster id each object belongs to, size num_objects
56 */
57 void search_centroid(Index *index,
58 const float* x, int n,
59 idx_t* centroid_ids);
60
61 /* Returns the cluster the embeddings belong to.
62 *
63 * @param index Index, which should be an IVF index
64 * (otherwise there are no clusters)
65 * @param query_centroid_ids
66 * centroid ids corresponding to the query vectors (size n)
67 * @param result_centroid_ids
68 * centroid ids corresponding to the results (size n * k)
69 * other arguments are the same as the standard search function
70 */
71 void search_and_return_centroids(Index *index,
72 size_t n,
73 const float* xin,
74 long k,
75 float *distances,
76 idx_t* labels,
77 idx_t* query_centroid_ids,
78 idx_t* result_centroid_ids);
79
80
81 /** A set of IndexIVFs concatenated together in a FIFO fashion.
82 * at each "step", the oldest index slice is removed and a new index is added.
83 */
84 struct SlidingIndexWindow {
85 /// common index that contains the sliding window
86 Index * index;
87
88 /// InvertedLists of index
89 ArrayInvertedLists *ils;
90
91 /// number of slices currently in index
92 int n_slice;
93
94 /// same as index->nlist
95 size_t nlist;
96
97 /// cumulative list sizes at each slice
98 std::vector<std::vector<size_t> > sizes;
99
100 /// index should be initially empty and trained
101 SlidingIndexWindow (Index *index);
102
103 /** Add one index to the current index and remove the oldest one.
104 *
105 * @param sub_index slice to swap in (can be NULL)
106 * @param remove_oldest if true, remove the oldest slices */
107 void step(const Index *sub_index, bool remove_oldest);
108
109 };
110
111
112 /// Get a subset of inverted lists [i0, i1)
113 ArrayInvertedLists * get_invlist_range (const Index *index,
114 long i0, long i1);
115
116 /// Set a subset of inverted lists
117 void set_invlist_range (Index *index, long i0, long i1,
118 ArrayInvertedLists * src);
119
120
121 // search an IndexIVF, possibly embedded in an IndexPreTransform
122 // with given parameters
123 void search_with_parameters (const Index *index,
124 idx_t n, const float *x, idx_t k,
125 float *distances, idx_t *labels,
126 IVFSearchParameters *params);
127
128 } } // namespace faiss::ivflib
129
130 #endif
faiss::ArrayInvertedLists
simple (default) implementation as an array of inverted lists
Definition: InvertedLists.h:168
faiss::ivflib::SlidingIndexWindow::nlist
size_t nlist
same as index-&gt;nlist
Definition: IVFlib.h:95
faiss::IVFSearchParameters
Definition: IndexIVF.h:61
faiss::ivflib::SlidingIndexWindow::ils
ArrayInvertedLists * ils
InvertedLists of index.
Definition: IVFlib.h:89
faiss::ivflib::SlidingIndexWindow::n_slice
int n_slice
number of slices currently in index
Definition: IVFlib.h:92
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:64
faiss::ivflib::SlidingIndexWindow
Definition: IVFlib.h:84
faiss::ivflib::SlidingIndexWindow::sizes
std::vector< std::vector< size_t > > sizes
cumulative list sizes at each slice
Definition: IVFlib.h:98
faiss::ivflib::SlidingIndexWindow::SlidingIndexWindow
SlidingIndexWindow(Index *index)
index should be initially empty and trained
Definition: IVFlib.cpp:160
faiss::ivflib::SlidingIndexWindow::step
void step(const Index *sub_index, bool remove_oldest)
Definition: IVFlib.cpp:191
faiss::Index
Definition: Index.h:62
faiss::ivflib::SlidingIndexWindow::index
Index * index
common index that contains the sliding window
Definition: IVFlib.h:86