Faiss

- Main Page
- Namespaces
- Classes
- Files

- File List

All Classes Namespaces Functions Variables Typedefs Enumerations Enumerator Friends
/data/users/matthijs/github_faiss/faiss/Clustering.cpp
1
2 /**
3 * Copyright (c) 2015-present, Facebook, Inc.
4 * All rights reserved.
5 *
6 * This source code is licensed under the CC-by-NC license found in the
7 * LICENSE file in the root directory of this source tree.
8 */
9
10 /* Copyright 2004-present Facebook. All Rights Reserved.
11 kmeans clustering routines
12 */
13
14 #include "Clustering.h"
15
16
17
18 #include <cmath>
19 #include <cstdio>
20 #include <cstring>
21
22 #include "utils.h"
23 #include "FaissAssert.h"
24 #include "IndexFlat.h"
25
26 namespace faiss {
27
28 ClusteringParameters::ClusteringParameters ():
29 niter(25),
30 nredo(1),
31 verbose(false), spherical(false),
32 update_index(false),
33 min_points_per_centroid(39),
34 max_points_per_centroid(256),
35 seed(1234)
36 {}
37 // 39 corresponds to 10000 / 256 -> to avoid warnings on PQ tests with randu10k
38
39
40 Clustering::Clustering (int d, int k):
41 d(d), k(k) {}
42
43 Clustering::Clustering (int d, int k, const ClusteringParameters &cp):
44 ClusteringParameters (cp), d(d), k(k) {}
45
46
47
48 static double imbalance_factor (int n, int k, long *assign) {
49 std::vector<int> hist(k, 0);
50 for (int i = 0; i < n; i++)
51 hist[assign[i]]++;
52
53 double tot = 0, uf = 0;
54
55 for (int i = 0 ; i < k ; i++) {
56 tot += hist[i];
57 uf += hist[i] * (double) hist[i];
58 }
59 uf = uf * k / (tot * tot);
60
61 return uf;
62 }
63
64
65
66
67 void Clustering::train (idx_t nx, const float *x_in, Index & index) {
68 FAISS_ASSERT (nx >= k ||
69 !"need at least as many training points as clusters");
70
71 double t0 = getmillisecs();
72
73 // yes it is the user's responsibility, but it may spare us some
74 // hard-to-debug reports.
75 for (size_t i = 0; i < nx * d; i++) {
76 FAISS_ASSERT (finite (x_in[i]) ||
77 !"input contains NaN's or Inf's");
78 }
79
80 const float *x = x_in;
81
82 if (nx > k * max_points_per_centroid) {
83 if (verbose)
84 printf("Sampling a subset of %ld / %ld for training\n",
85 k * max_points_per_centroid, nx);
86 int *perm = new int[nx];
87 rand_perm (perm, nx, seed);
88 nx = k * max_points_per_centroid;
89 float * x_new = new float [nx * d];
90 for (idx_t i = 0; i < nx; i++)
91 memcpy (x_new + i * d, x + perm[i] * d, sizeof(x_new[0]) * d);
92 delete [] perm;
93 x = x_new;
94 } else if (nx < k * min_points_per_centroid) {
95 fprintf (stderr,
96 "WARNING clustering %ld points to %ld centroids: "
97 "please provide at least %ld training points\n",
98 nx, k, idx_t(k) * min_points_per_centroid);
99 }
100
101 if (verbose)
102 printf("Clustering %d points in %ldD to %ld clusters, "
103 "redo %d times, %d iterations\n",
104 int(nx), d, k, nredo, niter);
105
106
107
108 idx_t * assign = new idx_t[nx];
109 float * dis = new float[nx];
110
111 float best_err = 1e50;
112 double t_search_tot = 0;
113 if (verbose) {
114 printf(" Preprocessing in %5g s\n",
115 (getmillisecs() - t0)/1000.);
116 }
117 t0 = getmillisecs();
118
119 for (int redo = 0; redo < nredo; redo++) {
120
121 std::vector<float> buf_centroids;
122
123 std::vector<float> &cur_centroids =
124 nredo == 1 ? centroids : buf_centroids;
125
126 if (verbose && nredo > 1) {
127 printf("Outer iteration %d / %d\n", redo, nredo);
128 }
129
130 if (cur_centroids.size() == 0) {
131 // initialize centroids with random points from the dataset
132 cur_centroids.resize (d * k);
133 int *perm = new int[nx];
134 rand_perm (perm, nx, seed + 1 + redo * 15486557L);
135 #pragma omp parallel for
136 for (int i = 0; i < k ; i++)
137 memcpy (&cur_centroids[i * d], x + perm[i] * d,
138 d * sizeof (float));
139 delete [] perm;
140 } else { // assume user provides some meaningful initialization
141 FAISS_ASSERT (cur_centroids.size() == d * k);
142 FAISS_ASSERT (nredo == 1 ||
143 !"will redo with same initialization");
144 }
145
146 if (spherical)
147 fvec_renorm_L2 (d, k, cur_centroids.data());
148
149 if (!index.is_trained)
150 index.train (k, cur_centroids.data());
151
152 FAISS_ASSERT (index.ntotal == 0 );
153 index.add (k, cur_centroids.data());
154 float err = 0;
155 for (int i = 0; i < niter; i++) {
156 double t0s = getmillisecs();
157 index.search (nx, x, 1, dis, assign);
158 t_search_tot += getmillisecs() - t0s;
159
160 err = 0;
161 for (int j = 0; j < nx; j++)
162 err += dis[j];
163 obj.push_back (err);
164
165 int nsplit = km_update_centroids (x, cur_centroids.data(),
166 assign, d, k, nx);
167
168 if (verbose) {
169 printf (" Iteration %d (%.2f s, search %.2f s): "
170 "objective=%g imbalance=%.3f nsplit=%d \r",
171 i, (getmillisecs() - t0) / 1000.0,
172 t_search_tot / 1000,
173 err, imbalance_factor (nx, k, assign),
174 nsplit);
175 fflush (stdout);
176 }
177
178 if (spherical)
179 fvec_renorm_L2 (d, k, cur_centroids.data());
180
181 index.reset ();
182 if (update_index)
183 index.train (k, cur_centroids.data());
184
185 assert (index.ntotal == 0);
186 index.add (k, centroids.data());
187 }
188 if (verbose) printf("\n");
189 if (nredo > 1) {
190 if (err < best_err) {
191 if (verbose)
192 printf ("Keep new clusters\n");
193 centroids = cur_centroids;
194 best_err = err;
195 }
196 }
197 }
198
199 delete [] assign;
200 delete [] dis;
201 if (x_in != x) delete [] x;
202 }
203
204 float kmeans_clustering (size_t d, size_t n, size_t k,
205 const float *x,
206 float *centroids)
207 {
208 Clustering clus (d, k);
209 clus.verbose = d * n * k > (1L << 30);
210 // display logs if > 1Gflop per iteration
211 IndexFlatL2 index (d);
212 clus.train (n, x, index);
213 memcpy(centroids, clus.centroids.data(), sizeof(*centroids) * d * k);
214 return clus.obj.back();
215 }
216
217 } // namespace faiss
faiss::ClusteringParameters::niter
int niter
clustering iterations
Definition: Clustering.h:26
faiss::km_update_centroids
int km_update_centroids(const float *x, float *centroids, long *assign, size_t d, size_t k, size_t n)
Definition: utils.cpp:1286
faiss::ClusteringParameters::nredo
int nredo
redo clustering this many times and keep best
Definition: Clustering.h:27
faiss::ClusteringParameters::ClusteringParameters
ClusteringParameters()
sets reasonable defaults
Definition: Clustering.cpp:28
faiss::Index::reset
virtual void reset()=0
removes all elements from the database.
faiss::IndexFlatL2
Definition: IndexFlat.h:81
faiss::Clustering::Clustering
Clustering(int d, int k)
the only mandatory parameters are k and d
Definition: Clustering.cpp:40
faiss::Clustering::k
size_t k
nb of centroids
Definition: Clustering.h:61
faiss::ClusteringParameters::seed
int seed
seed for the random number generator
Definition: Clustering.h:37
faiss::ClusteringParameters::min_points_per_centroid
int min_points_per_centroid
otherwise you get a warning
Definition: Clustering.h:34
faiss::Clustering
Definition: Clustering.h:58
faiss::Index::add
virtual void add(idx_t n, const float *x)=0
faiss::Clustering::obj
std::vector< float > obj
Definition: Clustering.h:68
faiss::kmeans_clustering
float kmeans_clustering(size_t d, size_t n, size_t k, const float *x, float *centroids)
Definition: Clustering.cpp:204
faiss::Index::ntotal
idx_t ntotal
total nb of indexed vectors
Definition: Index.h:67
faiss::getmillisecs
double getmillisecs()
ms elapsed since some arbitrary epoch
Definition: utils.cpp:71
faiss::Clustering::centroids
std::vector< float > centroids
centroids (k * d)
Definition: Clustering.h:64
faiss::Clustering::d
size_t d
dimension of the vectors
Definition: Clustering.h:60
faiss::Index::search
virtual void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const =0
faiss::ClusteringParameters::update_index
bool update_index
update index after each iteration?
Definition: Clustering.h:32
faiss::ClusteringParameters
Definition: Clustering.h:25
faiss::Index
Definition: Index.h:61
faiss::Clustering::train
virtual void train(idx_t n, const float *x, faiss::Index &index)
Index is used during the assignment stage.
Definition: Clustering.cpp:67
faiss::Index::is_trained
bool is_trained
set if the Index does not require training, or if training is done already
Definition: Index.h:71
faiss::Index::train
virtual void train(idx_t n, const float *x)
Definition: Index.h:92
faiss::ClusteringParameters::spherical
bool spherical
do we want normalized centroids?
Definition: Clustering.h:31
faiss::ClusteringParameters::max_points_per_centroid
int max_points_per_centroid
to limit size of dataset
Definition: Clustering.h:35