Faiss

- Main Page
- Namespaces
- Classes
- Files

- File List

All Classes Namespaces Functions Variables Typedefs Enumerations Enumerator Friends
/tmp/faiss/Clustering.cpp
1 /**
2 * Copyright (c) 2015-present, Facebook, Inc.
3 * All rights reserved.
4 *
5 * This source code is licensed under the BSD+Patents license found in the
6 * LICENSE file in the root directory of this source tree.
7 */
8
9 // -*- c++ -*-
10
11 #include "Clustering.h"
12
13
14 #include <cmath>
15 #include <cstdio>
16 #include <cstring>
17
18 #include "utils.h"
19 #include "FaissAssert.h"
20 #include "IndexFlat.h"
21
22 namespace faiss {
23
24 ClusteringParameters::ClusteringParameters ():
25 niter(25),
26 nredo(1),
27 verbose(false), spherical(false),
28 update_index(false),
29 frozen_centroids(false),
30 min_points_per_centroid(39),
31 max_points_per_centroid(256),
32 seed(1234)
33 {}
34 // 39 corresponds to 10000 / 256 -> to avoid warnings on PQ tests with randu10k
35
36
37 Clustering::Clustering (int d, int k):
38 d(d), k(k) {}
39
40 Clustering::Clustering (int d, int k, const ClusteringParameters &cp):
41 ClusteringParameters (cp), d(d), k(k) {}
42
43
44
45 static double imbalance_factor (int n, int k, long *assign) {
46 std::vector<int> hist(k, 0);
47 for (int i = 0; i < n; i++)
48 hist[assign[i]]++;
49
50 double tot = 0, uf = 0;
51
52 for (int i = 0 ; i < k ; i++) {
53 tot += hist[i];
54 uf += hist[i] * (double) hist[i];
55 }
56 uf = uf * k / (tot * tot);
57
58 return uf;
59 }
60
61
62
63
64 void Clustering::train (idx_t nx, const float *x_in, Index & index) {
65 FAISS_THROW_IF_NOT_FMT (nx >= k,
66 "Number of training points (%ld) should be at least "
67 "as large as number of clusters (%ld)", nx, k);
68
69 double t0 = getmillisecs();
70
71 // yes it is the user's responsibility, but it may spare us some
72 // hard-to-debug reports.
73 for (size_t i = 0; i < nx * d; i++) {
74 FAISS_THROW_IF_NOT_MSG (finite (x_in[i]),
75 "input contains NaN's or Inf's");
76 }
77
78 const float *x = x_in;
79 ScopeDeleter<float> del1;
80
81 if (nx > k * max_points_per_centroid) {
82 if (verbose)
83 printf("Sampling a subset of %ld / %ld for training\n",
84 k * max_points_per_centroid, nx);
85 std::vector<int> perm (nx);
86 rand_perm (perm.data (), nx, seed);
87 nx = k * max_points_per_centroid;
88 float * x_new = new float [nx * d];
89 for (idx_t i = 0; i < nx; i++)
90 memcpy (x_new + i * d, x + perm[i] * d, sizeof(x_new[0]) * d);
91 x = x_new;
92 del1.set (x);
93 } else if (nx < k * min_points_per_centroid) {
94 fprintf (stderr,
95 "WARNING clustering %ld points to %ld centroids: "
96 "please provide at least %ld training points\n",
97 nx, k, idx_t(k) * min_points_per_centroid);
98 }
99
100
101 if (nx == k) {
102 if (verbose) {
103 printf("Number of training points (%ld) same as number of "
104 "clusters, just copying\n", nx);
105 }
106 // this is a corner case, just copy training set to clusters
107 centroids.resize (d * k);
108 memcpy (centroids.data(), x_in, sizeof (*x_in) * d * k);
109 index.reset();
110 index.add(k, x_in);
111 return;
112 }
113
114
115 if (verbose)
116 printf("Clustering %d points in %ldD to %ld clusters, "
117 "redo %d times, %d iterations\n",
118 int(nx), d, k, nredo, niter);
119
120
121
122
123 idx_t * assign = new idx_t[nx];
124 ScopeDeleter<idx_t> del (assign);
125 float * dis = new float[nx];
126 ScopeDeleter<float> del2(dis);
127
128 // for redo
129 float best_err = HUGE_VALF;
130 std::vector<float> best_obj;
131 std::vector<float> best_centroids;
132
133 // support input centroids
134
135 FAISS_THROW_IF_NOT_MSG (
136 centroids.size() % d == 0,
137 "size of provided input centroids not a multiple of dimension");
138
139 size_t n_input_centroids = centroids.size() / d;
140
141 if (verbose && n_input_centroids > 0) {
142 printf (" Using %zd centroids provided as input (%sfrozen)\n",
143 n_input_centroids, frozen_centroids ? "" : "not ");
144 }
145
146 double t_search_tot = 0;
147 if (verbose) {
148 printf(" Preprocessing in %.2f s\n",
149 (getmillisecs() - t0)/1000.);
150 }
151 t0 = getmillisecs();
152
153 for (int redo = 0; redo < nredo; redo++) {
154
155 if (verbose && nredo > 1) {
156 printf("Outer iteration %d / %d\n", redo, nredo);
157 }
158
159
160 // initialize remaining centroids with random points from the dataset
161 centroids.resize (d * k);
162 std::vector<int> perm (nx);
163
164 rand_perm (perm.data(), nx, seed + 1 + redo * 15486557L);
165 for (int i = n_input_centroids; i < k ; i++)
166 memcpy (&centroids[i * d], x + perm[i] * d,
167 d * sizeof (float));
168
169 if (spherical) {
170 fvec_renorm_L2 (d, k, centroids.data());
171 }
172
173 if (index.ntotal != 0) {
174 index.reset();
175 }
176
177 if (!index.is_trained) {
178 index.train (k, centroids.data());
179 }
180
181 index.add (k, centroids.data());
182 float err = 0;
183 for (int i = 0; i < niter; i++) {
184 double t0s = getmillisecs();
185 index.search (nx, x, 1, dis, assign);
186 t_search_tot += getmillisecs() - t0s;
187
188 err = 0;
189 for (int j = 0; j < nx; j++)
190 err += dis[j];
191 obj.push_back (err);
192
193 int nsplit = km_update_centroids (
194 x, centroids.data(),
195 assign, d, k, nx, frozen_centroids ? n_input_centroids : 0);
196
197 if (verbose) {
198 printf (" Iteration %d (%.2f s, search %.2f s): "
199 "objective=%g imbalance=%.3f nsplit=%d \r",
200 i, (getmillisecs() - t0) / 1000.0,
201 t_search_tot / 1000,
202 err, imbalance_factor (nx, k, assign),
203 nsplit);
204 fflush (stdout);
205 }
206
207 if (spherical)
208 fvec_renorm_L2 (d, k, centroids.data());
209
210 index.reset ();
211 if (update_index)
212 index.train (k, centroids.data());
213
214 assert (index.ntotal == 0);
215 index.add (k, centroids.data());
216 }
217 if (verbose) printf("\n");
218 if (nredo > 1) {
219 if (err < best_err) {
220 if (verbose)
221 printf ("Objective improved: keep new clusters\n");
222 best_centroids = centroids;
223 best_obj = obj;
224 best_err = err;
225 }
226 index.reset ();
227 }
228 }
229 if (nredo > 1) {
230 centroids = best_centroids;
231 obj = best_obj;
232 index.reset();
233 index.add(k, best_centroids.data());
234 }
235
236 }
237
238 float kmeans_clustering (size_t d, size_t n, size_t k,
239 const float *x,
240 float *centroids)
241 {
242 Clustering clus (d, k);
243 clus.verbose = d * n * k > (1L << 30);
244 // display logs if > 1Gflop per iteration
245 IndexFlatL2 index (d);
246 clus.train (n, x, index);
247 memcpy(centroids, clus.centroids.data(), sizeof(*centroids) * d * k);
248 return clus.obj.back();
249 }
250
251 } // namespace faiss
faiss::km_update_centroids
int km_update_centroids(const float *x, float *centroids, long *assign, size_t d, size_t k, size_t n, size_t k_frozen)
Definition: utils.cpp:1066
faiss::ClusteringParameters::niter
int niter
clustering iterations
Definition: Clustering.h:24
faiss::ClusteringParameters::nredo
int nredo
redo clustering this many times and keep best
Definition: Clustering.h:25
faiss::ClusteringParameters::ClusteringParameters
ClusteringParameters()
sets reasonable defaults
Definition: Clustering.cpp:24
faiss::Index::reset
virtual void reset()=0
removes all elements from the database.
faiss::IndexFlatL2
Definition: IndexFlat.h:78
faiss::Clustering::Clustering
Clustering(int d, int k)
the only mandatory parameters are k and d
Definition: Clustering.cpp:37
faiss::Index::train
virtual void train(idx_t n, const float *x)
Definition: Index.cpp:24
faiss::Clustering::k
size_t k
nb of centroids
Definition: Clustering.h:59
faiss::ScopeDeleter
Definition: FaissException.h:39
faiss::ClusteringParameters::seed
int seed
seed for the random number generator
Definition: Clustering.h:35
faiss::ClusteringParameters::frozen_centroids
bool frozen_centroids
use the centroids provided as input and do not change them during iterations
Definition: Clustering.h:30
faiss::ClusteringParameters::min_points_per_centroid
int min_points_per_centroid
otherwise you get a warning
Definition: Clustering.h:32
faiss::Clustering
Definition: Clustering.h:56
faiss::Index::add
virtual void add(idx_t n, const float *x)=0
faiss::Clustering::obj
std::vector< float > obj
Definition: Clustering.h:66
faiss::kmeans_clustering
float kmeans_clustering(size_t d, size_t n, size_t k, const float *x, float *centroids)
Definition: Clustering.cpp:238
faiss::Index::ntotal
idx_t ntotal
total nb of indexed vectors
Definition: Index.h:67
faiss::getmillisecs
double getmillisecs()
ms elapsed since some arbitrary epoch
Definition: utils.cpp:70
faiss::Clustering::centroids
std::vector< float > centroids
centroids (k * d)
Definition: Clustering.h:62
faiss::Clustering::d
size_t d
dimension of the vectors
Definition: Clustering.h:58
faiss::Index::search
virtual void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const =0
faiss::ClusteringParameters::update_index
bool update_index
update index after each iteration?
Definition: Clustering.h:29
faiss::ClusteringParameters
Definition: Clustering.h:23
faiss::Index
Definition: Index.h:62
faiss::Clustering::train
virtual void train(idx_t n, const float *x, faiss::Index &index)
Index is used during the assignment stage.
Definition: Clustering.cpp:64
faiss::Index::is_trained
bool is_trained
set if the Index does not require training, or if training is done already
Definition: Index.h:71
faiss::ClusteringParameters::spherical
bool spherical
do we want normalized centroids?
Definition: Clustering.h:28
faiss::ClusteringParameters::max_points_per_centroid
int max_points_per_centroid
to limit size of dataset
Definition: Clustering.h:33