Faiss

- Main Page
- Namespaces
- Classes
- Files

- File List

All Classes Namespaces Functions Variables Typedefs Enumerations Enumerator Friends
WriteIndex.cpp
1 /**
2 * Copyright (c) 2015-present, Facebook, Inc.
3 * All rights reserved.
4 *
5 * This source code is licensed under the BSD+Patents license found in the
6 * LICENSE file in the root directory of this source tree.
7 */
8
9 // Copyright 2004-present Facebook. All Rights Reserved.
10
11 #include "../../IndexIVFFlat.h"
12 #include "../../IndexIVFPQ.h"
13 #include "../../IndexFlat.h"
14 #include "../../index_io.h"
15 #include "../test/TestUtils.h"
16 #include <vector>
17 #include <gflags/gflags.h>
18
19 // For IVFPQ:
20 DEFINE_bool(ivfpq, false, "use IVFPQ encoding");
21 DEFINE_int32(codes, 4, "number of PQ codes per vector");
22 DEFINE_int32(bits_per_code, 8, "number of bits per PQ code");
23
24 // For IVFFlat:
25 DEFINE_bool(l2, true, "use L2 metric (versus IP metric)");
26 DEFINE_bool(ivfflat, false, "use IVF flat encoding");
27
28 // For both:
29 DEFINE_string(out, "/home/jhj/local/index.out", "index file for output");
30 DEFINE_int32(dim, 128, "vector dimension");
31 DEFINE_int32(num_coarse, 100, "number of coarse centroids");
32 DEFINE_int32(num, 100000, "total database size");
33 DEFINE_int32(num_train, -1, "number of database vecs to train on");
34
35 template <typename T>
36 void fillAndSave(T& index, int numTrain, int num, int dim) {
37 auto trainVecs = faiss::gpu::randVecs(numTrain, dim);
38 index.train(numTrain, trainVecs.data());
39
40 constexpr int kAddChunk = 1000000;
41
42 for (int i = 0; i < num; i += kAddChunk) {
43 int numRemaining = (num - i) < kAddChunk ? (num - i) : kAddChunk;
44 auto vecs = faiss::gpu::randVecs(numRemaining, dim);
45
46 printf("adding at %d: %d\n", i, numRemaining);
47 index.add(numRemaining, vecs.data());
48 }
49
50 faiss::write_index(&index, FLAGS_out.c_str());
51 }
52
53 int main(int argc, char** argv) {
54 gflags::ParseCommandLineFlags(&argc, &argv, true);
55
56 // Either ivfpq or ivfflat must be set
57 if ((FLAGS_ivfpq && FLAGS_ivfflat) ||
58 (!FLAGS_ivfpq && !FLAGS_ivfflat)) {
59 printf("must specify either ivfpq or ivfflat\n");
60 return 1;
61 }
62
63 auto dim = FLAGS_dim;
64 auto numCentroids = FLAGS_num_coarse;
65 auto num = FLAGS_num;
66 auto numTrain = FLAGS_num_train;
67 numTrain = numTrain == -1 ? std::max((num / 4), 1) : numTrain;
68 numTrain = std::min(num, numTrain);
69
70 if (FLAGS_ivfpq) {
71 faiss::IndexFlatL2 quantizer(dim);
72 faiss::IndexIVFPQ index(&quantizer, dim, numCentroids,
73 FLAGS_codes, FLAGS_bits_per_code);
74 index.verbose = true;
75
76 printf("IVFPQ: codes %d bits per code %d\n",
77 FLAGS_codes, FLAGS_bits_per_code);
78 printf("Lists: %d\n", numCentroids);
79 printf("Database: dim %d num vecs %d trained on %d\n", dim, num, numTrain);
80 printf("output file: %s\n", FLAGS_out.c_str());
81
82 fillAndSave(index, numTrain, num, dim);
83 } else if (FLAGS_ivfflat) {
84 faiss::IndexFlatL2 quantizerL2(dim);
85 faiss::IndexFlatIP quantizerIP(dim);
86
87 faiss::IndexFlat* quantizer = FLAGS_l2 ?
88 (faiss::IndexFlat*) &quantizerL2 :
89 (faiss::IndexFlat*) &quantizerIP;
90
91 faiss::IndexIVFFlat index(quantizer, dim, numCentroids,
92 FLAGS_l2 ? faiss::METRIC_L2 :
93 faiss::METRIC_INNER_PRODUCT);
94
95 printf("IVFFlat: metric %s\n", FLAGS_l2 ? "L2" : "IP");
96 printf("Lists: %d\n", numCentroids);
97 printf("Database: dim %d num vecs %d trained on %d\n", dim, num, numTrain);
98 printf("output file: %s\n", FLAGS_out.c_str());
99
100 fillAndSave(index, numTrain, num, dim);
101 }
102
103 return 0;
104 }
faiss::IndexFlat
Definition: IndexFlat.h:23
faiss::IndexFlatL2
Definition: IndexFlat.h:79
faiss::IndexIVFFlat
Definition: IndexIVFFlat.h:25
faiss::IndexFlatIP
Definition: IndexFlat.h:73
faiss::IndexIVFPQ
Definition: IndexIVFPQ.h:29