diff --git a/CMakeLists.txt b/CMakeLists.txt index a8c0a43..93d94ac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -73,6 +73,8 @@ if(${UNIX}) set(CMAKE_CXX_STANDARD 11) # for std::unordered_set, std::unique_ptr set(CMAKE_CXX_STANDARD_REQUIRED ON) find_package(Threads REQUIRED) + find_package(BLAS REQUIRED) + find_package(LAPACK REQUIRED) endif() add_subdirectory("${PROJECT_SOURCE_DIR}/lib") diff --git a/README-jp.md b/README-jp.md index 4c986f2..d691348 100644 --- a/README-jp.md +++ b/README-jp.md @@ -10,6 +10,7 @@ Neighborhood Graph and Tree for Indexing High-dimensional Data ニュース ------- +- 2022/08/10 QBG(Quantized Blob Graph)およびQG(NGTQGの改良版)が利用可能となりました。ngtqおよびngtqgは[qbg](https://ghe.corp.yahoo.co.jp/NGT/NGT/blob/qbg/bin/qbg/README.md)で置き換えられました。 - 2022/02/04 FP16(半精度浮動小数点)が利用可能になりました。(v1.14.0) - 2021/03/12 READMEに量子化グラフの結果を追加しました。 - 2021/01/15 [量子化グラフ (NGTQG)](bin/ngtqg/README.md)を実装した NGT v1.13.0 をリリースしました。 @@ -20,21 +21,12 @@ Neighborhood Graph and Tree for Indexing High-dimensional Data - 2018/12/14 [NGTQ](bin/ngtq/README-jp.md) (NGT with Quantization) が利用可能になりました。(v1.5.0) - 2018/08/08 [ONNG](README-jp.md#onng)が利用可能になりました。(v1.4.0) -特徴 ----- -- OS：Linux、macOS -- データの追加削除が可能 -- [共有メモリ（マップドメモリ）](README-jp.md#共有メモリの利用)のオプションによるNGTではメモリサイズを超えるデータが利用可能 -- データ型：1バイト整数、4バイト単精度浮動小数点 -- 距離関数：L1、L2、コサイン類似度、角度、ハミング、ジャッカード、ポアンカレ、ローレンツ -- 対応言語：[Python](/python/README-jp.md)、[Ruby](https://github.com/ankane/ngt)、[Go](https://github.com/yahoojapan/gongt)、C、C++ -- 分散サーバ：[ngtd](https://github.com/yahoojapan/ngtd), [vald](https://github.com/vdaas/vald) -- 量子化版NGT（[NGTQ](bin/ngtq/README-jp.md)）は10億ものデータの検索が可能 - -ドキュメント ------------ - -- [NGT チュートリアル](https://github.com/yahoojapan/NGT/wiki) +手法 +--- +このリポジトリは次の手法を提供します。 +- NGT: Graph and tree-based method +- QG: Quantized graph-based method +- QBG: Quantized blob graph-based method インストール ----------- @@ -43,16 +35,34 @@ Neighborhood Graph and Tree for Indexing High-dimensional Data - [Releases](https://github.com/yahoojapan/NGT/releases) -### ビルド済み +### ビルド -#### macOS +#### Linux - $ brew install ngt + $ unzip NGT-x.x.x.zip + $ cd NGT-x.x.x + $ mkdir build + $ cd build + $ cmake .. + $ make + $ make install + $ ldconfig /usr/local/lib -### ビルド +#### CentOS -#### Linux + $ yum install blas-devel lapack-devel + $ unzip NGT-x.x.x.zip + $ cd NGT-x.x.x + $ mkdir build + $ cd build + $ cmake .. + $ make + $ make install + $ ldconfig /usr/local/lib +#### Ubuntu + + $ apt install libblas-dev liblapack-dev $ unzip NGT-x.x.x.zip $ cd NGT-x.x.x $ mkdir build @@ -66,9 +76,7 @@ Neighborhood Graph and Tree for Indexing High-dimensional Data $ /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" $ brew install cmake - $ brew install gcc@9 - $ export CXX=/usr/local/bin/g++-9 - $ export CC=/usr/local/bin/gcc-9 + $ brew install openblas $ unzip NGT-x.x.x.zip $ cd NGT-x.x.x $ mkdir build @@ -87,14 +95,45 @@ Neighborhood Graph and Tree for Indexing High-dimensional Data #### 大規模データの利用 -約500万以上のオブジェクトを登録する場合には、検索速度向上のために以下のパラメータを追加してください。 +約500万以上のオブジェクトをNGTに登録する場合には、検索速度向上のために以下のパラメータを追加してください。 $ cmake -DNGT_LARGE_DATASET=ON .. +#### QG and QBGの無効化 + +QBおよびQBGはBLASおよびLAPACKライブラリを必要とします。もし、これらのライブラリをインストールしたくなく、かつ、QGやQBGを利用しない場合には、QGおよびQBGを無効化できます。 + + $ cmake -DNGT_QBG_DISABLED=ON .. + +### ビルド済み + +#### macOS + + $ brew install ngt + +NGT (Graph and tree-based method) +================================= + +特徴 +---- +- OS：Linux、macOS +- データの追加削除が可能 +- [共有メモリ（マップドメモリ）](README-jp.md#共有メモリの利用)のオプションによるNGTではメモリサイズを超えるデータが利用可能 +- データ型：1バイト整数、4バイト単精度浮動小数点 +- 距離関数：L1、L2、コサイン類似度、角度、ハミング、ジャッカード、ポアンカレ、ローレンツ +- 対応言語：[Python](/python/README-jp.md)、[Ruby](https://github.com/ankane/ngt)、[Go](https://github.com/yahoojapan/gongt)、C、C++ +- 分散サーバ：[ngtd](https://github.com/yahoojapan/ngtd), [vald](https://github.com/vdaas/vald) + +ドキュメント +----------- + +- [NGT チュートリアル](https://github.com/yahoojapan/NGT/wiki) + + ユーティリティ ------------- -- コマンド : [ngt](/bin/ngt/README-jp.md#command), [ngtq](bin/ngtq/README-jp.md) +- コマンド : [ngt](/bin/ngt/README-jp.md#command) - サーバ : [ngtd](https://github.com/yahoojapan/ngtd), [vald](https://github.com/vdaas/vald) 対応言語 @@ -108,6 +147,48 @@ Neighborhood Graph and Tree for Indexing High-dimensional Data - C - C++([sample code](samples)) +QG (Quantized graph-based method) +================================= + +特徴 +---- +- NGTよりも高性能 +- OS：Linux、macOS +- 距離関数：L2、コサイン類似度 +- 対応言語：C++, C, Python + +ユーティリティ +------------- +- コマンド : [qbg](bin/qbg/README.md) + +対応言語 +-------- + +- C++ +- C +- Python （検索のみ対応） + +QBG (Quantized blob graph-based method) +======================================= + +特徴 +---- +- 10億ものオブジェクトの検索が可能 +- OS：Linux、macOS +- 距離関数：L2 +- 対応言語：C++, C, Python + +ユーティリティ +------------- +- コマンド : [qbg](bin/qbg/README.md) + +対応言語 +-------- + +- C++ +- C +- Python （検索のみ対応） + ベンチマーク結果 --------------- diff --git a/README.md b/README.md index b0bca1d..8524041 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,11 @@ Neighborhood Graph and Tree for Indexing High-dimensional Data [Home](/README.md) / [Installation](/README.md#Installation) / [Command](/bin/ngt/README.md#command) / [License](/README.md#license) / [Publications](/README.md#publications) / [About Us](http://research-lab.yahoo.co.jp/en/) / [日本語](/README-jp.md) -**NGT** provides commands and a library for performing high-speed approximate nearest neighbor searches against a large volume of data (several million to several 10 million items of data) in high dimensional vector data space (several ten to several thousand dimensions). +**NGT** provides commands and a library for performing high-speed approximate nearest neighbor searches against a large volume of data in high dimensional vector data space (several ten to several thousand dimensions). News ---- +- 08/10/2022 QBG (Quantized Blob Graph) and QG (renewed NGTQG) are now available. The command-line interface ngtq and ngtqg are now obsolete by replacing [qbg](bin/qbg/README.md). (v2.0.0) - 02/04/2022 FP16 (half-precision floating point) is now available. (v1.14.0) - 03/12/2021 The results for the quantized graph are added to this README. - 01/15/2021 NGT v1.13.0 to provide the [quantized graph (NGTQG)](bin/ngtqg/README.md) is released. @@ -20,39 +21,48 @@ News - 12/14/2018 [NGTQ](bin/ngtq/README.md) (NGT with Quantization) is now available. (v1.5.0) - 08/08/2018 [ONNG](README.md#onng) is now available. (v1.4.0) -Key Features ------------- -- Supported operating systems: Linux and macOS -- Object additional registration and removal are available. -- Objects beyond the memory size can be handled using [the shared memory (memory mapped file) option](README.md#shared-memory-use). -- Supported distance functions: L1, L2, Cosine similarity, Angular, Hamming, Jaccard, Poincare, and Lorentz -- Data Types: 4 byte floating point number and 1 byte unsigned integer -- Supported languages: [Python](/python/README.md), [Ruby](https://github.com/ankane/ngt), [Rust](https://crates.io/crates/ngt), [Go](https://github.com/yahoojapan/gongt), C, and C++ -- Distributed servers: [ngtd](https://github.com/yahoojapan/ngtd) and [vald](https://github.com/vdaas/vald) -- [NGTQ](bin/ngtq/README.md) can handle billions of objects. - -Documents ---------- - -- [NGT tutorial](https://github.com/yahoojapan/NGT/wiki) +Methods +------- +This repository the following methods. +- NGT: Graph and tree-based method +- QG: Quantized graph-based method +- QBG: Quantized blob graph-based method Installation ------------ -### Downloads +### Build -- [Releases](https://github.com/yahoojapan/NGT/releases) +#### Downloads -### Pre-Built +- [Releases](https://github.com/yahoojapan/NGT/releases) -#### On macOS +#### On Linux - $ brew install ngt + $ unzip NGT-x.x.x.zip + $ cd NGT-x.x.x + $ mkdir build + $ cd build + $ cmake .. + $ make + $ make install + $ ldconfig /usr/local/lib -### Build +#### On CentOS -#### On Linux + $ yum install blas-devel lapack-devel + $ unzip NGT-x.x.x.zip + $ cd NGT-x.x.x + $ mkdir build + $ cd build + $ cmake .. + $ make + $ make install + $ ldconfig /usr/local/lib + +#### On Ubuntu + $ apt install libblas-dev liblapack-dev $ unzip NGT-x.x.x.zip $ cd NGT-x.x.x $ mkdir build @@ -66,9 +76,7 @@ Installation $ /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" $ brew install cmake - $ brew install gcc@9 - $ export CXX=/usr/local/bin/g++-9 - $ export CC=/usr/local/bin/gcc-9 + $ brew install openblas $ unzip NGT-x.x.x.zip $ cd NGT-x.x.x $ mkdir build @@ -87,14 +95,63 @@ Note: Since there is no lock function, the index should be used only for referen #### Large-scale data use -When you insert more than about 5 million objects, please add the following parameter to improve the search time. +When you insert more than about 5 million objects for the graph-based method, please add the following parameter to improve the search time. $ cmake -DNGT_LARGE_DATASET=ON .. +#### Disable QG and QBG +QB and QBG requires BLAS and LAPACK libraries. If you would not like to install these libraries and do not use QG and QBG, you can disable QG and QBG. + + $ cmake -DNGT_QBG_DISABLED=ON .. + +### Pre-Built + +#### On macOS + + $ brew install ngt + + +License +------- + +Copyright (C) 2015 Yahoo Japan Corporation + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this software except in compliance with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and limitations under the License. + +Contributor License Agreement +----------------------------- + +This project requires contributors to accept the terms in the [Contributor License Agreement (CLA)](https://gist.github.com/yahoojapanoss/9bf8afd6ea67f32d29b4082abf220340). + +Please note that contributors to the NGT repository on GitHub (https://github.com/yahoojapan/NGT) shall be deemed to have accepted the CLA without individual written agreements. + +NGT (Graph and tree-based method) +================================= + +Key Features +------------ +- Supported operating systems: Linux and macOS +- Object additional registration and removal are available. +- Objects beyond the memory size can be handled using [the shared memory (memory mapped file) option](README.md#shared-memory-use). +- Supported distance functions: L1, L2, Cosine similarity, Angular, Hamming, Jaccard, Poincare, and Lorentz +- Data Types: 4 byte floating point number and 1 byte unsigned integer +- Supported languages: [Python](/python/README.md), [Ruby](https://github.com/ankane/ngt), [Rust](https://crates.io/crates/ngt), [Go](https://github.com/yahoojapan/gongt), C, and C++ +- Distributed servers: [ngtd](https://github.com/yahoojapan/ngtd) and [vald](https://github.com/vdaas/vald) + +Documents +--------- + +- [NGT tutorial](https://github.com/yahoojapan/NGT/wiki) + Utilities --------- -- Command : [ngt](/bin/ngt/README.md#command), [ngtq](bin/ngtq/README.md), [ngtqg](bin/ngtqg/README.md) +- Command : [ngt](/bin/ngt/README.md#command), [qbg](bin/qbg/README.md) - Server : [ngtd](https://github.com/yahoojapan/ngtd), [vald](https://github.com/vdaas/vald) Supported Programming Languages @@ -108,9 +165,50 @@ Supported Programming Languages - C - C++([sample code](samples)) +QG (Quantized graph-based method) +================================= + +Key Features +------------ +- Higher performance than the graph and tree-based method +- Supported operating systems: Linux and macOS +- Supported distance functions: L2 and Cosine similarity + +Utilities +--------- +- Command : [qbg](bin/qbg/README.md) + +Supported Programming Languages +------------------------------- + +- C++ +- C +- Python only for search + + +QBG (Quantized blob graph-based method) +======================================= + +Key Features +------------ +- [QBG](bin/qbg/README.md) can handle billions of objects. +- Supported operating systems: Linux and macOS +- Supported distance functions: L2 + +Utilities +--------- +- Command : [qbg](bin/qbg/README.md) + +Supported Programming Languages +------------------------------- + +- C++ +- C +- Python only for search + Benchmark Results ----------------- -The followings are the results of [ann benchmarks](https://github.com/erikbern/ann-benchmarks) for NGT v1.13.5 where the timeout is 5 hours on an AWS c5.4xlarge instance. +The followings are the results of [ann benchmarks](https://github.com/erikbern/ann-benchmarks) for NGT v2.0.0 where the timeout is 5 hours on an AWS c5.4xlarge instance. #### glove-100-angular @@ -127,25 +225,6 @@ The followings are the results of [ann benchmarks](https://github.com/erikbern/a #### sift-128-euclidean -License -------- - -Copyright (C) 2015 Yahoo Japan Corporation - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this software except in compliance with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and limitations under the License. - -Contributor License Agreement ------------------------------ - -This project requires contributors to accept the terms in the [Contributor License Agreement (CLA)](https://gist.github.com/yahoojapanoss/9bf8afd6ea67f32d29b4082abf220340). - -Please note that contributors to the NGT repository on GitHub (https://github.com/yahoojapan/NGT) shall be deemed to have accepted the CLA without individual written agreements. - Contact Person -------------- [masajiro](https://github.com/masajiro) diff --git a/VERSION b/VERSION index 9be7846..227cea2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.14.8 +2.0.0 diff --git a/bin/CMakeLists.txt b/bin/CMakeLists.txt index d56d78a..222d38c 100644 --- a/bin/CMakeLists.txt +++ b/bin/CMakeLists.txt @@ -1,7 +1,8 @@ -if( ${UNIX} ) +if(${UNIX}) include_directories("${PROJECT_SOURCE_DIR}/lib" "${PROJECT_BINARY_DIR}/lib/") link_directories("${PROJECT_BINARY_DIR}/lib/NGT") add_subdirectory("${PROJECT_SOURCE_DIR}/bin/ngt") - add_subdirectory("${PROJECT_SOURCE_DIR}/bin/ngtq") - add_subdirectory("${PROJECT_SOURCE_DIR}/bin/ngtqg") + if(NOT ${NGT_SHARED_MEMORY_ALLOCATOR}) + add_subdirectory("${PROJECT_SOURCE_DIR}/bin/qbg") + endif() endif() diff --git a/bin/ngt/ngt.cpp b/bin/ngt/ngt.cpp index b50ed5e..1403c8b 100644 --- a/bin/ngt/ngt.cpp +++ b/bin/ngt/ngt.cpp @@ -102,6 +102,8 @@ main(int argc, char **argv) ngt.optimizeNumberOfEdgesForANNG(args); } else if (command == "export-graph") { ngt.exportGraph(args); + } else if (command == "export-objects") { + ngt.exportObjects(args); #ifndef NGT_SHARED_MEMORY_ALLOCATOR } else if (command == "extract-query") { NGT::Optimizer::extractQueries(args); diff --git a/bin/ngtq/CMakeLists.txt b/bin/ngtq/CMakeLists.txt deleted file mode 100644 index 0e2ce3e..0000000 --- a/bin/ngtq/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -if( ${UNIX} ) - include_directories("${PROJECT_BINARY_DIR}/lib") - include_directories("${PROJECT_SOURCE_DIR}/lib") - link_directories("${PROJECT_SOURCE_DIR}/lib/NGT") - - add_executable(ngtq_exe ngtq.cpp) - add_dependencies(ngtq_exe ngt) - set_target_properties(ngtq_exe PROPERTIES OUTPUT_NAME ngtq) - target_link_libraries(ngtq_exe ngt pthread) - - install(TARGETS ngtq_exe RUNTIME DESTINATION bin) - -endif() diff --git a/bin/ngtq/README-jp.md b/bin/ngtq/README-jp.md deleted file mode 100644 index f1f6c92..0000000 --- a/bin/ngtq/README-jp.md +++ /dev/null @@ -1,122 +0,0 @@ -NGTQ -=== - -Neighborhood Graph and Tree for Indexing High-dimensional Data with Quantization - -Command -======= - - - -**ngtq** - 大規模高次元ベクトルデータの近傍検索 - - - - $ ngtq command [option] index [data] - -**注：** -CygWin といった POSIXLY_CORERECT が設定されている環境では、コマンドの前にオプションを指定しなければなりません。 - - $ ngtq [option] command index [data] - - - -十億件以上もの高次元ベクトルデータ（数十～数千次元）に対して高速な近傍検索を提供します。 - -**command** is one of: - -- *[create](#create)* -- *[append](#append)* -- *[search](#search)* - -### CREATE - -指定されたインデックスを生成した上で指定されたデータをインデックスに登録します。 - - $ ngtq create -d no_of_dimensions [-p no_of_threads] [-o object_type] [-n no_of_registration_data] - [-C global_codebook_size] [-c local_codebook_size] [-N no_of_divisions] - [-L local_centroid_creation_mode] - index registration_data - -*index* -生成するインデックス名を指定します。データを登録後、本インデックス名のディレクトリが生成されてその中に複数のファイルからなるインデックスが生成されます。 - -*registration\_data* -登録するベクトルデータを指定します。１行が１オブジェクト（データ）で構成され、各次元要素のデータはスペースまたはタブで区切られていなければなりません。距離関数はL2のみが利用可能です。 - -**-d** *no\_of\_dimensions* -登録データの次元数を指定します。 - -**-p** *no\_of\_threads* (default = 24, recomended value = number of cores) -生成時の並列処理時に利用するスレッド数を指定します。 - -**-o** *object\_type* -データオブジェクトの型を指定する。 -- __c__: 1バイト整数 (一部未実装) -- __f__: 4バイト浮動小数点（デフォルト、推奨） - -**-n** *no\_of\_registration\_data* -Specifies the number of data items to be registered. If not specified, all data in the specified file will be registered. - -**-C** *global\_codebook\_size* -グローバルコード（セントロイド）の数を指定します。 - -**-c** *local\_codebook\_size* -ローカルコード（セントロイド）の数を指定します。 - -**-N** *no\_of\_divisions* -ローカルベクトルデータ（残差データ）を生成するためのベクトルデータの分割数を指定します。 - -**-L** *local\_centroid\_creation\_mode* -ローカルセントロイドを生成するモードを指定します。 -- __d__: 指定された登録データの先頭をローカルセントロイドとして使用します。 -- __k__: kmeans を使用してローカルセントロイドを生成します。 - - -### APPEND - -指定された登録データを指定されたインデックスに追加登録します。 - - $ ngtq append [-n no_of_registration_data] index registration_data - - -*index* -既存のインデックスを指定します。 - -*registration\_data* -登録するベクトルデータを指定します。１行が１オブジェクト（データ）で、各次元のデータはスペース又はタブで区切られていなければなりません。 - -**-n** *no\_of\_registration\_data* -登録するデータ数を指定します。指定しない場合には指定されたファイル中のすべてのデータを登録します。 - -### SEARCH - -指定されたクエリデータを用いてインデックスを検索します。 - - $ ngtq search [-n no_of_search_results] [-e search_range_coefficient] [-m mode] - [-r search_radius] [-E approximate-expansion] index query_data - - -*index* -既存のインデックス名を指定します。 - -*query\_data* -クエリデータのファイル名を指定します。１行が１クエリデータであり、登録データと同様に各次元のデータはスペース又はタブで区切られていなければなりません。複数クエリを与えた場合には順次検索します。 - -**-n** *no\_of\_search\_results* (default: 20) -検索結果数を指定します。 - -**-e** *search\_range\_coefficient* (default = recomended value = 0.1) -グローバルコードブックを検索する時の探索範囲の拡大係数です。大きければ精度が高くなりますが遅くなり、小さければ精度は下がりますが速くなります。0～0.3の範囲内で調整することが望ましいですが、負の値も指定可能です。 - -**-m** *search\_mode* (__r__|__e__|__l__|__c__|__a__) -検索モードを -- __a__: 近似距離を用いて検索します。 -- __c__: 近似距離を用いて検索します。計算済みローカル距離がキャッシュされることで検索時間が削減されます。（推奨） -- __l__: ローカル距離のルックアップテーブルによる近似距離を用いて検索します。 -- __e__: 正確な距離を用いて検索します。ローカルコードブックを利用しません。 -- __r__: 近似距離を用いて絞り込んだ後に正確な距離を用いて検索します。（正確な距離が必要な場合には推奨） - -**-E** *approximate\_expansion* -検索結果に対する近似検索結果の割合を指定します。例えば、割合が10で検索結果数が20の場合近似検索結果数は200となります。 - diff --git a/bin/ngtq/README.md b/bin/ngtq/README.md deleted file mode 100644 index cc8db55..0000000 --- a/bin/ngtq/README.md +++ /dev/null @@ -1,124 +0,0 @@ -NGTQ -=== - -Neighborhood Graph and Tree for Indexing High-dimensional Data with Quantization - -Command -======= - - - -**ngtq** - proximity search for billions of high dimensional data - - - - $ ngtq command [option] index [data] - -**Note:** - -When the environment variable POSIXLY_CORERECT is set on some platforms such as Cygwin, you should specifiy options -before the command as follows. - - $ ngtq [option] command index [data] - - - -**ngtq** provides high-speed nearest neighbor searches against billions of data in high dimensional vector data space (several ten to several thousand dimensions). - -**command** is one of: - -- *[create](#create)* -- *[append](#append)* -- *[search](#search)* - -### CREATE - -Constructs the specified index with the specified data. - - $ ngtq create -d no_of_dimensions [-p no_of_threads] [-o object_type] [-n no_of_registration_data] - [-C global_codebook_size] [-c local_codebook_size] [-N no_of_divisions] - [-L local_centroid_creation_mode] - index registration_data - -*index* -Specifies the name of the directory for the index to be generated. After data registration, the directory consists of multiple files for the index. - -*registration\_data* -Specifies the vector data to be registered. These data shall consist of one object (data item) per line and each dimensional element shall be delimited by a space or tab. Note that L2 is only avilable as the distance function. - -**-d** *no\_of\_dimensions* -Specifies the number of dimensions of registration data. - -**-p** *no\_of\_threads* (default = 24, recomended value = number of cores) -Specifies the number of threads to be used for parallel processing at generation time. - -**-o** *object\_type* -Specifies the data object type. -- __c__: 1 byte unsigned integer (not fully implemented) -- __f__: 4 byte floating point number (default/recommended) - -**-n** *no\_of\_registration\_data* -Specifies the number of data items to be registered. If not specified, all data in the specified file will be registered. - -**-C** *global\_codebook\_size* -Specifies the number of the global codes (centroids). - -**-c** *local\_codebook\_size* -Specifies the number of the local codes (centroids). - -**-N** *no\_of\_divisions* -Specifies the number of division of the vector data for the local vector data (residual data). - -**-L** *local\_centroid\_creation\_mode* -Specifies the creation mode for the local centroids. -- __d__: The heads of the specified registration data are used as the local centroids. -- __k__: The local centoroids are generated by using kmeans. - - -### APPEND - -Adds the specified data to the specified index. - - $ ngtq append [-n no_of_registration_data] index registration_data - - -*index* -Specifies the name of the existing index. - -*registration\_data* -Specifies the vector data to be registered. These data shall consist of one object (data item) per line and each dimensional element shall be delimited by a space or tab. - -**-n** *no\_of\_registration\_data* -Specifies the number of data items to be registered. If not specified, all data in the specified file will be registered. - -### SEARCH - -Searches the index using the specified query data. - - $ ngtq search [-n no_of_search_results] [-e search_range_coefficient] [-m mode] - [-r search_radius] [-E approximate-expansion] index query_data - - -*index* -Specifies the name of the existing index. - -*query\_data* -Specifies the name of the file containing query data. This file shall consist of one item of query data per line and each dimensional element of that data item shall be delimited by a space or tab the same as registration data. Each search shall be sequentially performed when providing multiple queries. - -**-n** *no\_of\_search\_results* (default: 20) -Specifies the number of search results. - -**-e** *search\_range\_coefficient* (default = recomended value = 0.1) -Specifies the magnification coefficient of the search range to search for the global codebook. A larger value means greater accuracy but slower searching, while a smaller value means a drop in accuracy but faster searching. While it is desirable to adjust this value within the range of 0 - 0.3, a negative value may also be specified. - -**-m** *search\_mode* (__r__|__e__|__l__|__c__|__a__) -Specifies the search mode. -- __a__: searches using approximate distances. -- __c__: searches using approximate distances. Caching computed local distances reduces the query time. (recommended) -- __l__: searches using approximate distances with local distance lookup tables. -- __e__: searches using exact distances. The local codebooks are not used. -- __r__: searches using exact distances after screening by approximate distances. (recommended if you need exact distances) - -**-E** *approximate\_expansion* -Specifies the expansion ratio of the number of approximate search results to the number of search results. For example, when the ratio is 10 and the number of search results is 20, the number of the approximate search results is set to 200. - diff --git a/bin/ngtq/ngtq.cpp b/bin/ngtq/ngtq.cpp deleted file mode 100644 index 27ce4a0..0000000 --- a/bin/ngtq/ngtq.cpp +++ /dev/null @@ -1,30 +0,0 @@ -// -// Copyright (C) 2016 Yahoo Japan Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -#include "NGT/NGTQ/NGTQCommand.h" - -int -main(int argc, char **argv) -{ - NGT::Args args(argc, argv); - - NGTQ::Command ngtq; - - ngtq.execute(args); -} - - - diff --git a/bin/ngtqg/CMakeLists.txt b/bin/ngtqg/CMakeLists.txt deleted file mode 100644 index 6c61cfa..0000000 --- a/bin/ngtqg/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -if( ${UNIX} ) - include_directories("${PROJECT_BINARY_DIR}/lib") - include_directories("${PROJECT_SOURCE_DIR}/lib") - link_directories("${PROJECT_SOURCE_DIR}/lib/NGT") -##-# link_directories("${PROJECT_SOURCE_DIR}/lib/NGTQG") - - add_executable(ngtqg_exe ngtqg.cpp) - add_dependencies(ngtqg_exe ngt) - set_target_properties(ngtqg_exe PROPERTIES OUTPUT_NAME ngtqg) ## 名前をngtqgに -##-# target_link_libraries(ngtqg_exe ngtqg ngt pthread) - target_link_libraries(ngtqg_exe ngt pthread) - - install(TARGETS ngtqg_exe RUNTIME DESTINATION bin) - -endif() diff --git a/bin/ngtqg/README.md b/bin/ngtqg/README.md deleted file mode 100644 index 163c47d..0000000 --- a/bin/ngtqg/README.md +++ /dev/null @@ -1,138 +0,0 @@ -NGTQG -=== - -Neighborhood Graph and Tree for Indexing High-dimensional Data with Quantized Graph - -Command -======= - -## Name - -**ngtqg** - proximity search for high dimensional data with quantized graph - -## Synopsis - - $ ngtqg command [option] index [data] - -**Note:** - -When the environment variable POSIXLY_CORERECT is set on some platforms such as Cygwin, you should specifiy options -before the command as follows. - - $ ngtqg [option] command index [data] - -## Description - -**ngtqg** provides high-speed nearest neighbor searches against high dimensional data. - -**command** is one of: - -- *[quantize](#quantize)* -- *[search](#search)* - -### QUANTIZE - -Quantize the objects of the specified index and build a quantized graph into the index. - - $ ngtqg quantize [-E max_no_of_edges] [-Q dimension_of_subvector] index - -*index* -Specify the name of the directory for the existing index such as ANNG or ONNG to be quantized. The index only with L2 distance and normalized cosine similarity distance can be quantized. You should build the ANNG or ONNG with normalized cosine similarity in order to use cosine similarity for the quantized graph. - -**-E** *max_no_of_edges* -Specify the maximum number of edges to build a qunatized graph. Since every 16 objects that are associated with edges of each node are processed, the number should be a multiple of 16. - -**-Q** *dimension_of_subvector* -Specify dimension of a suvbector for quantized objects. The dimension should be a divisor of the dimension of the inserted objects. - -### SEARCH - -Search the index using the specified query data. - - $ ngtqg search [-n no_of_search_objects] [-e search_range_coefficient] [-p result_expansion] - [-r search_radius] index query_data - - -*index* -Specify the path of the existing quantized index. - -*query_data* -Specify the path of the file containing query data. This file shall consist of one item of query data per line and each dimensional element of that data item shall be delimited by a space or tab. Each search shall be sequentially performed when providing multiple queries. - -**-n** *no_of_search_objects* (default: 20) -Specify the number of search objects. - -**-e** *search_range_coefficient* (default = 0.02) -Specify the magnification coefficient (epsilon) of the search range. A larger value means higher accuracy but slower searching, while a smaller value means a drop in accuracy but faster searching. While it is desirable to adjust this value within the range of 0 - 0.1, a negative value (> -1.0) may also be specified. - -**-p** *result_expansion* (default = 3.0) -Specify the expansion ratio of the number of approximate inner search objects to the number of search objects. For example, when the ratio is 10 and the number of search objects is 20, the number of the approximate search objects is set to 200 inside the search processing. A larger value brings higher accuracy but slower searching. - -**-r** *search_radius* (default = infinite circle) -Specify the search range in terms of the radius of a circle. - -Examples of using the quantized graph -------------------------------------- - -### Setup data - - $ curl -L -O https://github.com/yahoojapan/NGT/raw/master/tests/datasets/ann-benchmarks/sift-128-euclidean.tsv - $ curl -L -O https://github.com/yahoojapan/NGT/raw/master/tests/datasets/ann-benchmarks/sift-128-euclidean_query.tsv - $ head -1 sift-128-euclidean_query.tsv > query.tsv - -### Build the quantized graph - -Build an ANNG for 128-dimensional, floating point data: - - $ ngt create -d 128 -o f -D 2 anng sift-128-euclidean.tsv - Data loading time=15.4804 (sec) 15480.4 (msec) - # of objects=1000000 - Processed 100000 objects. time= 4.26452 (sec) - ... - Processed 1000000 objects. time= 7.06745 (sec) - Index creation time=63.3504 (sec) 63350.4 (msec) - -Quantize the objects in the ANNG and build the quantized graph: - - $ ngtqg quantize anng - Clustering of the subvector is complete. anng/qg/local-0:17 - ... - Clustering of the subvector is complete. anng/qg/local-127:17 - Processed 100000 objects. - ... - Processed 1000000 objects. - -### Search with the quantized graph - -Search k nearest neighbors with the quantized graph: - - $ ngtqg search -n 20 -e 0.02 anng query.tsv - Query No.1 - Rank ID Distance - 1 932086 232.871 - 2 934877 234.715 - 3 561814 243.99 - ... - 20 2177 276.781 - Query Time= 0.0005034 (sec), 0.5034 (msec) - Average Query Time= 0.0005034 (sec), 0.5034 (msec), (0.0005034/1) - -Examples of building the quantized graph for higher performance ------------------------------------------------------------- - -Build an ANNG having more edges for higher performance: - - $ ngt create -d 128 -o f -D 2 -E 40 anng-40 sift-128-euclidean.tsv - -Build an ONNG: - - $ ngt reconstruct-graph -m S -E 64 -o 64 -i 120 anng-40 onng-40 - -Quantize the objects and build a quantized graph from the ONNG: - - $ ngtqg quantize onng-40 - -Search k nearest neighbors with the quantized graph: - - $ ngtqg search -n 20 -e 0.02 onng-40 query.tsv - diff --git a/bin/qbg/CMakeLists.txt b/bin/qbg/CMakeLists.txt new file mode 100644 index 0000000..fe8b7d2 --- /dev/null +++ b/bin/qbg/CMakeLists.txt @@ -0,0 +1,13 @@ +if( ${UNIX} ) + include_directories("${PROJECT_BINARY_DIR}/lib") + include_directories("${PROJECT_SOURCE_DIR}/lib") + link_directories("${PROJECT_SOURCE_DIR}/lib/NGT") + + add_executable(qbg_exe qbg.cpp) + add_dependencies(qbg_exe ngt) + set_target_properties(qbg_exe PROPERTIES OUTPUT_NAME qbg) ## 名前をqbgに + target_link_libraries(qbg_exe ngt pthread) + + install(TARGETS qbg_exe RUNTIME DESTINATION bin) + +endif() diff --git a/bin/qbg/README.md b/bin/qbg/README.md new file mode 100644 index 0000000..99a102f --- /dev/null +++ b/bin/qbg/README.md @@ -0,0 +1,287 @@ +QBG +=== + +Command-line interface for NGT with Quantization for indexing high-dimensional data + +Command +======= + + + +**qbg** - proximity search for high dimensional data with quantization + + + + $ qbg command [option] index [data] + +**Note:** + +When the environment variable POSIXLY_CORERECT is set on some platforms such as Cygwin, you should specifiy options +before the command as follows. + + $ qbg [option] command index [data] + + + +**qbg** handles two types of graphs with quantization: Quantized Graph (QG) and Quantized Blob Graph (QBG). + +**command** for the quantized graph is one of: + +- *[create-qg](#create-qg)* +- *[build-qg](#build-qg)* +- *[search-qg](#search-qg)* + +**command** for the quantized blob graph is one of: + +- *[create](#create)* +- *[append](#append)* +- *[build](#build)* +- *[search](#search)* + +### CREATE-QG +Make and initialize a QG directory for the quantized graph in the specified NGT index directory, and insert the data in the NGT index into the QG index. + + $ qbg create-qg [-P number_of_extended_dimensions] [-Q number_of_subvector_dimensions] index + +*index* +Specify the name of the directory for the existing index such as ANNG or ONNG to be quantized. The index only with L2 distance and normalized cosine similarity distance can be quantized. You should build the ANNG or ONNG with normalized cosine similarity in order to use cosine similarity for the quantized graph. + +**-P** *number_of_extended_dimensions* +Specify the number of the extended dimensions. The number should be greater than or equal to the number of the genuine dimensions, and also should be a multiple of 4. When this option is not specified, the smallest multiple of 4 that is greater than the dimension is set to the number of the extended dimensions. + +**-Q** *number_of_subvector_dimension* +Specify the number of the subvector dimensions. The number should be less than or equal to the the number of the extended dimensions, and also should be a divisor of the number of the extended dimensions. When this option is not specified, one is set to the number of the subvector dimensions. + + +### BUILD-QG + +Quantize the objects of the specified index and build a quantized graph into the index. + + $ qbg build-qg [-o number_of_objects_for_quantization] [-E max_number_of_edges] [-M number_of_trials] index + +*index* +Specify the name of the directory for the existing index such as ANNG or ONNG to be quantized. The index only with L2 distance and normalized cosine similarity distance can be quantized. You should build the ANNG or ONNG with normalized cosine similarity in order to use cosine similarity for the quantized graph. + +**-o** *number_of_objects_for_quantization* +Specify the number of object for quantization and optimization. The number should be less than or equal to the number of the registered objects. + +**-E** *max_number_of_edges* +Specify the maximum number of edges to build a qunatized graph. Since every 16 objects that are associated with edges of each node are processed, the number should be a multiple of 16. + +**-M** *number_of_trials* +Specify the number of trials to optimize the subvector quantization. + +### SEARCH-QG + +Search the index using the specified query data. + + $ qbg search-qg [-n number_of_search_objects] [-e search_range_coefficient] [-p result_expansion] + [-r search_radius] index query_data + + +*index* +Specify the path of the existing quantized index. + +*query_data* +Specify the path of the file containing query data. This file shall consist of one item of query data per line and each dimensional element of that data item shall be delimited by a space or tab. Each search shall be sequentially performed when providing multiple queries. + +**-n** *number_of_search_objects* (default: 20) +Specify the number of search objects. + +**-e** *search_range_coefficient* (default = 0.02) +Specify the magnification coefficient (epsilon) of the search range. A larger value means higher accuracy but slower searching, while a smaller value means a drop in accuracy but faster searching. While it is desirable to adjust this value within the range of 0 - 0.1, a negative value (> -1.0) may also be specified. + +**-p** *result_expansion* (default = 3.0) +Specify the expansion ratio of the number of approximate inner search objects to the number of search objects. For example, when the ratio is 10 and the number of search objects is 20, the number of the approximate search objects is set to 200 inside the search processing. A larger value brings higher accuracy but slower searching. + +### CREATE +Make and initialize a QBG directory for the quantized blob graph. + + $ qbg create [-d number_of_dimension] [-P number_of_extended_dimensions] [-O object_type] [-D distance_function] [-C number_of_blobs] index + +*index* +Specify the name of the directory for QBG. + +**-d** *number_of_dimensions* +Specify the number of dimensions of registration data. + +**-P** *number_of_extended_dimensions* +Specify the number of the extended dimensions. The number should be greater than or equal to the number of the genuine dimensions, and also should be a multiple of 4. When this option is not specified, the smallest multiple of 4 that is greater than the dimension is set to the number of the extended dimensions. + +**-O** *object_type* +Specify the data object type. +- __c__: 1 byte unsigned integer +- __f__: 4 byte floating point number (default) + +**-D** *distance_function* +Specify the distance function as follows. +- __2__: L2 distance (default) +- __c__: Cosine similarity + +**-C** *number_of_blobs* +Specify the number of blobs that should be less than or equal to the number of quantization clusters. + +**-N** *number_of_subvectors* +Specify the number of subvectors that should be a divisor of the number of the extended dimensions. + +### APPEND + +Append the specified data to the specified index. + + $ qbg append index registration_data + +### BUILD + +Quantize the objects of the specified index and build a quantized graph into the index. + + $ qbg build [-o number_of_objects_for_quantization] [-E max_number_of_edges] [-M number_of_trials] [-P rotation] index + +*index* +Specify the name of the directory for the existing index such as ANNG or ONNG to be quantized. The index only with L2 distance and normalized cosine similarity distance can be quantized. You should build the ANNG or ONNG with normalized cosine similarity in order to use cosine similarity for the quantized graph. + +**-o** *number_of_objects_for_quantization* +Specify the number of object for quantization and optimization. The number should be less than or equal to the number of the registered objects. + +**-P** *rotation* +Specify the transform matrix type for the inserted and query object to optimize the subvector quantization. +- __r__: Rotation matrix. +- __R__: Rotation and repositioning matrix. +- __p__: Repositioning matrix. +- __n__: No matrix. + +**-M** *number_of_trials* +Specify the number of trials to optimize the subvector quantization. + +### SEARCH + +Search the index using the specified query data. + + $ qbg search [-n number_of_search_objects] [-e search_range_coefficient] [-p result_expansion] + index query_data + + +*index* +Specify the path of the existing quantized index. + +*query_data* +Specify the path of the file containing query data. This file shall consist of one item of query data per line and each dimensional element of that data item shall be delimited by a space or tab. Each search shall be sequentially performed when providing multiple queries. + +**-n** *number_of_search_objects* (default: 20) +Specify the number of search objects. + +**-e** *search_range_coefficient* (default = 0.02) +Specify the magnification coefficient (epsilon) of the search range. A larger value means higher accuracy but slower searching, while a smaller value means a drop in accuracy but faster searching. While it is desirable to adjust this value within the range of 0 - 0.1, a negative value (> -1.0) may also be specified. + +**-B** *blob_search_range_coefficient* (default = 0.0) +Specify the magnification coefficient (epsilon) of the search range for the quantized blob graph. + +**-N** *number_of_explored_nodes* (default = 256) +Specify the number of the explored nodes in the graph. When the number of the explored nodes reached the specified number, the search is terminated. + +**-p** *result_expansion* (default = 0.0) +Specify the expansion ratio of the number of approximate inner search objects to the number of search objects. For example, when the ratio is 10 and the number of search objects is 20, the number of the approximate search objects is set to 200 inside the search processing. A larger value brings higher accuracy but slower searching. + + +Examples of using the quantized graph +------------------------------------- + +### Setup data + + $ curl -L -O https://github.com/yahoojapan/NGT/raw/master/tests/datasets/ann-benchmarks/sift-128-euclidean.tsv + $ curl -L -O https://github.com/yahoojapan/NGT/raw/master/tests/datasets/ann-benchmarks/sift-128-euclidean_query.tsv + $ head -1 sift-128-euclidean_query.tsv > query.tsv + +### Build the quantized graph + +Build an ANNG for 128-dimensional, floating point data: + + $ ngt create -d 128 -o f -D 2 anng sift-128-euclidean.tsv + Data loading time=15.4804 (sec) 15480.4 (msec) + # of objects=1000000 + Processed 100000 objects. time= 4.26452 (sec) + ... + Processed 1000000 objects. time= 7.06745 (sec) + Index creation time=63.3504 (sec) 63350.4 (msec) + +Create and initialize the quantized graph: + + $ qbg create-qg anng + creating... + appending... + +Build the quantized graph: + + $ qbg build-qg anng + optimizing... + building the inverted index... + building the quantized graph... + +### Search with the quantized graph + +Search k nearest neighbors with the quantized graph: + + $ qbg search-qg -n 20 -e 0.02 anng query.tsv + Query No.1 + Rank ID Distance + 1 932086 232.871 + 2 934877 234.715 + 3 561814 243.99 + ... + 20 2177 276.781 + Query Time= 0.0005034 (sec), 0.5034 (msec) + Average Query Time= 0.0005034 (sec), 0.5034 (msec), (0.0005034/1) + +Examples of building the quantized graph for higher performance +------------------------------------------------------------ + +Build an ANNG having more edges for higher performance: + + $ ngt create -d 128 -o f -D 2 -E 40 anng-40 sift-128-euclidean.tsv + +Build an ONNG: + + $ ngt reconstruct-graph -m S -E 64 -o 64 -i 120 anng-40 onng-40 + +Create and initialize the quantized graph: + + $ qbg create-qg onng-40 + +Build the quantized graph: + + $ qbg build-qg onng-40 + +Search k nearest neighbors with the quantized graph: + + $ qbg search -n 20 -e 0.02 onng-40 query.tsv + + +Examples of using the quantized blob graph +------------------------------------- + +### Setup data + + $ curl -L -O https://github.com/yahoojapan/NGT/raw/master/tests/datasets/ann-benchmarks/sift-128-euclidean.tsv + $ curl -L -O https://github.com/yahoojapan/NGT/raw/master/tests/datasets/ann-benchmarks/sift-128-euclidean_query.tsv + $ head -1 sift-128-euclidean_query.tsv > query.tsv + +### Build the quantized blob graph + +Create and initialize the quantized blob graph: + + $ qbg create -d 128 -D 2 -N 128 qbg-index + +Append objects: + + $ qbg append qbg-index sift-128-euclidean.tsv + +Build the quantized graph: + + $ qbg build qbg-index + +### Search with the quantized graph + +Search k nearest neighbors with the quantized graph: + + $ qbg search -n 20 -e 0.02 qbg-index query.tsv + + diff --git a/bin/ngtqg/ngtqg.cpp b/bin/qbg/qbg.cpp similarity index 92% rename from bin/ngtqg/ngtqg.cpp rename to bin/qbg/qbg.cpp index 0248c45..92281eb 100644 --- a/bin/ngtqg/ngtqg.cpp +++ b/bin/qbg/qbg.cpp @@ -14,14 +14,14 @@ // limitations under the License. // -#include "NGT/NGTQ/NGTQGCommand.h" +#include "NGT/NGTQ/QbgCli.h" int main(int argc, char **argv) { NGT::Args args(argc, argv); - NGTQG::Command ngt; + QBG::CLI ngt; ngt.execute(args); } diff --git a/lib/NGT/ArrayFile.h b/lib/NGT/ArrayFile.h index d48c610..5a1b68a 100644 --- a/lib/NGT/ArrayFile.h +++ b/lib/NGT/ArrayFile.h @@ -31,7 +31,7 @@ namespace NGT { template class ArrayFile { - private: + protected: struct FileHeadStruct { size_t recordSize; uint64_t extraData; // reserve diff --git a/lib/NGT/CMakeLists.txt b/lib/NGT/CMakeLists.txt index db4c436..fd93bde 100644 --- a/lib/NGT/CMakeLists.txt +++ b/lib/NGT/CMakeLists.txt @@ -21,12 +21,13 @@ if( ${UNIX} ) add_dependencies(ngt ngtstatic) if(${APPLE}) if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang") - target_link_libraries(ngt OpenMP::OpenMP_CXX) + target_link_libraries(ngt lapack blas OpenMP::OpenMP_CXX) else() - target_link_libraries(ngt gomp) + target_link_libraries(ngt lapack blas gomp) endif() else(${APPLE}) - target_link_libraries(ngt gomp rt) + #target_link_libraries(ngt gomp rt) + target_link_libraries(ngt gomp rt lapack blas) endif(${APPLE}) add_custom_command(OUTPUT command DEPENDS ${NGT_SOURCES} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMAND sh ${PROJECT_SOURCE_DIR}/utils/mk_version_defs_h.sh ${PROJECT_SOURCE_DIR} version_defs.h) diff --git a/lib/NGT/Clustering.h b/lib/NGT/Clustering.h index 77cfd21..8fe00d0 100644 --- a/lib/NGT/Clustering.h +++ b/lib/NGT/Clustering.h @@ -46,9 +46,12 @@ namespace NGT { public: enum InitializationMode { - InitializationModeHead = 0, - InitializationModeRandom = 1, - InitializationModeKmeansPlusPlus = 2 + InitializationModeHead = 0, + InitializationModeRandom = 1, + InitializationModeKmeansPlusPlus = 2, + InitializationModeRandomFixedSeed = 3, + InitializationModeKmeansPlusPlusFixedSeed = 4, + InitializationModeBest = 5 }; enum ClusteringType { @@ -92,14 +95,16 @@ namespace NGT { double radius; }; - Clustering(InitializationMode im = InitializationModeHead, ClusteringType ct = ClusteringTypeKmeansWithNGT, size_t mi = 100): - clusteringType(ct), initializationMode(im), maximumIteration(mi) { initialize(); } + Clustering(InitializationMode im = InitializationModeHead, ClusteringType ct = ClusteringTypeKmeansWithNGT, size_t mi = 10000, size_t nc = 0): + clusteringType(ct), initializationMode(im), numberOfClusters(nc), maximumIteration(mi) { initialize(); } void initialize() { epsilonFrom = 0.12; epsilonTo = epsilonFrom; epsilonStep = 0.04; resultSizeCoefficient = 5; + clusterSizeConstraint = false; + clusterSizeConstraintCoefficient = 0.0; } static void @@ -162,7 +167,23 @@ namespace NGT { } static void - saveVector(const std::string &file, std::vector &vectors) + loadVector(const std::string &file, std::vector &vectors) + { + std::ifstream is(file); + if (!is) { + throw std::runtime_error("loadVector::Cannot open " + file ); + } + std::string line; + while (true) { + size_t v; + is >> v; + if (is.eof()) break; + vectors.push_back(v); + } + } + + template static void + saveVector(const std::string &file, std::vector &vectors) { std::ofstream os(file); for (auto vit = vectors.begin(); vit != vectors.end(); ++vit) { @@ -249,7 +270,11 @@ namespace NGT { static void subtract(std::vector &a, std::vector &b) { - assert(a.size() == b.size()); + if (a.size() != b.size()) { + std::stringstream msg; + std::cerr << "Clustering::subtract: Mismatched dimensions. " << a.size() << "x" << b.size(); + NGTThrowException(msg); + } auto bit = b.begin(); for (auto ait = a.begin(); ait != a.end(); ++ait, ++bit) { *ait = *ait - *bit; @@ -267,34 +292,36 @@ namespace NGT { } static void - getInitialCentroidsRandomly(std::vector > &vectors, std::vector &clusters, size_t size, size_t seed) + getInitialCentroidsRandomly(std::vector > &vectors, std::vector &clusters, size_t size, size_t seed = 0) { + size = size > vectors.size() ? vectors.size() : size; clusters.clear(); - std::random_device rnd; if (seed == 0) { + std::random_device rnd; seed = rnd(); } std::mt19937 mt(seed); + std::uniform_int_distribution<> dist(0, vectors.size() - 1); for (size_t i = 0; i < size; i++) { - size_t idx = mt() * vectors.size() / mt.max(); - if (idx >= size) { - i--; - continue; - } + size_t idx = dist(mt); clusters.push_back(Cluster(vectors[idx])); } assert(clusters.size() == size); } static void - getInitialCentroidsKmeansPlusPlus(std::vector > &vectors, std::vector &clusters, size_t size) + getInitialCentroidsKmeansPlusPlus(std::vector > &vectors, std::vector &clusters, size_t size, size_t seed = 0) { size = size > vectors.size() ? vectors.size() : size; clusters.clear(); - std::random_device rnd; - std::mt19937 mt(rnd()); - size_t idx = (long long)mt() * (long long)vectors.size() / (long long)mt.max(); + if (seed == 0) { + std::random_device rnd; + seed = rnd(); + } + std::mt19937 mt(seed); + std::uniform_int_distribution<> dist(0, vectors.size() - 1); + size_t idx = dist(mt); clusters.push_back(Cluster(vectors[idx])); NGT::Timer timer; @@ -334,19 +361,26 @@ namespace NGT { static void - assign(std::vector > &vectors, std::vector &clusters, - size_t clusterSize = std::numeric_limits::max()) { + assign(std::vector> &vectors, std::vector &clusters, + size_t clusterSize = std::numeric_limits::max(), bool clear = true) { // compute distances to the nearest clusters, and construct heap by the distances. NGT::Timer timer; timer.start(); + size_t nOfVectors = 0; + if (!clear) { + for (auto &cluster : clusters) { + nOfVectors += cluster.members.size(); + } + } + std::vector sortedObjects(vectors.size()); -#pragma omp parallel for +#pragma omp parallel for for (size_t vi = 0; vi < vectors.size(); vi++) { auto vit = vectors.begin() + vi; { double mind = DBL_MAX; - size_t mincidx = -1; + int mincidx = -1; for (auto cit = clusters.begin(); cit != clusters.end(); ++cit) { double d = distanceL2(*vit, (*cit).centroid); if (d < mind) { @@ -354,22 +388,27 @@ namespace NGT { mincidx = distance(clusters.begin(), cit); } } - sortedObjects[vi] = Entry(vi, mincidx, mind); + if (mincidx == -1) { + std::cerr << "Clustering: Fatal error " << clusters.size() << std::endl; + std::cerr << vi << "/" << vectors.size() << std::endl; + abort(); + } + sortedObjects[vi] = Entry(vi + nOfVectors, mincidx, mind); } } std::sort(sortedObjects.begin(), sortedObjects.end()); - + // clear - for (auto cit = clusters.begin(); cit != clusters.end(); ++cit) { - (*cit).members.clear(); + if (clear) { + for (auto cit = clusters.begin(); cit != clusters.end(); ++cit) { + (*cit).members.clear(); + } } - - // distribute objects to the nearest clusters in the same size constraint. for (auto soi = sortedObjects.rbegin(); soi != sortedObjects.rend();) { Entry &entry = *soi; if (entry.centroidID >= clusters.size()) { - std::cerr << "Something wrong. " << entry.centroidID << ":" << clusters.size() << std::endl; + std::cerr << "Something wrong. (2) " << entry.centroidID << ":" << clusters.size() << std::endl; soi++; continue; } @@ -377,6 +416,7 @@ namespace NGT { clusters[entry.centroidID].members.push_back(entry); soi++; } else { +#if 0 double mind = DBL_MAX; size_t mincidx = -1; for (auto cit = clusters.begin(); cit != clusters.end(); ++cit) { @@ -389,6 +429,25 @@ namespace NGT { mincidx = distance(clusters.begin(), cit); } } +#else + std::vector ds(clusters.size()); +#pragma omp parallel for + for (size_t idx = 0; idx < clusters.size(); idx++) { + if (clusters[idx].members.size() >= clusterSize) { + ds[idx] = std::numeric_limits::max(); + continue; + } + ds[idx] = distanceL2(vectors[entry.vectorID], clusters[idx].centroid); + } + float mind = std::numeric_limits::max(); + size_t mincidx = -1; + for (size_t idx = 0; idx < clusters.size(); idx++) { + if (ds[idx] < mind) { + mind = ds[idx]; + mincidx = idx; + } + } +#endif entry = Entry(entry.vectorID, mincidx, mind); int pt = distance(sortedObjects.rbegin(), soi); std::sort(sortedObjects.begin(), soi.base()); @@ -493,7 +552,6 @@ namespace NGT { assignedObjectCount++; } } - //size_t notAssignedObjectCount = 0; vector notAssignedObjectIDs; notAssignedObjectIDs.reserve(dataSize - assignedObjectCount); @@ -503,7 +561,6 @@ namespace NGT { } } - if (clusterSize < std::numeric_limits::max()) { do { vector> notAssignedObjects(notAssignedObjectIDs.size()); @@ -572,7 +629,6 @@ namespace NGT { moveFartherObjectsToEmptyClusters(clusters); } - } @@ -621,25 +677,58 @@ namespace NGT { } - - double kmeansWithoutNGT(std::vector > &vectors, size_t numberOfClusters, - std::vector &clusters) + double kmeansWithoutNGT(std::vector > &vectors, std::vector &clusters, + size_t clusterSize) { - size_t clusterSize = std::numeric_limits::max(); - if (clusterSizeConstraint) { - clusterSize = ceil((double)vectors.size() / (double)numberOfClusters); - } - - double diff = 0; - for (size_t i = 0; i < maximumIteration; i++) { + NGT::Timer timer; + timer.start(); + double diff = std::numeric_limits::max(); + size_t stabilityLimit = 10; + size_t stabilityCount = 0; + size_t i; + for (i = 0; i < maximumIteration; i++) { assign(vectors, clusters, clusterSize); // centroid is recomputed. // diff is distance between the current centroids and the previous centroids. - diff = calculateCentroid(vectors, clusters); + auto d = calculateCentroid(vectors, clusters); + if (d == diff) { + stabilityCount++; + if (stabilityCount >= stabilityLimit) { + break; + } + } + if (d < diff) { + diff = d; + } if (diff == 0) { break; } } + return diff; + } + double kmeansWithoutNGT(std::vector > &vectors, size_t numberOfClusters, + std::vector &clusters) + { + size_t clusterSize = std::numeric_limits::max(); + + double diff = kmeansWithoutNGT(vectors, clusters, clusterSize); + + if (clusterSizeConstraint || clusterSizeConstraintCoefficient != 0.0) { + if (clusterSizeConstraintCoefficient >= 1.0) { + clusterSize = ceil((double)vectors.size() / (double)numberOfClusters) * clusterSizeConstraintCoefficient; + } else if (clusterSizeConstraintCoefficient != 0.0) { + std::stringstream msg; + msg << "kmeansWithoutNGT: clusterSizeConstraintCoefficient is invalid. " << clusterSizeConstraintCoefficient << " "; + throw std::runtime_error(msg.str()); + } else { + clusterSize = ceil((double)vectors.size() / (double)numberOfClusters); + } + } else { + return diff == 0; + } + + diff = kmeansWithoutNGT(vectors, clusters, clusterSize); + return diff == 0; } @@ -662,14 +751,18 @@ namespace NGT { double diff = 0.0; size_t resultSize; resultSize = resultSizeCoefficient * vectors.size() / clusters.size(); - for (size_t i = 0; i < maximumIteration; i++) { + size_t i; + for (i = 0; i < maximumIteration; i++) { assignWithNGT(index, vectors, clusters, resultSize, epsilon, clusterSize); // centroid is recomputed. // diff is distance between the current centroids and the previous centroids. + double prevDiff = diff; std::vector prevClusters = clusters; diff = calculateCentroid(vectors, clusters); - timer.stop(); - timer.start(); + if (prevDiff == diff) { + std::cerr << "epsilon=" << epsilon << "->" << epsilon * 1.1 << std::endl; + epsilon *= 1.1; + } diffHistory.push_back(diff); if (diff == 0) { @@ -783,7 +876,7 @@ namespace NGT { { double mse = 0.0; size_t count = 0; - for (auto cit = clusters.begin(); cit != clusters.end(); ++cit) { + for (auto cit = clusters.begin(); cit != clusters.end(); ++cit) { count += (*cit).members.size(); for (auto mit = (*cit).members.begin(); mit != (*cit).members.end(); ++mit) { mse += meanSumOfSquares((*cit).centroid, vectors[(*mit).vectorID]); @@ -843,7 +936,12 @@ namespace NGT { } case InitializationModeRandom: { - getInitialCentroidsRandomly(vectors, clusters, numberOfClusters, 0); + getInitialCentroidsRandomly(vectors, clusters, numberOfClusters); + break; + } + case InitializationModeRandomFixedSeed: + { + getInitialCentroidsRandomly(vectors, clusters, numberOfClusters, 1); break; } case InitializationModeKmeansPlusPlus: @@ -851,6 +949,11 @@ namespace NGT { getInitialCentroidsKmeansPlusPlus(vectors, clusters, numberOfClusters); break; } + case InitializationModeKmeansPlusPlusFixedSeed: + { + getInitialCentroidsKmeansPlusPlus(vectors, clusters, numberOfClusters, 1); + break; + } default: std::cerr << "proper initMode is not specified." << std::endl; exit(1); @@ -858,12 +961,26 @@ namespace NGT { } } + bool + kmeans(std::vector > &vectors, std::vector &clusters) { + return kmeans(vectors, numberOfClusters, clusters); + } + bool kmeans(std::vector > &vectors, size_t numberOfClusters, std::vector &clusters) { + if (vectors.size() == 0) { + std::stringstream msg; + msg << "Clustering::kmeans: No vector."; + NGTThrowException(msg); + } + if (vectors[0].size() == 0) { + std::stringstream msg; + msg << "Clustering::kmeans: No dimension."; + NGTThrowException(msg); + } setupInitialClusters(vectors, numberOfClusters, clusters); - switch (clusteringType) { case ClusteringTypeKmeansWithoutNGT: return kmeansWithoutNGT(vectors, numberOfClusters, clusters); @@ -912,10 +1029,13 @@ namespace NGT { } } + void setClusterSizeConstraintCoefficient(float v) { clusterSizeConstraintCoefficient = v; } + ClusteringType clusteringType; InitializationMode initializationMode; size_t numberOfClusters; bool clusterSizeConstraint; + float clusterSizeConstraintCoefficient; size_t maximumIteration; float epsilonFrom; float epsilonTo; diff --git a/lib/NGT/Command.cpp b/lib/NGT/Command.cpp index 495c806..1e8442d 100644 --- a/lib/NGT/Command.cpp +++ b/lib/NGT/Command.cpp @@ -1151,4 +1151,36 @@ using namespace std; #endif } + void NGT::Command::exportObjects(Args &args) { +#if defined(NGT_SHARED_MEMORY_ALLOCATOR) + std::cerr << "ngt: Error: exportObjects is not implemented." << std::endl; + abort(); +#else + std::string usage = "ngt export-objects index"; + string indexPath; + try { + indexPath = args.get("#1"); + } catch (...) { + cerr << "ngt::exportGraph: Index is not specified." << endl; + cerr << usage << endl; + return; + } + + NGT::Index index(indexPath); + auto &objectSpace = index.getObjectSpace(); + size_t size = objectSpace.getRepository().size(); + + for (size_t id = 1; id < size; ++id) { + std::vector object; + objectSpace.getObject(id, object); + for (auto v = object.begin(); v != object.end(); ++v) { + std::cout << *v; + if (v + 1 != object.end()) { + std::cout << "\t"; + } + } + std::cout << std::endl; + } +#endif + } diff --git a/lib/NGT/Command.h b/lib/NGT/Command.h index 4377a74..316d58a 100644 --- a/lib/NGT/Command.h +++ b/lib/NGT/Command.h @@ -127,7 +127,8 @@ class Command { void refineANNG(Args &args); void repair(Args &args); void exportGraph(Args &args); - + void exportObjects(Args &args); + void info(Args &args); void setDebugLevel(int level) { debugLevel = level; } int getDebugLevel() { return debugLevel; } diff --git a/lib/NGT/Common.h b/lib/NGT/Common.h index 99b9d4b..db4f1c8 100644 --- a/lib/NGT/Common.h +++ b/lib/NGT/Common.h @@ -53,17 +53,17 @@ namespace NGT { typedef half_float::half float16; #endif -#define NGTThrowException(MESSAGE) throw NGT::Exception(__FILE__, (size_t)__LINE__, MESSAGE) -#define NGTThrowSpecificException(MESSAGE, TYPE) throw NGT::TYPE(__FILE__, (size_t)__LINE__, MESSAGE) +#define NGTThrowException(MESSAGE) throw NGT::Exception(__FILE__, __FUNCTION__, (size_t)__LINE__, MESSAGE) +#define NGTThrowSpecificException(MESSAGE, TYPE) throw NGT::TYPE(__FILE__, __FUNCTION__, (size_t)__LINE__, MESSAGE) class Exception : public std::exception { public: Exception():message("No message") {} - Exception(const std::string &file, size_t line, std::stringstream &m) { set(file, line, m.str()); } - Exception(const std::string &file, size_t line, const std::string &m) { set(file, line, m); } - void set(const std::string &file, size_t line, const std::string &m) { + Exception(const std::string &file, const std::string &function, size_t line, std::stringstream &m) { set(file, function, line, m.str()); } + Exception(const std::string &file, const std::string &function, size_t line, const std::string &m) { set(file, function, line, m); } + void set(const std::string &file, const std::string &function, size_t line, const std::string &m) { std::stringstream ss; - ss << file << ":" << line << ": " << m; + ss << file << ":" << function << ":" << line << ": " << m; message = ss.str(); } ~Exception() throw() {} @@ -106,16 +106,17 @@ namespace NGT { } else if (opt.size() > 1 && opt[0] == '-') { if (opt.size() == 2) { key = opt[1]; - if (key == "h") { - value = ""; - } else { - ++i; - if (i != opts.end()) { - value = *i; - } else { + ++i; + if (i != opts.end()) { + if (((*i)[0] == '-') && ((*i)[1] != 0) ) { value = ""; --i; + } else { + value = *i; } + } else { + value = ""; + --i; } } else { key = opt[1]; @@ -178,6 +179,14 @@ namespace NGT { usedOptions.insert(ai->first); return ai->second; } + bool getBool(const char *s) { + try { + get(s); + } catch(...) { + return false; + } + return true; + } long getl(const char *s, long v) { char *e; long val; @@ -260,6 +269,28 @@ namespace NGT { } + template + static void extractVector(const std::string &textLine, const std::string &sep, T &object) { + std::vector tokens; + NGT::Common::tokenize(textLine, tokens, sep); + size_t idx; + for (idx = 0; idx < tokens.size(); idx++) { + if (tokens[idx].size() == 0) { + std::stringstream msg; + msg << "Common::extractVecotFromText: No data. " << textLine; + NGTThrowException(msg); + } + char *e; + double v = ::strtod(tokens[idx].c_str(), &e); + if (*e != 0) { + std::cerr << "ObjectSpace::readText: Warning! Not numerical value. [" << e << "]" << std::endl; + break; + } + object.push_back(v); + } + } + + static std::string getProcessStatus(const std::string &stat) { pid_t pid = getpid(); std::stringstream str; @@ -287,6 +318,24 @@ namespace NGT { static int getProcessVmSize() { return strtol(getProcessStatus("VmSize")); } static int getProcessVmPeak() { return strtol(getProcessStatus("VmPeak")); } static int getProcessVmRSS() { return strtol(getProcessStatus("VmRSS")); } + static std::string sizeToString(float size) { + char unit = 'K'; + if (size > 1024) { + size /= 1024; + unit = 'M'; + } + if (size > 1024) { + size /= 1024; + unit = 'G'; + } + size = round(size * 100) / 100; + std::stringstream str; + str << size << unit; + return str.str(); + } + static std::string getProcessVmSizeStr() { return sizeToString(getProcessVmSize()); } + static std::string getProcessVmPeakStr() { return sizeToString(getProcessVmPeak()); } + static std::string getProcessVmRSSStr() { return sizeToString(getProcessVmRSS()); } }; class StdOstreamRedirector { @@ -302,6 +351,11 @@ namespace NGT { void enable() { enabled = true; } void disable() { enabled = false; } + void set(bool e) { enabled = e; } + void bgin(bool e) { + set(e); + begin(); + } void begin() { if (!enabled) { return; @@ -1381,7 +1435,7 @@ namespace NGT { template class PersistentRepository { public: - typedef Vector ARRAY; + typedef Vector ARRAY; typedef TYPE ** iterator; PersistentRepository():array(0) {} @@ -1925,6 +1979,7 @@ namespace NGT { } } this->clear(); + this->shrink_to_fit(); #ifdef ADVANCED_USE_REMOVED_LIST while(!removedList.empty()){ removedList.pop(); } #endif @@ -2159,7 +2214,23 @@ namespace NGT { } friend std::ostream &operator<<(std::ostream &os, Timer &t) { - os << std::setprecision(6) << t.time << " (sec)"; + auto time = t.time; + if (time < 1.0) { + time *= 1000.0; + os << std::setprecision(6) << time << " (ms)"; + return os; + } + if (time < 60.0) { + os << std::setprecision(6) << time << " (s)"; + return os; + } + time /= 60.0; + if (time < 60.0) { + os << std::setprecision(6) << time << " (m)"; + return os; + } + time /= 60.0; + os << std::setprecision(6) << time << " (h)"; return os; } diff --git a/lib/NGT/Index.cpp b/lib/NGT/Index.cpp index 487a439..ed247c9 100644 --- a/lib/NGT/Index.cpp +++ b/lib/NGT/Index.cpp @@ -40,7 +40,7 @@ Index::getVersion() } #ifdef NGT_SHARED_MEMORY_ALLOCATOR -NGT::Index::Index(NGT::Property &prop, const string &database) { +NGT::Index::Index(NGT::Property &prop, const string &database):redirect(false) { if (prop.dimension == 0) { NGTThrowException("Index::Index. Dimension is not specified."); } @@ -62,7 +62,7 @@ NGT::Index::Index(NGT::Property &prop, const string &database) { path = ""; } #else -NGT::Index::Index(NGT::Property &prop) { +NGT::Index::Index(NGT::Property &prop):redirect(false) { if (prop.dimension == 0) { NGTThrowException("Index::Index. Dimension is not specified."); } @@ -512,6 +512,12 @@ NGT::GraphIndex::constructObjectSpace(NGT::Property &prop) { } } +void +NGT::GraphIndex::loadGraph(const string &ifile, NGT::GraphRepository &graph) { + ifstream isg(ifile + "/grp"); + graph.deserialize(isg); +} + void NGT::GraphIndex::loadIndex(const string &ifile, bool readOnly, bool graphDisabled) { objectSpace->deserialize(ifile + "/obj"); @@ -522,12 +528,10 @@ NGT::GraphIndex::loadIndex(const string &ifile, bool readOnly, bool graphDisable if (readOnly && property.indexType == NGT::Index::Property::IndexType::Graph) { GraphIndex::NeighborhoodGraph::loadSearchGraph(ifile); } else { - ifstream isg(ifile + "/grp"); - repository.deserialize(isg); + loadGraph(ifile, repository); } #else - ifstream isg(ifile + "/grp"); - repository.deserialize(isg); + loadGraph(ifile, repository); #endif } diff --git a/lib/NGT/Index.h b/lib/NGT/Index.h index 106f1cf..a702399 100644 --- a/lib/NGT/Index.h +++ b/lib/NGT/Index.h @@ -74,9 +74,12 @@ namespace NGT { pathAdjustmentInterval = 0; #ifdef NGT_SHARED_MEMORY_ALLOCATOR databaseType = DatabaseType::MemoryMappedFile; - graphSharedMemorySize = 512; // MB - treeSharedMemorySize = 512; // MB - objectSharedMemorySize = 512; // MB 512 is up to 50M objects. + //graphSharedMemorySize = 512; // MB + //treeSharedMemorySize = 512; // MB + //objectSharedMemorySize = 512; // MB 512 is up to 50M objects. + graphSharedMemorySize = 10240; // MB + treeSharedMemorySize = 10240; // MB + objectSharedMemorySize = 10240; // MB 512 is up to 50M objects. #else databaseType = DatabaseType::Memory; #endif @@ -368,13 +371,13 @@ namespace NGT { Index():index(0) {} #ifdef NGT_SHARED_MEMORY_ALLOCATOR - Index(NGT::Property &prop, const std::string &database); + Index(NGT::Property &prop, const std::string &database); #else - Index(NGT::Property &prop); + Index(NGT::Property &prop); #endif - Index(const std::string &database, bool rdOnly = false):index(0) { open(database, rdOnly); } - Index(const std::string &database, bool rdOnly, bool graphDisabled):index(0) { open(database, rdOnly, graphDisabled); } - Index(const std::string &database, NGT::Property &prop):index(0) { open(database, prop); } + Index(const std::string &database, bool rdOnly = false):index(0), redirect(false) { open(database, rdOnly); } + Index(const std::string &database, bool rdOnly, bool graphDisabled):index(0), redirect(false) { open(database, rdOnly, graphDisabled); } + Index(const std::string &database, NGT::Property &prop):index(0), redirect(false) { open(database, prop); } virtual ~Index() { close(); } void open(const std::string &database, NGT::Property &prop) { @@ -423,6 +426,7 @@ namespace NGT { virtual void load(const std::string &ifile, size_t dataSize) { getIndex().load(ifile, dataSize); } virtual void append(const std::string &ifile, size_t dataSize) { getIndex().append(ifile, dataSize); } virtual void append(const float *data, size_t dataSize) { + StdOstreamRedirector redirector(redirect); redirector.begin(); try { getIndex().append(data, dataSize); @@ -433,6 +437,7 @@ namespace NGT { redirector.end(); } virtual void append(const double *data, size_t dataSize) { + StdOstreamRedirector redirector(redirect); redirector.begin(); try { getIndex().append(data, dataSize); @@ -447,6 +452,7 @@ namespace NGT { virtual size_t getObjectRepositorySize() { return getIndex().getObjectRepositorySize(); } virtual size_t getGraphRepositorySize() { return getIndex().getGraphRepositorySize(); } virtual void createIndex(size_t threadNumber, size_t sizeOfRepository = 0) { + StdOstreamRedirector redirector(redirect); redirector.begin(); try { getIndex().createIndex(threadNumber, sizeOfRepository); @@ -475,6 +481,7 @@ namespace NGT { virtual void search(NGT::SearchContainer &sc) { getIndex().search(sc); } virtual void search(NGT::SearchQuery &sc) { getIndex().search(sc); } virtual void search(NGT::SearchContainer &sc, ObjectDistances &seeds) { getIndex().search(sc, seeds); } + virtual void getSeeds(NGT::SearchContainer &sc, ObjectDistances &seeds, size_t n) { getIndex().getSeeds(sc, seeds, n); } virtual void remove(ObjectID id, bool force = false) { getIndex().remove(id, force); } virtual void exportIndex(const std::string &file) { getIndex().exportIndex(file); } virtual void importIndex(const std::string &file) { getIndex().importIndex(file); } @@ -505,8 +512,8 @@ namespace NGT { } return *index; } - void enableLog() { redirector.disable(); } - void disableLog() { redirector.enable(); } + void enableLog() { redirect = true; } + void disableLog() { redirect = false; } static void destroy(const std::string &path) { #ifdef NGT_SHARED_MEMORY_ALLOCATOR @@ -562,7 +569,7 @@ namespace NGT { Index *index; std::string path; - StdOstreamRedirector redirector; + bool redirect; }; class GraphIndex : public Index, @@ -722,9 +729,9 @@ namespace NGT { } void saveProperty(const std::string &file); - void exportProperty(const std::string &file); + static void loadGraph(const std::string &ifile, NGT::GraphRepository &graph); virtual void loadIndex(const std::string &ifile, bool readOnly, bool graphDisabled); virtual void exportIndex(const std::string &ofile) { @@ -797,7 +804,14 @@ namespace NGT { } deleteObject(query); } - + void getSeeds(NGT::SearchContainer &sc, ObjectDistances &seeds, size_t n) { + getRandomSeeds(repository, seeds, n); + setupDistances(sc, seeds); + std::sort(seeds.begin(), seeds.end()); + if (seeds.size() > n) { + seeds.resize(n); + } + } // get randomly nodes as seeds. template void getRandomSeeds(REPOSITORY &repo, ObjectDistances &seeds, size_t seedSize) { // clear all distances to find the same object as a randomized object. @@ -1566,6 +1580,39 @@ namespace NGT { void createTreeIndex(); + void getSeeds(NGT::SearchContainer &sc, ObjectDistances &seeds, size_t n) { + DVPTree::SearchContainer tso(sc.object); + tso.mode = DVPTree::SearchContainer::SearchLeaf; + tso.radius = 0.0; + tso.size = 1; + tso.distanceComputationCount = 0; + tso.visitCount = 0; + try { + DVPTree::search(tso); + } catch (Exception &err) { + std::stringstream msg; + msg << "GraphAndTreeIndex::getSeeds: Cannot search for tree.:" << err.what(); + NGTThrowException(msg); + } + try { + DVPTree::getObjectIDsFromLeaf(tso.nodeID, seeds); + } catch (Exception &err) { + std::stringstream msg; + msg << "GraphAndTreeIndex::getSeeds: Cannot get a leaf.:" << err.what(); + NGTThrowException(msg); + } + sc.distanceComputationCount += tso.distanceComputationCount; + sc.visitCount += tso.visitCount; + if (seeds.size() < n) { + GraphIndex::getRandomSeeds(repository, seeds, n); + } + GraphIndex::setupDistances(sc, seeds); + std::sort(seeds.begin(), seeds.end()); + if (seeds.size() > n) { + seeds.resize(n); + } + } + // GraphAndTreeIndex void getSeedsFromTree(NGT::SearchContainer &sc, ObjectDistances &seeds) { DVPTree::SearchContainer tso(sc.object); diff --git a/lib/NGT/NGTQ/Capi.cpp b/lib/NGT/NGTQ/Capi.cpp index 508a640..40416f4 100644 --- a/lib/NGT/NGTQ/Capi.cpp +++ b/lib/NGT/NGTQ/Capi.cpp @@ -19,8 +19,14 @@ #include #include "NGT/Capi.h" +#include "NGT/NGTQ/Quantizer.h" #include "NGT/NGTQ/Capi.h" #include "NGT/NGTQ/QuantizedGraph.h" +#include "NGT/NGTQ/QuantizedBlobGraph.h" +#include "NGT/NGTQ/Optimizer.h" +#include "NGT/NGTQ/HierarchicalKmeans.h" + +#ifdef NGTQ_QBG static bool operate_error_string_(const std::stringstream &ss, NGTError error){ if(error != NULL){ @@ -119,7 +125,11 @@ void ngtqg_initialize_quantization_parameters(NGTQGQuantizationParameters *param bool ngtqg_quantize(const char *indexPath, NGTQGQuantizationParameters parameters, NGTError error) { try{ +#ifdef NGTQ_QBG + NGTQG::Index::quantize(indexPath, parameters.max_number_of_edges); +#else NGTQG::Index::quantize(indexPath, parameters.dimension_of_subvector, parameters.max_number_of_edges); +#endif return true; }catch(std::exception &err){ std::stringstream ss; @@ -129,3 +139,285 @@ bool ngtqg_quantize(const char *indexPath, NGTQGQuantizationParameters parameter } } + + +uint32_t qbg_get_result_size(QBGObjectDistances results, NGTError error) { + return ngt_get_result_size(results, error); +} + +NGTObjectDistance qbg_get_result(const QBGObjectDistances results, const uint32_t idx, NGTError error) { + return ngt_get_result(results, idx, error); +} + +void qbg_destroy_results(QBGObjectDistances results) { + ngt_destroy_results(results); +} + + +void qbg_initialize_construction_parameters(QBGConstructionParameters *parameters) +{ + parameters->extended_dimension = 0; + parameters->dimension = 0; + parameters->number_of_subvectors = 1; + parameters->number_of_blobs = 0; + parameters->internal_data_type = NGTQ::DataTypeFloat; + parameters->data_type = NGTQ::DataTypeFloat; + parameters->distance_type = NGTQ::DistanceType::DistanceTypeL2; +} + +bool qbg_create(const char *indexPath, QBGConstructionParameters *parameters, NGTQGError error) +{ + + try { + cerr << "qbgcapi: Create" << endl; + std::vector r; + NGTQ::Property property; + NGT::Property globalProperty; + NGT::Property localProperty; + property.dimension = parameters->extended_dimension; + if (property.dimension == 0) { + property.dimension = parameters->dimension; + } + property.genuineDimension = parameters->dimension; + property.globalRange = 0; + property.localRange = 0; + property.globalCentroidLimit = parameters->number_of_blobs; + property.localCentroidLimit = 16; + property.localDivisionNo = parameters->number_of_subvectors; + property.singleLocalCodebook = false; + property.centroidCreationMode = NGTQ::CentroidCreationModeStaticLayer; + property.localCentroidCreationMode = NGTQ::CentroidCreationModeStatic; + property.localIDByteSize = 1; + property.dataType = static_cast(parameters->internal_data_type); + property.genuineDataType = static_cast(parameters->data_type); + property.distanceType = static_cast(parameters->distance_type); + + globalProperty.edgeSizeForCreation = 10; + globalProperty.edgeSizeForSearch = 40; + globalProperty.indexType = NGT::Property::GraphAndTree; + globalProperty.insertionRadiusCoefficient = 1.1; + + localProperty.indexType = globalProperty.indexType; + localProperty.insertionRadiusCoefficient = globalProperty.insertionRadiusCoefficient; + + std::vector *rotation = 0; + const std::string objectPath; + QBG::Index::create(indexPath, property, globalProperty, localProperty, rotation, objectPath); + } catch(NGT::Exception &err) { + std::stringstream ss; + ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what(); + operate_error_string_(ss, error); + return false; + } + + return true; +} + +QBGIndex qbg_open_index(const char *index_path, QBGError error) { + try { + std::string index_path_str(index_path); + auto *index = new QBG::Index(index_path_str, true); + return static_cast(index); + } catch(std::exception &err){ + std::stringstream ss; + ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what(); + operate_error_string_(ss, error); + return NULL; + } +} + +void qbg_close_index(QBGIndex index) { + if (index == NULL) return; + (static_cast(index))->close(); + delete static_cast(index); + index = 0; +} + +bool qbg_save_index(QBGIndex index, QBGError error) { + if (index == NULL) return false; + try { + (static_cast(index))->save(); + } catch(std::exception &err) { + std::stringstream ss; + ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what(); + operate_error_string_(ss, error); + return false; + } + return true; +} + +ObjectID qbg_append_object(QBGIndex index, float *obj, uint32_t obj_dim, QBGError error) { + if (index == NULL || obj == NULL || obj_dim == 0){ + std::stringstream ss; + ss << "Capi : " << __FUNCTION__ << "() : parametor error: index = " << index << " obj = " << obj << " obj_dim = " << obj_dim; + operate_error_string_(ss, error); + return 0; + } + + try { + auto *pindex = static_cast(index); + std::vector vobj(&obj[0], &obj[obj_dim]); + return pindex->append(vobj); + } catch(std::exception &err) { + std::stringstream ss; + ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what(); + operate_error_string_(ss, error); + return 0; + } +} + +void qbg_initialize_build_parameters(QBGBuildParameters *parameters) { + parameters->hierarchical_clustering_init_mode = static_cast(NGT::Clustering::InitializationModeKmeansPlusPlus); + parameters->number_of_first_objects = 0; + parameters->number_of_first_clusters = 0; + parameters->number_of_second_objects = 0; + parameters->number_of_second_clusters = 0; + parameters->number_of_third_clusters = 0; + + parameters->number_of_objects = 0; + parameters->number_of_subvectors = 1; + parameters->optimization_clustering_init_mode = static_cast(NGT::Clustering::InitializationModeKmeansPlusPlus); + parameters->rotation_iteration = 2000; + parameters->subvector_iteration = 400; + parameters->number_of_matrices = 3; + parameters->rotation = true; + parameters->repositioning = false; +} + +bool qbg_build_index(const char *index_path, QBGBuildParameters *parameters, QBGError error) { + + QBG::HierarchicalKmeans hierarchicalKmeans; + + hierarchicalKmeans.maxSize = 1000; + hierarchicalKmeans.numOfClusters = 2; + hierarchicalKmeans.numOfTotalClusters = 0; + hierarchicalKmeans.numOfTotalBlobs = 0; + hierarchicalKmeans.clusterID = -1; + hierarchicalKmeans.initMode = static_cast(parameters->hierarchical_clustering_init_mode); + hierarchicalKmeans.numOfRandomObjects = 0; + hierarchicalKmeans.extractCentroid = false; + hierarchicalKmeans.numOfFirstObjects = parameters->number_of_first_objects; + hierarchicalKmeans.numOfFirstClusters = parameters->number_of_first_clusters; + hierarchicalKmeans.numOfSecondObjects = parameters->number_of_second_objects; + hierarchicalKmeans.numOfSecondClusters = parameters->number_of_second_clusters; + hierarchicalKmeans.numOfThirdClusters = parameters->number_of_third_clusters; + hierarchicalKmeans.numOfObjects = 0; + hierarchicalKmeans.threeLayerClustering = true; + hierarchicalKmeans.silence = true; + + try { + hierarchicalKmeans.clustering(index_path); + } catch (NGT::Exception &err) { + std::stringstream ss; + ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what(); + operate_error_string_(ss, error); + return false; + } + + NGTQ::Optimizer optimizer; + + optimizer.numberOfObjects = parameters->number_of_objects; + optimizer.numberOfClusters = 16; + optimizer.numberOfSubvectors = 0; + optimizer.clusteringType = NGT::Clustering::ClusteringTypeKmeansWithNGT; + optimizer.initMode = static_cast(parameters->optimization_clustering_init_mode); + optimizer.convergenceLimitTimes = 5; + optimizer.iteration = parameters->rotation_iteration; + optimizer.clusterIteration = parameters->subvector_iteration; + optimizer.clusterSizeConstraint = false; + optimizer.nOfMatrices = parameters->number_of_matrices; + optimizer.seedStartObjectSizeRate = 0.1; + optimizer.seedStep = 2; + optimizer.reject = 0.9; + optimizer.timelimit = 24 * 2; + optimizer.timelimit *= 60.0 * 60.0; + optimizer.rotation = parameters->rotation; + optimizer.repositioning = parameters->repositioning; + optimizer.globalType = NGTQ::Optimizer::GlobalTypeNone; + optimizer.silence = true; + + try { + bool random = true; + optimizer.optimize(index_path, random); + } catch (NGT::Exception &err) { + std::stringstream ss; + ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what(); + operate_error_string_(ss, error); + return false; + } + + try { + auto silence = true; + QBG::Index::build(index_path, silence); + } catch (NGT::Exception &err) { + std::stringstream ss; + ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what(); + operate_error_string_(ss, error); + return false; + } + return true; +} + +void qbg_initialize_query(QBGQuery *parameters) { + parameters->query = 0; + parameters->number_of_results = 20; + parameters->epsilon = 0.1; + parameters->blob_epsilon = 0.0; + parameters->result_expansion = 3.0; + parameters->number_of_explored_blobs = 256; + parameters->number_of_edges = 0; + parameters->radius = 0; +} + +static bool qbg_search_index_(QBG::Index* pindex, std::vector &query, QBGQuery ¶m, NGTObjectDistances results) { + // set search parameters. + if (param.radius < 0.0){ + param.radius = FLT_MAX; + } + + QBG::SearchContainer sc; + sc.setObjectVector(query); + sc.setResults(static_cast(results)); + if (param.result_expansion >= 1.0) { + sc.setSize(static_cast(param.number_of_results) * param.result_expansion); + sc.setExactResultSize(param.number_of_results); + } else { + sc.setSize(param.number_of_results); + sc.setExactResultSize(0); + } + sc.setEpsilon(param.epsilon); + sc.setBlobEpsilon(param.blob_epsilon); + sc.setEdgeSize(param.number_of_edges); + sc.setGraphExplorationSize(param.number_of_explored_blobs); + + pindex->searchBlobGraph(sc); + + return true; +} + +bool qbg_search_index(QBGIndex index, QBGQuery query, NGTObjectDistances results, QBGError error) { + if (index == NULL || query.query == NULL || results == NULL) { + std::stringstream ss; + ss << "Capi : " << __FUNCTION__ << "() : parametor error: index = " << index << " query = " << query.query << " results = " << results; + operate_error_string_(ss, error); + return false; + } + + auto *pindex = static_cast(index); + int32_t dim = pindex->getQuantizer().property.genuineDimension; + + try { + std::vector vquery(&query.query[0], &query.query[dim]); + qbg_search_index_(pindex, vquery, query, results); + } catch(std::exception &err) { + std::stringstream ss; + ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what(); + operate_error_string_(ss, error); + return false; + } + + return true; +} + + +#endif diff --git a/lib/NGT/NGTQ/Capi.h b/lib/NGT/NGTQ/Capi.h index 5a8b4e2..88b1a0c 100644 --- a/lib/NGT/NGTQ/Capi.h +++ b/lib/NGT/NGTQ/Capi.h @@ -14,85 +14,6 @@ // limitations under the License. // -/*** - { - // simple quantization and search example - - std::string indexPath = "onng_index"; // ONNG - std::string queryPath = "query.tsv"; // Query file. - NGTError err = ngt_create_error_object(); - - // quantize the specified existing index - // build quantized objects and a quantized graph - NGTQGQuantizationParameters quantizationParameters; - ngtqg_initialize_quantization_parameters(&quantizationParameters); - if (!ngtqg_quantize(indexPath.c_str(), quantizationParameters, err)) { - std::cerr << ngt_get_error_string(err) << std::endl; - return false; - } - - // open the index (ANNG or ONNG). - index = ngtqg_open_index(indexPath.c_str(), err); - if (index == NULL) { - std::cerr << ngt_get_error_string(err) << std::endl; - return false; - } - - std::ifstream is(queryPath); // open a query file. - if (!is) { - std::cerr << "Cannot open the specified file. " << queryPath << std::endl; - return false; - } - - // get the dimension of the index to check the dimension of the query - NGTProperty property = ngt_create_property(err); - ngt_get_property(index, property, err); - size_t dimension = ngt_get_property_dimension(property, err); - ngt_destroy_property(property); - - std::string line; - float queryVector[dimension]; - if (!getline(is, line)) { // read a query object from the query file. - std::cerr << "no data" << std::endl; - } - std::vector tokens; - NGT::Common::tokenize(line, tokens, " \t"); // split a string into words by the separators. - // create a query vector from the tokens. - if (tokens.size() != dimension) { - std::cerr << "dimension of the query is invalid. dimesion=" << tokens.size() << ":" << dimension << std::endl; - return false; - } - for (std::vector::iterator ti = tokens.begin(); ti != tokens.end(); ++ti) { - queryVector[distance(tokens.begin(), ti)] = NGT::Common::strtod(*ti); - } - // set search parameters. - NGTObjectDistances result = ngt_create_empty_results(err); - NGTQGQuery query; - ngtqg_initialize_query(&query); - query.query = queryVector; - query.size = 20; - query.epsilon = 0.03; - query.result_expansion = 2; - - // search with the quantized graph - bool status = ngtqg_search_index(index, query, result, err); - NGTObjectSpace objectSpace = ngt_get_object_space(index, err); - auto rsize = ngt_get_result_size(result, err); - // show resultant objects. - std::cout << "Rank\tID\tDistance\tObject" << std::endl; - for (size_t i = 0; i < rsize; i++) { - NGTObjectDistance object = ngt_get_result(result, i, err); - std::cout << i + 1 << "\t" << object.id << "\t" << object.distance << "\t"; - float *objectVector = ngt_get_object_as_float(objectSpace, object.id, err); - for (size_t i = 0; i < dimension; i++) { - std::cout << objectVector[i] << " "; - } - std::cout << std::endl; - } - ngt_destroy_results(result); - ngtqg_close_index(index); - } -***/ #pragma once @@ -106,34 +27,106 @@ extern "C" { #include "NGT/Capi.h" -typedef void* NGTQGIndex; -typedef NGTObjectDistance NGTObjectDistance; -typedef NGTError NGTQGError; + typedef void* NGTQGIndex; + typedef NGTObjectDistance NGTObjectDistance; + typedef NGTError NGTQGError; -typedef struct { - float *query; - size_t size; // # of returned objects - float epsilon; - float result_expansion; - float radius; -} NGTQGQuery; + typedef struct { + float *query; + size_t size; // # of returned objects + float epsilon; + float result_expansion; + float radius; + } NGTQGQuery; -typedef struct { - float dimension_of_subvector; - size_t max_number_of_edges; -} NGTQGQuantizationParameters; + typedef struct { + float dimension_of_subvector; + size_t max_number_of_edges; + } NGTQGQuantizationParameters; -NGTQGIndex ngtqg_open_index(const char *, NGTError); + NGTQGIndex ngtqg_open_index(const char *, NGTQGError); -void ngtqg_close_index(NGTQGIndex); + void ngtqg_close_index(NGTQGIndex); -void ngtqg_initialize_quantization_parameters(NGTQGQuantizationParameters *); + void ngtqg_initialize_quantization_parameters(NGTQGQuantizationParameters *); -bool ngtqg_quantize(const char *, NGTQGQuantizationParameters, NGTError); + bool ngtqg_quantize(const char *, NGTQGQuantizationParameters, NGTQGError); -void ngtqg_initialize_query(NGTQGQuery *); + void ngtqg_initialize_query(NGTQGQuery *); -bool ngtqg_search_index(NGTQGIndex, NGTQGQuery, NGTObjectDistances, NGTError); + bool ngtqg_search_index(NGTQGIndex, NGTQGQuery, NGTObjectDistances, NGTQGError); + + // QBG CAPI + + typedef void* QBGIndex; + typedef NGTError QBGError; + typedef NGTObjectDistances QBGObjectDistances; + + uint32_t qbg_get_result_size(QBGObjectDistances results, NGTError error); + + NGTObjectDistance qbg_get_result(const QBGObjectDistances results, const uint32_t idx, NGTError error); + + void qbg_destroy_results(QBGObjectDistances results); + + typedef struct { + size_t extended_dimension; + size_t dimension; + size_t number_of_subvectors; + size_t number_of_blobs; + int internal_data_type; + int data_type; + int distance_type; + } QBGConstructionParameters; + + typedef struct { + // hierarchical kmeans + int hierarchical_clustering_init_mode; + size_t number_of_first_objects; + size_t number_of_first_clusters; + size_t number_of_second_objects; + size_t number_of_second_clusters; + size_t number_of_third_clusters; + // optimization + size_t number_of_objects; + size_t number_of_subvectors; + int optimization_clustering_init_mode; + size_t rotation_iteration; + size_t subvector_iteration; + size_t number_of_matrices; + bool rotation; + bool repositioning; + } QBGBuildParameters; + + typedef struct { + float *query; + size_t number_of_results; + float epsilon; + float blob_epsilon; + float result_expansion; + size_t number_of_explored_blobs; + size_t number_of_edges; + float radius; + } QBGQuery; + + void qbg_initialize_construction_parameters(QBGConstructionParameters *parameters); + + bool qbg_create(const char *indexPath, QBGConstructionParameters *parameters, QBGError error); + + QBGIndex qbg_open_index(const char *index_path, QBGError error); + + void qbg_close_index(QBGIndex index); + + bool qbg_save_index(QBGIndex index, QBGError error); + + ObjectID qbg_append_object(QBGIndex index, float *obj, uint32_t obj_dim, QBGError error); + + void qbg_initialize_build_parameters(QBGBuildParameters *parameters); + + bool qbg_build_index(const char *index_path, QBGBuildParameters *parameters, QBGError error); + + void qbg_initialize_query(QBGQuery *parameters); + + bool qbg_search_index(QBGIndex index, QBGQuery query, NGTObjectDistances results, QBGError error); #ifdef __cplusplus } diff --git a/lib/NGT/NGTQ/HierarchicalKmeans.h b/lib/NGT/NGTQ/HierarchicalKmeans.h new file mode 100644 index 0000000..91b17f9 --- /dev/null +++ b/lib/NGT/NGTQ/HierarchicalKmeans.h @@ -0,0 +1,1243 @@ +// +// Copyright (C) 2021 Yahoo Japan Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#pragma once + +#include "Quantizer.h" + +namespace QBG { + class HierarchicalKmeans { + public: + typedef NGTQ::Quantizer::ObjectList QBGObjectList; + + class HKNode { + public: + bool leaf; + }; + + class HKLeafNode : public HKNode { + public: + HKLeafNode():id(0){ leaf = true; } + std::vector members; + uint32_t id; + }; + + class HKInternalNode : public HKNode { + public: + HKInternalNode() { leaf = false; } + std::vector>> children; + }; + + HierarchicalKmeans() { + silence = false; + } + + static int32_t searchLeaf(std::vector &nodes, int32_t rootID, float *object) { + auto nodeID = rootID; + while (true) { + auto *node = nodes[nodeID]; + if (node->leaf) { + return nodeID; + } else { + HKInternalNode &internalNode = static_cast(*node); + float min = std::numeric_limits::max(); + int32_t minid = 0; + for (auto &c : internalNode.children) { + auto d = NGT::PrimitiveComparator::compareL2(reinterpret_cast(&object[0]), + c.second.data(), c.second.size()); + if (d < min) { + min = d; + minid = c.first; + } + } + nodeID = minid; + } + } + return -1; + } + + static void aggregateObjects(HKLeafNode &leafNode, std::vector> &vectors, + NGT::ObjectSpace &objectSpace, QBGObjectList &objectList) + { + vectors.reserve(leafNode.members.size() + 1); + std::vector obj; + for (auto &m : leafNode.members) { + objectList.get(m, obj, &objectSpace); + vectors.push_back(obj); + } + } + + static void aggregateObjects(HKLeafNode &leafNode, std::vector> &vectors, + NGT::ObjectSpace &objectSpace, QBGObjectList &objectList, + std::vector &object) + { + aggregateObjects(leafNode, vectors, objectSpace, objectList); + vectors.push_back(std::move(object)); + } + + static void split(uint32_t id, std::vector> &vectors, + std::vector &nodes, int32_t leafNodeID, NGT::Clustering &clustering) + { + HKLeafNode &leafNode = static_cast(*nodes[leafNodeID]); + std::vector clusters; + clustering.kmeans(vectors, clusters); + auto *newNode = new HKInternalNode; + for (auto &cluster : clusters) { + auto centroid = std::move(cluster.centroid); + centroid.resize(vectors[0].size()); + newNode->children.push_back(std::make_pair(nodes.size(), std::move(centroid))); + auto *cnode = new HKLeafNode; + nodes.push_back(cnode); + for (auto &member : cluster.members) { + if (member.vectorID > leafNode.members.size()) { + std::cerr << "Fatal error. member:" << member.vectorID << ":" << leafNode.members.size() << std::endl; + abort(); + } + if (member.vectorID == leafNode.members.size()) { + cnode->members.push_back(id); + } else { + cnode->members.push_back(leafNode.members[member.vectorID]); + } + } + } + delete nodes[leafNodeID]; + nodes[leafNodeID] = newNode; + } + + static double computeError(std::vector &nodes, NGT::ObjectSpace &objectSpace, QBGObjectList &objectList) { + std::cerr << "node size=" << nodes.size() << std::endl; + double distance = 0.0; + size_t dcount = 0; + for (auto *node : nodes) { + if (node->leaf) { + } else { + HKInternalNode &internalNode = static_cast(*node); + std::vector obj; + for (auto &child : internalNode.children) { + if (nodes[child.first]->leaf) { + if (dcount % 100000 == 0) { + std::cerr << "Processed leaves=" << dcount << std::endl; + } + auto centroid = child.second; + HKLeafNode &leafNode = static_cast(*nodes[child.first]); + for (auto &m : leafNode.members) { + objectList.get(m, obj, &objectSpace); + distance += NGT::Clustering::distanceL2(centroid, obj); + dcount++; + } + } + } + } + } + distance /= dcount; + std::cout << "# of vectors=" << dcount << std::endl; + std::cout << "Quantization error=" << distance << std::endl; + return distance; + } + + static size_t extractCentroids(std::ostream &oStream, std::vector &nodes) { + std::cerr << "node size=" << nodes.size() << std::endl; + size_t clusterCount = 0; + size_t objectCount = 0; + size_t leafID = 0; + for (auto *node : nodes) { + if (node->leaf) { + HKLeafNode &leafNode = static_cast(*node); + objectCount += leafNode.members.size(); + } else { + HKInternalNode &internalNode = static_cast(*node); + for (auto &child : internalNode.children) { + if (nodes[child.first]->leaf) { + if (static_cast(nodes[child.first])->id == 0) { + static_cast(nodes[child.first])->id = leafID; + } else if (static_cast(nodes[child.first])->id != leafID) { + std::cerr << "leaf ID is invalid?" << std::endl; + } + leafID++; + size_t count = 0; + clusterCount++; + for (auto &v : child.second) { + oStream << v; + if (++count == child.second.size()) { + oStream << std::endl; + } else { + oStream << "\t"; + } + } + } + } + } + } + std::cerr << "# of clusters=" << clusterCount << std::endl; + return objectCount; + } + + static size_t extractIndex(std::ostream &oStream, std::vector &nodes, size_t numOfObjects) { + std::vector clusterID(numOfObjects, -1); + std::cerr << "numOfObjects=" << numOfObjects << std::endl; + std::cerr << "node size=" << nodes.size() << std::endl; + for (auto *node : nodes) { + if (node->leaf) { + HKLeafNode &leafNode = static_cast(*node); + for (auto &member : leafNode.members) { + if (member > numOfObjects) { + std::cerr << "output index: Internal fatal error. " << member << ":" << numOfObjects - 1 << std::endl; + abort(); + } + if (member == 0) { + std::cerr << "output index: Internal fatal error. Invalid ID" << std::endl; + abort(); + } + clusterID[member - 1] = leafNode.id; + } + } + } + std::cerr << "clusterID.size=" << clusterID.size() << std::endl; + size_t count = 0; + for (auto cid : clusterID) { + count++; + oStream << cid << std::endl; + } + std::cerr << "# of id=" << count << std::endl; + return count; + } + + static void extractBtoQAndQCentroid(std::ostream &btoqStream, std::ostream &qStream, + std::vector &nodes, size_t numOfThirdClusters) { + std::cerr << "extractBtoQ" << std::endl; + std::vector btoq(numOfThirdClusters); + std::cerr << "numOfThirdClusters=" << numOfThirdClusters << std::endl; + std::cerr << "node size=" << nodes.size() << std::endl; + size_t rootID = 0; + HKInternalNode &root = static_cast(*nodes[rootID]); + std::cerr << "first=" << root.children.size() << std::endl; + size_t secondCount = 0; + size_t thirdCount = 0; + size_t objectCount = 0; + size_t leafID = 0; + size_t qID = 0; + for (auto &c1 : root.children) { + HKInternalNode &node1 = static_cast(*nodes[c1.first]); + std::cerr << "second=" << node1.children.size() << std::endl; + secondCount += node1.children.size(); + for (auto &c2 : node1.children) { + HKInternalNode &node2 = static_cast(*nodes[c2.first]); + std::cerr << "third=" << node2.children.size() << std::endl; + thirdCount += node2.children.size(); + size_t count = 0; + for (auto &v : c2.second) { + qStream << v; + if (++count == c2.second.size()) { + qStream << std::endl; + } else { + qStream << "\t"; + } + } + for (auto &c3 : node2.children) { + btoqStream << qID << std::endl; + HKLeafNode &leaf = static_cast(*nodes[c3.first]); + objectCount += leaf.members.size(); + if (leaf.id != leafID++) { + std::cerr << "leaf is invalid" << leaf.id << ":" << leafID << std::endl; + abort(); + } + } + qID++; + } + } + std::cerr << "second=" << secondCount << std::endl; + std::cerr << "third=" << thirdCount << std::endl; + std::cerr << "object=" << objectCount << std::endl; + } + + static void extractRandomObjectsFromEachBlob(std::ostream &oStream, std::vector &nodes, size_t numOfObjects, + size_t numOfRandomObjects, NGTQ::QuantizerInstance& quantizer, bool extractCentroid) { + std::cerr << "node size=" << nodes.size() << std::endl; + std::vector>> randomObjects(numOfObjects); + std::vector> centroids(numOfObjects); + for (auto *node : nodes) { + if (node->leaf) { + HKLeafNode &leafNode = static_cast(*node); + std::vector randomObjectIDXs; + if (numOfRandomObjects >= leafNode.members.size()) { + randomObjectIDXs = leafNode.members; + while (randomObjectIDXs.size() < numOfRandomObjects) { + double random = ((double)rand() + 1.0) / ((double)RAND_MAX + 2.0); + uint32_t idx = floor(leafNode.members.size() * random); + if (idx >= leafNode.members.size()) { + std::cerr << "Internal error. " << idx << ":" << leafNode.members.size() << std::endl; + abort(); + } + randomObjectIDXs.push_back(leafNode.members[idx]); + } + } else { + srand(leafNode.id); + while (randomObjectIDXs.size() < numOfRandomObjects) { + uint32_t idx = 0; + do { + double random = ((double)rand() + 1.0) / ((double)RAND_MAX + 2.0); + idx = floor(leafNode.members.size() * random); + if (idx >= leafNode.members.size()) { + std::cerr << "Internal error. " << idx << ":" << leafNode.members.size() << std::endl; + abort(); + } + } while (std::find(randomObjectIDXs.begin(), randomObjectIDXs.end(), leafNode.members[idx]) != randomObjectIDXs.end()); + std::cerr << "IDX=" << idx << "/" << leafNode.members.size() << std::endl; + randomObjectIDXs.push_back(leafNode.members[idx]); + } + } + std::cerr << "randomObjectIDXs=" << randomObjectIDXs.size() << std::endl; + for (auto member : randomObjectIDXs) { + if (member == 0) { + std::cerr << "output index: Internal fatal error. Invalid ID. " << member << std::endl; + abort(); + } + std::vector object; + quantizer.objectList.get(member, object, &quantizer.globalCodebookIndex.getObjectSpace()); + if (leafNode.id >= numOfObjects) { + std::cerr << "Internal error! Wrong leaf ID. " << leafNode.id << ":" << numOfObjects << std::endl; + abort(); + } + randomObjects[leafNode.id].push_back(object); + } + } else { + if (extractCentroid) { + HKInternalNode &internalNode = static_cast(*node); + for (auto &child : internalNode.children) { + if (nodes[child.first]->leaf) { + HKLeafNode &leafNode = static_cast(*nodes[child.first]); + centroids[leafNode.id] = child.second; + } + } + } + } + } + for (size_t idx = 0; idx < centroids.size(); idx++) { + auto &c = centroids[idx]; + if (extractCentroid && c.empty()) { + std::cerr << "qbg: Fatal error! The centroid is empty." << std::endl; + abort(); + } + for (size_t i = 0; i < c.size(); i++) { + oStream << c[i]; + if (i + 1 != c.size()) { + oStream << "\t"; + } else { + oStream << std::endl;; + } + } + auto &ros = randomObjects[idx]; + for (auto &ro : ros) { + if (ro.empty()) { + std::cerr << "qbg: Fatal error! The random object vector is empty." << std::endl; + abort(); + } + for (size_t i = 0; i < ro.size(); i++) { + oStream << ro[i]; + if (i + 1 != ro.size()) { + oStream << "\t"; + } else { + oStream << std::endl;; + } + } + } + } + } + + static void extractBtoQIndex(std::ofstream &of, std::vector &nodes, std::vector &qNodeIDs) { + size_t leafID = 0; + for (size_t qnidx = 0; qnidx < qNodeIDs.size(); qnidx++) { + if (nodes[qNodeIDs[qnidx]]->leaf) { + std::cerr << "Fatal error. this should be an internal node." << std::endl; + abort(); + } + HKInternalNode &inode = static_cast(*nodes[qNodeIDs[qnidx]]); + for (auto &c : inode.children) { + if (!nodes[c.first]->leaf) { + std::cerr << "Fatal error. this should be a leaf." << std::endl; + abort(); + } + HKLeafNode &leaf = static_cast(*nodes[c.first]); + if (leaf.id == 0) { + leaf.id = leafID; + } + of << qnidx << std::endl; + leafID++; + } + } +} + + + static void hierarchicalKmeans(uint32_t id, int32_t rootID, std::vector &object, + QBGObjectList &objectList, NGT::ObjectSpace &objectSpace, + std::vector &nodes, NGT::Clustering &clustering, size_t maxSize) { + NGT::Timer timer; + objectList.get(id, object, &objectSpace); + int32_t nodeID = searchLeaf(nodes, rootID, reinterpret_cast(&object[0])); + if (nodeID < 0) { + std::cerr << "Fatal inner error! node ID=" << nodeID << std::endl; + exit(1); + } + auto *node = nodes[nodeID]; + HKLeafNode &leafNode = static_cast(*node); + if (leafNode.members.size() >= maxSize) { + NGT::Timer subtimer; + subtimer.start(); + std::vector> vectors; + aggregateObjects(leafNode, vectors, objectSpace, objectList, object); + subtimer.stop(); + std::cerr << "aggregate time=" << subtimer << std::endl; + subtimer.start(); + split(id, vectors, nodes, nodeID, clustering); + subtimer.stop(); + std::cerr << "split time=" << subtimer << std::endl; + } else { + leafNode.members.push_back(id); + } + } + + static void hierarchicalKmeansBatch(std::vector &batch, std::vector> &exceededLeaves, + int32_t rootID, std::vector &object, + QBGObjectList &objectList, NGT::ObjectSpace &objectSpace, + std::vector &nodes, NGT::Clustering &clustering, size_t maxSize, size_t &nleaves, + size_t maxExceededLeaves) { + + if (batch.size() == 0) { + return; + } + + int32_t nodeIDs[batch.size()]; + +#pragma omp parallel for + for (size_t idx = 0; idx < batch.size(); idx++) { + auto id = batch[idx]; +#pragma omp critical + objectList.get(id, object, &objectSpace); + int32_t nodeID = searchLeaf(nodes, rootID, reinterpret_cast(&object[0])); + if (nodeID < 0) { + std::cerr << "Fatal inner error! node ID=" << nodeID << std::endl; + exit(1); + } + nodeIDs[idx] = nodeID; + } + + + for (size_t idx = 0; idx < batch.size(); idx++) { + auto id = batch[idx]; + HKLeafNode &leafNode = static_cast(*nodes[nodeIDs[idx]]); + leafNode.members.push_back(id); + if (leafNode.members.size() > maxSize) { + auto i = exceededLeaves.begin(); + for (; i != exceededLeaves.end(); i++) { + if (static_cast((*i).second) == nodeIDs[idx]) break; + } + if (i == exceededLeaves.end()) { + exceededLeaves.push_back(std::make_pair(batch[idx], nodeIDs[idx])); + } + } + } + + batch.clear(); + + if (exceededLeaves.size() < maxExceededLeaves) { + return; + } + + std::vector> clusters(exceededLeaves.size()); +#pragma omp parallel for + for (size_t idx = 0; idx < exceededLeaves.size(); idx++) { + HKLeafNode &leafNode = static_cast(*nodes[exceededLeaves[idx].second]); + std::vector> vectors; +#pragma omp critical + aggregateObjects(leafNode, vectors, objectSpace, objectList); + clustering.kmeans(vectors, clusters[idx]); + } + + std::cerr << "exceeded leaves=" << exceededLeaves.size() << std::endl; + for (size_t idx = 0; idx < exceededLeaves.size(); idx++) { + auto leafNodeID = exceededLeaves[idx].second; + HKLeafNode &leafNode = static_cast(*nodes[leafNodeID]); + auto *newNode = new HKInternalNode; + for (auto &cluster : clusters[idx]) { + newNode->children.push_back(std::make_pair(nodes.size(), std::move(cluster.centroid))); + auto *cnode = new HKLeafNode; + nodes.push_back(cnode); + for (auto &member : cluster.members) { + cnode->members.push_back(leafNode.members[member.vectorID]); + } + } + nleaves += clusters[idx].size() - 1; + delete nodes[leafNodeID]; + nodes[leafNodeID] = newNode; + } + exceededLeaves.clear(); + + } + + static void hierarchicalKmeansWithNumberOfClusters(size_t numOfTotalClusters, size_t numOfObjects, size_t numOfLeaves, + QBGObjectList &objectList, NGT::ObjectSpace &objectSpace, + std::vector &nodes, NGT::Clustering::InitializationMode initMode){ + std::cerr << "numOfTotalClusters=" << numOfTotalClusters << std::endl; + std::cerr << "numOfLeaves=" << numOfLeaves << std::endl; + if (numOfLeaves > numOfTotalClusters) { + std::cerr << "# of clusters is invalid. " << numOfLeaves << ":" << numOfTotalClusters << std::endl; + abort(); + } + auto numOfRemainingClusters = numOfTotalClusters; + auto numOfRemainingVectors = numOfObjects; + size_t leafCount = 0; + size_t nodeSize = nodes.size(); + for (size_t nidx = 0; nidx < nodeSize; nidx++) { + if (nodes[nidx]->leaf) { + leafCount++; + if (numOfLeaves >= 100 && leafCount % (numOfLeaves / 100) == 0) { + std::cerr << "Processed leaves: " << leafCount << " " << leafCount * 100 / numOfLeaves << "%" << std::endl; + } + HKLeafNode &leafNode = static_cast(*nodes[nidx]); + std::vector> vectors; + aggregateObjects(leafNode, vectors, objectSpace, objectList); + size_t nClusters = round(static_cast(leafNode.members.size()) / numOfRemainingVectors * numOfRemainingClusters); + nClusters = nClusters == 0 ? 1 : nClusters; + numOfRemainingVectors -= leafNode.members.size(); + numOfRemainingClusters -= nClusters; + NGT::Clustering clustering(initMode, NGT::Clustering::ClusteringTypeKmeansWithoutNGT, 1000, nClusters); + NGT::Timer timer; + timer.start(); + split(0, vectors, nodes, nidx, clustering); + timer.stop(); + if (nodes[nidx]->leaf) { + std::cerr << "At this moment, the second node should be an internal" << std::endl; + abort(); + } + } + } + } + + static void hierarchicalKmeansWithNumberOfClustersInParallel(size_t numOfTotalClusters, size_t numOfObjects, size_t numOfLeaves, + QBGObjectList &objectList, NGT::ObjectSpace &objectSpace, + std::vector &nodes, NGT::Clustering::InitializationMode initMode){ + NGT::Timer timer; + timer.start(); + auto numOfRemainingClusters = numOfTotalClusters; + auto numOfRemainingVectors = numOfObjects; + size_t leafCount = 0; + + std::vector> leafNodes; + leafNodes.reserve(numOfLeaves); + for (size_t nidx = 0; nidx < nodes.size(); nidx++) { + if (nodes[nidx]->leaf) { + leafCount++; + { + size_t step = 10; + if (numOfLeaves >= step && leafCount % (numOfLeaves / step) == 0) { + std::cerr << "Processed leaves: " << leafCount << " " << leafCount * step / numOfLeaves << "%" << std::endl; + } + } + HKLeafNode &leafNode = static_cast(*nodes[nidx]); + size_t nClusters = round(static_cast(leafNode.members.size()) / numOfRemainingVectors * numOfRemainingClusters); + nClusters = nClusters == 0 ? 1 : nClusters; + numOfRemainingVectors -= leafNode.members.size(); + numOfRemainingClusters -= nClusters; + leafNodes.push_back(std::make_pair(nidx, nClusters)); + } + } + timer.stop(); + std::cerr << "hierarchicalKmeansWithNumberOfClustersInParallel: extract leaves. Time=" << timer << std::endl; + timer.start(); + + std::cerr << "start kmeans..." << std::endl; + std::vector> clusters(leafNodes.size()); +#pragma omp parallel for + for (size_t nidx = 0; nidx < leafNodes.size(); nidx++) { + HKLeafNode &leafNode = static_cast(*nodes[leafNodes[nidx].first]); + std::vector> vectors; +#pragma omp critical + aggregateObjects(leafNode, vectors, objectSpace, objectList); + NGT::Clustering clustering(initMode, NGT::Clustering::ClusteringTypeKmeansWithoutNGT, 1000, leafNodes[nidx].second); + clustering.kmeans(vectors, clusters[nidx]); + } + + timer.stop(); + std::cerr << "hierarchicalKmeansWithNumberOfClustersInParallel: kmeans. Time=" << timer << std::endl; + timer.start(); + + std::cerr << "add nodes..." << std::endl; + for (size_t idx = 0; idx < leafNodes.size(); idx++) { + auto leafNodeID = leafNodes[idx].first; + HKLeafNode &leafNode = static_cast(*nodes[leafNodeID]); + auto *newNode = new HKInternalNode; + for (auto &cluster : clusters[idx]) { + newNode->children.push_back(std::make_pair(nodes.size(), std::move(cluster.centroid))); + auto *cnode = new HKLeafNode; + nodes.push_back(cnode); + for (auto &member : cluster.members) { + cnode->members.push_back(leafNode.members[member.vectorID]); + } + } + delete nodes[leafNodeID]; + nodes[leafNodeID] = newNode; + } + timer.stop(); + std::cerr << "hierarchicalKmeansWithNumberOfClustersInParallel: add nodes. Time=" << timer << std::endl; + + } + + static void flattenClusters(std::vector &upperClusters, + std::vector> &lowerClusters, + size_t numOfLowerClusters, + std::vector &flatClusters) { + + + flatClusters.clear(); + flatClusters.reserve(numOfLowerClusters); + + for (size_t idx1 = 0; idx1 < lowerClusters.size(); idx1++) { + for (size_t idx2 = 0; idx2 < lowerClusters[idx1].size(); idx2++) { + for (auto &m : lowerClusters[idx1][idx2].members) { + m.vectorID = upperClusters[idx1].members[m.vectorID].vectorID; + } + flatClusters.push_back(lowerClusters[idx1][idx2]); + } + } + + } + +#ifndef MULTIPLE_OBJECT_LISTS + void subclustering(std::vector &upperClusters, size_t numOfLowerClusters, size_t numOfObjects, + NGT::ObjectSpace &objectSpace, QBGObjectList &objectList, + NGT::Clustering::InitializationMode initMode, std::vector> &lowerClusters) { + std::vector nPartialClusters(upperClusters.size()); + auto numOfRemainingClusters = numOfLowerClusters; + auto numOfRemainingVectors = numOfObjects; + size_t ts = 0; + for (size_t idx = 0; idx < upperClusters.size(); idx++) { + size_t ncs = round(static_cast(upperClusters[idx].members.size()) / numOfRemainingVectors * + numOfRemainingClusters); + ncs = ncs == 0 ? 1 : ncs; + numOfRemainingVectors -= upperClusters[idx].members.size(); + if (numOfRemainingClusters >= ncs) { + numOfRemainingClusters -= ncs; + } + nPartialClusters[idx] = ncs; + ts += ncs; + } + std::cerr << "numOfRemainingClusters=" << numOfRemainingClusters << std::endl; + std::cerr << "numOfRemainingVectors=" << numOfRemainingVectors << std::endl; + std::cerr << "upperClusters=" << upperClusters.size() << std::endl; + std::cerr << "total=" << ts << ":" << numOfLowerClusters << std::endl; + if (ts < numOfLowerClusters || numOfRemainingClusters != 0) { + std::cerr << "subclustering: Internal error! " << std::endl; + exit(1); + } + + lowerClusters.resize(upperClusters.size()); +#pragma omp parallel for schedule(dynamic) + for (size_t idx = 0; idx < upperClusters.size(); idx++) { + std::vector> partialVectors; + partialVectors.reserve(upperClusters[idx].members.size()); + std::vector obj; +#pragma omp critical + { + for (auto &m : upperClusters[idx].members) { + objectList.get(m.vectorID + 1, obj, &objectSpace); + partialVectors.push_back(obj); + } + } + if (upperClusters[idx].members.size() != partialVectors.size()) { + std::cerr << "the sizes of members are not consistent" << std::endl; + abort(); + } + NGT::Clustering lowerClustering(initMode, NGT::Clustering::ClusteringTypeKmeansWithoutNGT, 1000); + lowerClustering.kmeans(partialVectors, nPartialClusters[idx], lowerClusters[idx]); + if (nPartialClusters[idx] != lowerClusters[idx].size()) { + std::cerr << "the sizes of cluster members are not consistent" << std::endl; + abort(); + } + } + size_t nc = 0; + size_t mc = 0; + for (auto &cs : lowerClusters) { + nc += cs.size(); + for (auto &c : cs) { + mc += c.members.size(); + } + } + std::cerr << "# of clusters=" << nc << " # of members=" << mc << std::endl; + } + void subclustering(std::vector &upperClusters, size_t numOfLowerClusters, size_t numOfObjects, + NGT::ObjectSpace &objectSpace, QBGObjectList &objectList, + NGT::Clustering::InitializationMode initMode, std::vector &flatLowerClusters) { + + std::vector> lowerClusters; + subclustering(upperClusters, numOfLowerClusters, numOfObjects, objectSpace, objectList, initMode, lowerClusters); + + flattenClusters(upperClusters, lowerClusters, numOfLowerClusters, flatLowerClusters); + + } + +#else + static void subclustering(std::vector &upperClusters, size_t numOfLowerClusters, size_t numOfObjects, + NGT::ObjectSpace &objectSpace, QBGObjectList &objectList, + NGT::Clustering::InitializationMode initMode, std::vector> &lowerClusters) { + std::vector nPartialClusters(upperClusters.size()); + auto numOfRemainingClusters = numOfLowerClusters; + auto numOfRemainingVectors = numOfObjects; + size_t ts = 0; + for (size_t idx = 0; idx < upperClusters.size(); idx++) { + size_t ncs = round(static_cast(upperClusters[idx].members.size()) / numOfRemainingVectors * + numOfRemainingClusters); + ncs = ncs == 0 ? 1 : ncs; + numOfRemainingVectors -= upperClusters[idx].members.size(); + if (numOfRemainingClusters >= ncs) { + numOfRemainingClusters -= ncs; + } + nPartialClusters[idx] = ncs; + ts += ncs; + } + + std::cerr << "numOfRemainingClusters=" << numOfRemainingClusters << std::endl; + std::cerr << "numOfRemainingVectors=" << numOfRemainingVectors << std::endl; + std::cerr << "upperClusters=" << upperClusters.size() << std::endl; + std::cerr << "total=" << ts << ":" << numOfLowerClusters << std::endl; + if (ts < numOfLowerClusters || numOfRemainingClusters != 0) { + std::cerr << "subclustering: Internal error! " << std::endl; + exit(1); + } + + auto nthreads = omp_get_max_threads(); + if (!objectList.openMultipleStreams(nthreads)) { + std::cerr << "Cannot open multiple streams." << std::endl; + abort(); + } + + lowerClusters.resize(upperClusters.size()); +#pragma omp parallel for schedule(dynamic) + for (size_t idx = 0; idx < upperClusters.size(); idx++) { + std::vector> partialVectors; + partialVectors.reserve(upperClusters[idx].members.size()); + std::vector obj; + auto threadid = omp_get_thread_num(); + //#pragma omp critical + { + for (auto &m : upperClusters[idx].members) { + if (threadid >= nthreads) { + std::cerr << "inner fatal error. # of threads=" << nthreads << ":" << threadid << std::endl; + exit(1); + } + if (!objectList.get(threadid, m.vectorID + 1, obj, &objectSpace)) { + std::cerr << "subclustering: Fatal error! cannot get!!!! " << m.vectorID + 1 << std::endl; + abort(); + } + partialVectors.push_back(obj); + } + } + if (upperClusters[idx].members.size() != partialVectors.size()) { + std::cerr << "the sizes of members are not consistent" << std::endl; + abort(); + } + NGT::Clustering lowerClustering(initMode, NGT::Clustering::ClusteringTypeKmeansWithoutNGT, 1000); + lowerClustering.kmeans(partialVectors, nPartialClusters[idx], lowerClusters[idx]); + if (nPartialClusters[idx] != lowerClusters[idx].size()) { + std::cerr << "the sizes of cluster members are not consistent" << std::endl; + abort(); + } + } + size_t nc = 0; + size_t mc = 0; + for (auto &cs : lowerClusters) { + nc += cs.size(); + for (auto &c : cs) { + mc += c.members.size(); + } + } + std::cerr << "# of clusters=" << nc << " # of members=" << mc << std::endl; + } + + static void subclustering(std::vector &upperClusters, size_t numOfLowerClusters, size_t numOfObjects, + NGT::ObjectSpace &objectSpace, QBGObjectList &objectList, + NGT::Clustering::InitializationMode initMode, std::vector &flatLowerClusters) { + + std::vector> lowerClusters; + subclustering(upperClusters, numOfLowerClusters, numOfObjects, objectSpace, objectList, initMode, lowerClusters); + + flattenClusters(upperClusters, lowerClusters, numOfLowerClusters, flatLowerClusters); + + } + +#endif + + + static void assign(std::vector &clusters, size_t beginID, size_t endID, + NGT::ObjectSpace &objectSpace, QBGObjectList &objectList) { + +#ifdef MULTIPLE_OBJECT_LISTS + if (!objectList.openMultipleStreams(omp_get_max_threads())) { + std::cerr << "Cannot open multiple streams." << std::endl; + abort(); + } +#endif + + size_t count = 0; +#pragma omp parallel for + for (size_t id = beginID; id <= endID; id++) { + std::vector obj; + //#pragma omp critical +#ifdef MULTIPLE_OBJECT_LISTS + objectList.get(omp_get_thread_num(), id, obj, &objectSpace); +#else + objectList.get(id, obj, &objectSpace); +#endif + float min = std::numeric_limits::max(); + int minidx = -1; + for (size_t cidx = 0; cidx != clusters.size(); cidx++) { + auto d = NGT::PrimitiveComparator::compareL2(reinterpret_cast(obj.data()), + clusters[cidx].centroid.data(), obj.size()); + if (d < min) { + min = d; + minidx = cidx; + } + } + if (minidx < 0) { + std::cerr << "assign: Fatal error!" << std::endl; + abort(); + } +#pragma omp critical + { + clusters[minidx].members.push_back(NGT::Clustering::Entry(id - 1, minidx, min)); + count++; + if (count % 1000000 == 0) { + std::cerr << "# of assigned objects=" << count << std::endl; + } + } + } + + } + + static void assignWithNGT(std::vector &clusters, size_t beginID, size_t endID, + NGT::ObjectSpace &objectSpace, QBGObjectList &objectList) { + if (beginID > endID) { + std::cerr << "assignWithNGT::Warning. beginID:" << beginID << " > endID:" << endID << std::endl; + return; + } + + NGT::Property prop; + prop.dimension = objectSpace.getDimension(); + prop.objectType = NGT::Index::Property::ObjectType::Float; + prop.distanceType = NGT::Property::DistanceType::DistanceTypeL2; + prop.edgeSizeForCreation = 10; + prop.edgeSizeForSearch = 40; + +#ifdef NGT_SHARED_MEMORY_ALLOCATOR + NGT::Index index(prop, "dummy"); +#else + NGT::Index index(prop); +#endif + for (size_t cidx = 0; cidx < clusters.size(); cidx++) { + if (cidx % 100000 == 0) { + std::cerr << "# of appended cluster objects=" << cidx << std::endl; + } + index.append(clusters[cidx].centroid); + } + std::cerr << "createIndex..." << std::endl; + index.createIndex(500); + + std::cerr << "assign with NGT..." << std::endl; + endID++; +#ifdef MULTIPLE_OBJECT_LISTS + if (!objectList.openMultipleStreams(omp_get_max_threads())) { + std::cerr << "Cannot open multiple streams." << std::endl; + abort(); + } +#endif + std::vector> clusterIDs(endID - beginID); +#pragma omp parallel for + for (size_t id = beginID; id < endID; id++) { + std::vector obj; +#ifdef MULTIPLE_OBJECT_LISTS + objectList.get(omp_get_thread_num(), id, obj, &objectSpace); +#else + objectList.get(id, obj, &objectSpace); +#endif + NGT::SearchQuery sc(obj); + NGT::ObjectDistances objects; + sc.setResults(&objects); + sc.setSize(10); + sc.setEpsilon(0.12); + index.search(sc); + //index.linearSearch(sc); + clusterIDs[id - beginID] = make_pair(objects[0].id - 1, objects[0].distance); + } + std::cerr << "pushing..." << std::endl; + for (size_t id = beginID; id < endID; id++) { + auto cid = clusterIDs[id - beginID].first; + auto cdistance = clusterIDs[id - beginID].second; + clusters[cid].members.push_back(NGT::Clustering::Entry(id - 1, cid, cdistance)); + } + } + +#ifdef NGTQ_QBG + void treeBasedTopdownClustering(std::string prefix, QBG::Index &index, uint32_t rootID, std::vector &object, std::vector &nodes, NGT::Clustering &clustering) { + auto &quantizer = static_cast&>(index.getQuantizer()); + auto &objectSpace = quantizer.globalCodebookIndex.getObjectSpace(); + QBGObjectList &objectList = quantizer.objectList; + NGT::Timer timer; + timer.start(); + std::vector batch; + std::vector> exceededLeaves; + size_t nleaves = 1; + size_t nOfThreads = 32; + for (size_t id = 1; id <= numOfObjects; id++) { + if (id % (numOfObjects / 100) == 0) { + timer.stop(); + std::cerr << "# of processed objects=" << id << " " << id * 100 / numOfObjects << "% " << timer << " # of leaves=" << nleaves << std::endl; + timer.start(); + } + batch.push_back(id); + if (batch.size() > 100000) { + size_t kmeansBatchSize = nleaves < nOfThreads ? nleaves : nOfThreads; + hierarchicalKmeansBatch(batch, exceededLeaves, rootID, object, objectList, objectSpace, nodes, + clustering, maxSize, nleaves, kmeansBatchSize); + + } + } + hierarchicalKmeansBatch(batch, exceededLeaves, rootID, object, objectList, objectSpace, nodes, + clustering, maxSize, nleaves, 0); + + if (numOfTotalClusters != 0) { + NGT::Timer timer; + timer.start(); + size_t numOfLeaves = 0; + for (auto node : nodes) { + if (node->leaf) { + numOfLeaves++; + } + } + std::cerr << "# of nodes=" << nodes.size() << std::endl; + std::cerr << "# of leaves=" << numOfLeaves << std::endl; + std::cerr << "clustering for quantization." << std::endl; + hierarchicalKmeansWithNumberOfClustersInParallel(numOfTotalClusters, numOfObjects, numOfLeaves, + objectList, objectSpace, nodes, initMode); + if (numOfTotalBlobs != 0) { + NGT::Timer timer; + timer.start(); + size_t numOfLeaves = 0; + for (auto node : nodes) { + if (node->leaf) { + numOfLeaves++; + } + } + std::cerr << "# of leaves=" << numOfLeaves << ":" << numOfTotalClusters << std::endl; + if (numOfLeaves != numOfTotalClusters) { + std::cerr << "# of leaves is invalid " << numOfLeaves << ":" << numOfTotalClusters << std::endl; + abort(); + } + { + std::ofstream of(prefix + "_qcentroid.tsv"); + extractCentroids(of, nodes); + } + std::vector qNodeIDs; + for (uint32_t nid = 0; nid < nodes.size(); nid++) { + if (nodes[nid]->leaf) { + qNodeIDs.push_back(nid); + } + } + std::cerr << "clustering to make blobs." << std::endl; + hierarchicalKmeansWithNumberOfClustersInParallel(numOfTotalBlobs, numOfObjects, numOfTotalClusters, + objectList, objectSpace, nodes, initMode); + { + std::ofstream of(prefix + "_btoq_index.tsv"); + extractBtoQIndex(of, nodes, qNodeIDs); + } + } + } + + } + + void multilayerClustering(std::string prefix, QBG::Index &index) { + auto &quantizer = static_cast&>(index.getQuantizer()); + auto &objectSpace = quantizer.globalCodebookIndex.getObjectSpace(); + { + std::cerr << "Three layer clustering..." << std::endl; + std::cerr << "HiearchicalKmeans::clustering: # of clusters=" << numOfThirdClusters << ":" << index.getQuantizer().property.globalCentroidLimit << std::endl; + if (index.getQuantizer().objectList.size() <= 1) { + NGTThrowException("optimize: No objects"); + } + if (numOfThirdClusters == 0) { + if (index.getQuantizer().property.globalCentroidLimit == 0) { + numOfThirdClusters = index.getQuantizer().objectList.size() / 1000; + numOfThirdClusters = numOfThirdClusters == 0 ? 1 : numOfThirdClusters; + numOfThirdClusters = numOfThirdClusters > 1000000 ? 1000000 : numOfThirdClusters; + } else { + numOfThirdClusters = index.getQuantizer().property.globalCentroidLimit; + } + } + if (numOfThirdClusters != 0 && index.getQuantizer().property.globalCentroidLimit != 0 && + numOfThirdClusters != index.getQuantizer().property.globalCentroidLimit) { + } + auto &quantizer = static_cast&>(index.getQuantizer()); + QBGObjectList &objectList = quantizer.objectList; + if (numOfObjects == 0) { + numOfObjects = objectList.size() - 1; + } + + std::cerr << "The first layer. " << numOfFirstClusters << ":" << numOfFirstObjects << std::endl; + if (numOfThirdClusters == 0 || numOfObjects == 0) { + NGTThrowException("numOfThirdClusters or numOfObjects are zero"); + } + numOfSecondClusters = numOfSecondClusters == 0 ? numOfThirdClusters : numOfSecondClusters; + numOfFirstClusters = numOfFirstClusters == 0 ? static_cast(sqrt(numOfSecondClusters)) : numOfFirstClusters; + numOfSecondObjects = numOfSecondClusters * 100; + numOfSecondObjects = numOfSecondObjects > numOfObjects ? numOfObjects : numOfSecondObjects; + numOfFirstObjects = numOfFirstClusters * 2000; + numOfFirstObjects = numOfFirstObjects > numOfSecondObjects ? numOfSecondObjects : numOfFirstObjects; + if (numOfFirstObjects < numOfFirstClusters) { + std::stringstream msg; + msg << "# of objects for the first should be larger than # of the first clusters. " << numOfFirstObjects << ":" << numOfFirstClusters; + NGTThrowException(msg); + } + if (numOfFirstClusters > numOfSecondClusters) { + std::stringstream msg; + msg << "# of the first clusters should be larger than or equal to # of the second clusters. " << numOfFirstClusters << ":" << numOfSecondClusters; + NGTThrowException(msg); + } + if (numOfSecondClusters > numOfThirdClusters) { + std::stringstream msg; + msg << "# of the second clusters should be larger than or equal to # of the second clusters. " << numOfSecondClusters << ":" << numOfThirdClusters; + NGTThrowException(msg); + } + if (numOfFirstClusters > numOfSecondClusters) { + std::stringstream msg; + msg << "# of the second clusters should be larger than # of the first clusters. " << numOfFirstClusters << ":" << numOfSecondClusters; + NGTThrowException(msg); + } + + std::cerr << "Three layer clustering:" << numOfFirstClusters << ":" << numOfFirstObjects << "," << numOfSecondClusters << ":" << numOfSecondObjects << "," << numOfThirdClusters << ":" << numOfObjects << std::endl; + + NGT::Clustering firstClustering(initMode, NGT::Clustering::ClusteringTypeKmeansWithoutNGT, 300); + float clusterSizeConstraint = 5.0; + firstClustering.setClusterSizeConstraintCoefficient(clusterSizeConstraint); + std::cerr << "size constraint=" << clusterSizeConstraint << std::endl; + std::vector> vectors; + vectors.reserve(numOfFirstObjects); + std::vector obj; + for (size_t id = 1; id <= numOfFirstObjects; id++) { + if (id % 1000000 == 0) { + std::cerr << "# of prcessed objects is " << id << std::endl; + } + if (!objectList.get(id, obj, &objectSpace)) { + std::stringstream msg; + msg << "qbg: Cannot get object. ID=" << id; + NGTThrowException(msg); + } + vectors.push_back(obj); + } + std::cerr << "Kmeans... " << vectors.size() << " to " << numOfFirstClusters << std::endl; + std::vector firstClusters; + NGT::Timer timer; + + timer.start(); + firstClustering.kmeans(vectors, numOfFirstClusters, firstClusters); + timer.stop(); + std::cerr << "# of clusters=" << firstClusters.size() << " time=" << timer << std::endl; + + std::vector> otherVectors; + timer.start(); + std::cerr << "Assign for the second. (" << numOfFirstObjects << "-" << numOfSecondObjects << ")..." << std::endl; + assign(firstClusters, numOfFirstObjects + 1, numOfSecondObjects, objectSpace, objectList); + timer.stop(); + std::cerr << "Assign(1) time=" << timer << std::endl; + + std::cerr << "subclustering for the second." << std::endl; + std::vector secondClusters; + timer.start(); + subclustering(firstClusters, numOfSecondClusters, numOfSecondObjects, objectSpace, objectList, initMode, secondClusters); + timer.stop(); + std::cerr << "subclustering(1) time=" << timer << std::endl; + std::cerr << "save quantization centroid" << std::endl; + NGT::Clustering::saveClusters(prefix + "_qcentroid.tsv", secondClusters); + timer.start(); + std::cerr << "Assign for the third. (" << numOfSecondObjects << "-" << numOfObjects << ")..." << std::endl; + assignWithNGT(secondClusters, numOfSecondObjects + 1, numOfObjects, objectSpace, objectList); + timer.stop(); + std::cerr << "Assign(2) time=" << timer << std::endl; + std::cerr << "subclustering for the third." << std::endl; + std::vector> thirdClusters; + timer.start(); + subclustering(secondClusters, numOfThirdClusters, numOfObjects, objectSpace, objectList, initMode, thirdClusters); + timer.stop(); + std::cerr << "subclustering(2) time=" << timer << std::endl; + { + std::vector bqindex; + for (size_t idx1 = 0; idx1 < thirdClusters.size(); idx1++) { + for (size_t idx2 = 0; idx2 < thirdClusters[idx1].size(); idx2++) { + bqindex.push_back(idx1); + } + } + std::cerr << "save bqindex..." << std::endl; + NGT::Clustering::saveVector(prefix + "_bqindex.tsv", bqindex); + } + + std::vector thirdFlatClusters; + flattenClusters(secondClusters, thirdClusters, numOfThirdClusters, thirdFlatClusters); + + std::cerr << "save centroid..." << std::endl; + NGT::Clustering::saveClusters(prefix + "_centroid.tsv", thirdFlatClusters); + + { + std::vector cindex(numOfObjects); + for (size_t cidx = 0; cidx < thirdFlatClusters.size(); cidx++) { + for (auto mit = thirdFlatClusters[cidx].members.begin(); mit != thirdFlatClusters[cidx].members.end(); ++mit) { + size_t vid = (*mit).vectorID; + cindex[vid] = cidx; + } + } + std::cerr << "save index... " << cindex.size() << std::endl; + NGT::Clustering::saveVector(prefix + "_index.tsv", cindex); + } + std::cerr << "end of clustering" << std::endl; + return; + } + + } + + void clustering(std::string indexPath, std::string prefix = "", std::string objectIDsFile = "") { + NGT::StdOstreamRedirector redirector(silence); + redirector.begin(); + + QBG::Index index(indexPath, true); + if (threeLayerClustering) { + + if (prefix.empty()) { + std::cerr << "Prefix is not specified." << std::endl; + prefix = indexPath + "/" + QBG::Index::getWorkspaceName(); + try { + NGT::Index::mkdir(prefix); + } catch(...) {} + prefix +="/kmeans-cluster"; + std::cerr << prefix << " is used" << std::endl; + } + auto &quantizer = static_cast&>(index.getQuantizer()); + auto &objectSpace = quantizer.globalCodebookIndex.getObjectSpace(); + size_t paddedDimension = objectSpace.getPaddedDimension(); + size_t dimension = objectSpace.getDimension(); + if (paddedDimension != dimension) { + std::cerr << "HierarachicalKmeans: Warning! Dimensions are inconsistent. Dimension=" << paddedDimension << ":" << dimension << std::endl; + } + multilayerClustering(prefix, index); + redirector.end(); + return; + } + + NGT::Clustering::ClusteringType clusteringType = NGT::Clustering::ClusteringTypeKmeansWithoutNGT; + + uint32_t rootID = 0; + std::vector nodes; + nodes.push_back(new HKLeafNode); + + std::vector object; + size_t iteration = 1000; + NGT::Clustering clustering(initMode, clusteringType, iteration, numOfClusters); + auto &quantizer = static_cast&>(index.getQuantizer()); + QBGObjectList &objectList = quantizer.objectList; + if (objectIDsFile.empty()) { + treeBasedTopdownClustering(prefix, index, rootID, object, nodes, clustering); + } else { + std::cerr << "Cluster ID=" << clusterID << std::endl; + if (clusterID < 0) { + std::stringstream msg; + msg << "Any target cluster ID is not specified."; + NGTThrowException(msg); + } + std::ifstream objectIDs(objectIDsFile); + if (!objectIDs) { + std::stringstream msg; + msg << "Cannot open the object id file. " << objectIDsFile; + NGTThrowException(msg); + } + auto &objectSpace = quantizer.globalCodebookIndex.getObjectSpace(); + uint32_t id = 1; + int32_t cid; + size_t ccount = 0; + while (objectIDs >> cid) { + std::cerr << cid << std::endl; + if (id % 100000 == 0) { + std::cerr << "# of processed objects=" << id << std::endl; + } + if (cid == -1) { + continue; + } + if (cid == clusterID) { + ccount++; + hierarchicalKmeans(id, rootID, object, objectList, objectSpace, nodes, clustering, maxSize); + } + id++; + } + } + size_t objectCount = 0; + if (prefix.empty()) { + objectCount = extractCentroids(std::cout, nodes); + } else { + { + std::ofstream of(prefix + "_centroid.tsv"); + objectCount = extractCentroids(of, nodes); + } + { + std::ofstream of(prefix + "_index.tsv"); + extractIndex(of, nodes, numOfObjects); + } + if (numOfFirstObjects > 0) { + std::ofstream btoqof(prefix + "_btoq.tsv"); + std::ofstream qcof(prefix + "_qcentroid.tsv"); + extractBtoQAndQCentroid(btoqof, qcof, nodes, numOfThirdClusters); + } + if (numOfRandomObjects > 0) { + std::ofstream of(prefix + "_random_object.tsv"); + if (extractCentroid) { + extractRandomObjectsFromEachBlob(of, nodes, numOfObjects, numOfRandomObjects - 1, quantizer, extractCentroid); + } else { + extractRandomObjectsFromEachBlob(of, nodes, numOfObjects, numOfRandomObjects, quantizer, extractCentroid); + } + } + } + if (objectCount != numOfObjects) { + std::cerr << "# of objects is invalid. " << objectCount << ":" << numOfObjects << std::endl; + } + redirector.end(); + } +#endif + + size_t maxSize; + size_t numOfObjects; + size_t numOfClusters; + size_t numOfTotalClusters; + size_t numOfTotalBlobs; + int32_t clusterID; + + NGT::Clustering::InitializationMode initMode; + + size_t numOfRandomObjects; + + size_t numOfFirstObjects; + size_t numOfFirstClusters; + size_t numOfSecondObjects; + size_t numOfSecondClusters; + size_t numOfThirdClusters; + bool extractCentroid; + + bool threeLayerClustering; + bool silence; + }; +} diff --git a/lib/NGT/NGTQ/Matrix.h b/lib/NGT/NGTQ/Matrix.h new file mode 100644 index 0000000..042968a --- /dev/null +++ b/lib/NGT/NGTQ/Matrix.h @@ -0,0 +1,687 @@ +// +// Copyright (C) 2021 Yahoo Japan Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" { + // svd + void dgesvd_(char* jobu, char* jobvt, int* m, int* n, double* a, + int* lda, double* s, double* u, int* ldu, double* vt, int* ldvt, + double* work, int* lwork, int* info); + void sgesvd_(char* jobu, char* jobvt, int* m, int* n, float* a, + int* lda, float* s, float* u, int* ldu, float* vt, int* ldvt, + float* work, int* lwork, int* info); + // multiplication + void dgemm_(char *transa, char *transb, int *m, int *n, int *k, + double *alpha, double *a, int *lda, double *b, int *ldb, + double *beta , double *c, int *ldc); + void sgemm_(char *transa, char *transb, int *m, int *n, int *k, + float *alpha, float *a, int *lda, float *b, int *ldb, + float *beta , float *c, int *ldc); + // {D|S}GEQRF computes a QR factorization of a M-by-N matrix A: A = Q * R. + // {D|S}ORGQR return Q = H(1) H(2) . . . H(k) as returned by {D|S}GEQRF. + void dgeqrf_(int *m, int *n, double *a, int *lda, double *tau, double *work, int *lwork, int *info); + void dorgqr_(int *m, int *n, int *k, double *a, int *lda, double *tau, double *work, int *lwork, int *info); + + void sgeqrf_(int *m, int *n, float *a, int *lda, float *tau, float *work, int *lwork, int *info); + void sorgqr_(int *m, int *n, int *k, float *a, int *lda, float *tau, float *work, int *lwork, int *info); +} + + + +template class Matrix { +public: + Matrix(size_t r = 0, size_t c = 0, const float *v = 0): row(r), col(c), matrix(0) { construct(r, c, v); } + Matrix(const Matrix &m): row(0), col(0), matrix(0) { *this = m; } + Matrix(const std::vector> &v): row(0), col(0), matrix(0) { construct(v); } + + ~Matrix() { delete[] matrix; } + + Matrix &operator=(const Matrix &m) { + allocate(m.row, m.col); + std::memcpy(matrix, m.matrix, row * col * sizeof(T)); + return *this; + } + + void construct(size_t r, size_t c, const double *v) { + allocate(r, c); + set(v); + } + + void construct(size_t r, size_t c, const float *v) { + allocate(r, c); + set(v); + } + + void construct(const std::vector> &v) { +#if !defined(NGT_DISABLE_BLAS) + allocate(v[0].size(), v.size()); +#else + allocate(v.size(), v[0].size()); +#endif + set(v); + } + + void allocate(size_t r, size_t c) { + if (matrix != 0) { + delete[] matrix; + } + row = r; + col = c; + if (r == 0 && c == 0) { + matrix = 0; + } else { + matrix = new T[r * c]; + } + } + + bool isEmpty() { return (col == 0) && (row == 0); } + + static void + tokenize(const std::string &str, std::vector &token, const std::string seps) { + std::string::size_type current = 0; + std::string::size_type next; + while ((next = str.find_first_of(seps, current)) != std::string::npos) { + token.push_back(str.substr(current, next - current)); + current = next + 1; + } + std::string t = str.substr(current); + token.push_back(t); + } + + void set(const double *v) { + if (v == 0) { + return; + } + size_t l = row * col; + for (size_t p = 0; p < l; p++) { + matrix[p] = *v++; + } + } + + void set(const float *v) { + if (v == 0) { + return; + } + size_t l = row * col; + for (size_t p = 0; p < l; p++) { + matrix[p] = *v++; + } + } + + void set(const std::vector> &v) { + T *m = matrix; +#if !defined(NGT_DISABLE_BLAS) + assert(row == v[0].size()); + assert(col == v.size()); + for (size_t c = 0; c < col; c++) { + for (size_t r = 0; r < row; r++) { + *m++ = v[c][r]; + } + } +#else + assert(row == v.size()); + assert(col == v[0].size()); + for (size_t r = 0; r < row; r++) { + for (size_t c = 0; c < col; c++) { + *m++ = v[r][c]; + } + } +#endif + } + + void set(size_t pr, size_t pc, T v) { +#if !defined(NGT_DISABLE_BLAS) + matrix[pc * row + pr] = v; +#else + matrix[pr * col + pc] = v; +#endif + } + + void put(size_t pr, size_t pc, const Matrix &m) { + for (size_t r = 0; r < m.row; r++) { + if (pr + r < row) { + for (size_t c = 0; c < m.col; c++) { + if (pc + c < col) { + matrix[(pr + r) * col + (pc + c)] = m.matrix[r * m.col + c]; + } + } + } + } + } + + void horzcat(const Matrix &m){ + assert(row == m.row); + size_t nc = col + m.col; + T *mtx = new T[row * nc]; + for (size_t r = 0; r < row; r++) { + for (size_t c = 0; c < col; c++) { + mtx[r * nc + c] = matrix[r * col + c]; + } + } + put(0, col, m); + col = nc; + delete[] matrix; + matrix = mtx; + } + + void vert(const Matrix &m) { + if (row == 0 && col == 0) { + construct(m.row, m.col, m.matrix); + return; + } + assert(col == m.col); + size_t nr = row + m.row; + T *mtx = new T[nr * col]; + for (size_t r = 0; r < row; r++) { + for (size_t c = 0; c < col; c++) { + mtx[r * col + c] = matrix[r * col + c]; + } + } + put(row, 0, m); + row = nr; + delete[] matrix; + matrix = mtx; + } + + void zero(size_t r, size_t c = 0) { + allocate(r, c); + zero(); + } + + void zero() { + size_t l = row * col; + for (size_t p = 0; p < l; p++) { + matrix[p] = 0.0; + } + } + + void random(size_t r, size_t c = 0) { + allocate(r, c); + random(); + } + + void random() { + std::random_device seed_gen; + std::mt19937 engine(seed_gen()); + + auto maxNum = engine.max(); + size_t l = row * col; + for (size_t p = 0; p < l; p++) { + auto randomNum = engine(); + matrix[p] = static_cast(randomNum) / maxNum; + } + } + + static void random(std::vector &a) { + std::random_device seed_gen; + std::mt19937 engine(seed_gen()); + auto maxNum = engine.max(); + for (auto &i : a) { + auto randomNum = engine(); + i = static_cast(randomNum) / maxNum; + } + } + + void transpose() { + T *m = new T[col * row]; + T *msrc = matrix; + size_t nr = col; + size_t nc = row; +#if !defined(NGT_DISABLE_BLAS) + for (size_t r = 0; r < nr; r++) { + for (size_t c = 0; c < nc; c++) { + m[c * nr + r] = *msrc++; + //std::cerr << r * nc + c << std::endl; + } + } +#else + for (size_t c = 0; c < nc; c++) { + for (size_t r = 0; r < nr; r++) { + m[r * nc + c] = *msrc++; + //std::cerr << r * nc + c << std::endl; + } + } +#endif + row = nr; + col = nc; + delete[] matrix; + matrix = m; + } + +#if !defined(NGT_DISABLE_BLAS) + void mul(const std::vector> &v) { + Matrix m(v); + mulBlas(m, true); + } + + + void mul(const Matrix &mtx) { + mulBlas(mtx); + } + + + void mulBlas(const Matrix &mtx, bool transpose = false) { + char transa = 'N'; + char transb = 'N'; + int m = row; + int n = mtx.col; + int k = col; + if (transpose) { + transb = 'T'; + if (row != mtx.row) { + std::cerr << "mul:" << row << "x" << mtx.row << std::endl; + } + assert(row == mtx.row); + n = mtx.row; + row = m; + col = mtx.row; + } else { + if (col != mtx.row) { + std::cerr << "mul:" << col << "x" << mtx.row << std::endl; + } + assert(col == mtx.row); + row = m; + col = n; + } + float alpha = 1.0; + float beta = 0.0; + T *tmpmtx = new T[m * n]; + if (transpose) { + int ldb = mtx.row; + gemm(&transa, &transb, &m, &n, &k, &alpha, matrix, &m, mtx.matrix, &ldb, &beta, tmpmtx, &m); + } else { + gemm(&transa, &transb, &m, &n, &k, &alpha, matrix, &m, mtx.matrix, &k, &beta, tmpmtx, &m); + } + delete[] matrix; + matrix = tmpmtx; + } +#else + void mul(const Matrix &mtx) { + mulNaive(mtx); + } +#endif + + + void mulNaive(const Matrix &mtx) { +#ifdef MATRIX_TRACE + cerr << row << "x" << col << " mtx=" << mtx.row << "x" << mtx.col << std::endl; + std::cerr << mtx << std::endl; +#endif + if (col != mtx.row) { + std::cerr << "mul:" << col << "x" << mtx.row << std::endl; + } + assert(col == mtx.row); + size_t nr = row; + size_t nc = mtx.col; + T *tmpmtx = new T[nr * nc]; + for (size_t r = 0; r < nr; r++) { + for (size_t c = 0; c < nc; c++) { +#if !defined(NGT_DISABLE_BLAS) + T &sum = tmpmtx[c * nr + r]; + sum = 0; + for (size_t p = 0; p < col; p++) { + sum += matrix[p * row + r] * mtx.matrix[c * mtx.row + p]; + } +#else + T &sum = tmpmtx[r * nc + c]; + sum = 0; + for (size_t p = 0; p < col; p++) { + sum += matrix[r * col + p] * mtx.matrix[p * mtx.col + c]; + } +#endif + } + } + row = nr; + col = nc; + delete[] matrix; + matrix = tmpmtx; + } + + void diag(const Matrix &m) { + if (m.row != 1 && m.col != 1) { + std::cerr << "Error : not vector. " << m.row << "x" << m.col << std::endl; + return; + } + size_t length = m.row > m.col ? m.row : m.col; + zero(length, length); + for (size_t i = 0; i < length; i++) { +#if !defined(NGT_DISABLE_BLAS) + matrix[i * row + i] = m.matrix[i]; +#else + matrix[i * col + i] = m.matrix[i]; +#endif + } + } + + void reshape(size_t r, size_t c) { + if (r == row && c == col) { + return; + } + size_t l = r * c; + T *m = new T[l]; + for (size_t i = 0; i < l; i++) { + m[i] = 0.0; + } +#if !defined(NGT_DISABLE_BLAS) + for (size_t sc = 0; sc < col; sc++) { + for (size_t sr = 0; sr < row; sr++) { + m[sc * r + sr] = matrix[sc * row + sr]; + } + } +#else + for (size_t sr = 0; sr < row; sr++) { + for (size_t sc = 0; sc < col; sc++) { + m[sr * c + sc] = matrix[sr * col + sc]; + } + } +#endif + row = r; + col = c; + delete[] matrix; + matrix = m; + } + + static void mulSquare(std::vector &a, Matrix &b) { + if (b.col != b.row) { + std::stringstream msg; + msg << "mulSquare : Invalid # of cols and rows. " << b.col << ":" << b.row << std::endl; + throw std::runtime_error(msg.str().c_str()); + } + if (a.size() != b.row) { + std::stringstream msg; + msg << "mulSquare : Invalid # of rows and size. " << a.size() << ":" << b.row << std::endl; + throw std::runtime_error(msg.str().c_str()); + } + + std::vector vec; +#if !defined(NGT_DISABLE_BLAS) + for (size_t c = 0; c < a.size(); c++) { + T sum = 0; + for (size_t p = 0; p < b.col; p++) { + sum += a[p] * b.matrix[c * b.row + p]; + } + vec.push_back(sum); + } +#else + for (size_t c = 0; c < a.size(); c++) { + T sum = 0; + for (size_t p = 0; p < b.col; p++) { + sum += a[p] * b.matrix[p * b.col + c]; + } + vec.push_back(sum); + } +#endif + a = vec; + + } + + static void mulSquare(std::vector> &a, Matrix &b) { + assert(b.col == b.row); + for (size_t r = 0; r < a.size(); r++) { + mulSquare(a[r], b); + } + } + + static void gesvd(char* jobu, char* jobvt, int* m, int* n, double* a, + int* lda, double* s, double* u, int* ldu, double* vt, int* ldvt, + double* work, int* lwork, int* info) { + dgesvd_(jobu, jobvt, m, n, a, lda, s, u, ldu, vt, ldvt, work, lwork, info); + } + + static void gesvd(char* jobu, char* jobvt, int* m, int* n, float* a, + int* lda, float* s, float* u, int* ldu, float* vt, int* ldvt, + float* work, int* lwork, int* info) { + sgesvd_(jobu, jobvt, m, n, a, lda, s, u, ldu, vt, ldvt, work, lwork, info); + } + + static void gemm(char *transa, char *transb, int *m, int *n, int *k, + double *alpha, double *a, int *lda, double *b, int *ldb, + double *beta , double *c, int *ldc) { + dgemm_(transa, transb, m, n, k, alpha, a, lda, b, ldb, beta , c, ldc); + } + + static void gemm(char *transa, char *transb, int *m, int *n, int *k, + float *alpha, float *a, int *lda, float *b, int *ldb, + float *beta , float *c, int *ldc) { + sgemm_(transa, transb, m, n, k, alpha, a, lda, b, ldb, beta , c, ldc); + } + + static void geqrf(int *m, int *n, double *a, int *lda, double *tau, double *work, int *lwork, int *info) { + dgeqrf_(m, n, a, lda, tau, work, lwork, info); + } + + static void geqrf(int *m, int *n, float *a, int *lda, float *tau, float *work, int *lwork, int *info) { + sgeqrf_(m, n, a, lda, tau, work, lwork, info); + } + + static void orgqr(int *m, int *n, int *k, double *a, int *lda, double *tau, double *work, int *lwork, int *info) { + dorgqr_(m, n, k, a, lda, tau, work, lwork, info); + } + + static void orgqr(int *m, int *n, int *k, float *a, int *lda, float *tau, float *work, int *lwork, int *info) { + sorgqr_(m, n, k, a, lda, tau, work, lwork, info); + } + + static void svd(Matrix &a, Matrix &u, Matrix &s, Matrix &v) { + Matrix svda(a); +#if !defined(NGT_DISABLE_BLAS) + int m = svda.row; + int n = svda.col; +#else + svda.transpose(); + int n = svda.row; + int m = svda.col; +#endif + char jobu = 'A'; + char jobvt = 'A'; + int lda = m, ldu = m, ldvt = n, info; + int max, min; + if (m > n) { + max = m; + min = n; + } else { + max = n; + min = m; + } + int v1 = 3 * min + max; + int v2 = 5 * min; + int lwork = v1 > v2 ? v1 : v2; + T work[lwork]; + + Matrix sd; + sd.allocate(m, 1); + u.allocate(m, m); + v.allocate(n, n); + // S U VT + gesvd(&jobu, &jobvt, &m, &n, svda.matrix, &lda, sd.matrix, u.matrix, &ldu, v.matrix, &ldvt, work, &lwork, &info); + s.diag(sd); + s.reshape(m, n); +#if !defined(NGT_DISABLE_BLAS) + v.transpose(); +#else + u.transpose(); +#endif + } + + void eye(size_t d) { + zero(d, d); + for (size_t p = 0; p < row; p++) { + matrix[p * col + p] = 1.0; + } + } + + void randomRotation(size_t d) { + random(d, d); + qr(d); + } + + void qr(size_t d) { + auto di = static_cast(d); + T tau[di]; + std::vector work(1); + int lwork = -1; + int info; + geqrf(&di, &di, matrix, &di, &tau[0], work.data(), &lwork, &info); + work.resize(static_cast(work[0])); + + geqrf(&di, &di, matrix, &di, tau, work.data(), &lwork, &info); + orgqr(&di, &di, &di, matrix, &di, tau, work.data(), &lwork, &info); + } + + void printmat() { + T mtmp; + printf("[ "); + for (size_t i = 0; i < row; i++) { + printf("[ "); + for (size_t j = 0; j < col; j++) { + mtmp = matrix[i + j * col]; + printf("%5.2e", mtmp); + if (j < col - 1) printf(", "); + } + if (i < row - 1) printf("]; "); + else printf("] "); + } + printf("]"); + std::cout << std::endl; + } + + static void save(const std::string &file, const Matrix &m) { + std::ofstream os(file); + for (size_t r = 0; r < m.row; r++) { + for (size_t c = 0; c < m.col; c++) { +#if !defined(NGT_DISABLE_BLAS) + os << m.matrix[c * m.row + r]; +#else + os << m.matrix[r * m.col + c]; +#endif + if (c + 1 != m.col) { + os << "\t"; + } + } + os << std::endl; + } + } + + static void + convert(std::vector &strings, std::vector &vector) { + vector.clear(); + for (auto it = strings.begin(); it != strings.end(); ++it) { + try { + vector.push_back(stod(*it)); + } catch(...) { + break; + } + } + } + + static void + extractVector(const std::string &str, std::vector &vec) + { + std::vector tokens; + tokenize(str, tokens, " \t"); + convert(tokens, vec); + } + +#if !defined(NGT_DISABLE_BLAS) + static + void load(const std::string &file, Matrix &m) + { + loadVectors(file, m); + m.transpose(); + } + + static + void loadVectors(const std::string &file, Matrix &m) +#else + static + void load(const std::string &file, Matrix &m) +#endif + { + std::ifstream is(file); + if (!is) { + std::stringstream msg; + msg << "Matrix::load: Cannot load. " << file; + throw std::runtime_error(msg.str().c_str()); + } + std::string line; + size_t row = 0, col = 0; + std::vector tmpv; + while (getline(is, line)) { + std::vector v; + extractVector(line, v); +#if !defined(NGT_DISABLE_BLAS) + if (row == 0) { + row = v.size(); + } else if (row != v.size()) { + std::cerr << "somthing wrong." << std::endl; + abort(); + } + col++; +#else + if (col == 0) { + col = v.size(); + } else if (col != v.size()) { + std::cerr << "somthing wrong." << std::endl; + abort(); + } + row++; +#endif + for (size_t i = 0; i < v.size(); i++) { + tmpv.push_back(v[i]); + } + } + m.construct(row, col, &tmpv[0]); + } + + friend std::ostream& operator<<(std::ostream &os, const Matrix &m) { + os << m.row << " x " << m.col << "=" << std::endl; + os << "["; + for (size_t r = 0; r < m.row; r++) { + os << r << ":["; + for (size_t c = 0; c < m.col; c++) { +#if !defined(NGT_DISABLE_BLAS) + os << m.matrix[c * m.row + r]; +#else + os << m.matrix[r * m.col + c]; +#endif + if (c + 1 != m.col) { + os << "\t"; + } + } + os << "]"; + if (r + 1 != m.row) { + os << std::endl; + } + } + os << "]"; + return os; + } + + size_t row; + size_t col; + T *matrix; +}; + + diff --git a/lib/NGT/NGTQ/NGTQCommand.h b/lib/NGT/NGTQ/NGTQCommand.h deleted file mode 100644 index b54649b..0000000 --- a/lib/NGT/NGTQ/NGTQCommand.h +++ /dev/null @@ -1,612 +0,0 @@ -// -// Copyright (C) 2016 Yahoo Japan Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -#include "NGT/NGTQ/Quantizer.h" - -#define NGTQ_SEARCH_CODEBOOK_SIZE_FLUCTUATION - -namespace NGTQ { - -class Command { -public: - class CreateParameters { - public: - CreateParameters() {} - CreateParameters(NGT::Args &args, char centroidCreationMode = 'd', char localCentroidCreationMode = 'd') { - try { - index = args.get("#1"); - } catch (...) { - std::stringstream msg; - msg << "Command::CreateParameters: Error: An index is not specified."; - NGTThrowException(msg); - } - try { - objectPath = args.get("#2"); - } catch (...) {} - - char objectType = args.getChar("o", 'f'); - char distanceType = args.getChar("D", '2'); - numOfObjects = args.getl("n", 0); - - property.threadSize = args.getl("p", 24); - property.dimension = args.getl("d", 0); - property.globalRange = args.getf("R", 0); - property.localRange = args.getf("r", 0); - property.globalCentroidLimit = args.getl("C", 1000000); - property.localCentroidLimit = args.getl("c", 65000); - property.localDivisionNo = args.getl("N", 8); - property.batchSize = args.getl("b", 1000); - property.localClusteringSampleCoefficient = args.getl("s", 10); - { - char localCentroidType = args.getChar("T", 'f'); - property.singleLocalCodebook = localCentroidType == 't' ? true : false; - } - { - centroidCreationMode = args.getChar("M", centroidCreationMode); - switch(centroidCreationMode) { - case 'd': property.centroidCreationMode = NGTQ::CentroidCreationModeDynamic; break; - case 's': property.centroidCreationMode = NGTQ::CentroidCreationModeStatic; break; - default: - std::stringstream msg; - msg << "Command::CreateParameters: Error: Invalid centroid creation mode. " << centroidCreationMode; - NGTThrowException(msg); - } - } - { - localCentroidCreationMode = args.getChar("L", localCentroidCreationMode); - switch(localCentroidCreationMode) { - case 'd': property.localCentroidCreationMode = NGTQ::CentroidCreationModeDynamic; break; - case 's': property.localCentroidCreationMode = NGTQ::CentroidCreationModeStatic; break; - case 'k': property.localCentroidCreationMode = NGTQ::CentroidCreationModeDynamicKmeans; break; - default: - std::stringstream msg; - msg << "Command::CreateParameters: Error: Invalid centroid creation mode. " << localCentroidCreationMode; - NGTThrowException(msg); - } - } - - globalProperty.edgeSizeForCreation = args.getl("E", 10); - globalProperty.edgeSizeForSearch = args.getl("S", 40); - { - char indexType = args.getChar("i", 't'); - globalProperty.indexType = indexType == 't' ? NGT::Property::GraphAndTree : NGT::Property::Graph; - localProperty.indexType = globalProperty.indexType; - } - globalProperty.insertionRadiusCoefficient = args.getf("e", 0.1) + 1.0; - localProperty.insertionRadiusCoefficient = globalProperty.insertionRadiusCoefficient; - - - switch (objectType) { - case 'f': property.dataType = NGTQ::DataTypeFloat; break; - case 'c': property.dataType = NGTQ::DataTypeUint8; break; - default: - std::stringstream msg; - msg << "Command::CreateParameters: Error: Invalid object type. " << objectType; - NGTThrowException(msg); - } - - switch (distanceType) { - case '2': property.distanceType = NGTQ::DistanceTypeL2; break; - case '1': property.distanceType = NGTQ::DistanceTypeL1; break; - case 'a': property.distanceType = NGTQ::DistanceTypeAngle; break; - case 'C': property.distanceType = NGTQ::DistanceTypeNormalizedCosine; break; - case 'E': property.distanceType = NGTQ::DistanceTypeL2; break; - default: - std::stringstream msg; - msg << "Command::CreateParameters: Error: Invalid distance type. " << distanceType; - NGTThrowException(msg); - } - } - - std::string index; - std::string objectPath; - size_t numOfObjects; - NGTQ::Property property; - NGT::Property globalProperty; - NGT::Property localProperty; - }; - - Command():debugLevel(0) {} - - void - create(NGT::Args &args) - { - const string usage = "Usage: ngtq create " - " -d dimension [-o object-type (f:float|c:unsigned char)] [-D distance-function] [-n data-size] " - "[-p #-of-thread] [-R global-codebook-range] [-r local-codebook-range] " - "[-C global-codebook-size-limit] [-c local-codebook-size-limit] [-N local-division-no] " - "[-T single-local-centroid (t|f)] [-e epsilon] [-i index-type (t:Tree|g:Graph)] " - "[-M global-centroid-creation-mode (d|s)] [-L local-centroid-creation-mode (d|k|s)] " - "[-s local-sample-coefficient] " - "index(output) data.tsv(input)"; - - try { - NGTQ::Command::CreateParameters createParameters(args); - - if (debugLevel >= 1) { - cerr << "epsilon=" << createParameters.globalProperty.insertionRadiusCoefficient << endl; - cerr << "data size=" << createParameters.numOfObjects << endl; - cerr << "dimension=" << createParameters.property.dimension << endl; - cerr << "thread size=" << createParameters.property.threadSize << endl; - cerr << "batch size=" << createParameters.localProperty.batchSizeForCreation << endl;; - cerr << "index type=" << createParameters.globalProperty.indexType << endl; - } - - cerr << "ngtq: Create" << endl; - NGTQ::Index::create(createParameters.index, createParameters.property, createParameters.globalProperty, createParameters.localProperty); - - cerr << "ngtq: Append" << endl; - NGTQ::Index::append(createParameters.index, createParameters.objectPath, createParameters.numOfObjects); - } catch(NGT::Exception &err) { - std::cerr << err.what() << std::endl; - cerr << usage << endl; - } - } - - void - rebuild(NGT::Args &args) - { - const string usage = "Usage: ngtq rebuild " - "[-o object-type (f:float|c:unsigned char)] [-D distance-function] [-n data-size] " - "[-p #-of-thread] [-d dimension] [-R global-codebook-range] [-r local-codebook-range] " - "[-C global-codebook-size-limit] [-c local-codebook-size-limit] [-N local-division-no] " - "[-T single-local-centroid (t|f)] [-e epsilon] [-i index-type (t:Tree|g:Graph)] " - "[-M centroid-creation_mode (d|s)] " - "index(output) data.tsv(input)"; - string srcIndex; - try { - srcIndex = args.get("#1"); - } catch (...) { - cerr << "DB is not specified." << endl; - cerr << usage << endl; - return; - } - string rebuiltIndex = srcIndex + ".tmp"; - - - NGTQ::Property property; - NGT::Property globalProperty; - NGT::Property localProperty; - - { - NGTQ::Index index(srcIndex); - property = index.getQuantizer().property; - index.getQuantizer().globalCodebook.getProperty(globalProperty); - index.getQuantizer().getLocalCodebook(0).getProperty(localProperty); - } - - property.globalRange = args.getf("R", property.globalRange); - property.localRange = args.getf("r", property.localRange); - property.globalCentroidLimit = args.getl("C", property.globalCentroidLimit); - property.localCentroidLimit = args.getl("c", property.localCentroidLimit); - property.localDivisionNo = args.getl("N", property.localDivisionNo); - { - char localCentroidType = args.getChar("T", '-'); - if (localCentroidType != '-') { - property.singleLocalCodebook = localCentroidType == 't' ? true : false; - } - } - { - char centroidCreationMode = args.getChar("M", '-'); - if (centroidCreationMode != '-') { - property.centroidCreationMode = centroidCreationMode == 'd' ? - NGTQ::CentroidCreationModeDynamic : NGTQ::CentroidCreationModeStatic; - } - } - - cerr << "global range=" << property.globalRange << endl; - cerr << "local range=" << property.localRange << endl; - cerr << "global centroid limit=" << property.globalCentroidLimit << endl; - cerr << "local centroid limit=" << property.localCentroidLimit << endl; - cerr << "local division no=" << property.localDivisionNo << endl; - - NGTQ::Index::create(rebuiltIndex, property, globalProperty, localProperty); - cerr << "created a new db" << endl; - cerr << "start rebuilding..." << endl; - NGTQ::Index::rebuild(srcIndex, rebuiltIndex); - { - string src = srcIndex; - string dst = srcIndex + ".org"; - if (std::rename(src.c_str(), dst.c_str()) != 0) { - stringstream msg; - msg << "ngtq::rebuild: Cannot rename. " << src << "=>" << dst ; - NGTThrowException(msg); - } - } - { - string src = rebuiltIndex; - string dst = srcIndex; - if (std::rename(src.c_str(), dst.c_str()) != 0) { - stringstream msg; - msg << "ngtq::rebuild: Cannot rename. " << src << "=>" << dst ; - NGTThrowException(msg); - } - } - } - - - void - append(NGT::Args &args) - { - const string usage = "Usage: ngtq append [-n data-size] " - "index(output) data.tsv(input)"; - string index; - try { - index = args.get("#1"); - } catch (...) { - cerr << "DB is not specified." << endl; - cerr << usage << endl; - return; - } - string data; - try { - data = args.get("#2"); - } catch (...) { - cerr << "Data is not specified." << endl; - } - - size_t dataSize = args.getl("n", 0); - - if (debugLevel >= 1) { - cerr << "data size=" << dataSize << endl; - } - - NGTQ::Index::append(index, data, dataSize); - } - - void - search(NGT::Args &args) - { - const string usage = "Usage: ngtq search [-i g|t|s] [-n result-size] [-e epsilon] [-m mode(r|l|c|a)] " - "[-E edge-size] [-o output-mode] [-b result expansion(begin:end:[x]step)] " - "index(input) query.tsv(input)"; - string database; - try { - database = args.get("#1"); - } catch (...) { - cerr << "DB is not specified" << endl; - cerr << usage << endl; - return; - } - - string query; - try { - query = args.get("#2"); - } catch (...) { - cerr << "Query is not specified" << endl; - cerr << usage << endl; - return; - } - - int size = args.getl("n", 20); - char outputMode = args.getChar("o", '-'); - float epsilon = 0.1; - - char mode = args.getChar("m", '-'); - NGTQ::AggregationMode aggregationMode; - switch (mode) { - case 'r': aggregationMode = NGTQ::AggregationModeExactDistanceThroughApproximateDistance; break; // refine - case 'e': aggregationMode = NGTQ::AggregationModeExactDistance; break; // refine - case 'l': aggregationMode = NGTQ::AggregationModeApproximateDistanceWithLookupTable; break; // lookup - case 'c': aggregationMode = NGTQ::AggregationModeApproximateDistanceWithCache; break; // cache - case '-': - case 'a': aggregationMode = NGTQ::AggregationModeApproximateDistance; break; // cache - default: - cerr << "Invalid aggregation mode. " << mode << endl; - cerr << usage << endl; - return; - } - - if (args.getString("e", "none") == "-") { - // linear search - epsilon = FLT_MAX; - } else { - epsilon = args.getf("e", 0.1); - } - - size_t beginOfResultExpansion, endOfResultExpansion, stepOfResultExpansion; - bool mulStep = false; - { - beginOfResultExpansion = stepOfResultExpansion = 1; - endOfResultExpansion = 0; - string str = args.getString("b", "16"); - vector tokens; - NGT::Common::tokenize(str, tokens, ":"); - if (tokens.size() >= 1) { beginOfResultExpansion = NGT::Common::strtod(tokens[0]); } - if (tokens.size() >= 2) { endOfResultExpansion = NGT::Common::strtod(tokens[1]); } - if (tokens.size() >= 3) { - if (tokens[2][0] == 'x') { - mulStep = true; - stepOfResultExpansion = NGT::Common::strtod(tokens[2].substr(1)); - } else { - stepOfResultExpansion = NGT::Common::strtod(tokens[2]); - } - } - } - if (debugLevel >= 1) { - cerr << "size=" << size << endl; - cerr << "result expansion=" << beginOfResultExpansion << "->" << endOfResultExpansion << "," << stepOfResultExpansion << endl; - } - - NGTQ::Index index(database); - try { - ifstream is(query); - if (!is) { - cerr << "Cannot open the specified file. " << query << endl; - return; - } - if (outputMode == 's') { cout << "# Beginning of Evaluation" << endl; } - string line; - double totalTime = 0; - int queryCount = 0; - while(getline(is, line)) { - NGT::Object *query = index.allocateObject(line, " \t", 0); - queryCount++; - size_t resultExpansion = 0; - for (size_t base = beginOfResultExpansion; - resultExpansion <= endOfResultExpansion; - base = mulStep ? base * stepOfResultExpansion : base + stepOfResultExpansion) { - resultExpansion = base; - NGT::ObjectDistances objects; - NGT::Timer timer; - timer.start(); - // size : # of final resultant objects - // resultExpansion : # of resultant objects by using codebook search - index.search(query, objects, size, resultExpansion, aggregationMode, epsilon); - timer.stop(); - - totalTime += timer.time; - if (outputMode == 'e') { - cout << "# Query No.=" << queryCount << endl; - cout << "# Query=" << line.substr(0, 20) + " ..." << endl; - cout << "# Index Type=" << "----" << endl; - cout << "# Size=" << size << endl; - cout << "# Epsilon=" << epsilon << endl; - cout << "# Result expansion=" << resultExpansion << endl; - cout << "# Distance Computation=" << index.getQuantizer().distanceComputationCount << endl; - cout << "# Query Time (msec)=" << timer.time * 1000.0 << endl; - } else { - cout << "Query No." << queryCount << endl; - cout << "Rank\tIN-ID\tID\tDistance" << endl; - } - - for (size_t i = 0; i < objects.size(); i++) { - cout << i + 1 << "\t" << objects[i].id << "\t"; - cout << objects[i].distance << endl; - } - - if (outputMode == 'e') { - cout << "# End of Search" << endl; - } else { - cout << "Query Time= " << timer.time << " (sec), " << timer.time * 1000.0 << " (msec)" << endl; - } - } - if (outputMode == 'e') { - cout << "# End of Query" << endl; - } - index.deleteObject(query); - } - if (outputMode == 'e') { - cout << "# Average Query Time (msec)=" << totalTime * 1000.0 / (double)queryCount << endl; - cout << "# Number of queries=" << queryCount << endl; - cout << "# End of Evaluation" << endl; - } else { - cout << "Average Query Time= " << totalTime / (double)queryCount << " (sec), " - << totalTime * 1000.0 / (double)queryCount << " (msec), (" - << totalTime << "/" << queryCount << ")" << endl; - } - } catch (NGT::Exception &err) { - cerr << "Error " << err.what() << endl; - cerr << usage << endl; - } catch (...) { - cerr << "Error" << endl; - cerr << usage << endl; - } - index.close(); - } - - void - remove(NGT::Args &args) - { - const string usage = "Usage: ngtq remove [-d object-ID-type(f|d)] index(input) object-ID(input)"; - string database; - try { - database = args.get("#1"); - } catch (...) { - cerr << "DB is not specified" << endl; - cerr << usage << endl; - return; - } - try { - args.get("#2"); - } catch (...) { - cerr << "ID is not specified" << endl; - cerr << usage << endl; - return; - } - char dataType = args.getChar("d", 'f'); - if (debugLevel >= 1) { - cerr << "dataType=" << dataType << endl; - } - - try { - vector objects; - if (dataType == 'f') { - string ids; - try { - ids = args.get("#2"); - } catch (...) { - cerr << "Data file is not specified" << endl; - cerr << usage << endl; - return; - } - ifstream is(ids); - if (!is) { - cerr << "Cannot open the specified file. " << ids << endl; - return; - } - string line; - int count = 0; - while(getline(is, line)) { - count++; - vector tokens; - NGT::Common::tokenize(line, tokens, "\t "); - if (tokens.size() == 0 || tokens[0].size() == 0) { - continue; - } - char *e; - size_t id; - try { - id = strtol(tokens[0].c_str(), &e, 10); - objects.push_back(id); - } catch (...) { - cerr << "Illegal data. " << tokens[0] << endl; - } - if (*e != 0) { - cerr << "Illegal data. " << e << endl; - } - cerr << "removed ID=" << id << endl; - } - } else { - size_t id = args.getl("#2", 0); - cerr << "removed ID=" << id << endl; - objects.push_back(id); - } - NGT::Index::remove(database, objects); - } catch (NGT::Exception &err) { - cerr << "Error " << err.what() << endl; - cerr << usage << endl; - } catch (...) { - cerr << "Error" << endl; - cerr << usage << endl; - } - } - - - void - info(NGT::Args &args) - { - const string usage = "Usage: ngtq info index"; - string database; - try { - database = args.get("#1"); - } catch (...) { - cerr << "DB is not specified" << endl; - cerr << usage << endl; - return; - } - NGTQ::Index index(database); - index.info(cout, args.getChar("m", '-')); - - } - - void - validate(NGT::Args &args) - { - const string usage = "parameter"; - string database; - try { - database = args.get("#1"); - } catch (...) { - cerr << "DB is not specified" << endl; - cerr << usage << endl; - return; - } - NGTQ::Index index(database); - - index.getQuantizer().validate(); - - } - - - -#ifdef NGTQ_SHARED_INVERTED_INDEX - void - compress(NGT::Args &args) - { - const string usage = "Usage: ngtq compress index)"; - string database; - try { - database = args.get("#1"); - } catch (...) { - cerr << "DB is not specified" << endl; - cerr << usage << endl; - return; - } - try { - NGTQ::Index::compress(database); - } catch (NGT::Exception &err) { - cerr << "Error " << err.what() << endl; - cerr << usage << endl; - } catch (...) { - cerr << "Error" << endl; - cerr << usage << endl; - } - } -#endif - - void help() { - cerr << "Usage : ngtq command database data" << endl; - cerr << " command : create search remove append export import" << endl; - } - - void execute(NGT::Args args) { - string command; - try { - command = args.get("#0"); - } catch(...) { - help(); - return; - } - - debugLevel = args.getl("X", 0); - - try { - if (debugLevel >= 1) { - cerr << "ngt::command=" << command << endl; - } - if (command == "search") { - search(args); - } else if (command == "create") { - create(args); - } else if (command == "append") { - append(args); - } else if (command == "remove") { - remove(args); - } else if (command == "info") { - info(args); - } else if (command == "validate") { - validate(args); - } else if (command == "rebuild") { - rebuild(args); -#ifdef NGTQ_SHARED_INVERTED_INDEX - } else if (command == "compress") { - compress(args); -#endif - } else { - cerr << "Illegal command. " << command << endl; - } - } catch(NGT::Exception &err) { - cerr << "ngt: Fatal error: " << err.what() << endl; - } - } - - int debugLevel; - -}; - -}; - diff --git a/lib/NGT/NGTQ/NGTQGCommand.cpp b/lib/NGT/NGTQ/NGTQGCommand.cpp deleted file mode 100644 index 0306f8e..0000000 --- a/lib/NGT/NGTQ/NGTQGCommand.cpp +++ /dev/null @@ -1,296 +0,0 @@ -// -// Copyright (C) 2020 Yahoo Japan Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - - -#include "NGT/NGTQ/NGTQGCommand.h" - -#if !defined(NGT_SHARED_MEMORY_ALLOCATOR) && !defined(NGTQ_SHARED_INVERTED_INDEX) - -using namespace std; - - -void -NGTQG::Command::create(NGT::Args &args) -{ - - const string usage = "Usage: ngtqg create " - "[-D distance-function] " - "[-p #-of-thread] [-d dimension] [-R global-codebook-range] [-r local-codebook-range] " - "[-C global-codebook-size-limit] [-c local-codebook-size-limit] " - "[-Q dimension-of-subvector] [-i index-type (t:Tree|g:Graph)] " - "[-M global-centroid-creation-mode (d|s)] [-l global-centroid-creation-mode (d|k|s)] " - "[-s local-sample-coefficient] " - "index(output)"; - - try { - NGT::Command::CreateParameters createParameters(args); - - switch (createParameters.indexType) { - case 't': - NGT::Index::createGraphAndTree(createParameters.index, createParameters.property, createParameters.objectPath, createParameters.numOfObjects); - break; - case 'g': - NGT::Index::createGraph(createParameters.index, createParameters.property, createParameters.objectPath, createParameters.numOfObjects); - break; - } - } catch(NGT::Exception &err) { - std::cerr << err.what() << std::endl; - cerr << usage << endl; - } - - NGTQG::Command::CreateParameters createParameters(args); - - try { - char localCentroidCreationMode = args.getChar("l", 'd'); - switch(localCentroidCreationMode) { - case 'd': createParameters.property.localCentroidCreationMode = NGTQ::CentroidCreationModeDynamic; break; - case 's': createParameters.property.localCentroidCreationMode = NGTQ::CentroidCreationModeStatic; break; - case 'k': createParameters.property.localCentroidCreationMode = NGTQ::CentroidCreationModeDynamicKmeans; break; - default: - cerr << "ngt: Invalid centroid creation mode. " << localCentroidCreationMode << endl; - cerr << usage << endl; - return; - } - - createParameters.index += "/qg"; - - cerr << "ngtqg: Create" << endl; - NGTQ::Index::create(createParameters.index, createParameters.property, createParameters.globalProperty, createParameters.localProperty); - } catch(NGT::Exception &err) { - std::cerr << err.what() << std::endl; - cerr << usage << endl; - } - - -} - -void -NGTQG::Command::build(NGT::Args &args) -{ - NGT::Command::append(args); -} - -void -NGTQG::Command::quantize(NGT::Args &args) -{ - const std::string usage = "Usage: ngtqg quantize [-Q dimension-of-subvector] [-E max-number-of-edges] index"; - string indexPath; - try { - indexPath = args.get("#1"); - } catch (...) { - cerr << "An index is not specified." << endl; - cerr << usage << endl; - return; - } - size_t maxNumOfEdges = args.getl("E", 128); - size_t dimensionOfSubvector = args.getl("Q", 0); - NGTQG::Index::quantize(indexPath, dimensionOfSubvector, maxNumOfEdges); -} - -void -search(NGTQG::Index &index, NGTQG::Command::SearchParameters &searchParameters, ostream &stream) -{ - - std::ifstream is(searchParameters.query); - if (!is) { - std::cerr << "Cannot open the specified file. " << searchParameters.query << std::endl; - return; - } - - if (searchParameters.outputMode[0] == 'e') { - stream << "# Beginning of Evaluation" << endl; - } - - string line; - double totalTime = 0; - size_t queryCount = 0; - - while(getline(is, line)) { - if (searchParameters.querySize > 0 && queryCount >= searchParameters.querySize) { - break; - } - vector query; - stringstream linestream(line); - while (!linestream.eof()) { - float value; - linestream >> value; - if (linestream.fail()) { - NGTThrowException("NGTQG: invalid stream."); - } - query.push_back(value); - } - queryCount++; - float beginOfParam = 0; - float endOfParam = 0; - float stepOfParam = FLT_MAX; - bool resultExpansionEnabled = true; - if (searchParameters.beginOfResultExpansion != searchParameters.endOfResultExpansion) { - beginOfParam = searchParameters.beginOfResultExpansion; - endOfParam = searchParameters.endOfResultExpansion; - stepOfParam = searchParameters.stepOfResultExpansion; - resultExpansionEnabled = true; - } else if (searchParameters.beginOfEpsilon != searchParameters.endOfEpsilon) { - beginOfParam = searchParameters.beginOfEpsilon; - endOfParam = searchParameters.endOfEpsilon; - stepOfParam = searchParameters.stepOfEpsilon; - resultExpansionEnabled = false; - } - for (float param = beginOfParam; param <= endOfParam; param += stepOfParam) { - NGTQG::SearchQuery searchQuery(query); - NGT::ObjectDistances objects; - searchQuery.setResults(&objects); - searchQuery.setSize(searchParameters.size); - searchQuery.setRadius(searchParameters.radius); - float epsilon, resultExpansion; - if (stepOfParam == FLT_MAX) { - resultExpansion = searchParameters.beginOfResultExpansion; - epsilon = searchParameters.beginOfEpsilon; - } else if (resultExpansionEnabled) { - resultExpansion = param; - epsilon = searchParameters.beginOfEpsilon; - } else { - resultExpansion = searchParameters.beginOfResultExpansion; - epsilon = param; - } - searchQuery.setResultExpansion(resultExpansion); - searchQuery.setEpsilon(epsilon); - searchQuery.setEdgeSize(searchParameters.edgeSize); - NGT::Timer timer; - switch (searchParameters.indexType) { - case 't': timer.start(); index.NGTQG::Index::search(searchQuery); timer.stop(); break; - case 's': timer.start(); index.linearSearch(searchQuery); timer.stop(); break; - } - totalTime += timer.time; - if (searchParameters.outputMode[0] == 'e') { - stream << "# Query No.=" << queryCount << endl; - stream << "# Query=" << line.substr(0, 20) + " ..." << endl; - stream << "# Index Type=" << searchParameters.indexType << endl; - stream << "# Size=" << searchParameters.size << endl; - stream << "# Radius=" << searchParameters.radius << endl; - stream << "# Epsilon*=" << epsilon << endl; - stream << "# Result Expansion*=" << resultExpansion << endl; - stream << "# Factor=" << param << endl; - stream << "# Query Time (msec)=" << timer.time * 1000.0 << endl; - stream << "# Distance Computation=" << searchQuery.distanceComputationCount << endl; - stream << "# Visit Count=" << searchQuery.visitCount << endl; - } else { - stream << "Query No." << queryCount << endl; - stream << "Rank\tID\tDistance" << endl; - } - for (size_t i = 0; i < objects.size(); i++) { - stream << i + 1 << "\t" << objects[i].id << "\t"; - stream << objects[i].distance << endl; - } - if (searchParameters.outputMode[0] == 'e') { - stream << "# End of Search" << endl; - } else { - stream << "Query Time= " << timer.time << " (sec), " << timer.time * 1000.0 << " (msec)" << endl; - } - } - if (searchParameters.outputMode[0] == 'e') { - stream << "# End of Query" << endl; - } - } - if (searchParameters.outputMode[0] == 'e') { - stream << "# Average Query Time (msec)=" << totalTime * 1000.0 / (double)queryCount << endl; - stream << "# Number of queries=" << queryCount << endl; - stream << "# End of Evaluation" << endl; - } else { - stream << "Average Query Time= " << totalTime / (double)queryCount << " (sec), " - << totalTime * 1000.0 / (double)queryCount << " (msec), (" - << totalTime << "/" << queryCount << ")" << endl; - } -} - - -void -NGTQG::Command::search(NGT::Args &args) { - const string usage = "Usage: ngtqg search [-i index-type(g|t|s)] [-n result-size] [-e epsilon] [-E edge-size] " - "[-o output-mode] [-p result-expansion] index(input) query.tsv(input)"; - - string indexPath; - try { - indexPath = args.get("#1"); - } catch (...) { - cerr << "An index is not specified." << endl; - cerr << usage << endl; - return; - } - NGTQG::Command::SearchParameters searchParameters(args); - - bool readOnly = true; - NGTQG::Index index(indexPath, 128, readOnly); - - if (debugLevel >= 1) { - cerr << "indexType=" << searchParameters.indexType << endl; - cerr << "size=" << searchParameters.size << endl; - cerr << "edgeSize=" << searchParameters.edgeSize << endl; - cerr << "epsilon=" << searchParameters.beginOfEpsilon << "<->" << searchParameters.endOfEpsilon << "," - << searchParameters.stepOfEpsilon << endl; - } - - try { - ::search(index, searchParameters, std::cout); - } catch (NGT::Exception &err) { - cerr << "ngtqg: Error " << err.what() << endl; - cerr << usage << endl; - } catch (std::exception &err) { - cerr << "ngtqg: Error " << err.what() << endl; - cerr << usage << endl; - } catch (...) { - cerr << "ngtqg: Error" << endl; - cerr << usage << endl; - } - -} - - -void -NGTQG::Command::info(NGT::Args &args) -{ - const string usage = "Usage: ngt info [-E #-of-edges] [-m h|e] index"; - - cerr << "NGT version: " << NGT::Index::getVersion() << endl; - - string database; - try { - database = args.get("#1"); - } catch (...) { - cerr << "ngt: Error: DB is not specified" << endl; - cerr << usage << endl; - return; - } - - size_t edgeSize = args.getl("E", UINT_MAX); - char mode = args.getChar("m", '-'); - - try { - NGT::Index index(database); - NGT::GraphIndex::showStatisticsOfGraph(static_cast(index.getIndex()), mode, edgeSize); - if (mode == 'v') { - vector status; - index.verify(status); - } - } catch (NGT::Exception &err) { - cerr << "ngt: Error " << err.what() << endl; - cerr << usage << endl; - } catch (...) { - cerr << "ngt: Error" << endl; - cerr << usage << endl; - } -} - -#endif diff --git a/lib/NGT/NGTQ/NGTQGCommand.h b/lib/NGT/NGTQ/NGTQGCommand.h deleted file mode 100644 index 4f39767..0000000 --- a/lib/NGT/NGTQ/NGTQGCommand.h +++ /dev/null @@ -1,117 +0,0 @@ -// -// Copyright (C) 2020 Yahoo Japan Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -#pragma once - -#include "NGT/NGTQ/QuantizedGraph.h" -#include "NGT/Command.h" -#include "NGT/NGTQ/NGTQCommand.h" - -namespace NGTQG { - - class Command : public NGT::Command { - public: - class CreateParameters : public NGTQ::Command::CreateParameters { - public: - CreateParameters(NGT::Args &args): NGTQ::Command::CreateParameters(args, 's', 'k') { - setup(args, property.dimension); - } - CreateParameters(NGT::Args &args, int dimension): NGTQ::Command::CreateParameters(args, 's', 'k') { - setup(args, dimension); - } - void setup(NGT::Args &args, size_t dimension) { - property.globalCentroidLimit = args.getl("C", 1); - property.localCentroidLimit = args.getl("c", 16); - property.localClusteringSampleCoefficient = args.getl("s", 100); - size_t dimensionOfSubvector = args.getl("Q", 0); - property.localDivisionNo = NGTQG::Index::getNumberOfSubvectors(dimension, dimensionOfSubvector); - property.dimension = dimension; - } - }; - - class SearchParameters : public NGT::Command::SearchParameters { - public: - SearchParameters(NGT::Args &args): NGT::Command::SearchParameters(args, "0.02") { - stepOfResultExpansion = 2; - std::string resultExpansion = args.getString("p", "3.0"); - std::vector tokens; - NGT::Common::tokenize(resultExpansion, tokens, ":"); - if (tokens.size() >= 1) { beginOfResultExpansion = endOfResultExpansion = NGT::Common::strtod(tokens[0]); } - if (tokens.size() >= 2) { endOfResultExpansion = NGT::Common::strtod(tokens[1]); } - if (tokens.size() >= 3) { stepOfResultExpansion = NGT::Common::strtod(tokens[2]); } - } - float beginOfResultExpansion; - float endOfResultExpansion; - float stepOfResultExpansion; - }; - -#if defined(NGT_SHARED_MEMORY_ALLOCATOR) || defined(NGTQ_SHARED_INVERTED_INDEX) - void create(NGT::Args &args) {}; - void build(NGT::Args &args) {}; - void quantize(NGT::Args &args) {}; - void search(NGT::Args &args) {}; - void info(NGT::Args &args) {}; -#else - void create(NGT::Args &args); - void build(NGT::Args &args); - void quantize(NGT::Args &args); - void search(NGT::Args &args); - void info(NGT::Args &args); -#endif - - void setDebugLevel(int level) { debugLevel = level; } - int getDebugLevel() { return debugLevel; } - - void help() { - cerr << "Usage : ngtqg command database [data]" << endl; - cerr << " command : create build quantize search" << endl; - } - - void execute(NGT::Args args) { - string command; - try { - command = args.get("#0"); - } catch(...) { - help(); - return; - } - - debugLevel = args.getl("X", 0); - - try { - if (debugLevel >= 1) { - cerr << "ngt::command=" << command << endl; - } - if (command == "search") { - search(args); - } else if (command == "create") { - create(args); - } else if (command == "build") { - build(args); - } else if (command == "quantize") { - quantize(args); - } else if (command == "info") { - info(args); - } else { - cerr << "Illegal command. " << command << endl; - } - } catch(NGT::Exception &err) { - cerr << "ngtqg: Fatal error: " << err.what() << endl; - } - } - - }; - -}; // NGTQG diff --git a/lib/NGT/NGTQ/ObjectFile.h b/lib/NGT/NGTQ/ObjectFile.h new file mode 100644 index 0000000..3beeb19 --- /dev/null +++ b/lib/NGT/NGTQ/ObjectFile.h @@ -0,0 +1,606 @@ +// +// Copyright (C) 2021 Yahoo Japan Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace NGT { + class ObjectSpace; +}; + +class ObjectFile : public ArrayFile { + public: + enum DataType { + DataTypeUint8 = 0, + DataTypeFloat = 1, + DataTypeFloat16 = 2 + }; + + ObjectFile():objectSpace(0) {} + ObjectFile(const ObjectFile &of):objectSpace(0) { + fileName = of.fileName; + dataType = of.dataType; + distanceType = of.distanceType; + pseudoDimension = of.pseudoDimension; + } + ~ObjectFile() { + closeMultipleStreams(); + close(); + delete objectSpace; + } + + bool open() { + if (!ArrayFile::open(fileName)) { + return false; + } + switch (dataType) { + case DataTypeFloat: + genuineDimension = ArrayFile::_fileHead.recordSize / sizeof(float); + objectSpace = new NGT::ObjectSpaceRepository(genuineDimension, typeid(float), distanceType); + break; + case DataTypeUint8: + genuineDimension = ArrayFile::_fileHead.recordSize / sizeof(uint8_t); + objectSpace = new NGT::ObjectSpaceRepository(genuineDimension, typeid(uint8_t), distanceType); + break; +#ifdef NGT_HALF_FLOAT + case DataTypeFloat16: + genuineDimension = ArrayFile::_fileHead.recordSize / sizeof(NGT::float16); + objectSpace = new NGT::ObjectSpaceRepository(genuineDimension, typeid(NGT::float16), distanceType); + break; +#endif + default: + stringstream msg; + msg << "ObjectFile::Invalid Object Type in the property. " << dataType; + NGTThrowException(msg); + break; + } + return true; + } + + bool open(const std::string &file, DataType datat, NGT::Index::Property::DistanceType distt, size_t pseudoDim) { + dataType = datat; + fileName = file; + distanceType = distt; + pseudoDimension = pseudoDim; + return open(); + } + + bool openMultipleStreams(const size_t nOfStreams) { + if (!isOpen()) { + return false; + } + if (!objectFiles.empty()) { + std::cerr << "ObjectFile::openMultipleStreams : already opened multiple streams. close and reopen. # of streams=" << nOfStreams << std::endl; + closeMultipleStreams(); + } + for (size_t i = 0; i < nOfStreams; i++) { + auto *of = new ObjectFile(*this); + if (!of->open()) { + std::cerr << "ObjectFile::openMultipleStreams: Cannot open. " << fileName << std::endl; + return false; + } + objectFiles.push_back(of); + } + return true; + } + + void closeMultipleStreams() { + for (auto *i : objectFiles) { + i->close(); + delete i; + i = 0; + } + objectFiles.clear(); + } + + bool get(const size_t streamID, size_t id, std::vector &data, NGT::ObjectSpace *objectSpace) { + if (streamID >= objectFiles.size()) { + std::cerr << "ObjectFile::streamID is invalid. " << streamID << ":" << objectFiles.size() << std::endl; + return false; + } + if (!objectFiles[streamID]->get(id, data, objectSpace)) { + return false; + } + return true; + } + + bool get(const size_t id, std::vector &data, NGT::ObjectSpace *os = 0) { + if (objectSpace == 0) { + stringstream msg; + msg << "ObjectFile::Fatal Error. objectSpace is not set." << std::endl; + NGTThrowException(msg); + } + NGT::Object *object = objectSpace->allocateObject(); + if (!ArrayFile::get(id, *object, objectSpace)) { + objectSpace->deleteObject(object); + return false; + } + const std::type_info &otype = objectSpace->getObjectType(); + size_t dim = objectSpace->getDimension(); + data.resize(pseudoDimension, 0); + if (otype == typeid(uint8_t)) { + auto *v = static_cast(object->getPointer()); + for (size_t i = 0; i < dim; i++) { + data[i] = v[i]; + } + } else if (otype == typeid(NGT::float16)) { + auto *v = static_cast(object->getPointer()); + for (size_t i = 0; i < dim; i++) { + data[i] = v[i]; + } + } else if (otype == typeid(float)) { + auto *v = static_cast(object->getPointer()); + memcpy(data.data(), v, sizeof(float) * dim); + } + objectSpace->deleteObject(object); + return true; + } + + void put(const size_t id, std::vector &data, NGT::ObjectSpace *os = 0) { + if (objectSpace == 0) { + stringstream msg; + msg << "ObjectFile::Fatal Error. objectSpace is not set." << std::endl; + NGTThrowException(msg); + } + if (objectSpace->getDimension() != data.size()) { + stringstream msg; + msg << "ObjectFile::Dimensions are inconsistency. " << objectSpace->getDimension() << ":" << data.size(); + NGTThrowException(msg); + } + NGT::Object *object = objectSpace->allocateObject(); + const std::type_info &otype = objectSpace->getObjectType(); + if (otype == typeid(uint8_t)) { + auto *v = static_cast(object->getPointer()); + for (size_t i = 0; i < data.size(); i++) { + v[i] = data[i]; + } + } else if (otype == typeid(NGT::float16)) { + auto *v = static_cast(object->getPointer()); + for (size_t i = 0; i < data.size(); i++) { + v[i] = data[i]; + } + } else if (otype == typeid(float)) { + auto *v = static_cast(object->getPointer()); + memcpy(v, data.data(), sizeof(float) * data.size()); + } + ArrayFile::put(id, *object, objectSpace); + objectSpace->deleteObject(object); + return; + } + + void put(const size_t id, NGT::Object &data, NGT::ObjectSpace *os = 0) { + return ArrayFile::put(id, data, objectSpace); + } + + bool get(size_t id, NGT::Object &data, NGT::ObjectSpace *os = 0) { + return ArrayFile::get(id, data, objectSpace); + } + + public: + std::string fileName; + size_t pseudoDimension; + size_t genuineDimension; + DataType dataType; + NGT::ObjectSpace::DistanceType distanceType; + NGT::ObjectSpace *objectSpace; + std::vector objectFiles; +}; + +template +class StaticObjectFile { + private: + enum Type { + TypeFloat = 0, + TypeUint8 = 1, + TypeInt8 = 2 + }; + struct FileHeadStruct { + uint32_t noOfObjects; + uint32_t noOfDimensions; + }; + + bool _isOpen; + std::ifstream _stream; + FileHeadStruct _fileHead; + + uint32_t _recordSize; + uint32_t _sizeOfElement; + std::string _typeName; + Type _type; + std::string _objectPath; + std::string _fileName; + + bool _readFileHead(); + + size_t _pseudoDimension; + std::vector*> _objectFiles; + + public: + StaticObjectFile(); + ~StaticObjectFile(); + bool create(const std::string &file, const std::string &objectPath); + bool open(const std::string &file, const size_t pseudoDimension = 0); + void close(); + size_t insert(TYPE &data, NGT::ObjectSpace *objectSpace = 0) {std::cerr << "insert: not implemented."; abort();} + void put(const size_t id, TYPE &data, NGT::ObjectSpace *objectSpace = 0) {std::cerr << "put: not implemented."; abort();} + bool get(size_t id, std::vector &data, NGT::ObjectSpace *objectSpace = 0); + bool get(const size_t streamID, size_t id, std::vector &data, NGT::ObjectSpace *objectSpace = 0); + bool get(size_t id, TYPE &data, NGT::ObjectSpace *objectSpace = 0); + bool get(const size_t streamID, size_t id, TYPE &data, NGT::ObjectSpace *objectSpace = 0); + void remove(const size_t id) {std::cerr << "remove: not implemented."; abort();} + bool isOpen() const; + size_t size(); + size_t getRecordSize() { return _recordSize; } + bool openMultipleStreams(const size_t nOfStreams); + void closeMultipleStreams(); +}; + +class StaticObjectFileLoader { +public: + StaticObjectFileLoader(const std::string &path, std::string t = "") { + if (path.find(".u8bin") != std::string::npos || t == "uint8") { + std::cerr << "type=u8bin" << std::endl; + type = "u8"; + sizeOfObject = 1; + } else if (path.find(".i8bin") != std::string::npos || t == "int8") { + std::cerr << "type=i8bin" << std::endl; + type = "i8"; + sizeOfObject = 1; + } else if (path.find(".fbin") != std::string::npos || t == "float32") { + std::cerr << "type=fbin" << std::endl; + type = "f"; + sizeOfObject = 4; + } else { + std::cerr << "Fatal error!!!" << std::endl; + exit(1); + } + stream.open(path, std::ios::in | std::ios::binary); + if (!stream) { + std::cerr << "qbg: Error! " << path << std::endl; + return; + } + stream.read(reinterpret_cast(&noOfObjects), sizeof(noOfObjects)); + stream.read(reinterpret_cast(&noOfDimensions), sizeof(noOfDimensions)); + sizeOfObject *= noOfDimensions; + std::cerr << "# of objects=" << noOfObjects << std::endl; + std::cerr << "# of dimensions=" << noOfDimensions << std::endl; + counter = 0; + } + + void seek(size_t id) { + if (noOfObjects <= id) { + id = noOfObjects - 1; + } + size_t headerSize = sizeof(noOfObjects) + sizeof(noOfDimensions); + stream.seekg(id * sizeOfObject + headerSize, ios_base::beg); + counter = id; + return; + } + + std::vector getObject() { + vector object; + if (isEmpty()) { + return object; + } + if (type == "u8") { + for (uint32_t dimidx = 0; dimidx < noOfDimensions; dimidx++) { + uint8_t v; + stream.read(reinterpret_cast(&v), sizeof(v)); + if (stream.eof()) { + std::cerr << "something wrong" << std::endl; + return object; + } + object.push_back(v); + } + } else if (type == "i8") { + for (uint32_t dimidx = 0; dimidx < noOfDimensions; dimidx++) { + int8_t v; + stream.read(reinterpret_cast(&v), sizeof(v)); + if (stream.eof()) { + std::cerr << "something wrong" << std::endl; + return object; + } + object.push_back(v); + } + } else if (type == "f") { + for (uint32_t dimidx = 0; dimidx < noOfDimensions; dimidx++) { + float v; + stream.read(reinterpret_cast(&v), sizeof(v)); + if (stream.eof()) { + std::cerr << "something wrong" << std::endl; + return object; + } + object.push_back(v); + } + } else { + std::cerr << "Fatal error!!!" << std::endl; + exit(1); + } + counter++; + return object; + } + bool isEmpty() { + return noOfObjects <= counter; + } + std::ifstream stream; + uint32_t noOfObjects; + uint32_t noOfDimensions; + uint32_t sizeOfObject; + uint32_t counter; + std::string type; +}; + + +// constructor +template +StaticObjectFile::StaticObjectFile() + : _isOpen(false) { +} + +// destructor +template +StaticObjectFile::~StaticObjectFile() { + close(); +} + +template +bool StaticObjectFile::create(const std::string &file, const std::string &objectPath) { + { + _stream.open(objectPath, std::ios::in); + if (!_stream) { + std::cerr << "Cannot open " << objectPath << std::endl; + return false; + } + bool ret = _readFileHead(); + if (ret == false) { + return false; + } + _stream.close(); + } + { + std::ofstream tmpstream; + tmpstream.open(file); + std::vector tokens; + NGT::Common::tokenize(objectPath, tokens, "."); + tmpstream << _fileHead.noOfObjects << std::endl; + tmpstream << _fileHead.noOfDimensions << std::endl; + if (tokens.size() <= 1) { + std::cerr << "The specifiled object file name has no proper extensions. " << objectPath << " use fbin." << std::endl; + tmpstream << "fbin" << std::endl; + } else { + tmpstream << tokens.back() << std::endl; + } + tmpstream << objectPath << std::endl; + } + return true; +} + +template +bool StaticObjectFile::open(const std::string &file, size_t pseudoDimension) { + _pseudoDimension = pseudoDimension; + uint32_t noOfObjects; + uint32_t noOfDimensions; + _fileName = file; + { + std::ifstream tmpstream; + tmpstream.open(file); + if (!tmpstream) { + std::cerr << "Cannot open " << file << std::endl; + abort(); + } + tmpstream >> noOfObjects; + tmpstream >> noOfDimensions; + tmpstream >> _typeName; + tmpstream >> _objectPath; + } + + if (_typeName == "fbin") { + _sizeOfElement = 4; + _type = TypeFloat; + } else if (_typeName == "u8bin") { + _sizeOfElement = 1; + _type = TypeUint8; + } else if (_typeName == "i8bin") { + _sizeOfElement = 1; + _type = TypeInt8; + } else { + _sizeOfElement = 4; + _type = TypeFloat; + } + _stream.open(_objectPath, std::ios::in); + if(!_stream){ + _isOpen = false; + return false; + } + _isOpen = true; + + bool ret = _readFileHead(); + if (_fileHead.noOfObjects != noOfObjects) { + stringstream msg; + msg << "Invalid # of objects=" << _fileHead.noOfObjects << ":" << noOfObjects; + NGTThrowException(msg); + } + if (_fileHead.noOfDimensions != noOfDimensions) { + stringstream msg; + msg << "Invalid # of dimensions=" << _fileHead.noOfDimensions << ":" << noOfDimensions; + NGTThrowException(msg); + } + _recordSize = _sizeOfElement * _fileHead.noOfDimensions; + return ret; +} + +template +bool StaticObjectFile::openMultipleStreams(const size_t nOfStreams) { + if (!isOpen()) { + return false; + } + if (!_objectFiles.empty()) { + std::cerr << "StaticObjectFile : already opened multiple streams. close and reopen. # of streams=" << nOfStreams << std::endl; + closeMultipleStreams(); + } + for (size_t i = 0; i < nOfStreams; i++) { + auto *of = new StaticObjectFile; + if (!of->open(_fileName, _pseudoDimension)) { + return false; + } + _objectFiles.push_back(of); + } + return true; +} + +template +bool StaticObjectFile::get(const size_t streamID, size_t id, std::vector &data, NGT::ObjectSpace *objectSpace) { + if (streamID >= _objectFiles.size()) { + std::cerr << "streamID is invalid. " << streamID << ":" << _objectFiles.size() << std::endl; + return false; + } + if (!_objectFiles[streamID]->get(id, data, objectSpace)) { + return false; + } + return true; +} + +template +bool StaticObjectFile::get(const size_t streamID, size_t id, TYPE &data, NGT::ObjectSpace *objectSpace) { + if (streamID >= _objectFiles.size()) { + std::cerr << "streamID is invalid. " << streamID << ":" << _objectFiles.size() << std::endl; + return false; + } + if (!_objectFiles[streamID]->get(id, data, objectSpace)) { + return false; + } + return true; +} + +template +void StaticObjectFile::close(){ + _stream.close(); + _isOpen = false; + closeMultipleStreams(); +} + +template +void StaticObjectFile::closeMultipleStreams() { + for (auto *i : _objectFiles) { + i->close(); + delete i; + i = 0; + } + _objectFiles.clear(); +} + + +template +bool StaticObjectFile::get(size_t id, TYPE &data, NGT::ObjectSpace *objectSpace) { + std::vector record; + bool stat = get(id, record, objectSpace); + std::stringstream object; + object.write(reinterpret_cast(record.data()), record.size() * sizeof(float)); + data.deserialize(object, objectSpace); + return stat; +} + +template +bool StaticObjectFile::get(size_t id, std::vector &data, NGT::ObjectSpace *objectSpace) { + id--; + if( size() <= id ){ + return false; + } + //uint64_t offset_pos = (id * (sizeof(RecordStruct) + _fileHead.recordSize)) + sizeof(FileHeadStruct); + uint64_t offset_pos = id * _recordSize + sizeof(FileHeadStruct); + //offset_pos += sizeof(RecordStruct); + _stream.seekg(offset_pos, std::ios::beg); + if (!_stream.fail()) { + switch (_type) { + case TypeFloat: + { + auto dim = _pseudoDimension; + if (dim == 0) { + dim = _recordSize / sizeof(float); + } + if (_recordSize > dim * sizeof(float)) { + abort(); + } + data.resize(dim, 0.0); + _stream.read(reinterpret_cast(data.data()), _recordSize); + break; + } + case TypeUint8: + case TypeInt8: + { + auto dim = _pseudoDimension; + if (dim == 0) { + dim = _recordSize; + } + uint8_t src[_recordSize]; + _stream.read(reinterpret_cast(src), _recordSize); + data.resize(dim, 0.0); + if (_type == TypeUint8) { + for (size_t i = 0; i < _recordSize; i++) { + data[i] = static_cast(src[i]); + } + } else { + int8_t *intsrc = reinterpret_cast(&src[0]); + for (size_t i = 0; i < _recordSize; i++) { + data[i] = static_cast(intsrc[i]); + } + } + } + break; + } + } else { + std::cerr << "StaticObjectFile::get something wrong! id=" << id << " type=" << _type << std::endl; + abort(); + } + + return true; +} + +template +bool StaticObjectFile::isOpen() const +{ + return _isOpen; +} + +template +size_t StaticObjectFile::size() +{ + _stream.seekg(0, std::ios::end); + int64_t offset_pos = _stream.tellg(); + offset_pos -= sizeof(FileHeadStruct); + size_t num = offset_pos / _recordSize; + num++; + return num; +} + +template +bool StaticObjectFile::_readFileHead() { + _stream.seekg(0, std::ios::beg); + _stream.read((char *)(&_fileHead), sizeof(FileHeadStruct)); + if(_stream.bad()){ + return false; + } + return true; +} + diff --git a/lib/NGT/NGTQ/Optimizer.h b/lib/NGT/NGTQ/Optimizer.h new file mode 100644 index 0000000..583560f --- /dev/null +++ b/lib/NGT/NGTQ/Optimizer.h @@ -0,0 +1,771 @@ +// +// Copyright (C) 2021 Yahoo Japan Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#pragma once + +#define NGT_CLUSTERING + +#include "QuantizedBlobGraph.h" +#ifdef NGT_CLUSTERING +#include "NGT/Clustering.h" +#else +#include "Cluster.h" +#endif + +#ifndef NGTQ_BLAS_FOR_ROTATION +#define NGT_DISABLE_BLAS +#endif + +#include "Matrix.h" + +namespace NGTQ { + class Optimizer { + public: + enum GlobalType { + GlobalTypeNone = 0, + GlobalTypeZero = 1, + GlobalTypeMean = 2 + }; + + Optimizer() { + numberOfClusters = 0; + numberOfSubvectors = 0; + clusteringType = NGT::Clustering::ClusteringTypeKmeansWithNGT; + initMode = NGT::Clustering::InitializationModeRandom; + convergenceLimitTimes = 5; + iteration = 100; + clusterIteration = 100; + clusterSizeConstraint = false; + iteration = 100; + repositioning = false; + rotation = true; + globalType = GlobalTypeNone; + silence = false; + } + + static void + extractSubvector(vector> &vectors, vector> &subvectors, size_t start , size_t size) + { + size_t vsize = vectors.size(); + subvectors.clear(); + subvectors.resize(vsize); + for (size_t vidx = 0; vidx < vsize; vidx++) { + subvectors[vidx].reserve(size); + for (size_t i = 0; i < size; i++) { + subvectors[vidx].push_back(vectors[vidx][start + i]); + } + } + } + + static void + catSubvector(vector> &vectors, vector> &subvectors) + { + if (vectors.size() == 0) { + vectors.resize(subvectors.size()); + } + assert(vectors.size() == subvectors.size()); + size_t vsize = vectors.size(); + size_t subvsize = subvectors[0].size(); + size_t size = vectors[0].size() + subvsize; + for (size_t vidx = 0; vidx < vsize; vidx++) { + subvectors[vidx].reserve(size); + for (size_t i = 0; i < subvsize; i++) { + vectors[vidx].push_back(subvectors[vidx][i]); + } + } + } + + static void +#ifdef NGT_CLUSTERING + extractQuantizedVector(vector> &qvectors, vector &clusters) +#else + extractQuantizedVector(vector> &qvectors, vector &clusters) +#endif + { + for (size_t cidx = 0; cidx < clusters.size(); ++cidx) { + for (auto mit = clusters[cidx].members.begin(); mit != clusters[cidx].members.end(); ++mit) { + size_t vid = (*mit).vectorID; + if (vid >= qvectors.size()) { + qvectors.resize(vid + 1); + } + qvectors[vid] = clusters[cidx].centroid; + } + } + } + + static double + squareDistance(vector> &va, vector> &vb) + { + assert(va.size() == vb.size()); + size_t vsize = va.size(); + assert(va[0].size() == vb[0].size()); + size_t dim = va[0].size(); + double distance = 0; + for (size_t vidx = 0; vidx < vsize; vidx++) { + for (size_t i = 0; i < dim; i++) { + double d = va[vidx][i] - vb[vidx][i]; + distance += d * d; + } + } + distance /= dim * vsize; + return distance; + } + + void evaluate(string global, vector> &vectors, char clusteringType, string &ofile, size_t &numberOfSubvectors, size_t &subvectorSize) + { + vector> residualVectors; + { + // compute residual vectors by global centroids. +#ifdef NGT_CLUSTERING + vector globalCentroid; +#else + vector globalCentroid; +#endif + if (!global.empty()) { + cerr << "generate residual vectors." << endl; + try { +#ifdef NGT_CLUSTERING + NGT::Clustering::loadClusters(global, globalCentroid); +#else + loadClusters(global, globalCentroid); +#endif + } catch (...) { + cerr << "Cannot load vectors. " << global << endl; + return; + } + if (clusteringType == 'k') { +#ifdef NGT_CLUSTERING + NGT::Clustering::assign(vectors, globalCentroid); +#else + assign(vectors, globalCentroid); +#endif + } else { + cerr << "Using NGT" << endl; +#ifdef NGT_CLUSTERING + std::cerr << "Not implemented" << std::endl; + abort(); +#else + assignWithNGT(vectors, globalCentroid); +#endif + } + residualVectors.resize(vectors.size()); + cerr << "global centroid size=" << globalCentroid.size() << endl; + for (size_t cidx = 0; cidx < globalCentroid.size(); ++cidx) { + for (auto mit = globalCentroid[cidx].members.begin(); mit != globalCentroid[cidx].members.end(); ++mit) { + size_t vid = (*mit).vectorID; + residualVectors[vid] = vectors[vid]; +#ifdef NGT_CLUSTERING + NGT::Clustering::subtract(residualVectors[vid], globalCentroid[cidx].centroid); +#else + subtract(residualVectors[vid], globalCentroid[cidx].centroid); +#endif + } + } + } + } + + Matrix R; + Matrix::load(ofile + "_R.tsv", R); + vector> qv(vectors.size()); // quantized vector + vector> xp; // residual vector + if (residualVectors.empty()) { + xp = vectors; + } else { + xp = residualVectors; + } + Matrix::mulSquare(xp, R); + for (size_t m = 0; m < numberOfSubvectors; m++) { +#ifdef NGT_CLUSTERING + vector subClusters; +#else + vector subClusters; +#endif + stringstream str; + str << ofile << "-" << m << ".tsv"; +#ifdef NGT_CLUSTERING + NGT::Clustering::loadClusters(str.str(), subClusters); +#else + loadClusters(str.str(), subClusters); +#endif + vector> subVectors; + extractSubvector(xp, subVectors, m * subvectorSize, subvectorSize); + if (clusteringType == 'k') { +#ifdef NGT_CLUSTERING + NGT::Clustering::assign(subVectors, subClusters); +#else + assign(subVectors, subClusters); +#endif + } else { + cerr << "Using NGT for subvector" << endl; +#ifdef NGT_CLUSTERING + std::cerr << "not implemented" << std::endl; + abort(); +#else + assignWithNGT(subVectors, subClusters); +#endif + } +#ifdef NGT_CLUSTERING + double distortion = NGT::Clustering::calculateML2(subVectors, subClusters); +#else + double distortion = calculateML2(subVectors, subClusters); +#endif + cout << "distortion[" << m << "]=" << distortion << endl; + vector> subCentroids(vectors.size()); + for (size_t cidx = 0; cidx < subClusters.size(); ++cidx) { +#ifdef NGT_CLUSTERING + vector &members = subClusters[cidx].members; +#else + vector &members = subClusters[cidx].members; +#endif + for (size_t eidx = 0; eidx < members.size(); ++eidx) { +#ifdef NGT_CLUSTERING + NGT::Clustering::Entry &entry = members[eidx]; +#else + Entry &entry = members[eidx]; +#endif + assert(cidx == entry.centroidID); + subCentroids[entry.vectorID] = subClusters[cidx].centroid; + } + } + catSubvector(qv, subCentroids); + } +#ifdef NGT_CLUSTERING + double distortion = NGT::Clustering::distanceL2(qv, xp); +#else + double distortion = distanceL2(qv, xp); +#endif + cout << "distortion=" << distortion << endl; + return; + } + + void evaluate(vector> &vectors, string &ofile, size_t &numberOfSubvectors, size_t &subvectorSize) { + cerr << "Evaluate" << endl; + Matrix R; + Matrix::load(ofile + "_R.tsv", R); + vector> xp = vectors; + Matrix::mulSquare(xp, R); + for (size_t m = 0; m < numberOfSubvectors; m++) { +#ifdef NGT_CLUSTERING + vector subClusters; +#else + vector subClusters; +#endif + stringstream str; + str << ofile << "-" << m << ".tsv"; +#ifdef NGT_CLUSTERING + NGT::Clustering::loadClusters(str.str(), subClusters); +#else + loadClusters(str.str(), subClusters); +#endif + vector> subVectors; + extractSubvector(xp, subVectors, m * subvectorSize, subvectorSize); +#ifdef NGT_CLUSTERING + NGT::Clustering::assign(subVectors, subClusters); + double distortion = NGT::Clustering::calculateML2(subVectors, subClusters); +#else + assign(subVectors, subClusters); + double distortion = calculateML2(subVectors, subClusters); +#endif + cout << "distortion[" << m << "]=" << distortion << endl; + for (size_t cidx = 0; cidx < subClusters.size(); ++cidx) { + cout << " members[" << cidx << "]=" << subClusters[cidx].members.size() << endl; + } + } + return; + } + + void + generateResidualObjects(string global, vector> &vectors) + { +#if defined(NGT_SHARED_MEMORY_ALLOCATOR) + std::cerr << "generateResidualObjects: Not implemented." << std::endl; + abort(); +#else + if (global.empty()) { + NGTThrowException("A global codebook is not specified!"); + } + vector> residualVectors; + vector> globalCentroid; + try { + NGT::Clustering::loadVectors(global, globalCentroid); + } catch (...) { + std::stringstream msg; + msg << "Optimizer::generateResidualObjects: Cannot load global vectors. " << global; + NGTThrowException(msg); + } + + NGT::Property property; + property.objectType = NGT::Index::Property::ObjectType::Float; + property.distanceType = NGT::Index::Property::DistanceType::DistanceTypeL2; + if (globalCentroid[0].size() != vectors[0].size()) { + std::cerr << "optimizer: Warning. The dimension is inconsistency. " << globalCentroid[0].size() << ":" << vectors[0].size() << std::endl; + } + property.dimension = vectors[0].size(); + NGT::Index index(property); + for (auto &c : globalCentroid) { + if (static_cast(c.size()) != property.dimension) { + c.resize(property.dimension); + } + index.append(c); + } + index.createIndex(100); + + for (size_t idx = 0; idx < vectors.size(); idx++) { + auto &v = vectors[idx]; + if ((idx + 1) % 10000 == 0) { + } + NGT::ObjectDistances gc; + NGT::SearchQuery query(v); + query.setResults(&gc); + query.setSize(10); + query.setEpsilon(0.1); + index.search(query); + if (gc.empty()) { + std::cerr << "inner fatal error. no results! something wrong." << std::endl; + abort(); + } + if (gc[0].id == 0 || gc[0].id > globalCentroid.size()) { + std::cerr << "wrong id " << gc[0].id << ":" << globalCentroid.size() << std::endl; + abort(); + } + auto gcidx = gc[0].id - 1; + try { + NGT::Clustering::subtract(v, globalCentroid[gcidx]); + } catch (NGT::Exception &err) { + std::cerr << err.what() << ":" << v.size() << "x" << globalCentroid[gcidx].size() << std::endl; + abort(); + } + } +#endif + } + + static void optimizeRotation( + size_t iteration, + vector> &vectors, + Matrix &xt, + Matrix &R, + Matrix &minR, + vector> &minLocalClusters, + NGT::Clustering::ClusteringType clusteringType, + NGT::Clustering::InitializationMode initMode, + size_t numberOfClusters, + size_t numberOfSubvectors, + size_t subvectorSize, + size_t clusterIteration, + bool clusterSizeConstraint, + size_t convergenceLimitTimes, + double &minDistortion, + NGT::Timer &timelimitTimer, float timelimit, + bool rotation + ) { + + minDistortion = DBL_MAX; + + int minIt = 0; + for (size_t it = 0; it < iteration; it++) { + vector> xp = vectors; + Matrix::mulSquare(xp, R); + float distance = 0.0; +#ifdef NGT_CLUSTERING + vector> localClusters(numberOfSubvectors); +#else + vector> localClusters(numberOfSubvectors); +#endif + vector> subQuantizedVectors[numberOfSubvectors]; +#define ERROR_CALCULATION +#ifdef ERROR_CALCULATION + vector subvectorDistances(numberOfSubvectors); +#endif +#pragma omp parallel for + for (size_t m = 0; m < numberOfSubvectors; m++) { + vector> subVectors; + extractSubvector(xp, subVectors, m * subvectorSize, subvectorSize); +#ifdef NGT_CLUSTERING + vector &clusters = localClusters[m]; +#else + vector &clusters = localClusters[m]; +#endif +#ifdef NGT_CLUSTERING + NGT::Clustering clustering(initMode, clusteringType, clusterIteration); + //clustering.setClusterSizeConstraintCoefficient(1.2); + clustering.clusterSizeConstraint = clusterSizeConstraint; + clustering.kmeans(subVectors, numberOfClusters, clusters); +#else + size_t reassign; + if (clusteringType == 'k') { + reassign = kmeansClustering(initMode, subVectors, numberOfClusters, clusters, clusterSizeConstraint, 0); + } else { + reassign = kmeansClusteringWithNGT(initMode, subVectors, numberOfClusters, clusters, clusterSizeConstraint, 0); + } +#endif + extractQuantizedVector(subQuantizedVectors[m], clusters); + assert(subQuantizedVectors[m].size() == vectors.size()); + assert(subQuantizedVectors[m][0].size() == subvectorSize); + // 入力部分ベクトルと量子化部分ベクトルを比較して量子化誤差の計算 +#ifdef ERROR_CALCULATION + double d = squareDistance(subQuantizedVectors[m], subVectors); + subvectorDistances[m] = d; +#endif + } + distance = 0.0; +#ifdef ERROR_CALCULATION + for (size_t m = 0; m < numberOfSubvectors; m++) { + distance += subvectorDistances[m]; + } +#endif + vector> quantizedVectors; + for (size_t m = 0; m < numberOfSubvectors; m++) { + catSubvector(quantizedVectors, subQuantizedVectors[m]); + } + distance = sqrt(distance / numberOfSubvectors); + if (minDistortion > distance) { + minDistortion = distance; + minR = R; + minIt = it; + minLocalClusters = localClusters; + } + if (it + 1 > iteration || it - minIt > convergenceLimitTimes) { + break; + } + timelimitTimer.stop(); + if (timelimitTimer.time > timelimit) { + std::cerr << "Optimizer: Warning. The elapsed time exceeded the limit-time. " << timelimit << ":" << timelimitTimer.time << std::endl; + timelimitTimer.restart(); + break; + } + timelimitTimer.restart(); + if (rotation) { + Matrix a(xt); + a.mul(quantizedVectors); + Matrix u, s, v; + Matrix::svd(a, u, s, v); + v.transpose(); + u.mul(v); + R = u; + } + } + } + + void optimize(vector> &vectors, + string global, + string ofile, + Matrix &reposition, + vector> &rs, + vector>> &localClusters, + vector &errors + ) { + if (vectors.size() == 0) { + NGTThrowException("the vector is empty"); + } + generateResidualObjects(global, vectors); + if (!reposition.isEmpty()) { + Matrix::mulSquare(vectors, reposition); + } + Matrix xt(vectors); +#ifndef NGTQ_BLAS_FOR_ROTATION + xt.transpose(); +#endif + localClusters.resize(rs.size()); + errors.resize(rs.size()); + NGT::Timer timer; + for (size_t ri = 0; ri < rs.size(); ri++) { + auto imode = initMode; + if (imode == NGT::Clustering::InitializationModeBest) { + imode = ri % 2 == 0 ? NGT::Clustering::InitializationModeRandom : NGT::Clustering::InitializationModeKmeansPlusPlus; + } + timer.start(); + Matrix optr; + optimizeRotation( + iteration, + vectors, + xt, + rs[ri], + optr, + localClusters[ri], + clusteringType, + imode, + numberOfClusters, + numberOfSubvectors, + subvectorSize, + clusterIteration, + clusterSizeConstraint, + convergenceLimitTimes, + errors[ri], + timelimitTimer, timelimit, + rotation + ); + timer.stop(); + rs[ri] = optr; + } + } + +#ifdef NGTQ_QBG + void optimize(const std::string indexPath, bool random, size_t threadSize = 0) { + NGT::StdOstreamRedirector redirector(silence); + redirector.begin(); + { + QBG::Index index(indexPath); + if (index.getQuantizer().objectList.size() <= 1) { + NGTThrowException("optimize: No objects"); + } + if (numberOfObjects == 0) { + numberOfObjects = index.getQuantizer().objectList.size() - 1; + } + std::cerr << "optimize: # of clusters=" << index.getQuantizer().property.localCentroidLimit << ":" << numberOfClusters << std::endl; + if (index.getQuantizer().property.localCentroidLimit == 0 && numberOfClusters == 0) { + std::stringstream msg; + msg << "optimize: # of clusters is illegal. " << index.getQuantizer().property.localCentroidLimit << ":" << numberOfClusters; + NGTThrowException(msg); + } + if (index.getQuantizer().property.localCentroidLimit != 0 && numberOfClusters != 0 && + index.getQuantizer().property.localCentroidLimit != numberOfClusters) { + std::cerr << "optimize: warning! # of clusters is already specified. " << index.getQuantizer().property.localCentroidLimit << ":" << numberOfClusters << std::endl; + } + if (numberOfClusters == 0) { + numberOfClusters = index.getQuantizer().property.localCentroidLimit; + } + + if (numberOfSubvectors == 0 && index.getQuantizer().property.localDivisionNo == 0) { + std::stringstream msg; + msg << "optimize: # of subvectors is illegal. " << numberOfSubvectors << ":" << index.getQuantizer().property.localDivisionNo; + NGTThrowException(msg); + } + if (numberOfSubvectors != 0 && index.getQuantizer().property.localDivisionNo != 0 && + numberOfSubvectors != index.getQuantizer().property.localDivisionNo) { + std::cerr << "optimize: warning! # of subvectros is already specified. " << numberOfSubvectors << ":" << index.getQuantizer().property.localDivisionNo << std::endl; + } + if (numberOfSubvectors == 0) { + numberOfSubvectors = index.getQuantizer().property.localDivisionNo; + } + + const std::string ws = indexPath + "/" + QBG::Index::getWorkspaceName(); + try { + NGT::Index::mkdir(ws); + } catch(...) {} + const std::string object = QBG::Index::getTrainObjectFileName(indexPath); + std::ofstream ofs; + ofs.open(object); + index.extract(ofs, numberOfObjects, random); + if (globalType == GlobalTypeZero) { + assert(index.getQuantizer().objectList.pseudoDimension != 0); + std::vector> global(1); + global[0].resize(index.getQuantizer().property.dimension, 0.0); + NGT::Clustering::saveVectors(QBG::Index::getQuantizerCodebookFileName(indexPath), global); + } else if (globalType == GlobalTypeMean) { + std::vector> vectors; + std::string objects = QBG::Index::getTrainObjectFileName(indexPath); +#ifdef NGT_CLUSTERING + NGT::Clustering::loadVectors(objects, vectors); +#else + loadVectors(objects, vectors); +#endif + if (vectors.size() == 0 || vectors[0].size() == 0) { + NGTThrowException("Optimizer::optimize: invalid input vectors"); + } + std::vector> global(1); + global[0].resize(index.getQuantizer().property.dimension, 0); + for (auto v = vectors.begin(); v != vectors.end(); ++v) { + for (size_t i = 0; i < (*v).size(); i++) { + global[0][i] += (*v)[i]; + } + } + for (size_t i = 0; i < global[0].size(); i++) { + global[0][i] /= vectors.size(); + } + NGT::Clustering::saveVectors(QBG::Index::getQuantizerCodebookFileName(indexPath), global); + } + } + + optimize(indexPath); + + if (globalType == GlobalTypeNone) { + QBG::Index::load(indexPath, "", "", "", threadSize); + } else { + QBG::Index::load(indexPath, QBG::Index::getQuantizerCodebookFileName(indexPath), "", "", threadSize); + } + redirector.end(); + } +#endif + +#ifdef NGTQ_QBG + void optimize(std::string indexPath) { + std::string object; + std::string pq; + std::string global; + { + object = QBG::Index::getTrainObjectFileName(indexPath); + pq = QBG::Index::getPQFileName(indexPath); + global = QBG::Index::getQuantizerCodebookFileName(indexPath); + } + + try { + NGT::Index::mkdir(pq); + } catch(...) {} + pq += "/opt.tsv"; + optimize(object, pq, global); + } +#endif + + void optimize(std::string invector, std::string ofile, std::string global) { + vector> vectors; + + +#ifdef NGT_CLUSTERING + NGT::Clustering::loadVectors(invector, vectors); +#else + loadVectors(invector, vectors); +#endif + + if (vectors.size() == 0) { + std::cerr << "Optimizer: error! the specified vetor file is empty. " << invector << ". the size=" << vectors.size() << std::endl; + exit(1); + } + + dim = vectors[0].size(); + subvectorSize = dim / numberOfSubvectors; + if (dim % numberOfSubvectors != 0) { + std::stringstream msg; + msg << "# of subspaces (m) is illegal. " << dim << ":" << numberOfSubvectors; + NGTThrowException(msg); + } + + + timelimitTimer.start(); + + Matrix reposition; + if (repositioning) { + reposition.zero(dim, dim); + size_t dstidx = 0; + for (size_t didx = 0; didx < numberOfSubvectors; didx++) { + for (size_t sdidx = 0; sdidx < subvectorSize; sdidx++) { + size_t srcidx = didx + sdidx * numberOfSubvectors; + auto col = dstidx; + auto row = srcidx; + reposition.set(row, col, 1.0); + dstidx++; + } + } + std::cerr << "Optimizer: Each axis was repositioned." << std::endl; + } + + + vector>> localClusters; + vector errors; + + bool useEye = false; + nOfMatrices = nOfMatrices == 0 ? 1 : nOfMatrices; + if (!rotation) { + iteration = 1; + seedStartObjectSizeRate = 1.0; + seedStep = 2; + } + useEye = !rotation; + vector> rs(nOfMatrices); + for (auto &r: rs) { + if (useEye) { + r.eye(dim); + } else { + r.randomRotation(dim); + } + } + + for (size_t vsize = static_cast(vectors.size()) * seedStartObjectSizeRate; ; vsize *= seedStep) { + auto partialVectors = vectors; + if (vsize < vectors.size()) { + partialVectors.resize(vsize); + } + + optimize(partialVectors, + global, + ofile, + reposition, + rs, + localClusters, + errors); + if (rs.size() > 1) { + nOfMatrices = static_cast(nOfMatrices) * (1.0 - reject); + nOfMatrices = nOfMatrices == 0 ? 1 : nOfMatrices; + vector*, vector>*>>> sortedErrors; + for (size_t idx = 0; idx < errors.size(); idx++) { + sortedErrors.emplace_back(make_pair(errors[idx], make_pair(&rs[idx], &localClusters[idx]))); + } + sort(sortedErrors.begin(), sortedErrors.end()); + vector> tmpMatrix; + vector>> tmpLocalClusters; + for (size_t idx = 0; idx < nOfMatrices; idx++) { + tmpMatrix.emplace_back(*sortedErrors[idx].second.first); + tmpLocalClusters.emplace_back(*sortedErrors[idx].second.second); + } + if (tmpMatrix.size() != nOfMatrices) { + std::cerr << "something strange. " << tmpMatrix.size() << ":" << nOfMatrices << std::endl; + } + rs = std::move(tmpMatrix); + localClusters = std::move(tmpLocalClusters); + } + if (vsize >= vectors.size()) { + break; + } + } + + if (rs.size() != 1) { + std::cerr << "Optimizer: Warning. rs.size=" << rs.size() << std::endl; + } + auto minR = std::move(rs[0]); + auto minLocalClusters = std::move(localClusters[0]); + size_t pos = std::distance(std::find(ofile.rbegin(), ofile.rend(), '.'), ofile.rend()) - 1; + string of = ofile.substr(0, pos); + if (repositioning) { + Matrix repositionedR(reposition); + repositionedR.mul(minR); + Matrix::save(of + "_R.tsv", repositionedR); + } else { + Matrix::save(of + "_R.tsv", minR); + } + for (size_t m = 0; m < numberOfSubvectors; m++) { + stringstream str; + str << of << "-" << m << ".tsv"; +#ifdef NGT_CLUSTERING + NGT::Clustering::saveClusters(str.str(), minLocalClusters[m]); +#else + saveClusters(str.str(), minLocalClusters[m]); +#endif + } + + } + + NGT::Clustering::ClusteringType clusteringType; + NGT::Clustering::InitializationMode initMode; + + NGT::Timer timelimitTimer; + float timelimit; + size_t iteration; + size_t clusterIteration; + bool clusterSizeConstraint; + size_t convergenceLimitTimes; + size_t dim; + size_t numberOfObjects; + size_t numberOfClusters; + size_t numberOfSubvectors; + size_t subvectorSize; + size_t nOfMatrices; + float seedStartObjectSizeRate; + size_t seedStep; + float reject; + bool repositioning; + bool rotation; + GlobalType globalType; + bool silence; + }; +} // namespace NGTQ diff --git a/lib/NGT/NGTQ/QbgCli.cpp b/lib/NGT/NGTQ/QbgCli.cpp new file mode 100644 index 0000000..d08c1c7 --- /dev/null +++ b/lib/NGT/NGTQ/QbgCli.cpp @@ -0,0 +1,1565 @@ +// +// Copyright (C) 2020 Yahoo Japan Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include "Quantizer.h" + +#if defined(NGTQ_QBG) && !defined(NGTQ_SHARED_INVERTED_INDEX) + +#include "NGT/NGTQ/QbgCli.h" +#include "NGT/NGTQ/Optimizer.h" +#include "NGT/NGTQ/HierarchicalKmeans.h" + +typedef NGTQ::Quantizer::ObjectList QBGObjectList; + + +class CreateParameters { +public: + CreateParameters(NGT::Args &args) { + try { + index = args.get("#1"); + } catch (...) { + std::stringstream msg; + msg << "Command::CreateParameters: Error: An index is not specified."; + NGTThrowException(msg); + } + try { + objectPath = args.get("#2"); + } catch (...) {} + + char objectType = args.getChar("o", 'f'); + char distanceType = args.getChar("D", '2'); + numOfObjects = args.getl("n", 0); + + property.threadSize = args.getl("p", 24); + property.dimension = args.getl("d", 0); + property.globalRange = args.getf("R", 0); + property.localRange = args.getf("r", 0); + property.globalCentroidLimit = args.getl("C", 0); +#ifdef NGTQ_QBG + property.localCentroidLimit = args.getl("c", 16); +#else + property.localCentroidLimit = args.getl("c", 65000); +#endif + property.localDivisionNo = args.getl("N", 8); + property.batchSize = args.getl("b", 1000); + property.localClusteringSampleCoefficient = args.getl("s", 10); + { + char localCentroidType = args.getChar("T", 'f'); + property.singleLocalCodebook = localCentroidType == 't' ? true : false; + } + { + char centroidCreationMode = args.getChar("M", 'l'); + switch(centroidCreationMode) { + case 'd': property.centroidCreationMode = NGTQ::CentroidCreationModeDynamic; break; + case 's': property.centroidCreationMode = NGTQ::CentroidCreationModeStatic; break; + case 'l': property.centroidCreationMode = NGTQ::CentroidCreationModeStaticLayer; break; + default: + std::stringstream msg; + msg << "Command::CreateParameters: Error: Invalid centroid creation mode. " << centroidCreationMode; + NGTThrowException(msg); + } + } + { + char localCentroidCreationMode = args.getChar("L", 's'); + switch(localCentroidCreationMode) { + case 'd': property.localCentroidCreationMode = NGTQ::CentroidCreationModeDynamic; break; + case 's': property.localCentroidCreationMode = NGTQ::CentroidCreationModeStatic; break; + case 'k': property.localCentroidCreationMode = NGTQ::CentroidCreationModeDynamicKmeans; break; + default: + std::stringstream msg; + msg << "Command::CreateParameters: Error: Invalid centroid creation mode. " << localCentroidCreationMode; + NGTThrowException(msg); + } + } +#ifdef NGTQ_QBG + property.localIDByteSize = args.getl("B", 1); +#endif + + globalProperty.edgeSizeForCreation = args.getl("E", 10); + globalProperty.edgeSizeForSearch = args.getl("S", 40); + { + char indexType = args.getChar("i", 't'); + globalProperty.indexType = indexType == 't' ? NGT::Property::GraphAndTree : NGT::Property::Graph; + localProperty.indexType = globalProperty.indexType; + } + globalProperty.insertionRadiusCoefficient = args.getf("e", 0.1) + 1.0; + localProperty.insertionRadiusCoefficient = globalProperty.insertionRadiusCoefficient; + + + switch (objectType) { + case 'f': property.dataType = NGTQ::DataTypeFloat; break; +#ifdef NGT_HALF_FLOAT + case 'h': property.dataType = NGTQ::DataTypeFloat16; break; +#endif + case 'c': property.dataType = NGTQ::DataTypeUint8; break; + default: + std::stringstream msg; + msg << "Command::CreateParameters: Error: Invalid object type. " << objectType; + NGTThrowException(msg); + } + + switch (distanceType) { + case '2': property.distanceType = NGTQ::DistanceType::DistanceTypeL2; break; + case '1': property.distanceType = NGTQ::DistanceType::DistanceTypeL1; break; + case 'a': property.distanceType = NGTQ::DistanceType::DistanceTypeAngle; break; + case 'C': property.distanceType = NGTQ::DistanceType::DistanceTypeNormalizedCosine; break; + case 'E': property.distanceType = NGTQ::DistanceType::DistanceTypeL2; break; + default: + std::stringstream msg; + msg << "Command::CreateParameters: Error: Invalid distance type. " << distanceType; + NGTThrowException(msg); + } +#ifdef NGTQ_QBG + property.genuineDimension = property.dimension; + property.dimension = args.getl("P", property.genuineDimension); + { + char objectType = args.getChar("O", 'f'); + switch (objectType) { + case 'f': property.genuineDataType = ObjectFile::DataTypeFloat; break; +#ifdef NGT_HALF_FLOAT + case 'h': property.genuineDataType = ObjectFile::DataTypeFloat16; break; +#endif + case 'c': property.genuineDataType = ObjectFile::DataTypeUint8; break; + default: + std::stringstream msg; + msg << "Command::CreateParameters: Error: Invalid genuine object type. " << objectType; + NGTThrowException(msg); + } + } +#endif + + } + + std::string index; + std::string objectPath; + size_t numOfObjects; + NGTQ::Property property; + NGT::Property globalProperty; + NGT::Property localProperty; + +}; + +class SearchParameters : public NGT::Command::SearchParameters { +public: + SearchParameters(NGT::Args &args): NGT::Command::SearchParameters(args, "0.02") { + stepOfResultExpansion = 2; + std::string resultExpansion = args.getString("p", "3.0"); + std::vector tokens; + NGT::Common::tokenize(resultExpansion, tokens, ":"); + if (tokens.size() >= 1) { beginOfResultExpansion = endOfResultExpansion = NGT::Common::strtod(tokens[0]); } + if (tokens.size() >= 2) { endOfResultExpansion = NGT::Common::strtod(tokens[1]); } + if (tokens.size() >= 3) { stepOfResultExpansion = NGT::Common::strtod(tokens[2]); } + } + float beginOfResultExpansion; + float endOfResultExpansion; + float stepOfResultExpansion; +}; + +static void optimizationParameters(NGT::Args &args, NGTQ::Optimizer &optimizer) { + optimizer.numberOfObjects = args.getl("o", 1000); + optimizer.numberOfClusters = args.getl("n", 0); + optimizer.numberOfSubvectors = args.getl("m", 0); + +#ifdef NGT_CLUSTERING + string cType; + try { + cType = args.getString("C", "k"); + } catch(...) {} + + optimizer.clusteringType = NGT::Clustering::ClusteringTypeKmeansWithNGT; + if (cType == "k") { + optimizer.clusteringType = NGT::Clustering::ClusteringTypeKmeansWithoutNGT; + } else if (cType == "KS") { + optimizer.clusteringType = NGT::Clustering::ClusteringTypeKmeansWithNGT; + } else if (cType == "i") { + optimizer.clusteringType = NGT::Clustering::ClusteringTypeKmeansWithIteration; + } else { + std::stringstream msg; + msg << "invalid clustering type. " << cType; + NGTThrowException(msg); + } +#else + char clusteringType; + try { + clusteringType = args.getChar("C", 'k'); + } catch(...) {} +#endif + +#ifdef NGT_CLUSTERING + char iMode = args.getChar("i", '-'); + optimizer.initMode = NGT::Clustering::InitializationModeKmeansPlusPlus; + switch (iMode) { + case 'h': optimizer.initMode = NGT::Clustering::InitializationModeHead; break; + case 'r': optimizer.initMode = NGT::Clustering::InitializationModeRandom; break; + case 'p': optimizer.initMode = NGT::Clustering::InitializationModeKmeansPlusPlus; break; + case 'R': optimizer.initMode = NGT::Clustering::InitializationModeRandomFixedSeed; break; + case 'P': optimizer.initMode = NGT::Clustering::InitializationModeKmeansPlusPlusFixedSeed; break; + default: + case '-': + case 'b': optimizer.initMode = NGT::Clustering::InitializationModeBest; break; + } +#else + optimizer.initMode = args.getChar("i", '-'); +#endif + + optimizer.convergenceLimitTimes = args.getl("c", 5); + optimizer.iteration = args.getl("t", 100); + optimizer.clusterIteration = args.getl("I", 100); + + optimizer.clusterSizeConstraint = false; + if (args.getChar("s", 'f') == 't') { + optimizer.clusterSizeConstraint = true; + } + + optimizer.nOfMatrices = args.getl("M", 2); + optimizer.seedStartObjectSizeRate = args.getf("S", 0.1); + optimizer.seedStep = args.getl("X", 2); + optimizer.reject = args.getf("R", 0.9); + optimizer.timelimit = args.getf("L", 24 * 1); + optimizer.timelimit *= 60.0 * 60.0; +#ifdef NGTQG_NO_ROTATION + char positionMode = args.getChar("P", 'n'); +#else + char positionMode = args.getChar("P", 'r'); +#endif + switch (positionMode) { + case 'r': + optimizer.rotation = true; + optimizer.repositioning = false; + break; + case 'R': + optimizer.rotation = true; + optimizer.repositioning = true; + break; + case 'p': + optimizer.rotation = false; + optimizer.repositioning = true; + break; + case 'n': + default: + optimizer.rotation = false; + optimizer.repositioning = false; + } + char globalType = args.getChar("G", '-'); + switch (globalType) { + case 'z': + optimizer.globalType = NGTQ::Optimizer::GlobalTypeZero; break; + case 'm': + optimizer.globalType = NGTQ::Optimizer::GlobalTypeMean; break; + default: + case 'n': + optimizer.globalType = NGTQ::Optimizer::GlobalTypeNone; break; + break; + } +} + +void +QBG::CLI::buildQG(NGT::Args &args) +{ + const std::string usage = "Usage: qbg build-qg [-Q dimension-of-subvector] [-E max-number-of-edges] index"; + + string indexPath; + try { + indexPath = args.get("#1"); + } catch (...) { + cerr << "An index is not specified." << endl; + cerr << usage << endl; + return; + } + + size_t phase = args.getl("p", 0); + size_t maxNumOfEdges = args.getl("E", 128); + bool silence = !args.getBool("v"); + + const std::string qgPath = indexPath + "/qg"; + + if (phase == 0 || phase == 1) { + NGTQ::Optimizer optimizer; + optimizer.globalType = NGTQ::Optimizer::GlobalTypeZero; + try { + optimizationParameters(args, optimizer); + } catch(NGT::Exception &err) { + std::cerr << err.what() << std::endl; + cerr << usage << endl; + } + +#ifdef NGTQG_NO_ROTATION + if (optimizer.rotation || optimizer.repositioning) { + std::cerr << "build-qg: Warning! Although rotation or repositioning is specified, turn off rotation and repositioning because of unavailable options." << std::endl; + optimizer.rotation = false; + optimizer.repositioning = false; + } +#endif + + if (optimizer.globalType == NGTQ::Optimizer::GlobalTypeNone) { + std::cerr << "build-qg: Warning! None is unavailable for the global type. Zero is set to the global type." << std::endl; + optimizer.globalType = NGTQ::Optimizer::GlobalTypeZero; + } + + std::cerr << "optimizing..." << std::endl; + bool random = true; + optimizer.silence = silence; + optimizer.optimize(qgPath, random); + } + if (phase == 0 || phase == 2) { + std::cerr << "building the inverted index..." << std::endl; + bool silence = true; + QBG::Index::buildNGTQ(qgPath, silence); + } + if (phase == 0 || phase == 3) { + std::cerr << "building the quantized graph... " << std::endl; + NGTQG::Index::quantize(indexPath, maxNumOfEdges, silence); + } +} + +void +searchQG(NGTQG::Index &index, SearchParameters &searchParameters, ostream &stream) +{ + + std::ifstream is(searchParameters.query); + if (!is) { + std::cerr << "Cannot open the specified file. " << searchParameters.query << std::endl; + return; + } + + if (searchParameters.outputMode[0] == 'e') { + stream << "# Beginning of Evaluation" << endl; + } + + string line; + double totalTime = 0; + size_t queryCount = 0; + + while(getline(is, line)) { + if (searchParameters.querySize > 0 && queryCount >= searchParameters.querySize) { + break; + } + vector query; + stringstream linestream(line); + while (!linestream.eof()) { + float value; + linestream >> value; + if (linestream.fail()) { + NGTThrowException("NGTQG: invalid stream."); + } + query.push_back(value); + } + queryCount++; + float beginOfParam = 0; + float endOfParam = 0; + float stepOfParam = FLT_MAX; + bool resultExpansionEnabled = true; + if (searchParameters.beginOfResultExpansion != searchParameters.endOfResultExpansion) { + beginOfParam = searchParameters.beginOfResultExpansion; + endOfParam = searchParameters.endOfResultExpansion; + stepOfParam = searchParameters.stepOfResultExpansion; + resultExpansionEnabled = true; + } else if (searchParameters.beginOfEpsilon != searchParameters.endOfEpsilon) { + beginOfParam = searchParameters.beginOfEpsilon; + endOfParam = searchParameters.endOfEpsilon; + stepOfParam = searchParameters.stepOfEpsilon; + resultExpansionEnabled = false; + } + for (float param = beginOfParam; param <= endOfParam; param += stepOfParam) { + NGTQG::SearchQuery searchQuery(query); + NGT::ObjectDistances objects; + searchQuery.setResults(&objects); + searchQuery.setSize(searchParameters.size); + searchQuery.setRadius(searchParameters.radius); + float epsilon, resultExpansion; + if (stepOfParam == FLT_MAX) { + resultExpansion = searchParameters.beginOfResultExpansion; + epsilon = searchParameters.beginOfEpsilon; + } else if (resultExpansionEnabled) { + resultExpansion = param; + epsilon = searchParameters.beginOfEpsilon; + } else { + resultExpansion = searchParameters.beginOfResultExpansion; + epsilon = param; + } + searchQuery.setResultExpansion(resultExpansion); + searchQuery.setEpsilon(epsilon); + searchQuery.setEdgeSize(searchParameters.edgeSize); + NGT::Timer timer; + switch (searchParameters.indexType) { + case 't': timer.start(); index.NGTQG::Index::search(searchQuery); timer.stop(); break; + case 's': timer.start(); index.linearSearch(searchQuery); timer.stop(); break; + } + totalTime += timer.time; + if (searchParameters.outputMode[0] == 'e') { + stream << "# Query No.=" << queryCount << endl; + stream << "# Query=" << line.substr(0, 20) + " ..." << endl; + stream << "# Index Type=" << searchParameters.indexType << endl; + stream << "# Size=" << searchParameters.size << endl; + stream << "# Radius=" << searchParameters.radius << endl; + stream << "# Epsilon*=" << epsilon << endl; + stream << "# Result Expansion*=" << resultExpansion << endl; + stream << "# Factor=" << param << endl; + stream << "# Query Time (msec)=" << timer.time * 1000.0 << endl; + stream << "# Distance Computation=" << searchQuery.distanceComputationCount << endl; + stream << "# Visit Count=" << searchQuery.visitCount << endl; + } else { + stream << "Query No." << queryCount << endl; + stream << "Rank\tID\tDistance" << endl; + } + for (size_t i = 0; i < objects.size(); i++) { + stream << i + 1 << "\t" << objects[i].id << "\t"; + stream << objects[i].distance << endl; + } + if (searchParameters.outputMode[0] == 'e') { + stream << "# End of Search" << endl; + } else { + stream << "Query Time= " << timer.time << " (sec), " << timer.time * 1000.0 << " (msec)" << endl; + } + } + if (searchParameters.outputMode[0] == 'e') { + stream << "# End of Query" << endl; + } + } + if (searchParameters.outputMode[0] == 'e') { + stream << "# Average Query Time (msec)=" << totalTime * 1000.0 / (double)queryCount << endl; + stream << "# Number of queries=" << queryCount << endl; + stream << "# End of Evaluation" << endl; + } else { + stream << "Average Query Time= " << totalTime / (double)queryCount << " (sec), " + << totalTime * 1000.0 / (double)queryCount << " (msec), (" + << totalTime << "/" << queryCount << ")" << endl; + } +} + +void +QBG::CLI::searchQG(NGT::Args &args) { + const string usage = "Usage: ngtqg search-qg [-i index-type(g|t|s)] [-n result-size] [-e epsilon] [-E edge-size] " + "[-o output-mode] [-p result-expansion] index(input) query.tsv(input)"; + + string indexPath; + try { + indexPath = args.get("#1"); + } catch (...) { + cerr << "An index is not specified." << endl; + cerr << usage << endl; + return; + } + + SearchParameters searchParameters(args); + + bool readOnly = true; + NGTQG::Index index(indexPath, 128, readOnly); + + if (debugLevel >= 1) { + cerr << "indexType=" << searchParameters.indexType << endl; + cerr << "size=" << searchParameters.size << endl; + cerr << "edgeSize=" << searchParameters.edgeSize << endl; + cerr << "epsilon=" << searchParameters.beginOfEpsilon << "<->" << searchParameters.endOfEpsilon << "," + << searchParameters.stepOfEpsilon << endl; + } + + try { + ::searchQG(index, searchParameters, std::cout); + } catch (NGT::Exception &err) { + cerr << "qbg: Error " << err.what() << endl; + cerr << usage << endl; + } catch (std::exception &err) { + cerr << "qbg: Error " << err.what() << endl; + cerr << usage << endl; + } catch (...) { + cerr << "qbg: Error" << endl; + cerr << usage << endl; + } + +} + + +void +QBG::CLI::createQG(NGT::Args &args) +{ + const std::string usage = "Usage: qbg create-qbg [-Q dimension-of-subvector] index"; + string indexPath; + try { + indexPath = args.get("#1"); + } catch (...) { + cerr << "An index is not specified." << endl; + cerr << usage << endl; + return; + } + char mode = args.getChar("m", '-'); + size_t dimensionOfSubvector = args.getl("Q", 0); + size_t seudoDimension = args.getl("P", 0); + bool silence = !args.getBool("v"); + + std::cerr << "creating..." << std::endl; + NGTQG::Index::create(indexPath, dimensionOfSubvector, seudoDimension); + + if (mode == '-') { + std::cerr << "appending..." << std::endl; + QBG::Index::appendFromObjectRepository(indexPath, indexPath + "/qg", silence); + } +} + +void +QBG::CLI::appendQG(NGT::Args &args) +{ + const std::string usage = "Usage: qbg append-qbg ngt-index"; + string indexPath; + try { + indexPath = args.get("#1"); + } catch (...) { + cerr << "An index is not specified." << endl; + cerr << usage << endl; + return; + } + QBG::Index::appendFromObjectRepository(indexPath, indexPath + "/qg", false); +} + + +void +QBG::CLI::quantizeQG(NGT::Args &args) +{ +#ifdef NGTQ_QBG + const std::string usage = "Usage: ngtqg quantize [-E max-number-of-edges] index"; +#else + const std::string usage = "Usage: ngtqg quantize [-Q dimension-of-subvector] [-E max-number-of-edges] index"; +#endif + string indexPath; + try { + indexPath = args.get("#1"); + } catch (...) { + cerr << "An index is not specified." << endl; + cerr << usage << endl; + return; + } + size_t maxNumOfEdges = args.getl("E", 128); + +#ifdef NGTQ_QBG + NGTQG::Index::quantize(indexPath, maxNumOfEdges); +#else + size_t dimensionOfSubvector = args.getl("Q", 0); + NGTQG::Index::quantize(indexPath, dimensionOfSubvector, maxNumOfEdges); +#endif +} + +void +QBG::CLI::create(NGT::Args &args) +{ + const string usage = "Usage: qbg create " + " -d dimension [-o object-type (f:float|c:unsigned char)] [-D distance-function] [-n data-size] " + "[-p #-of-thread] [-R global-codebook-range] [-r local-codebook-range] " + "[-C global-codebook-size-limit] [-c local-codebook-size-limit] [-N local-division-no] " + "[-T single-local-centroid (t|f)] [-e epsilon] [-i index-type (t:Tree|g:Graph)] " + "[-M global-centroid-creation-mode (d|s)] [-L local-centroid-creation-mode (d|k|s)] " + "[-s local-sample-coefficient] " + "index(OUT) data.tsv(IN) rotation(IN)"; + + try { + CreateParameters createParameters(args); + + cerr << "qbg: Create" << endl; +#ifdef NGTQ_QBG + std::vector r; + auto *rotation = &r; + { + try { + std::string rotationPath = args.get("#3"); + cerr << "rotation is " << rotationPath << "." << endl; + std::ifstream stream(rotationPath); + if (!stream) { + std::cerr << "Cannot open the rotation. " << rotationPath << std::endl; + cerr << usage << endl; + return; + } + std::string line; + while (getline(stream, line)) { + std::vector tokens; + NGT::Common::tokenize(line, tokens, " \t"); + for (auto &token : tokens) { + r.push_back(NGT::Common::strtof(token)); + } + } + } catch (...) { + rotation = 0; + } + std::cerr << "rotation matrix size=" << r.size() << std::endl; + } + + QBG::Index::create(createParameters.index, createParameters.property, createParameters.globalProperty, + createParameters.localProperty, rotation, createParameters.objectPath); +#else + QBG::Index::create(createParameters.index, createParameters.property, createParameters.globalProperty, + createParameters.localProperty); +#endif + +#ifndef NGTQ_QBG + if (!createParameters.objectPath.empty()) { + cerr << "qbg: Append" << endl; + QBG::Index::append(createParameters.index, createParameters.objectPath, createParameters.numOfObjects); + } +#endif + } catch(NGT::Exception &err) { + std::cerr << err.what() << std::endl; + cerr << usage << endl; + } +} + + +void +QBG::CLI::load(NGT::Args &args) +{ + const string usage = "Usage: qbg load "; + + int threadSize = args.getl("p", 50); + + std::string indexPath; + try { + indexPath = args.get("#1"); + } catch (...) { + std::cerr << "Not specified the index." << std::endl; + std::cerr << usage << std::endl; + return; + } + + std::cerr << "qbg: loading the specified blobs..." << std::endl; + std::string blobs; + try { + blobs = args.get("#2"); + } catch(...) {} + + std::string localCodebooks; + try { + localCodebooks = args.get("#3"); + } catch (...) {} + + std::string rotationPath; + try { + rotationPath = args.get("#4"); + } catch (...) {} + cerr << "rotation is " << rotationPath << "." << endl; + + QBG::Index::load(indexPath, blobs, localCodebooks, rotationPath, threadSize); +} + +void +QBG::CLI::search(NGT::Args &args) +{ + + const string usage = "Usage: qbg search [-i g|t|s] [-n result-size] [-e epsilon] [-m mode(r|l|c|a)] " + "[-E edge-size] [-o output-mode] [-b result expansion(begin:end:[x]step)] " + "index(input) query.tsv(input)"; + string indexPath; + try { + indexPath = args.get("#1"); + } catch (...) { + cerr << "DB is not specified" << endl; + cerr << usage << endl; + return; + } + + string query; + try { + query = args.get("#2"); + } catch (...) { + cerr << "Query is not specified" << endl; + cerr << usage << endl; + return; + } + + size_t size = args.getl("n", 20); + char outputMode = args.getChar("o", '-'); + float epsilon = 0.1; + + char searchMode = args.getChar("M", 'g'); + if (args.getString("e", "none") == "-") { + // linear search + epsilon = FLT_MAX; + } else { + epsilon = args.getf("e", 0.1); + } + float blobEpsilon = args.getf("B", 0.0); + size_t edgeSize = args.getl("E", 0); + float cutback = args.getf("C", 0.0); + size_t explorationSize = args.getf("N", 256); + size_t nOfProbes = args.getl("P", 10); + + float beginOfResultExpansion, endOfResultExpansion, stepOfResultExpansion; + bool mulStep = false; + { + beginOfResultExpansion = 0.0; + endOfResultExpansion = 0.0; + stepOfResultExpansion = 1; + string str = args.getString("p", "0.0"); + vector tokens; + NGT::Common::tokenize(str, tokens, ":"); + if (tokens.size() >= 1) { + beginOfResultExpansion = NGT::Common::strtod(tokens[0]); + endOfResultExpansion = beginOfResultExpansion; + } + if (tokens.size() >= 2) { endOfResultExpansion = NGT::Common::strtod(tokens[1]); } + if (tokens.size() >= 3) { + if (tokens[2][0] == 'x') { + mulStep = true; + stepOfResultExpansion = NGT::Common::strtod(tokens[2].substr(1)); + } else { + stepOfResultExpansion = NGT::Common::strtod(tokens[2]); + } + } + } + if (debugLevel >= 1) { + cerr << "size=" << size << endl; + cerr << "result expansion=" << beginOfResultExpansion << "->" << endOfResultExpansion << "," << stepOfResultExpansion << endl; + } + + QBG::Index index(indexPath, true); + std::cerr << "qbg::The index is open." << std::endl; + std::cerr << " vmsize==" << NGT::Common::getProcessVmSizeStr() << std::endl; + std::cerr << " peak vmsize==" << NGT::Common::getProcessVmPeakStr() << std::endl; + auto dimension = index.getQuantizer().globalCodebookIndex.getObjectSpace().getDimension(); + try { + ifstream is(query); + if (!is) { + cerr << "Cannot open the specified file. " << query << endl; + return; + } + if (outputMode == 's') { cout << "# Beginning of Evaluation" << endl; } + string line; + double totalTime = 0; + int queryCount = 0; + while(getline(is, line)) { + vector queryVector; + stringstream linestream(line); + while (!linestream.eof()) { + float value; + linestream >> value; + queryVector.push_back(value); + } + queryVector.resize(dimension); + queryCount++; + for (auto resultExpansion = beginOfResultExpansion; + resultExpansion <= endOfResultExpansion; + resultExpansion = mulStep ? resultExpansion * stepOfResultExpansion : + resultExpansion + stepOfResultExpansion) { + NGT::ObjectDistances objects; + NGT::Timer timer; + timer.start(); + QBG::SearchContainer searchContainer; + auto query = queryVector; + searchContainer.setObjectVector(query); + searchContainer.setResults(&objects); + if (resultExpansion >= 1.0) { + searchContainer.setSize(static_cast(size) * resultExpansion); + searchContainer.setExactResultSize(size); + } else { + searchContainer.setSize(size); + searchContainer.setExactResultSize(0); + } + searchContainer.setEpsilon(epsilon); + searchContainer.setBlobEpsilon(blobEpsilon); + searchContainer.setEdgeSize(edgeSize); + searchContainer.setCutback(cutback); + searchContainer.setGraphExplorationSize(explorationSize); + searchContainer.setNumOfProbes(nOfProbes); + switch (searchMode) { + case 'b': + index.searchBlobGraphNaively(searchContainer); + break; + case 'n': + index.searchBlobNaively(searchContainer); + break; + case 'g': + default: + index.searchBlobGraph(searchContainer); + break; + } + if (objects.size() > size) { + objects.resize(size); + } + timer.stop(); + totalTime += timer.time; + if (outputMode == 'e') { + cout << "# Query No.=" << queryCount << endl; + cout << "# Query=" << line.substr(0, 20) + " ..." << endl; + cout << "# Index Type=" << "----" << endl; + cout << "# Size=" << size << endl; + cout << "# Epsilon=" << epsilon << endl; + cout << "# Result expansion=" << resultExpansion << endl; + cout << "# Distance Computation=" << index.getQuantizer().distanceComputationCount << endl; + cout << "# Query Time (msec)=" << timer.time * 1000.0 << endl; + } else { + cout << "Query No." << queryCount << endl; + cout << "Rank\tIN-ID\tID\tDistance" << endl; + } + + for (size_t i = 0; i < objects.size(); i++) { + cout << i + 1 << "\t" << objects[i].id << "\t"; + cout << objects[i].distance << endl; + } + + if (outputMode == 'e') { + cout << "# End of Search" << endl; + } else { + cout << "Query Time= " << timer.time << " (sec), " << timer.time * 1000.0 << " (msec)" << endl; + } + } + if (outputMode == 'e') { + cout << "# End of Query" << endl; + } + } + if (outputMode == 'e') { + cout << "# Average Query Time (msec)=" << totalTime * 1000.0 / (double)queryCount << endl; + cout << "# Number of queries=" << queryCount << endl; + cout << "# End of Evaluation" << endl; + } else { + cout << "Average Query Time= " << totalTime / (double)queryCount << " (sec), " + << totalTime * 1000.0 / (double)queryCount << " (msec), (" + << totalTime << "/" << queryCount << ")" << endl; + } + } catch (NGT::Exception &err) { + cerr << "Error " << err.what() << endl; + cerr << usage << endl; + } catch (...) { + cerr << "Error" << endl; + cerr << usage << endl; + } + std::cerr << "qbg::The end of search" << std::endl; + std::cerr << " vmsize==" << NGT::Common::getProcessVmSizeStr() << std::endl; + std::cerr << " peak vmsize==" << NGT::Common::getProcessVmPeakStr() << std::endl; + index.close(); +} + + +void +QBG::CLI::append(NGT::Args &args) +{ + const string usage = "Usage: qbg append [-n data-size] index(output) data.tsv(input)"; + string index; + try { + index = args.get("#1"); + } catch (...) { + cerr << "DB is not specified." << endl; + cerr << usage << endl; + return; + } + string data; + try { + data = args.get("#2"); + } catch (...) { + cerr << "Data is not specified." << endl; + } + + size_t dataSize = args.getl("n", 0); + char mode = args.getChar("m", '-'); + + if (mode == 'b') { + QBG::Index::appendBinary(index, data, dataSize); + } else { + QBG::Index::append(index, data, dataSize); + } +} + +static void hierarchicalKmeansParameters(NGT::Args &args, QBG::HierarchicalKmeans &hierarchicalKmeans) +{ + hierarchicalKmeans.maxSize = args.getl("r", 1000); + hierarchicalKmeans.numOfObjects = args.getl("O", 0); + hierarchicalKmeans.numOfClusters = args.getl("E", 2); + try { + hierarchicalKmeans.numOfTotalClusters = args.getl("C", 0); + } catch (...) { + hierarchicalKmeans.numOfTotalClusters = 0; + } + hierarchicalKmeans.numOfTotalBlobs = args.getl("b", 0); + hierarchicalKmeans.clusterID = args.getl("c", -1); + + char iMode = args.getChar("i", '-'); + hierarchicalKmeans.initMode = NGT::Clustering::InitializationModeKmeansPlusPlus; + switch (iMode) { + case 'l': + case 'h': hierarchicalKmeans.initMode = NGT::Clustering::InitializationModeHead; break; + case 'r': hierarchicalKmeans.initMode = NGT::Clustering::InitializationModeRandom; break; + case 'R': hierarchicalKmeans.initMode = NGT::Clustering::InitializationModeRandomFixedSeed; break; + case 'P': hierarchicalKmeans.initMode = NGT::Clustering::InitializationModeKmeansPlusPlusFixedSeed; break; + default: + case '-': + case 'p': hierarchicalKmeans.initMode = NGT::Clustering::InitializationModeKmeansPlusPlus; break; + } + + hierarchicalKmeans.numOfRandomObjects = args.getl("r", 0); + char rmode = args.getChar("R", '-'); + if (rmode == 'c') { + hierarchicalKmeans.extractCentroid = true; + } else { + hierarchicalKmeans.extractCentroid = false; + } + + hierarchicalKmeans.numOfFirstObjects = 0; + hierarchicalKmeans.numOfFirstClusters = 0; + hierarchicalKmeans.numOfSecondObjects = 0; + hierarchicalKmeans.numOfSecondClusters = 0; + hierarchicalKmeans.numOfThirdClusters = 0; + + hierarchicalKmeans.threeLayerClustering = true; + + std::string blob = args.getString("B", ""); + if (blob == "-") { + hierarchicalKmeans.threeLayerClustering = false; + } else { + hierarchicalKmeans.threeLayerClustering = true; + } + if (hierarchicalKmeans.threeLayerClustering) { + std::vector tokens; + NGT::Common::tokenize(blob, tokens, ","); + if (tokens.size() > 0) { + std::vector ftokens; + NGT::Common::tokenize(tokens[0], ftokens, ":"); + if (ftokens.size() >= 1) { + hierarchicalKmeans.numOfFirstObjects = NGT::Common::strtof(ftokens[0]); + } + if (ftokens.size() >= 2) { + hierarchicalKmeans.numOfFirstClusters = NGT::Common::strtof(ftokens[1]); + } + } + if (tokens.size() > 1) { + std::vector ftokens; + NGT::Common::tokenize(tokens[1], ftokens, ":"); + if (ftokens.size() >= 1) { + hierarchicalKmeans.numOfSecondObjects = NGT::Common::strtof(ftokens[0]); + } + if (ftokens.size() >= 2) { + hierarchicalKmeans.numOfSecondClusters = NGT::Common::strtof(ftokens[1]); + } + } + if (tokens.size() > 2) { + std::vector ftokens; + NGT::Common::tokenize(tokens[2], ftokens, ":"); + if (ftokens.size() >= 1) { + hierarchicalKmeans.numOfObjects = NGT::Common::strtof(ftokens[0]); + } + if (ftokens.size() >= 2) { + hierarchicalKmeans.numOfThirdClusters = NGT::Common::strtof(ftokens[1]); + } + } + std::cerr << "blob param=:" << std::endl; + std::cerr << "numOfFirstObjects=" << hierarchicalKmeans.numOfFirstObjects << std::endl; + std::cerr << "numOfFirstClusters=" << hierarchicalKmeans.numOfFirstClusters << std::endl; + std::cerr << "numOfSecondClusters=" << hierarchicalKmeans.numOfSecondClusters << std::endl; + std::cerr << "numOfSecondObjects=" << hierarchicalKmeans.numOfSecondObjects << std::endl; + std::cerr << "numOfThirdClusters=" << hierarchicalKmeans.numOfThirdClusters << std::endl; + std::cerr << "numOfThirdObjects=" << hierarchicalKmeans.numOfObjects << std::endl; + } + +} + +void +QBG::CLI::buildIndex(NGT::Args &args) +{ + const std::string usage = "Usage: qbg build-index [-Q dimension-of-subvector] [-E max-number-of-edges] index"; + string indexPath; + try { + indexPath = args.get("#1"); + } catch (...) { + cerr << "An index is not specified." << endl; + cerr << usage << endl; + return; + } + char mode = args.getChar("m", '-'); + + size_t beginID = args.getl("s", 1); + size_t size = args.getl("n", 0); + size_t endID = beginID + size - 1; + + std::vector> quantizerCodebook; + std::vector codebookIndex; + std::vector objectIndex; + + if (mode == 'q' || mode == '-') { + { + try { + std::string codebookPath; + try { + codebookPath = args.get("#2"); + } catch(...) { + codebookPath = indexPath + "/ws/kmeans-cluster_qcentroid.tsv"; + } + std::ifstream stream(codebookPath); + if (!stream) { + std::cerr << "Cannot open the codebook. " << codebookPath << std::endl; + cerr << usage << endl; + return; + } + std::string line; + while (getline(stream, line)) { + std::vector tokens; + NGT::Common::tokenize(line, tokens, " \t"); + std::vector object; + for (auto &token : tokens) { + object.push_back(NGT::Common::strtof(token)); + } + if (!quantizerCodebook.empty() && quantizerCodebook[0].size() != object.size()) { + cerr << "The specified quantizer codebook is invalid. " << quantizerCodebook[0].size() + << ":" << object.size() << ":" << quantizerCodebook.size() << ":" << line << endl; + cerr << usage << endl; + return; + } + if (!object.empty()) { + quantizerCodebook.push_back(object); + } + } + } catch (...) {} + } + + { + try { + std::string codebookIndexPath; + try { + codebookIndexPath = args.get("#3"); + } catch (...) { + codebookIndexPath = indexPath + "/ws/kmeans-cluster_bqindex.tsv"; + } + cerr << "codebook index is " << codebookIndexPath << "." << endl; + std::ifstream stream(codebookIndexPath); + if (!stream) { + std::cerr << "Cannot open the codebook index. " << codebookIndexPath << std::endl; + cerr << usage << endl; + return; + } + std::string line; + while (getline(stream, line)) { + std::vector tokens; + NGT::Common::tokenize(line, tokens, " \t"); + std::vector object; + if (tokens.size() != 1) { + cerr << "The specified codebook index is invalid. " << line << std::endl; + cerr << usage << endl; + return; + } + codebookIndex.push_back(NGT::Common::strtol(tokens[0])); + } + + } catch (...) {} + } + + { + try { + std::string objectIndexPath; + try { + objectIndexPath = args.get("#4"); + } catch (...) { + objectIndexPath = indexPath + "/ws/kmeans-cluster_index.tsv"; + } + std::ifstream stream(objectIndexPath); + if (!stream) { + std::cerr << "Cannot open the codebook index. " << objectIndexPath << std::endl; + cerr << usage << endl; + return; + } + std::string line; + while (getline(stream, line)) { + std::vector tokens; + NGT::Common::tokenize(line, tokens, " \t"); + std::vector object; + if (tokens.size() != 1) { + cerr << "The specified codebook index is invalid. " << line << std::endl; + cerr << usage << endl; + return; + } + objectIndex.push_back(NGT::Common::strtol(tokens[0])); + } + + } catch (...) {} + } + + std::cerr << "quantizer codebook size=" << quantizerCodebook.size() << std::endl; + std::cerr << "codebook index size=" << codebookIndex.size() << std::endl; + std::cerr << "object index size=" << objectIndex.size() << std::endl; + + if (mode == 'q') { + QBG::Index::buildNGTQ(indexPath, quantizerCodebook, codebookIndex, objectIndex, beginID, endID); + return; + } + } + + if (mode == 'g') { + QBG::Index::buildQBG(indexPath); + return; + } + + QBG::Index::build(indexPath, quantizerCodebook, codebookIndex, objectIndex, beginID, endID); + +} + +void +QBG::CLI::build(NGT::Args &args) +{ + const std::string usage = "Usage: qbg build [-Q dimension-of-subvector] [-E max-number-of-edges] index"; + string indexPath; + try { + indexPath = args.get("#1"); + } catch (...) { + cerr << "An index is not specified." << endl; + cerr << usage << endl; + return; + } + bool silence = !args.getBool("v"); + + HierarchicalKmeans hierarchicalKmeans; + + try { + hierarchicalKmeansParameters(args, hierarchicalKmeans); + } catch (NGT::Exception &err) { + cerr << "Error " << err.what() << endl; + cerr << usage << endl; + return; + } + + hierarchicalKmeans.clustering(indexPath); + + NGTQ::Optimizer optimizer; + + try { + optimizationParameters(args, optimizer); + } catch(NGT::Exception &err) { + std::cerr << err.what() << std::endl; + cerr << usage << endl; + } + + std::cerr << "optimizing..." << std::endl; + bool random = true; + optimizer.silence = silence; + optimizer.optimize(indexPath, random); + + std::cerr << "building..." << std::endl; + QBG::Index::build(indexPath, silence); +} + + + +void +QBG::CLI::hierarchicalKmeans(NGT::Args &args) +{ + const std::string usage = "qbg kmeans -O #-of-objects -B x1:y1,x2,y2,x3 index [prefix] [object-ID-file]"; + std::string indexPath; + + try { + indexPath = args.get("#1"); + } catch (...) { + cerr << "Index is not specified" << endl; + cerr << usage << endl; + return; + } + + std::string prefix; + try { + prefix = args.get("#2"); + std::cerr << "prefix=" << prefix << std::endl; + } catch (...) { + std::cerr << "Prefix is not specified." << std::endl; + } + + std::string objectIDsFile; + try { + objectIDsFile = args.get("#3"); + std::cerr << "object IDs=" << objectIDsFile << std::endl; + } catch (...) { + cerr << "Object ID file is not specified" << endl; + } + + HierarchicalKmeans hierarchicalKmeans; + + try { + hierarchicalKmeansParameters(args, hierarchicalKmeans); + } catch (NGT::Exception &err) { + cerr << "Error " << err.what() << endl; + cerr << usage << endl; + return; + } + + hierarchicalKmeans.clustering(indexPath, prefix, objectIDsFile); +} + +void +QBG::CLI::assign(NGT::Args &args) +{ + const std::string usage = "qbg assign"; + std::string indexPath; + + try { + indexPath = args.get("#1"); + } catch (...) { + cerr << "Any index is not specified" << endl; + cerr << usage << endl; + return; + } + + std::string queryPath; + + try { + queryPath = args.get("#2"); + } catch (...) { + cerr << "Any query is not specified" << endl; + cerr << usage << endl; + return; + } + + auto epsilon = args.getf("e", 0.1); + auto numOfObjects = args.getl("n", 10); + auto mode = args.getChar("m", '-'); + + try { + NGT::Index index(indexPath); + NGT::Property property; + index.getProperty(property); + ifstream is(queryPath); + if (!is) { + std::cerr << "Cannot open the query file. " << queryPath << std::endl; + return; + } + string line; + while (getline(is, line)) { + vector query; + { + stringstream linestream(line); + while (!linestream.eof()) { + float value; + linestream >> value; + query.push_back(value); + } + if (static_cast(property.dimension) != query.size()) { + std::cerr << "Dimension is invallid. " << property.dimension << ":" << query.size() << std::endl; + return; + } + } + NGT::SearchQuery sc(query); + NGT::ObjectDistances objects; + sc.setResults(&objects); + sc.setSize(numOfObjects); + sc.setEpsilon(epsilon); + + index.search(sc); + if (objects.size() == 0) { + std::cerr << "The result is empty. Something wrong." << std::endl; + return; + } + if (mode == '-') { + std::cout << objects[0].id - 1 << std::endl; + } else { + std::cout << objects[0].id << std::endl; + } + } + } catch (NGT::Exception &err) { + cerr << "Error " << err.what() << endl; + return; + } catch (...) { + cerr << "Error" << endl; + return; + } +} + +void +QBG::CLI::extract(NGT::Args &args) +{ + + const string usage = "Usage: qbg extract binary-file|index [output-file]"; + + std::string objectPath; + try { + objectPath = args.get("#1"); + } catch (...) { + std::cerr << "Object file is not specified." << std::endl; + std::cerr << usage << std::endl; + return; + } + + std::ostream *os; + std::ofstream ofs; + std::string outputFile; + + size_t n = args.getl("n", 100); + size_t dim = args.getl("d", 0); + std::string type = args.getString("t", "float32"); + char mode = args.getChar("m", 'r'); + + try { + QBG::Index index(objectPath, true); + + try { + outputFile = args.get("#2"); + if (outputFile == "-") { + os = &std::cout; + } else { + ofs.open(outputFile); + os = &ofs; + } + } catch (...) { + ofs.open(objectPath + "/ws/base.10000.u8.tsv"); + os = &ofs; + } + index.extract(*os, n, mode == 'r'); + } catch (NGT::Exception &err) { + try { + outputFile = args.get("#2"); + ofs.open(outputFile); + os = &ofs; + } catch (...) { + os = &std::cout; + } + try { + StaticObjectFileLoader loader(objectPath, type); + if (mode == 'r') { + struct timeval randTime; + gettimeofday(&randTime, 0); + srand(randTime.tv_usec); + for (size_t cnt = 0; cnt < n; cnt++) { + double random = ((double)rand() + 1.0) / ((double)RAND_MAX + 2.0); + size_t id = 0; + do { + id = floor(loader.noOfObjects * random); + } while (id >= loader.noOfObjects); + loader.seek(id); + auto object = loader.getObject(); + if (cnt + 1 % 100000 == 0) { + std::cerr << "loaded " << static_cast(cnt + 1) / 1000000.0 << "M objects." << std::endl; + } + if (dim != 0) { + object.resize(dim, 0.0); + } + for (auto v = object.begin(); v != object.end(); ++v) { + if (v + 1 != object.end()) { + *os << *v << "\t"; + } else { + *os << *v << std::endl;; + } + } + } + } else { + size_t cnt = 0; + while (!loader.isEmpty()) { + auto object = loader.getObject(); + cnt++; + if (cnt % 100000 == 0) { + std::cerr << "loaded " << static_cast(cnt) / 1000000.0 << "M objects." << std::endl; + } + if (dim != 0) { + object.resize(dim, 0.0); + } + for (auto v = object.begin(); v != object.end(); ++v) { + if (v + 1 != object.end()) { + *os << *v << "\t"; + } else { + *os << *v << std::endl;; + } + } + if (n > 0 && cnt >= n) { + break; + } + } + } + } catch (...) { + cerr << "Error" << endl; + return; + } + } + return; +} + +void +QBG::CLI::gt(NGT::Args &args) +{ + string path; + + try { + path = args.get("#1"); + } catch (...) { + std::cerr << "An object file is not specified." << std::endl; + return; + } + + std::ifstream stream; + stream.open(path, std::ios::in | std::ios::binary); + if (!stream) { + std::cerr << "Error!" << std::endl; + return; + } + uint32_t numQueries; + uint32_t k; + + stream.read(reinterpret_cast(&numQueries), sizeof(numQueries)); + stream.read(reinterpret_cast(&k), sizeof(k)); + std::cerr << "# of queries=" << numQueries << std::endl; + std::cerr << "k=" << k << std::endl; + + { + std::ofstream idf; + idf.open(path + "_gt.tsv"); + for (uint32_t qidx = 0; qidx < numQueries; qidx++) { + for (uint32_t rank = 0; rank < k; rank++) { + uint32_t id; + stream.read(reinterpret_cast(&id), sizeof(id)); + idf << id; + if (rank + 1 == k) { + idf << std::endl; + } else { + idf << "\t"; + } + } + } + } + + { + std::ofstream df; + df.open(path + "_gtdist.tsv"); + for (uint32_t qidx = 0; qidx < numQueries; qidx++) { + for (uint32_t rank = 0; rank < k; rank++) { + float distance; + stream.read(reinterpret_cast(&distance), sizeof(distance)); + df << distance; + if (rank + 1 == k) { + df << std::endl; + } else { + df << "\t"; + } + } + } + } + +} + +void +QBG::CLI::gtRange(NGT::Args &args) +{ + string path; + + try { + path = args.get("#1"); + } catch (...) { + std::cerr << "An object file is not specified." << std::endl; + return; + } + + std::ifstream stream; + stream.open(path, std::ios::in | std::ios::binary); + if (!stream) { + std::cerr << "Error!" << std::endl; + return; + } + uint32_t numQueries; + uint32_t totalRes; + + stream.read(reinterpret_cast(&numQueries), sizeof(numQueries)); + stream.read(reinterpret_cast(&totalRes), sizeof(totalRes)); + std::cerr << "# of queries=" << numQueries << std::endl; + std::cerr << "totalRes=" << totalRes << std::endl; + + std::vector numResultsPerQuery(numQueries); + for (size_t qidx = 0; qidx < numQueries; qidx++) { + uint32_t v; + stream.read(reinterpret_cast(&v), sizeof(v)); + //std::cerr << qidx << ":" << v << std::endl; + numResultsPerQuery[qidx] = v; + } + { + std::ofstream idf; + idf.open(path + "_gt.tsv"); + std::ofstream df; + df.open(path + "_gtdist.tsv"); + size_t count = 0; + for (size_t qidx = 0; qidx < numQueries; qidx++) { + if (numResultsPerQuery[qidx] == 0) { + idf << std::endl; + df << std::endl; + continue; + } + for (size_t rank = 0; rank < numResultsPerQuery[qidx]; rank++) { + uint32_t v; + stream.read(reinterpret_cast(&v), sizeof(v)); + idf << v; + df << 0.0; + count++; + if (rank + 1 == numResultsPerQuery[qidx]) { + idf << std::endl; + df << std::endl; + } else { + idf << "\t"; + df << "\t"; + } + } + } + if (count != totalRes) { + std::cerr << "Fatal error. " << count << ":" << totalRes << std::endl; + } + } +} + + +void +QBG::CLI::optimize(NGT::Args &args) +{ + + string usage = "Usage: qbg optimize -n number-of-clusters -m number-of subspaces [-O t|f] [-s t|f] [-I cluster-iteration] [-t R-max-iteration] [-c convergence-limit-times] vector-file [output-file-prefix]\n" + " qbg optimize -e E -n number-of-clusters -m number-of index [subspaces] [vector-file] [local-centroid-file] [global-centroid-file]"; + + std::string indexPath; + try { + indexPath = args.get("#1"); + } catch(...) { + cerr << "qbg: index is not specified." << endl; + cerr << usage << endl; + return; + } + + string invector; + try { + invector = args.get("#2"); + } catch(...) {} + + string ofile; + try { + ofile = args.get("#3"); + } catch(...) {} + + string global; + try { + global = args.get("#4"); + } catch(...) {} + + NGTQ::Optimizer optimizer; + + try { + optimizationParameters(args, optimizer); + } catch(NGT::Exception &err) { + std::cerr << err.what() << std::endl; + cerr << usage << endl; + } + + if (optimizer.numberOfClusters == 0) { + cerr << "-n adequate number-of-clusters should be specified." << endl; + cerr << usage << endl; + return; + }; + + if (optimizer.numberOfSubvectors == 0) { + cerr << "-m adequate number-of-subspaces should be specified." << endl; + cerr << usage << endl; + return; + }; + + if (invector.empty() || ofile.empty() || global.empty()) { + optimizer.optimize(indexPath); + } else { + optimizer.optimize(invector, ofile, global); + } + + return; + +} + +#endif diff --git a/lib/NGT/NGTQ/QbgCli.h b/lib/NGT/NGTQ/QbgCli.h new file mode 100644 index 0000000..1d849cb --- /dev/null +++ b/lib/NGT/NGTQ/QbgCli.h @@ -0,0 +1,136 @@ +// +// Copyright (C) 2021 Yahoo Japan Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#pragma once + +#include "NGT/NGTQ/QuantizedBlobGraph.h" +#include "NGT/Command.h" + +namespace QBG { + + class CLI { + public: + + int debugLevel; + +#if !defined(NGTQ_QBG) || defined(NGTQ_SHARED_INVERTED_INDEX) + void create(NGT::Args &args) {}; + void load(NGT::Args &args) {}; + void append(NGT::Args &args) {}; + void buildIndex(NGT::Args &args) {}; + void hierarchicalKmeans(NGT::Args &args) {}; + void search(NGT::Args &args) {}; + void assign(NGT::Args &args) {}; + void extract(NGT::Args &args) {}; + void gt(NGT::Args &args) {}; + void gtRange(NGT::Args &args) {}; + void optimize(NGT::Args &args) {}; + void build(NGT::Args &args) {}; + void quantizeQG(NGT::Args &args) {}; + void createQG(NGT::Args &args) {}; + void buildQG(NGT::Args &args) {}; + void appendQG(NGT::Args &args) {}; + void searchQG(NGT::Args &args) {}; + void info(NGT::Args &args) {}; +#else + void create(NGT::Args &args); + void load(NGT::Args &args); + void append(NGT::Args &args); + void buildIndex(NGT::Args &args); + void hierarchicalKmeans(NGT::Args &args); + void search(NGT::Args &args); + void assign(NGT::Args &args); + void extract(NGT::Args &args); + void gt(NGT::Args &args); + void gtRange(NGT::Args &args); + void optimize(NGT::Args &args); + void build(NGT::Args &args); + void quantizeQG(NGT::Args &args); + void createQG(NGT::Args &args); + void buildQG(NGT::Args &args); + void appendQG(NGT::Args &args); + void searchQG(NGT::Args &args); + void info(NGT::Args &args); +#endif + + void setDebugLevel(int level) { debugLevel = level; } + int getDebugLevel() { return debugLevel; } + + void help() { + cerr << "Usage : qbg command database [data]" << endl; + cerr << " command : create build quantize search" << endl; + } + + void execute(NGT::Args args) { + string command; + try { + command = args.get("#0"); + } catch(...) { + help(); + return; + } + + debugLevel = args.getl("X", 0); + + try { + if (debugLevel >= 1) { + cerr << "ngt::command=" << command << endl; + } + if (command == "search") { + search(args); + } else if (command == "create") { + create(args); + } else if (command == "load") { + load(args); + } else if (command == "append") { + append(args); + } else if (command == "build-index") { + buildIndex(args); + } else if (command == "kmeans") { + hierarchicalKmeans(args); + } else if (command == "assign") { + assign(args); + } else if (command == "extract") { + extract(args); + } else if (command == "gt") { + gt(args); + } else if (command == "gt-range") { + gtRange(args); + } else if (command == "optimize") { + optimize(args); + } else if (command == "build") { + build(args); + } else if (command == "create-qg") { + createQG(args); + } else if (command == "quantize-qg") { + quantizeQG(args); + } else if (command == "build-qg") { + buildQG(args); + } else if (command == "append-qg") { + appendQG(args); + } else if (command == "search-qg") { + searchQG(args); + } else { + cerr << "Illegal command. " << command << endl; + } + } catch(NGT::Exception &err) { + cerr << "qbg: Fatal error: " << err.what() << endl; + } + } + + }; + +}; // NGTQBG diff --git a/lib/NGT/NGTQ/QuantizedBlobGraph.h b/lib/NGT/NGTQ/QuantizedBlobGraph.h new file mode 100644 index 0000000..27151b9 --- /dev/null +++ b/lib/NGT/NGTQ/QuantizedBlobGraph.h @@ -0,0 +1,1111 @@ +// +// Copyright (C) 2021 Yahoo Japan Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#pragma once + +#include "NGT/Index.h" +#include "NGT/NGTQ/Quantizer.h" + +#ifdef NGTQ_QBG +#include "NGT/NGTQ/QuantizedGraph.h" + +#include + + +namespace QBG { + class SearchContainer : public NGT::SearchContainer { + public: + SearchContainer(NGT::Object &q): NGT::SearchContainer(q), + cutback(0.0), graphExplorationSize(50), exactResultSize(0), + blobExplorationCoefficient(0.0), numOfProbes(0) {} + SearchContainer(): NGT::SearchContainer(*reinterpret_cast(0)), + cutback(0.0), graphExplorationSize(50), exactResultSize(0), + blobExplorationCoefficient(0.0), numOfProbes(0) {} + SearchContainer(SearchContainer &sc, NGT::Object &q): NGT::SearchContainer(q) { + QBG::SearchContainer::operator=(sc); + } + SearchContainer &operator=(SearchContainer &sc) { + NGT::SearchContainer::operator=(sc); + cutback = sc.cutback; + graphExplorationSize = sc.graphExplorationSize; + exactResultSize = sc.exactResultSize; + blobExplorationCoefficient = sc.blobExplorationCoefficient; + numOfProbes = sc.numOfProbes; + objectVector = sc.objectVector; + return *this; + } + void setCutback(float c) { cutback = c; } + void setGraphExplorationSize(size_t size) { graphExplorationSize = size; } + void setExactResultSize(size_t esize) { exactResultSize = esize; } + void setBlobEpsilon(float c) { blobExplorationCoefficient = c + 1.0; } + void setNumOfProbes(size_t p) { numOfProbes = p; } + void setObjectVector(std::vector &query) { objectVector = std::move(query); } + float cutback; + size_t graphExplorationSize; + size_t exactResultSize; + float blobExplorationCoefficient; + size_t numOfProbes; + std::vector objectVector; + }; + + class QuantizedBlobGraphRepository : public NGTQG::QuantizedGraphRepository { + public: + QuantizedBlobGraphRepository(NGTQ::Index &quantizedIndex): NGTQG::QuantizedGraphRepository(quantizedIndex){ + } + + void construct(NGTQ::Index &quantizedIndex) { +#if defined(NGT_SHARED_MEMORY_ALLOCATOR) + std::cerr << "construct: Not implemented" << std::endl; + abort(); +#else + + (*this).resize(quantizedIndex.getGlobalCodebookSize()); + NGT::Timer timer; + timer.start(); + for (size_t gid = 1; gid < quantizedIndex.getGlobalCodebookSize(); gid++) { + if (gid % 100000 == 0) { + timer.stop(); + std::cerr << "The number of processed blobs=" << gid << " VmSize=" << NGT::Common::getProcessVmSizeStr() << " Elapsed time=" << timer << std::endl; + timer.restart(); + } + NGTQ::InvertedIndexEntry invertedIndexObjects(numOfSubspaces); + quantizedIndex.getQuantizer().extractInvertedIndexObject(invertedIndexObjects, gid); + quantizedIndex.getQuantizer().eraseInvertedIndexObject(gid); + NGTQ::QuantizedObjectProcessingStream quantizedStream(quantizedIndex.getQuantizer(), invertedIndexObjects.size()); + (*this)[gid].ids.reserve(invertedIndexObjects.size()); + for (size_t oidx = 0; oidx < invertedIndexObjects.size(); oidx++) { + (*this)[gid].ids.push_back(invertedIndexObjects[oidx].id); + for (size_t idx = 0; idx < numOfSubspaces; idx++) { +#ifdef NGTQ_UINT8_LUT +#ifdef NGTQ_SIMD_BLOCK_SIZE + size_t dataNo = oidx; +#if defined(NGT_SHARED_MEMORY_ALLOCATOR) + abort(); +#else + quantizedStream.arrangeQuantizedObject(dataNo, idx, invertedIndexObjects[oidx].localID[idx] - 1); +#endif +#else + objectData[idx * noobjs + dataNo] = invertedIndexObjects[oidx].localID[idx] - 1; +#endif +#else + objectData[idx * noobjs + dataNo] = invertedIndexObjects[oidx].localID[idx]; +#endif + } + } + + (*this)[gid].subspaceID = invertedIndexObjects.subspaceID; + (*this)[gid].objects = quantizedStream.compressIntoUint4(); + } +#endif + } + + }; + + class Index : public NGTQ::Index { + public: + Index(const std::string &indexPath, bool readOnly = false, bool silence = false) : + NGTQ::Index(indexPath, readOnly), path(indexPath), quantizedBlobGraph(*this) { + NGT::StdOstreamRedirector redirector(silence); + redirector.begin(); + try { + load(); + } catch (NGT::Exception &err) { + if (readOnly) { + } else { + quantizedBlobGraph.construct(*this); + } + } + redirector.end(); + } + + ~Index() {} + + bool &getSilence() { return silence; }; + + NGT::Object *allocateObject(std::vector &objectVector) { + auto &globalIndex = getQuantizer().globalCodebookIndex; + auto dim = getQuantizer().property.dimension; + objectVector.resize(dim, 0); + return globalIndex.allocateObject(objectVector); + } + + static void create(const std::string &index, NGTQ::Property &property, + NGT::Property &globalProperty, +#ifdef NGTQ_QBG + NGT::Property &localProperty, + std::vector *rotation, + const std::string &objectFile) { +#else + NGT::Property &localProperty) { +#endif + property.quantizerType = NGTQ::QuantizerTypeQBG; +#ifdef NGTQ_QBG + NGTQ::Index::create(index, property, globalProperty, localProperty, rotation, objectFile); + try { + NGT::Index::mkdir(index + getWorkspaceName()); + } catch(...) {} +#else + NGTQ::Index::create(index, property, globalProperty, localProperty); +#endif + } + + static void load(const std::string &indexPath, const std::vector &rotation) { + NGTQ::Index index(indexPath); + index.getQuantizer().saveRotation(rotation); + } + + void insert(const size_t id, std::vector &object) { + getQuantizer().objectList.put(id, object, &getQuantizer().globalCodebookIndex.getObjectSpace()); + } + + NGT::ObjectID append(std::vector &object) { + NGT::ObjectID id = getQuantizer().objectList.size(); + id = id == 0 ? 1 : id; + std::cerr << "ID=" << getQuantizer().objectList.size(); + getQuantizer().objectList.put(id, object, &getQuantizer().globalCodebookIndex.getObjectSpace()); + std::cerr << ":" << getQuantizer().objectList.size() << std::endl; + return id; + } + + static void append(const std::string &indexName, // index file + const std::string &data, // data file + size_t dataSize = 0, // data size + bool silence = false + ) { + NGT::StdOstreamRedirector redirector(silence); + redirector.begin(); + QBG::Index index(indexName); + istream *is; + if (data == "-") { + is = &cin; + } else { + ifstream *ifs = new ifstream; + ifs->ifstream::open(data); + if (!(*ifs)) { + cerr << "Cannot open the specified file. " << data << endl; + return; + } + is = ifs; + } + string line; + vector > objects; + size_t count = 0; + // extract objects from the file and insert them to the object list. + while(getline(*is, line)) { + count++; + std::vector object; + NGT::Common::extractVector(line, " ,\t", object); + if (object.empty()) { + cerr << "An empty line or invalid value: " << line << endl; + continue; + } + index.insert(count, object); + + if (count % 100000 == 0) { + cerr << "Processed " << count; + cerr << endl; + } + } + if (data != "-") { + delete is; + } + + index.save(); + index.close(); + redirector.end(); + } + + static void appendBinary(const std::string &indexName, // index file + const std::string &data, // data file + size_t dataSize = 0, // data size + bool silence = false + ) { + NGT::StdOstreamRedirector redirector(silence); + redirector.begin(); + QBG::Index index(indexName); + std::vector tokens; + NGT::Common::tokenize(data, tokens, "."); + if (tokens.size() < 2) { + std::stringstream msg; + msg << "Invalid file name format"; + NGTThrowException(msg); + } + StaticObjectFileLoader loader(data, tokens[tokens.size() - 1]); + size_t idx = 0; + while (!loader.isEmpty()) { + idx++; + if (dataSize > 0 && idx > dataSize) { + break; + } + auto object = loader.getObject(); + index.insert(idx, object); + if (idx % 100000 == 0) { + std::cerr << "loaded " << static_cast(idx) / 1000000.0 << "M objects." << std::endl; + } + } + index.save(); + index.close(); + redirector.end(); + } + + static void appendFromObjectRepository(const std::string &ngtIndex, // QG + const std::string &qgIndex, // NGT + bool silence = false) { + NGT::StdOstreamRedirector redirector(silence); + redirector.begin(); + + NGT::Index ngt(ngtIndex); + QBG::Index qg(qgIndex); + auto &objectSpace = ngt.getObjectSpace(); + size_t size = objectSpace.getRepository().size(); + for (size_t id = 1; id < size; ++id) { + std::vector object; + try { + objectSpace.getObject(id, object); + } catch(...) { + std::cerr << "append: Info: removed object. " << id << std::endl; + } + qg.insert(id, object); + } + cerr << "end of insertion." << endl; + qg.save(); + qg.close(); + redirector.end(); + } + + void getSeeds(NGT::Index &index, NGT::Object *object, NGT::ObjectDistances &seeds, size_t noOfSeeds) { + auto &graph = static_cast(index.getIndex()); + NGT::SearchContainer sc(*object); + sc.setResults(&seeds); + sc.setSize(noOfSeeds); + sc.setEpsilon(0.0); + sc.setEdgeSize(-2); + graph.search(sc); + } + + NGT::Distance getDistance(void *objects, std::vector &distances, size_t noOfObjects, NGTQ::QuantizedObjectDistance::DistanceLookupTableUint8 &lut + ) { + auto &quantizedObjectDistance = getQuantizer().getQuantizedObjectDistance(); +#ifdef NGTQBG_MIN + auto min = quantizedObjectDistance(objects, distances.data(), noOfObjects, lut); +#else + quantizedObjectDistance(objects, distances.data(), noOfObjects, lut); +#endif +#ifdef NGTQBG_MIN + return min; +#endif + } + + std::tuple + judge(NGTQG::QuantizedNode &ivi, size_t k, NGT::Distance radius, + NGTQ::QuantizedObjectDistance::DistanceLookupTableUint8 &lut, + NGT::NeighborhoodGraph::ResultSet &result, size_t &foundCount + ) { + auto noOfObjects = ivi.ids.size(); + float distances[NGTQ::QuantizedObjectProcessingStream::getNumOfAlignedObjects(noOfObjects)]; + auto &quantizedObjectDistance = getQuantizer().getQuantizedObjectDistance(); +#ifdef NGTQBG_MIN + float distance = quantizedObjectDistance(ivi.objects, &distances[0], noOfObjects, lut); +#else + quantizedObjectDistance(ivi.objects, &distances[0], noOfObjects, lut); +#endif + +#ifdef NGTQBG_MIN + if (distance >= radius) { + return std::make_pair(distance, radius); + } +#endif + bool found = false; + for (size_t i = 0; i < noOfObjects; i++) { + if (distances[i] <= radius) { + result.push(NGT::ObjectDistance(ivi.ids[i], distances[i])); + found = true; + if (result.size() > k) { + result.pop(); + } + } + } + if (result.size() >= k) { + radius = result.top().distance; + } + if (found) foundCount++; +#ifdef NGTQBG_MIN + return std::make_pair(distance, radius); +#else + return std::make_pair(0.0, radius); +#endif + } + + /////////////////// ///-/ + void searchBlobGraphNaively(QBG::SearchContainer &searchContainer) { + NGT::Object *query = &searchContainer.object; + + auto &quantizer = getQuantizer(); + auto &globalIndex = quantizer.globalCodebookIndex; + auto &globalGraph = static_cast(globalIndex.getIndex()); + NGT::ObjectDistances seeds; + getSeeds(globalIndex, query, seeds, 5); + + if (seeds.empty()) { + std::cerr << "something wrong." << std::endl; + return; + } + size_t seedBlobID = seeds[0].id; + auto &quantizedObjectDistance = quantizer.getQuantizedObjectDistance(); +#ifdef NGTQG_ZERO_GLOBAL + NGTQ::QuantizedObjectDistance::DistanceLookupTableUint8 lut; + quantizedObjectDistance.initialize(lut); + quantizedObjectDistance.createDistanceLookup(*query, 1, lut); +#else +#if defined(NGTQG_ROTATION) + quantizedObjectDistance.rotation->mul(static_cast(query->getPointer())); +#endif + uint32_t subspaceID = quantizedBlobGraph[seedBlobID].subspaceID; + + std::unordered_map luts; + luts.insert(std::make_pair(subspaceID, NGTQ::QuantizedObjectDistance::DistanceLookupTableUint8())); + auto lutfi = luts.find(subspaceID); + quantizedObjectDistance.initialize((*lutfi).second); + quantizedObjectDistance.createDistanceLookup(*query, subspaceID, (*lutfi).second); + +#endif + size_t visitCount = 1; + size_t foundCount = 0; + + size_t k = searchContainer.size; + NGT::Distance radius = FLT_MAX; + NGT::Distance distance; + NGT::NeighborhoodGraph::ResultSet result; +#ifdef NGTQG_ZERO_GLOBAL + std::tie(distance, radius) = judge(quantizedBlobGraph[seedBlobID], k, radius, lut, result, foundCount); +#else + std::tie(distance, radius) = judge(quantizedBlobGraph[seedBlobID], k, radius, (*lutfi).second, result, foundCount); +#endif + NGT::NeighborhoodGraph::UncheckedSet uncheckedBlobs; + NGT::NeighborhoodGraph::DistanceCheckedSet distanceCheckedBlobs(globalGraph.repository.size()); + distanceCheckedBlobs.insert(seedBlobID); + if (globalGraph.searchRepository.size() == 0) { + std::cerr << "graph is empty! Is it read only?" << std::endl; + abort(); + } + auto *nodes = globalGraph.searchRepository.data(); + uncheckedBlobs.push(NGT::ObjectDistance(seedBlobID, distance)); + float explorationRadius = radius * searchContainer.explorationCoefficient; + while (!uncheckedBlobs.empty()) { + auto targetBlob = uncheckedBlobs.top(); + if (targetBlob.distance > explorationRadius) { + break; + } + uncheckedBlobs.pop(); + auto *neighbors = nodes[targetBlob.id].data(); + auto noOfEdges = (searchContainer.edgeSize == 0 || searchContainer.edgeSize > static_cast(nodes[targetBlob.id].size())) ? + nodes[targetBlob.id].size() : searchContainer.edgeSize; + auto neighborend = neighbors + noOfEdges; +; + for (auto neighbor = neighbors; neighbor < neighborend; neighbor++) { + NGT::ObjectID neighborID = neighbor->first; + if (distanceCheckedBlobs[neighborID]) { + continue; + } + visitCount++; + distanceCheckedBlobs.insert(neighborID); +#ifdef NGTQG_ZERO_GLOBAL + std::tie(distance, radius) = judge(quantizedBlobGraph[neighborID], k, radius, lut, result, foundCount); +#else + auto luti = luts.find(subspaceID); + if (luti == luts.end()) { + luts.insert(std::make_pair(subspaceID, NGTQ::QuantizedObjectDistance::DistanceLookupTableUint8())); + luti = luts.find(subspaceID); + quantizedObjectDistance.initialize((*luti).second); + quantizedObjectDistance.createDistanceLookup(*query, subspaceID, (*luti).second); + } + std::tie(distance, radius) = judge(quantizedBlobGraph[neighborID], k, radius, (*luti).second, result, foundCount); +#endif + if (static_cast(foundCount) / visitCount < searchContainer.cutback) { + uncheckedBlobs = NGT::NeighborhoodGraph::UncheckedSet(); + break; + } + if (distance <= explorationRadius) { + uncheckedBlobs.push(NGT::ObjectDistance(neighborID, distance)); + } + } + } + + if (searchContainer.resultIsAvailable()) { + searchContainer.getResult().clear(); + searchContainer.getResult().moveFrom(result); + } else { + searchContainer.workingResult = result; + } + + + } + + + void searchBlobNaively(QBG::SearchContainer &searchContainer) { + NGT::ObjectDistances blobs; + NGT::SearchContainer sc(searchContainer); + sc.setResults(&blobs); + sc.setSize(searchContainer.numOfProbes); + + auto &quantizer = getQuantizer(); + auto &globalIndex = quantizer.globalCodebookIndex; + auto &objectSpace = globalIndex.getObjectSpace(); + globalIndex.search(sc); + + + if (blobs.empty()) { + std::cerr << "something wrong." << std::endl; + return; + } + + auto &quantizedObjectDistance = quantizer.getQuantizedObjectDistance(); + NGT::Object rotatedQuery(&objectSpace); + objectSpace.copy(rotatedQuery, searchContainer.object); + +#if defined(NGTQG_ROTATION) + quantizedObjectDistance.rotation->mul(static_cast(rotatedQuery.getPointer())); +#endif + std::unordered_map luts; + size_t foundCount = 0; + + size_t k = searchContainer.size; + NGT::Distance radius = FLT_MAX; + NGT::Distance distance; + NGT::NeighborhoodGraph::ResultSet result; + + for (size_t idx = 0; idx < blobs.size(); idx++) { + auto blobID = blobs[idx].id; + auto subspaceID = quantizedBlobGraph[blobID].subspaceID; + auto luti = luts.find(subspaceID); + if (luti == luts.end()) { + luts.insert(std::make_pair(subspaceID, NGTQ::QuantizedObjectDistance::DistanceLookupTableUint8())); + luti = luts.find(subspaceID); + quantizedObjectDistance.initialize((*luti).second); + quantizedObjectDistance.createDistanceLookup(rotatedQuery, subspaceID, (*luti).second); + } + std::tie(distance, radius) = judge(quantizedBlobGraph[blobID], k, radius, (*luti).second, result, foundCount); + + } + + if (searchContainer.resultIsAvailable()) { + searchContainer.getResult().clear(); + searchContainer.getResult().moveFrom(result); + } else { + searchContainer.workingResult = result; + } + + + } + + void searchBlobGraph(QBG::SearchContainer &searchContainer) { + auto &globalIndex = getQuantizer().globalCodebookIndex; + auto &globalGraph = static_cast(globalIndex.getIndex()); + NGT::ObjectDistances seeds; + NGT::Object *query = allocateObject(searchContainer.objectVector); + SearchContainer sc(searchContainer, *query); + globalGraph.getSeedsFromTree(sc, seeds); + if (seeds.empty()) { + globalGraph.getRandomSeeds(globalGraph.repository, seeds, 20); + } + searchBlobGraph(sc, seeds); + globalIndex.deleteObject(query); + searchContainer.workingResult = std::move(sc.workingResult); + } + + void searchBlobGraph(QBG::SearchContainer &searchContainer, NGT::ObjectDistances &seeds) { +#if defined(NGT_SHARED_MEMORY_ALLOCATOR) + std::cerr << "searchBlobGraph: Not implemented. " << std::endl; + abort(); +#else + + auto &quantizer = getQuantizer(); + auto &globalIndex = quantizer.globalCodebookIndex; + auto &globalGraph = static_cast(globalIndex.getIndex()); + auto &objectSpace = globalIndex.getObjectSpace(); + + if (searchContainer.explorationCoefficient == 0.0) { + searchContainer.explorationCoefficient = NGT_EXPLORATION_COEFFICIENT; + } + + const auto requestedSize = searchContainer.size; + searchContainer.size = std::numeric_limits::max(); + + // setup edgeSize + size_t edgeSize = globalGraph.getEdgeSize(searchContainer); + + NGT::NeighborhoodGraph::UncheckedSet untracedNodes; + + NGT::NeighborhoodGraph::DistanceCheckedSet distanceChecked(globalGraph.searchRepository.size()); + NGT::NeighborhoodGraph::ResultSet results; + + if (objectSpace.getObjectType() == typeid(float)) { + globalGraph.setupDistances(searchContainer, seeds, NGT::PrimitiveComparator::L2Float::compare); +#ifdef NGT_HALF_FLOAT + } else if (objectSpace.getObjectType() == typeid(NGT::float16)) { + globalGraph.setupDistances(searchContainer, seeds, NGT::PrimitiveComparator::L2Float16::compare); + } +#endif + std::sort(seeds.begin(), seeds.end()); + NGT::ObjectDistance currentNearestBlob = seeds.front(); + NGT::Distance explorationRadius = searchContainer.blobExplorationCoefficient * currentNearestBlob.distance; + std::priority_queue, std::greater> discardedObjects; + untracedNodes.push(seeds.front()); + distanceChecked.insert(seeds.front().id); + for (size_t i = 1; i < seeds.size(); i++) { + untracedNodes.push(seeds[i]); + distanceChecked.insert(seeds[i].id); + discardedObjects.push(seeds[i]); + } + size_t explorationSize = 1; + auto &quantizedObjectDistance = quantizer.getQuantizedObjectDistance(); + std::unordered_map luts; + std::vector &rotatedQuery = searchContainer.objectVector; + if (objectSpace.getObjectType() == typeid(float)) { + memcpy(rotatedQuery.data(), searchContainer.object.getPointer(), rotatedQuery.size() * sizeof(float)); +#ifdef NGT_HALF_FLOAT + } else if (objectSpace.getObjectType() == typeid(NGT::float16)) { + auto *ptr = static_cast(searchContainer.object.getPointer()); + for (size_t i = 0; i < rotatedQuery.size(); i++) { + rotatedQuery[i] = ptr[i]; + } +#endif + } else { + std::cerr << "Fatal inner error! Invalid object type." << std::endl; + } + quantizedObjectDistance.rotation->mul(rotatedQuery.data()); + NGT::Distance radius = searchContainer.radius; + if (requestedSize >= std::numeric_limits::max()) { + radius *= searchContainer.explorationCoefficient; + } + const size_t dimension = objectSpace.getPaddedDimension(); + if (globalGraph.searchRepository.empty()) { + NGTThrowException("QBG:Index: searchRepository is empty."); + } + NGT::ReadOnlyGraphNode *nodes = globalGraph.searchRepository.data(); + NGT::ReadOnlyGraphNode *neighbors = 0; + NGT::ObjectDistance target; + const size_t prefetchSize = objectSpace.getPrefetchSize(); + const size_t prefetchOffset = objectSpace.getPrefetchOffset(); + pair *neighborptr; + pair *neighborendptr; + for (;;) { + if (untracedNodes.empty() || untracedNodes.top().distance > explorationRadius) { + explorationSize++; + auto blobID = currentNearestBlob.id; + auto subspaceID = quantizedBlobGraph[blobID].subspaceID; + auto luti = luts.find(subspaceID); + if (luti == luts.end()) { + luts.insert(std::make_pair(subspaceID, NGTQ::QuantizedObjectDistance::DistanceLookupTableUint8())); + luti = luts.find(subspaceID); + quantizedObjectDistance.initialize((*luti).second); + quantizedObjectDistance.createDistanceLookup(rotatedQuery.data(), subspaceID, (*luti).second); + } + NGT::Distance blobDistance; + size_t foundCount; + std::tie(blobDistance, radius) = judge(quantizedBlobGraph[blobID], requestedSize, + radius, (*luti).second, results, foundCount); +#ifdef NGTQBG_MIN + if (blobDistance > radius * searchContainer.explorationCoefficient) { + break; + } +#endif + if (explorationSize > searchContainer.graphExplorationSize) { + break; + } + if (discardedObjects.empty()) { + break; + } + currentNearestBlob = discardedObjects.top(); + discardedObjects.pop(); + explorationRadius = searchContainer.blobExplorationCoefficient * currentNearestBlob.distance; + continue; + } + target = untracedNodes.top(); + untracedNodes.pop(); + + neighbors = &nodes[target.id]; + neighborptr = &(*neighbors)[0]; + size_t neighborSize = neighbors->size() < edgeSize ? neighbors->size() : edgeSize; + neighborendptr = neighborptr + neighborSize; + + pair* nsPtrs[neighborSize]; + size_t nsPtrsSize = 0; +#ifndef PREFETCH_DISABLE + for (; neighborptr < neighborendptr; ++neighborptr) { +#ifdef NGT_VISIT_COUNT + searchContainer.visitCount++; +#endif + if (!distanceChecked[(*(neighborptr)).first]) { + distanceChecked.insert((*(neighborptr)).first); + nsPtrs[nsPtrsSize] = neighborptr; + if (nsPtrsSize < prefetchOffset) { + unsigned char *ptr = reinterpret_cast((*(neighborptr)).second); + NGT::MemoryCache::prefetch(ptr, prefetchSize); + } + nsPtrsSize++; + } + } +#endif +#ifdef PREFETCH_DISABLE + for (; neighborptr < neighborendptr; ++neighborptr) { +#else + for (size_t idx = 0; idx < nsPtrsSize; idx++) { +#endif +#ifdef PREFETCH_DISABLE + if (distanceChecked[(*(neighborptr)).first]) { + continue; + } + distanceChecked.insert((*(neighborptr)).first); +#else + neighborptr = nsPtrs[idx]; + if (idx + prefetchOffset < nsPtrsSize) { + unsigned char *ptr = reinterpret_cast((*(nsPtrs[idx + prefetchOffset])).second); + NGT::MemoryCache::prefetch(ptr, prefetchSize); + } +#endif +#ifdef NGT_DISTANCE_COMPUTATION_COUNT + searchContainer.distanceComputationCount++; +#endif + NGT::Distance distance = 0.0; + if (objectSpace.getObjectType() == typeid(float)) { + distance = NGT::PrimitiveComparator::L2Float::compare(searchContainer.object.getPointer(), + neighborptr->second->getPointer(), dimension); +#ifdef NGT_HALF_FLOAT + } else if (objectSpace.getObjectType() == typeid(NGT::float16)) { + distance = NGT::PrimitiveComparator::L2Float16::compare(searchContainer.object.getPointer(), + neighborptr->second->getPointer(), dimension); +#endif + } else { + assert(false); + } + NGT::ObjectDistance r; + r.set(neighborptr->first, distance); + untracedNodes.push(r); + if (distance < currentNearestBlob.distance) { + discardedObjects.push(currentNearestBlob); + currentNearestBlob = r; + explorationRadius = searchContainer.blobExplorationCoefficient * currentNearestBlob.distance; + } else { + discardedObjects.push(r); + } + } + } + + if (searchContainer.resultIsAvailable()) { + if (searchContainer.exactResultSize > 0) { + NGT::ObjectDistances &qresults = searchContainer.getResult(); + auto threadid = omp_get_thread_num(); + auto paddedDimension = getQuantizer().globalCodebookIndex.getObjectSpace().getPaddedDimension(); + NGT::ResultPriorityQueue rs; + std::vector object; + qresults.resize(results.size()); + size_t idx = results.size(); + while (!results.empty()) { + auto r = results.top(); + results.pop(); +#ifdef MULTIPLE_OBJECT_LISTS + quantizer.objectList.get(threadid, r.id, object, &quantizer.globalCodebookIndex.getObjectSpace()); +#else + quantizer.objectList.get(r.id, object, &quantizer.globalCodebookIndex.getObjectSpace()); +#endif + if (objectSpace.getObjectType() == typeid(float)) { + r.distance = NGT::PrimitiveComparator::compareL2(static_cast(searchContainer.object.getPointer()), + static_cast(object.data()), paddedDimension); +#ifdef NGT_HALF_FLOAT + } else if (objectSpace.getObjectType() == typeid(NGT::float16)) { + r.distance = NGT::PrimitiveComparator::compareL2(reinterpret_cast(searchContainer.object.getPointer()), + reinterpret_cast(object.data()), paddedDimension); +#endif + } + qresults[--idx] = r; + } + std::sort(qresults.begin(), qresults.end()); + qresults.resize(searchContainer.exactResultSize); + } else { + NGT::ObjectDistances &qresults = searchContainer.getResult(); + qresults.moveFrom(results); + } + } else { + if (searchContainer.exactResultSize > 0) { + auto threadid = omp_get_thread_num(); + auto paddedDimension = getQuantizer().globalCodebookIndex.getObjectSpace().getPaddedDimension(); + NGT::ResultPriorityQueue rs; + std::vector object; + while (!results.empty()) { + auto r = results.top(); + results.pop(); +#ifdef MULTIPLE_OBJECT_LISTS + quantizer.objectList.get(threadid, r.id, object, &quantizer.globalCodebookIndex.getObjectSpace()); +#else + quantizer.objectList.get(r.id, object, &quantizer.globalCodebookIndex.getObjectSpace()); +#endif + if (objectSpace.getObjectType() == typeid(float)) { + r.distance = NGT::PrimitiveComparator::compareL2(static_cast(searchContainer.object.getPointer()), + static_cast(object.data()), paddedDimension); +#ifdef NGT_HALF_FLOAT + } else if (objectSpace.getObjectType() == typeid(NGT::float16)) { + r.distance = NGT::PrimitiveComparator::compareL2(reinterpret_cast(searchContainer.object.getPointer()), + reinterpret_cast(object.data()), paddedDimension); +#endif + } + rs.push(r); + } + results = std::move(rs); + } else { + searchContainer.workingResult = std::move(results); + } + } +#endif + } + + void save() { + quantizedBlobGraph.save(path); + } + + void load() { + if (quantizedBlobGraph.stat(path)) { + quantizedBlobGraph.load(path); + } else { + NGTThrowException("Not found the rearranged inverted index. [" + path + "]"); + } + } + + static void build(const std::string &indexPath, bool silence = false) { + build(indexPath, "", "", "", 1, 0, silence); + } + + static void build(const std::string &indexPath, + std::string quantizerCodebookFile = "", + std::string codebookIndexFile = "", + std::string objectIndexFile = "", + size_t beginID = 1, size_t endID = 0, bool silence = false) { + buildNGTQ(indexPath, quantizerCodebookFile, codebookIndexFile, objectIndexFile, beginID, endID, silence); + buildQBG(indexPath, silence); + std::cerr << "NGTQ and NGTQBG indices are completed." << std::endl; + std::cerr << " vmsize=" << NGT::Common::getProcessVmSizeStr() << std::endl; + std::cerr << " peak vmsize=" << NGT::Common::getProcessVmPeakStr() << std::endl; + + } + + static void build(const std::string &indexPath, + std::vector> &quantizerCodebook, + std::vector &codebookIndex, + std::vector &objectIndex, + size_t beginID = 1, size_t endID = 0) { + buildNGTQ(indexPath, quantizerCodebook, codebookIndex, objectIndex, beginID, endID); + buildQBG(indexPath); + std::cerr << "NGTQ and NGTQBG indices are completed." << std::endl; + std::cerr << " vmsize=" << NGT::Common::getProcessVmSizeStr() << std::endl; + std::cerr << " peak vmsize=" << NGT::Common::getProcessVmPeakStr() << std::endl; + } + + static void buildNGTQ(const std::string &indexPath, bool silence = false) { + buildNGTQ(indexPath, "", "-", "-", 1, 0, silence); + } + + static void buildNGTQ(const std::string &indexPath, + std::string quantizerCodebookFile = "", + std::string codebookIndexFile = "", + std::string objectIndexFile = "", + size_t beginID = 1, size_t endID = 0, bool silence = false) { + std::vector> quantizerCodebook; + std::vector codebookIndex; + std::vector objectIndex; + { + std::string codebookPath = quantizerCodebookFile; + if (codebookPath.empty()) { + codebookPath = QBG::Index::getQuantizerCodebookFileName(indexPath); + } + std::ifstream stream(codebookPath); + if (!stream) { + std::stringstream msg; + msg << "Cannot open the codebook. " << codebookPath; + NGTThrowException(msg); + } + std::string line; + while (getline(stream, line)) { + std::vector tokens; + NGT::Common::tokenize(line, tokens, " \t"); + std::vector object; + for (auto &token : tokens) { + object.push_back(NGT::Common::strtof(token)); + } + if (!quantizerCodebook.empty() && quantizerCodebook[0].size() != object.size()) { + std::stringstream msg; + msg << "The specified quantizer codebook is invalid. " << quantizerCodebook[0].size() + << ":" << object.size() << ":" << quantizerCodebook.size() << ":" << line; + NGTThrowException(msg); + } + if (!object.empty()) { + quantizerCodebook.push_back(object); + } + } + } + { + std::string codebookIndexPath = codebookIndexFile; + if (codebookIndexPath.empty()) { + codebookIndexPath = QBG::Index::getCodebookIndexFileName(indexPath); + } + if (codebookIndexPath != "-") { + cerr << "buildNGTQ: codebook index is " << codebookIndexPath << "." << endl; + std::ifstream stream(codebookIndexPath); + if (!stream) { + std::stringstream msg; + msg << "Cannot open the codebook index. " << codebookIndexPath; + NGTThrowException(msg); + } + std::string line; + while (getline(stream, line)) { + std::vector tokens; + NGT::Common::tokenize(line, tokens, " \t"); + std::vector object; + if (tokens.size() != 1) { + std::stringstream msg; + msg << "The specified codebook index is invalid. " << line; + NGTThrowException(msg); + } + codebookIndex.push_back(NGT::Common::strtol(tokens[0])); + } + } + } + { + std::string objectIndexPath = objectIndexFile; + if (objectIndexPath.empty()) { + objectIndexPath = QBG::Index::getObjectIndexFileName(indexPath); + } + if (objectIndexPath != "-") { + std::ifstream stream(objectIndexPath); + if (!stream) { + std::stringstream msg; + msg << "Cannot open the codebook index. " << objectIndexPath; + NGTThrowException(msg); + } + std::string line; + while (getline(stream, line)) { + std::vector tokens; + NGT::Common::tokenize(line, tokens, " \t"); + std::vector object; + if (tokens.size() != 1) { + std::stringstream msg; + msg << "The specified codebook index is invalid. " << line; + NGTThrowException(msg); + } + objectIndex.push_back(NGT::Common::strtol(tokens[0])); + } + } + } + buildNGTQ(indexPath, quantizerCodebook, codebookIndex, objectIndex, beginID, endID, silence); + } + + static void buildNGTQ(const std::string &indexPath, + std::vector> &quantizerCodebook, + std::vector &codebookIndex, + std::vector &objectIndex, + size_t beginID = 1, size_t endID = 0, bool silence = false) { + NGT::StdOstreamRedirector redirector(silence); + //redirector.begin(); + NGT::Timer timer; + timer.start(); + NGTQ::Index index(indexPath); + if (quantizerCodebook.size() == 0) { + index.createIndex(beginID, endID); + } else { + if (codebookIndex.size() == 0) { + codebookIndex.resize(quantizerCodebook.size()); + } + if (objectIndex.size() == 0) { + size_t size = index.getQuantizer().objectList.size(); + size = size == 0 ? 0 : size - 1; + objectIndex.resize(size); + } + if ((quantizerCodebook.size() == 0) || (codebookIndex.size() == 0)) { + stringstream msg; + msg << "The specified codebooks or indexes are invalild " << quantizerCodebook.size() << ":" << codebookIndex.size(); + NGTThrowException(msg); + } + index.createIndex(quantizerCodebook, codebookIndex, objectIndex, beginID, endID); + } + timer.stop(); + std::cerr << "NGTQ index is completed." << std::endl; + std::cerr << " time=" << timer << std::endl; + std::cerr << " vmsize=" << NGT::Common::getProcessVmSizeStr() << std::endl; + std::cerr << " peak vmsize=" << NGT::Common::getProcessVmPeakStr() << std::endl; + std::cerr << "saving..." << std::endl; + index.save(); + redirector.end(); + } + + static void buildQBG(const std::string &indexPath, bool silence = false) { + std::cerr << "build QBG" << std::endl; + NGT::Timer timer; + timer.start(); + QBG::Index index(indexPath, false, silence); + timer.stop(); + std::cerr << "QBG index is completed." << std::endl; + std::cerr << " time=" << timer << std::endl; + std::cerr << " vmsize=" << NGT::Common::getProcessVmSizeStr() << std::endl; + std::cerr << " peak vmsize=" << NGT::Common::getProcessVmPeakStr() << std::endl; + std::cerr << "saving..." << std::endl; + index.save(); + } + + void extract(std::ostream &os, size_t n, bool random = true) { + if (n == 0) { + NGTThrowException("QuantizedBlobGraph::extract # of objects is zero."); + } + auto &quantizer = getQuantizer(); + size_t dim = quantizer.property.dimension; + std::vector object; + if (random) { + struct timeval randTime; + gettimeofday(&randTime, 0); + srand(randTime.tv_usec); + for (size_t cnt = 0; cnt < n; cnt++) { + double random = ((double)rand() + 1.0) / ((double)RAND_MAX + 2.0); + size_t id = 0; + while (true) { + do { + id = floor(quantizer.objectList.size() * random); + } while (id >= quantizer.objectList.size()); + if (quantizer.objectList.get(id, object, &quantizer.globalCodebookIndex.getObjectSpace())) { + break; + } else { + std::cerr << "Cannot get the object. " << id << std::endl; + } + } + if (cnt + 1 % 100000 == 0) { + std::cerr << "loaded " << static_cast(cnt + 1) / 1000000.0 << "M objects." << std::endl; + } + if (dim != 0) { + object.resize(dim, 0.0); + } + for (auto v = object.begin(); v != object.end(); ++v) { + if (v + 1 != object.end()) { + os << *v << "\t"; + } else { + os << *v << std::endl;; + } + } + } + } else { + for (size_t cnt = 1; cnt <= n; cnt++) { + if (!quantizer.objectList.get(cnt, object, &quantizer.globalCodebookIndex.getObjectSpace())) { + std::cerr << "Cannot get the object. " << cnt << std::endl; + continue; + } + if (cnt % 100000 == 0) { + std::cerr << "loaded " << static_cast(cnt) / 1000000.0 << "M objects." << std::endl; + } + if (dim != 0) { + object.resize(dim, 0.0); + } + for (auto v = object.begin(); v != object.end(); ++v) { + if (v + 1 != object.end()) { + os << *v << "\t"; + } else { + os << *v << std::endl;; + } + } + if (n > 0 && cnt >= n) { + break; + } + } + } + } + + + static void + load(std::string indexPath, std::string blobs = "", std::string localCodebooks = "", std::string rotationPath = "", int threadSize = 0) + { + if (blobs.empty()) { + blobs = QBG::Index::getBlobFileName(indexPath); + } + if (localCodebooks.empty()) { + localCodebooks = QBG::Index::getPQFileName(indexPath) + "/opt-@.tsv"; + } + if (rotationPath.empty()) { + rotationPath = QBG::Index::getRotationFileName(indexPath); + } + + threadSize = threadSize == 0 ? std::thread::hardware_concurrency() : threadSize; + assert(threadSize != 0); + + size_t dataSize = 0; + NGT::Index::append(indexPath + "/global", blobs, threadSize, dataSize); + + std::cerr << "qbg: loading the local codebooks..." << std::endl; + NGTQ::Property property; + property.load(indexPath); + + std::vector tokens; + NGT::Common::tokenize(localCodebooks, tokens, "@"); + if (tokens.size() != 2) { + NGTThrowException("No @ in the specified local codebook string."); + } + for (size_t no = 0; no < property.localDivisionNo; no++) { + std::stringstream data; + data << tokens[0] << no << tokens[1]; + std::stringstream localCodebook; + localCodebook << indexPath << "/local-" << no; + std::cerr << data.str() << "->" << localCodebook.str() << std::endl; + NGT::Index::append(localCodebook.str(), data.str(), threadSize, dataSize); + } + + cerr << "qbg: loading the rotation..." << endl; +#ifdef NGTQ_QBG + std::vector rotation; + + std::ifstream stream(rotationPath); + if (!stream) { + std::stringstream msg; + msg << "Cannot open the rotation. " << rotationPath; + NGTThrowException(msg); + } + std::string line; + while (getline(stream, line)) { + std::vector tokens; + NGT::Common::tokenize(line, tokens, " \t"); + for (auto &token : tokens) { + rotation.push_back(NGT::Common::strtof(token)); + } + } + std::cerr << "rotation matrix size=" << rotation.size() << std::endl; + QBG::Index::load(indexPath, rotation); +#endif + } + + static const std::string getTrainObjectFileName(std::string indexPath) { return indexPath + "/" + getWorkspaceName() + "/objects.tsv"; } + static const std::string getPQFileName(std::string indexPath) { return indexPath + "/" + getWorkspaceName() + "/kmeans-cluster_opt"; } + static const std::string getBlobFileName(std::string indexPath) { return indexPath + "/" + getWorkspaceName() + "/kmeans-cluster_centroid.tsv"; } + + static const std::string getQuantizerCodebookFileName(std::string indexPath) { return indexPath + "/" + getWorkspaceName() + "/kmeans-cluster_qcentroid.tsv"; } + static const std::string getCodebookIndexFileName(std::string indexPath) { return indexPath + "/" + getWorkspaceName() + "/kmeans-cluster_bqindex.tsv"; } + static const std::string getObjectIndexFileName(std::string indexPath) { return indexPath + "/" + getWorkspaceName() + "/kmeans-cluster_index.tsv"; } + static const std::string getRotationFileName(std::string indexPath) { return getPQFileName(indexPath) + "/opt_R.tsv"; } + + static const std::string getWorkspaceName() { return "ws"; } + + const std::string path; + + QuantizedBlobGraphRepository quantizedBlobGraph; + + + }; + +} + +#endif diff --git a/lib/NGT/NGTQ/QuantizedGraph.h b/lib/NGT/NGTQ/QuantizedGraph.h index 6f4c425..fdecb4f 100644 --- a/lib/NGT/NGTQ/QuantizedGraph.h +++ b/lib/NGT/NGTQ/QuantizedGraph.h @@ -16,13 +16,18 @@ #pragma once - #include "NGT/Index.h" #include "NGT/NGTQ/Quantizer.h" +#ifdef NGTQ_QBG #define GLOBAL_SIZE 1 +#ifdef NGT_SHARED_MEMORY_ALLOCATOR +#undef NGTQG_BLOB_GRAPH +#endif + + namespace NGTQG { class SearchContainer : public NGT::SearchContainer { public: @@ -43,6 +48,7 @@ namespace NGTQG { ~QuantizedNode() { delete[] static_cast(objects); } + uint32_t subspaceID; std::vector ids; void *objects; }; @@ -60,18 +66,25 @@ namespace NGTQG { std::vector& getIDs(size_t id) { return PARENT::at(id).ids; } - void construct(NGT::Index &ngtindex, NGTQ::Index &quantizedIndex, size_t maxNoOfEdges) { - NGTQ::InvertedIndexEntry invertedIndexObjects(numOfSubspaces); - quantizedIndex.getQuantizer().extractInvertedIndexObject(invertedIndexObjects); - NGT::GraphAndTreeIndex &index = static_cast(ngtindex.getIndex()); NGT::NeighborhoodGraph &graph = static_cast(index); - NGT::GraphRepository &graphRepository = graph.repository; + construct(graphRepository, quantizedIndex, maxNoOfEdges); + } + + void construct(NGT::GraphRepository &graphRepository, NGTQ::Index &quantizedIndex, size_t maxNoOfEdges) { + NGTQ::InvertedIndexEntry invertedIndexObjects(numOfSubspaces); + quantizedIndex.getQuantizer().extractInvertedIndexObject(invertedIndexObjects); + quantizedIndex.getQuantizer().eraseInvertedIndexObject(); + + PARENT::resize(graphRepository.size()); for (size_t id = 1; id < graphRepository.size(); id++) { + if (id % 100000 == 0) { + std::cerr << "# of processed objects=" << id << "/" << graphRepository.size() << std::endl; + } NGT::GraphNode &node = *graphRepository.VECTOR::get(id); size_t numOfEdges = node.size() < maxNoOfEdges ? node.size() : maxNoOfEdges; (*this)[id].ids.reserve(numOfEdges); @@ -120,6 +133,8 @@ namespace NGTQG { n = PARENT::size(); NGT::Serializer::write(os, n); for (auto i = PARENT::begin(); i != PARENT::end(); ++i) { + uint32_t sid = (*i).subspaceID; + NGT::Serializer::write(os, sid); NGT::Serializer::write(os, (*i).ids); size_t streamSize = quantizedObjectProcessingStream.getUint4StreamSize((*i).ids.size()); NGT::Serializer::write(os, static_cast((*i).objects), streamSize); @@ -135,6 +150,9 @@ namespace NGTQG { NGT::Serializer::read(is, n); PARENT::resize(n); for (auto i = PARENT::begin(); i != PARENT::end(); ++i) { + uint32_t sid; + NGT::Serializer::read(is, sid); + (*i).subspaceID = sid; NGT::Serializer::read(is, (*i).ids); size_t streamSize = quantizedObjectProcessingStream.getUint4StreamSize((*i).ids.size()); uint8_t *objectStream = new uint8_t[streamSize]; @@ -148,7 +166,16 @@ namespace NGTQG { } } + bool stat(const string &path) { + struct stat st; + const std::string p(path + "/grp"); + return ::stat(p.c_str(), &st) == 0; + } + void save(const string &path) { + if (PARENT::size() == 0) { + return; + } const std::string p(path + "/grp"); std::ofstream os(p); serialize(os); @@ -164,6 +191,7 @@ namespace NGTQG { }; + class Index : public NGT::Index { public: Index(const std::string &indexPath, size_t maxNoOfEdges = 128, bool rdOnly = false) : @@ -201,12 +229,27 @@ namespace NGTQG { NGTQ::Quantizer &quantizer = quantizedIndex.getQuantizer(); NGTQ::QuantizedObjectDistance &quantizedObjectDistance = quantizer.getQuantizedObjectDistance(); + +#ifdef NGTQ_QBG + NGTQ::QuantizedObjectDistance::DistanceLookupTableUint8 cache[GLOBAL_SIZE]; +#else NGTQ::QuantizedObjectDistance::DistanceLookupTableUint8 cache[GLOBAL_SIZE + 1]; +#endif + auto rotatedQuery = graph.getObjectSpace().getObject(sc.object); + if (quantizer.property.dimension > rotatedQuery.size()) { + rotatedQuery.resize(quantizer.property.dimension); + } +#ifndef NGTQG_NO_ROTATION + quantizedObjectDistance.rotation->mul(rotatedQuery.data()); +#endif +#ifdef NGTQ_QBG + for (int i = 0; i < GLOBAL_SIZE; i++) { +#else for (int i = 1; i < GLOBAL_SIZE + 1; i++) { +#endif quantizedObjectDistance.initialize(cache[i]); - quantizedObjectDistance.createDistanceLookup(sc.object, i, cache[i]); + quantizedObjectDistance.createDistanceLookup(rotatedQuery.data(), i, cache[i]); } - if (sc.explorationCoefficient == 0.0) { sc.explorationCoefficient = NGT_EXPLORATION_COEFFICIENT; } @@ -240,8 +283,12 @@ namespace NGTQG { NGT::MemoryCache::prefetch(lid, size); } -#endif //// NGTQG_PREFETCH +#endif +#ifdef NGTQ_QBG + quantizedObjectDistance(quantizedGraph.get(target.id), ds, neighborSize, cache[0]); +#else quantizedObjectDistance(quantizedGraph.get(target.id), ds, neighborSize, cache[1]); +#endif for (size_t idx = 0;idx < neighborSize; idx++) { NGT::Distance distance = ds[idx]; auto objid = neighborIDs[idx]; @@ -401,19 +448,38 @@ namespace NGTQG { return dimension / dimensionOfSubvector; } - static void buildQuantizedGraph(const std::string indexPath, size_t maxNumOfEdges) { - NGTQG::Index index(indexPath, maxNumOfEdges); - index.save(); + static void buildQuantizedGraph(const std::string indexPath, size_t maxNumOfEdges = 128) { + const std::string qgPath(indexPath + "/qg"); + NGTQ::Index quantizedIndex(qgPath, false); + NGTQG::QuantizedGraphRepository quantizedGraph(quantizedIndex); + { + struct stat st; + std::string qgGraphPath(qgPath + "/grp"); + if (stat(qgGraphPath.c_str(), &st) == 0) { + std::cerr << "already exists" << std::endl; + abort(); + } else { + NGT::GraphRepository graph; + NGT::GraphIndex::loadGraph(indexPath, graph); + quantizedGraph.construct(graph, quantizedIndex, maxNumOfEdges); + quantizedGraph.save(qgPath); + } + } + + std::cerr << "Quantized graph is completed." << std::endl; + std::cerr << " vmsize=" << NGT::Common::getProcessVmSizeStr() << std::endl; + std::cerr << " peak vmsize=" << NGT::Common::getProcessVmPeakStr() << std::endl; } - static void buildQuantizedObjects(const std::string quantizedIndexPath, NGT::ObjectSpace &objectSpace) { +#ifndef NGTQ_QBG + static void buildQuantizedObjects(const std::string quantizedIndexPath, NGT::ObjectSpace &objectSpace, bool insertion = false) { NGTQ::Index quantizedIndex(quantizedIndexPath); NGTQ::Quantizer &quantizer = quantizedIndex.getQuantizer(); { std::vector meanObject(objectSpace.getDimension(), 0); - quantizedIndex.getQuantizer().globalCodebook.insert(meanObject); - quantizedIndex.getQuantizer().globalCodebook.createIndex(8); + quantizedIndex.getQuantizer().globalCodebookIndex.insert(meanObject); + quantizedIndex.getQuantizer().globalCodebookIndex.createIndex(8); } vector > objects; @@ -425,6 +491,7 @@ namespace NGTQG { try { objectSpace.getObject(id, object); } catch(...) { + std::cerr << "NGTQG::buildQuantizedObjects: Warning! Cannot get the object. " << id << std::endl; continue; } quantizer.insert(object, objects, id); @@ -436,8 +503,13 @@ namespace NGTQG { quantizedIndex.save(); quantizedIndex.close(); } +#endif - static void constructQuantizedGraphFrame(const std::string quantizedIndexPath, size_t dimension, size_t dimensionOfSubvector) { +#ifdef NGTQ_QBG + static void createQuantizedGraphFrame(const std::string quantizedIndexPath, size_t dimension, size_t pseudoDimension, size_t dimensionOfSubvector) { +#else + static void createQuantizedGraphFrame(const std::string quantizedIndexPath, size_t dimension, size_t dimensionOfSubvector) { +#endif NGTQ::Property property; NGT::Property globalProperty; NGT::Property localProperty; @@ -446,18 +518,30 @@ namespace NGTQG { property.globalRange = 0; property.localRange = 0; property.dataType = NGTQ::DataTypeFloat; - property.distanceType = NGTQ::DistanceTypeL2; + property.distanceType = NGTQ::DistanceType::DistanceTypeL2; property.singleLocalCodebook = false; property.batchSize = 1000; property.centroidCreationMode = NGTQ::CentroidCreationModeStatic; +#ifdef NGTQ_QBG + property.localCentroidCreationMode = NGTQ::CentroidCreationModeStatic; +#else property.localCentroidCreationMode = NGTQ::CentroidCreationModeDynamicKmeans; +#endif property.globalCentroidLimit = 1; property.localCentroidLimit = 16; property.localClusteringSampleCoefficient = 100; - property.localDivisionNo = NGTQG::Index::getNumberOfSubvectors(dimension, dimensionOfSubvector); +#ifdef NGTQ_QBG + property.genuineDimension = dimension; + if (pseudoDimension == 0) { + property.dimension = dimension; + } else { + property.dimension = pseudoDimension; + } +#else property.dimension = dimension; - +#endif + property.localDivisionNo = NGTQG::Index::getNumberOfSubvectors(property.dimension, dimensionOfSubvector); globalProperty.edgeSizeForCreation = 10; globalProperty.edgeSizeForSearch = 40; globalProperty.indexType = NGT::Property::GraphAndTree; @@ -470,25 +554,82 @@ namespace NGTQG { } - static void quantize(const std::string indexPath, float dimensionOfSubvector, size_t maxNumOfEdges) { + static void create(const std::string indexPath, size_t dimensionOfSubvector, size_t pseudoDimension) { NGT::Index index(indexPath); - NGT::ObjectSpace &objectSpace = index.getObjectSpace(); + std::string quantizedIndexPath = indexPath + "/qg"; + struct stat st; + if (stat(quantizedIndexPath.c_str(), &st) == 0) { + std::stringstream msg; + msg << "QuantizedGraph::create: Quantized graph is already existed. " << indexPath; + NGTThrowException(msg); + } + NGT::Property ngtProperty; + index.getProperty(ngtProperty); + //NGTQG::Command::CreateParameters createParameters(args, property.dimension); +#ifdef NGTQ_QBG + if (pseudoDimension == 0) { + pseudoDimension = ((ngtProperty.dimension - 1) / 4 + 1) * 4; + } + if (ngtProperty.dimension > static_cast(pseudoDimension)) { + std::stringstream msg; + msg << "QuantizedGraph::quantize: the specified pseudo dimension is smaller than the genuine dimension. " + << ngtProperty.dimension << ":" << pseudoDimension << std::endl; + NGTThrowException(msg); + } + if (pseudoDimension % 4 != 0) { + std::stringstream msg; + msg << "QuantizedGraph::quantize: the specified pseudo dimension should be a multiple of 4. " + << pseudoDimension << std::endl; + NGTThrowException(msg); + } + createQuantizedGraphFrame(quantizedIndexPath, ngtProperty.dimension, pseudoDimension, dimensionOfSubvector); +#else + createQuantizedGraphFrame(quantizedIndexPath, ngtProperty.dimension, dimensionOfSubvector); +#endif + return; + } +#ifdef NGTQ_QBG + static void quantize(const std::string indexPath, size_t maxNumOfEdges, bool silence = false) { +#else + static void quantize(const std::string indexPath, float dimensionOfSubvector, size_t maxNumOfEdges, bool silence = false) { +#endif + NGT::StdOstreamRedirector redirector(silence); + redirector.begin(); +#ifdef NGTQ_QBG + { + std::string quantizedIndexPath = indexPath + "/qg"; + struct stat st; + if (stat(quantizedIndexPath.c_str(), &st) != 0) { + std::stringstream msg; + msg << "QuantizedGraph::quantize: Quantized graph is already existed. " << quantizedIndexPath; + NGTThrowException(msg); + } + if (maxNumOfEdges == 0) { + NGTThrowException("QuantizedGraph::quantize: The maximum number of edges is zero."); + } + buildQuantizedGraph(indexPath, maxNumOfEdges); + } +#else { + NGT::Index index(indexPath); + NGT::ObjectSpace &objectSpace = index.getObjectSpace(); std::string quantizedIndexPath = indexPath + "/qg"; struct stat st; if (stat(quantizedIndexPath.c_str(), &st) != 0) { NGT::Property ngtProperty; index.getProperty(ngtProperty); //NGTQG::Command::CreateParameters createParameters(args, property.dimension); - constructQuantizedGraphFrame(quantizedIndexPath, ngtProperty.dimension, dimensionOfSubvector); + createQuantizedGraphFrame(quantizedIndexPath, ngtProperty.dimension, dimensionOfSubvector); buildQuantizedObjects(quantizedIndexPath, objectSpace); if (maxNumOfEdges != 0) { buildQuantizedGraph(indexPath, maxNumOfEdges); } } } +#endif + redirector.end(); } const bool readOnly; @@ -502,3 +643,4 @@ namespace NGTQG { } +#endif diff --git a/lib/NGT/NGTQ/Quantizer.h b/lib/NGT/NGTQ/Quantizer.h index 3a00318..20de1dd 100644 --- a/lib/NGT/NGTQ/Quantizer.h +++ b/lib/NGT/NGTQ/Quantizer.h @@ -19,10 +19,34 @@ #include "NGT/Index.h" #include "NGT/ArrayFile.h" #include "NGT/Clustering.h" +#include +#include "NGT/NGTQ/ObjectFile.h" +#if defined(NGT_SHARED_MEMORY_ALLOCATOR) || defined(NGT_QBG_DISABLED) +#undef NGTQ_QBG +#else +#define NGTQ_QBG +#endif + +#define NGTQ_VECTOR_OBJECT + +#ifdef NGTQ_STATIC_OBJECT_FILE +#define MULTIPLE_OBJECT_LISTS +#endif + +#define NGTQBG_MIN + +#define MULTIPLE_OBJECT_LISTS +#define NGTQG_ROTATION +#define NGTQ_BLAS_FOR_ROTATION +#define NGTQG_ROTATED_GLOBAL_CODEBOOKS + +#define NGTQ_UINT8_LUT #define NGTQ_SIMD_BLOCK_SIZE 16 #define NGTQ_BATCH_SIZE 2 +#define NGTQ_UINT4_OBJECT +#define NGTQ_TOTAL_SCALE_OFFSET_COMPRESSION #define NGTQG_PREFETCH #if defined(NGT_AVX512) #define NGTQG_AVX512 @@ -36,12 +60,260 @@ #endif + #ifdef NGT_SHARED_MEMORY_ALLOCATOR -#define NGTQ_SHARED_INVERTED_INDEX +#define NGTQ_SHARED_INVERTED_INDEX #endif +extern "C" { + void sgemv_(char *trans, int *m, int *n, + float *alpha, float *A, int *ldA, float *x, int *incx, + float *beta , float *y, int *incy); +} + namespace NGTQ { +class Rotation : public std::vector { + typedef std::vector PARENT; + public: + Rotation& operator=(const std::vector &r) { + PARENT::operator=(r); + dim = sqrt(PARENT::size()); + if ((dim * dim) != PARENT::size()) { + std::cerr << "Rotation: Fatal inner error! Invalid data. " << dim * dim << ":" << PARENT::size() << std::endl; + abort(); + } + return *this; + } +#ifdef NGTQ_BLAS_FOR_ROTATION + void mulBlas(float *a) { + char trans = 'N'; + int m = dim; + float alpha = 1.0; + float *matrix = PARENT::data(); + int incx = 1; + float beta = 0.0; + float *y = new float[dim]; + int incy = 1; + sgemv_(&trans, &m, &m, &alpha, matrix, &m, a, &incx, &beta, y, &incy); + memcpy(a, y, sizeof(float) * dim); + delete[] y; + } + void mulBlas(NGT::float16 *a) { + float *floatv = new float[dim]; + for (size_t d = 0; d < dim; d++) { + floatv[d] = a[d]; + } + char trans = 'N'; + int m = dim; + float alpha = 1.0; + float *matrix = PARENT::data(); + int incx = 1; + float beta = 0.0; + float *y = new float[dim]; + int incy = 1; + sgemv_(&trans, &m, &m, &alpha, matrix, &m, floatv, &incx, &beta, y, &incy); + for (size_t d = 0; d < dim; d++) { + a[d] = y[d]; + } + delete[] y; + delete[] floatv; + } +#endif + template + void mul(T *a) { + if (PARENT::size() == 0) { + return; + } +#ifdef NGTQ_BLAS_FOR_ROTATION + mulBlas(a); + return; +#else + float *matrix = PARENT::data(); + T vec[dim]; + for (int c = 0; c < dim; c++) { + double sum = 0; + for (int p = 0; p < dim; p++) { + sum += a[p] * matrix[p * dim + c]; + } + vec[c] = sum; + } + memcpy(a, vec, sizeof(T) * dim); +#endif + } + void mul(std::vector &a) { + if (a.size() != dim) { + std::cerr << "Fatal inner error! " << a.size() << ":" << dim << std::endl; + abort(); + } + mul(a.data()); + } + void serialize(std::ofstream &os) { + uint32_t v = PARENT::size(); + NGT::Serializer::write(os, v); + os.write(reinterpret_cast(PARENT::data()), static_cast(PARENT::size()) * sizeof(float)); + } + + void deserialize(std::ifstream &is) { + uint32_t v; + NGT::Serializer::read(is, v); + PARENT::resize(v); + dim = sqrt(v); + if (dim * dim != v) { + std::cerr << "rotation::deserialize: Fatal inner error. Invalid data. " << dim << ":" << dim * dim << ":" << v << std::endl; + abort(); + } + + is.read(reinterpret_cast(PARENT::data()), PARENT::size() * sizeof(float)); + + } + + void show() { + std::cerr << "R=" << std::endl; + for (size_t i = 0; i < dim; i++) { + for (size_t j = 0; j < dim; j++) { + std::cerr << (*this)[i * dim + j] << " "; + } + std::cerr << std::endl; + } + std::cerr << std::endl; + } + + uint32_t dim; +}; + +template +class QuantizationCodebook : public std::vector { + typedef std::vector PARENT; + public: + QuantizationCodebook(): dimension(0), paddedDimension(0), index(0) {} + QuantizationCodebook& operator=(const std::vector> &qc) { + if (qc.empty()) { + NGTThrowException("NGTQ::QuantizationCodebook::operator=: codebook is empty."); + } + if (paddedDimension == 0) { + NGTThrowException("NGTQ::QuantizationCodebook::operator=: paddedDimension is unset."); + } + dimension = qc[0].size(); + std::cerr << "dim=" << dimension << ":" << qc.size() << ":" << paddedDimension << std::endl; + PARENT::resize(paddedDimension * qc.size()); + for (size_t i = 0; i < qc.size(); i++) { + if (qc[i].size() != dimension) { + std::stringstream msg; + msg << "NGTQ::QuantizationCodebook::operator=: paddedDimension is invalid. " << i << ":" << qc[i].size() << ":" << dimension; + NGTThrowException(msg); + } + memcpy(PARENT::data() + i * paddedDimension, qc[i].data(), dimension * sizeof(T)); + } + buildIndex(); + return *this; + } + ~QuantizationCodebook() { delete index; } + void setPaddedDimension(size_t pd) { paddedDimension = pd; } + T at(size_t id, size_t dim) { + return *(PARENT::data() + id * paddedDimension + dim); + } + T *data(size_t id) { + return PARENT::data() + id * paddedDimension; + } + size_t size() { return PARENT::size() == 0 ? 0 : PARENT::size() / paddedDimension; } + void buildIndex() { + if (PARENT::size() == 0) { + return; + } + std::cerr << "QuantizationCodebook::buildIndex" << std::endl; + if (index != 0) { + std::cerr << "Quantization codebook: something wrong?" << std::endl; + delete index; + } + NGT::Property property; + property.dimension = dimension; + property.distanceType = NGT::Index::Property::DistanceType::DistanceTypeL2; +#ifdef NGTQ_SHARED_INVERTED_INDEX + index = new NGT::Index("dummy", property); + std::cerr << "Not implemented" << std::endl; + abort(); +#else + index = new NGT::Index(property); +#endif + size_t noOfCentroids = PARENT::size() / paddedDimension; + std::cerr << "QuantizationCodebook::buildIndex # of the centroids=" << noOfCentroids << std::endl; + for (size_t idx = 0; idx < noOfCentroids; idx++) { + if ((idx + 1) % 100000 == 0) { + std::cerr << "QuantizationCodebook::buildIndex processed objects=" << idx << std::endl; + } + index->append(data(idx), 1); + } + index->createIndex(50); + } + size_t search(NGT::Object &object) { + if (index == 0) { + std::cerr << "QuantizeationCodebook: Fatal Error! The index is not available." << std::endl; + abort(); + } + NGT::ObjectDistances result; + NGT::SearchContainer sc(object); + sc.setResults(&result); + sc.setSize(10); + sc.radius = FLT_MAX; + sc.setEpsilon(0.1); + index->search(sc); + if (result.size() == 0) { + std::cerr << "QuantizeationCodebook: Fatal Error! Cannot search." << std::endl; + abort(); + } else { + return result[0].id - 1; + } + } + size_t find(NGT::Object &object, NGT::ObjectSpace::Comparator &comparator) { + size_t noOfCentroids = PARENT::size() / paddedDimension; + auto mind = std::numeric_limits::max(); + size_t minidx = 0; + for (size_t idx = 0; idx < noOfCentroids; idx++) { + auto d = NGT::PrimitiveComparator::compareL2(static_cast(object.getPointer()), data(idx), paddedDimension); + if (mind > d) { + mind = d; + minidx = idx; + } + } + return minidx; + } + + void rotate(Rotation &r) { + size_t noOfCentroids = PARENT::size() / paddedDimension; + for (size_t idx = 0; idx < noOfCentroids; idx++) { + r.mul(data(idx)); + } + } + + void serialize(std::ofstream &os) { + uint32_t v = PARENT::size(); + NGT::Serializer::write(os, v); + v = dimension; + NGT::Serializer::write(os, v); + v = paddedDimension; + NGT::Serializer::write(os, v); + os.write(reinterpret_cast(PARENT::data()), static_cast(PARENT::size()) * sizeof(T)); + } + + void deserialize(std::ifstream &is, bool readOnly) { + uint32_t v; + NGT::Serializer::read(is, v); + PARENT::resize(v); + NGT::Serializer::read(is, v); + dimension = v; + NGT::Serializer::read(is, v); + paddedDimension = v; + is.read(reinterpret_cast(PARENT::data()), PARENT::size() * sizeof(T)); + if (!readOnly) { + buildIndex(); + } + } + uint32_t dimension; + uint32_t paddedDimension; + NGT::Index *index; +}; + template class InvertedIndexObject { public: @@ -70,14 +342,22 @@ class InvertedIndexObject { }; template -class InvertedIndexEntry : public NGT::DynamicLengthVector > { +class InvertedIndexEntry : public NGT::DynamicLengthVector> { public: typedef NGT::DynamicLengthVector> PARENT; #ifdef NGTQ_SHARED_INVERTED_INDEX - InvertedIndexEntry(size_t n, NGT::ObjectSpace *os = 0):numOfLocalIDs(n) { + InvertedIndexEntry(size_t n, NGT::ObjectSpace *os = 0):numOfLocalIDs(n) +#ifdef NGTQ_QBG + , subspaceID(std::numeric_limits::max()) +#endif + { PARENT::elementSize = getSizeOfElement(); } - InvertedIndexEntry(size_t n, SharedMemoryAllocator &allocator, NGT::ObjectSpace *os = 0):numOfLocalIDs(n) { + InvertedIndexEntry(size_t n, SharedMemoryAllocator &allocator, NGT::ObjectSpace *os = 0):numOfLocalIDs(n) +#ifdef NGTQ_QBG + , subspaceID(std::numeric_limits::max()) +#endif + { PARENT::elementSize = getSizeOfElement(); } void pushBack(SharedMemoryAllocator &allocator) { @@ -89,8 +369,16 @@ class InvertedIndexEntry : public NGT::DynamicLengthVector::max()) +#endif + {} + InvertedIndexEntry(size_t n, NGT::ObjectSpace *os = 0):numOfLocalIDs(n) +#ifdef NGTQ_QBG + , subspaceID(std::numeric_limits::max()) +#endif + { PARENT::elementSize = getSizeOfElement(); } void pushBack() { @@ -111,22 +399,36 @@ class InvertedIndexEntry : public NGT::DynamicLengthVector(PARENT::vector), PARENT::size() * PARENT::elementSize); } void deserialize(std::ifstream &is, NGT::ObjectSpace *objectspace = 0) { uint32_t sz; uint16_t nids; +#ifdef NGTQ_QBG + int32_t ssid; +#endif try { NGT::Serializer::read(is, sz); NGT::Serializer::read(is, nids); +#ifdef NGTQ_QBG + NGT::Serializer::read(is, ssid); +#endif } catch(NGT::Exception &err) { std::stringstream msg; msg << "InvertedIndexEntry::deserialize: It might be caused by inconsistency of the valuable type of the inverted index size. " << err.what(); NGTThrowException(msg); } numOfLocalIDs = nids; +#ifdef NGTQ_QBG + subspaceID = ssid; +#endif PARENT::elementSize = getSizeOfElement(); + PARENT::reserve(sz); PARENT::resize(sz); is.read(reinterpret_cast(PARENT::vector), sz * PARENT::elementSize); } @@ -139,15 +441,25 @@ class InvertedIndexEntry : public NGT::DynamicLengthVector::headerSize() + dsize; } - size_t numOfLocalIDs; + uint32_t numOfLocalIDs; +#ifdef NGTQ_QBG + uint32_t subspaceID; +#endif }; class LocalDatam { public: LocalDatam(){}; - LocalDatam(size_t iii, size_t iil) : iiIdx(iii), iiLocalIdx(iil) {} +#ifdef NGTQ_QBG + LocalDatam(size_t iii, size_t iil, uint32_t ssID = 0) : iiIdx(iii), iiLocalIdx(iil), subspaceID(ssID) {} +#else + LocalDatam(size_t iii, size_t iil) : iiIdx(iii), iiLocalIdx(iil) {} +#endif size_t iiIdx; size_t iiLocalIdx; +#ifdef NGTQ_QBG + uint32_t subspaceID; +#endif }; template @@ -159,23 +471,19 @@ class SerializableObject : public NGT::Object { enum DataType { DataTypeUint8 = 0, DataTypeFloat = 1 +#ifdef NGT_HALF_FLOAT + , + DataTypeFloat16 = 2 +#endif }; - enum DistanceType { - DistanceTypeNone = 0, - DistanceTypeL1 = 1, - DistanceTypeL2 = 2, - DistanceTypeHamming = 3, - DistanceTypeAngle = 4, - DistanceTypeNormalizedCosine = 5, - DistanceTypeNormalizedL2 = 6, - DistanceTypeCosine = 7 - }; + typedef NGT::ObjectSpace::DistanceType DistanceType; enum CentroidCreationMode { CentroidCreationModeDynamic = 0, CentroidCreationModeStatic = 1, CentroidCreationModeDynamicKmeans = 2, + CentroidCreationModeStaticLayer = 3, CentroidCreationModeNone = 9 }; @@ -187,6 +495,12 @@ class SerializableObject : public NGT::Object { AggregationModeExactDistance = 4 }; + enum QuantizerType { + QuantizerTypeNone = 0, + QuantizerTypeQG = 1, + QuantizerTypeQBG = 2 + }; + class Property { public: Property() { @@ -197,9 +511,13 @@ class SerializableObject : public NGT::Object { globalCentroidLimit = 10000000; localCentroidLimit = 1000000; dimension = 0; +#ifdef NGTQ_QBG + genuineDimension = 0; + genuineDataType = ObjectFile::DataTypeFloat; +#endif dataSize = 0; dataType = DataTypeFloat; - distanceType = DistanceTypeNone; + distanceType = DistanceType::DistanceTypeNone; singleLocalCodebook = false; localDivisionNo = 8; batchSize = 1000; @@ -208,6 +526,7 @@ class SerializableObject : public NGT::Object { localIDByteSize = 0; // finally decided by localCentroidLimit localCodebookState = false; // not completed localClusteringSampleCoefficient = 10; + quantizerType = QuantizerTypeNone; #ifdef NGT_SHARED_MEMORY_ALLOCATOR invertedIndexSharedMemorySize = 512; // MB #endif @@ -221,6 +540,10 @@ class SerializableObject : public NGT::Object { prop.set("GlobalCentroidLimit", (long)globalCentroidLimit); prop.set("LocalCentroidLimit", (long)localCentroidLimit); prop.set("Dimension", (long)dimension); +#ifdef NGTQ_QBG + prop.set("GenuineDimension", (long)genuineDimension); + prop.set("GenuineDataType", (long)genuineDataType); +#endif prop.set("DataSize", (long)dataSize); prop.set("DataType", (long)dataType); prop.set("DistanceType", (long)distanceType); @@ -232,6 +555,7 @@ class SerializableObject : public NGT::Object { prop.set("LocalIDByteSize", (long)localIDByteSize); prop.set("LocalCodebookState", (long)localCodebookState); prop.set("LocalSampleCoefficient", (long)localClusteringSampleCoefficient); + prop.set("QuantizerType", (long)quantizerType); #ifdef NGT_SHARED_MEMORY_ALLOCATOR prop.set("InvertedIndexSharedMemorySize", (long)invertedIndexSharedMemorySize); #endif @@ -252,7 +576,11 @@ class SerializableObject : public NGT::Object { } else { } } +#ifdef NGTQ_QBG + if (localIDByteSize != 1 && localIDByteSize != 2 && localIDByteSize != 4) { +#else if (localIDByteSize != 2 && localIDByteSize != 4) { +#endif NGTThrowException("NGTQ::Property: Fatal internal error! localIDByteSize should be 2 or 4."); } } @@ -266,6 +594,10 @@ class SerializableObject : public NGT::Object { globalCentroidLimit = prop.getl("GlobalCentroidLimit", globalCentroidLimit); localCentroidLimit = prop.getl("LocalCentroidLimit", localCentroidLimit); dimension = prop.getl("Dimension", dimension); +#ifdef NGTQ_QBG + genuineDimension = prop.getl("GenuineDimension", genuineDimension); + genuineDataType = static_cast(prop.getl("GenuineDataType", genuineDataType)); +#endif dataSize = prop.getl("DataSize", dataSize); dataType = (DataType)prop.getl("DataType", dataType); distanceType = (DistanceType)prop.getl("DistanceType", distanceType); @@ -278,6 +610,7 @@ class SerializableObject : public NGT::Object { localCodebookState = prop.getl("LocalCodebookState", localCodebookState); localClusteringSampleCoefficient = prop.getl("LocalSampleCoefficient", localClusteringSampleCoefficient); setupLocalIDByteSize(); + quantizerType = (QuantizerType)prop.getl("QuantizerType", quantizerType); #ifdef NGT_SHARED_MEMORY_ALLOCATOR invertedIndexSharedMemorySize = prop.getl("InvertedIndexSharedMemorySize", invertedIndexSharedMemorySize); @@ -285,23 +618,39 @@ class SerializableObject : public NGT::Object { } void setup(const Property &p) { - threadSize = p.threadSize; - globalRange = p.globalRange; - localRange = p.localRange; - globalCentroidLimit = p.globalCentroidLimit; - localCentroidLimit = p.localCentroidLimit; - distanceType = p.distanceType; - singleLocalCodebook = p.singleLocalCodebook; - localDivisionNo = p.localDivisionNo; - batchSize = p.batchSize; - centroidCreationMode = p.centroidCreationMode; - localCentroidCreationMode = p.localCentroidCreationMode; - localIDByteSize = p.localIDByteSize; - localCodebookState = p.localCodebookState; - localClusteringSampleCoefficient = p.localClusteringSampleCoefficient; -#ifdef NGT_SHARED_MEMORY_ALLOCATOR - invertedIndexSharedMemorySize = p.invertedIndexSharedMemorySize; + *this = p; +#ifdef NGTQ_QBG + switch (genuineDataType) { +#else + switch (dataType) { #endif + case DataTypeUint8: +#ifdef NGTQ_QBG + dataSize = sizeof(uint8_t) * genuineDimension; +#else + dataSize = sizeof(uint8_t) * dimension; +#endif + break; + case DataTypeFloat: +#ifdef NGTQ_QBG + dataSize = sizeof(float) * genuineDimension; +#else + dataSize = sizeof(float) * dimension; +#endif + break; + case DataTypeFloat16: +#ifdef NGTQ_QBG + dataSize = sizeof(float) * genuineDimension; +#else + dataSize = sizeof(float) * dimension; +#endif + break; + default: + NGTThrowException("Quantizer constructor: Inner error. Invalid data type."); + break; + } + setupLocalIDByteSize(); + localDivisionNo = getLocalCodebookNo(); } inline size_t getLocalCodebookNo() { return singleLocalCodebook ? 1 : localDivisionNo; } @@ -312,6 +661,10 @@ class SerializableObject : public NGT::Object { size_t globalCentroidLimit; size_t localCentroidLimit; size_t dimension; +#ifdef NGTQ_QBG + size_t genuineDimension; + ObjectFile::DataType genuineDataType; +#endif size_t dataSize; DataType dataType; DistanceType distanceType; @@ -323,6 +676,7 @@ class SerializableObject : public NGT::Object { size_t localIDByteSize; bool localCodebookState; size_t localClusteringSampleCoefficient; + QuantizerType quantizerType; #ifdef NGT_SHARED_MEMORY_ALLOCATOR size_t invertedIndexSharedMemorySize; #endif @@ -350,9 +704,21 @@ class QuantizedObjectDistance { localDistanceLookup = 0; } } - bool isValid(size_t idx) { return flag[idx]; } + bool isValid(size_t idx) { +#ifdef NGTQ_QBG + std::cerr << "isValid() is not implemented" << std::endl; + abort(); +#else + return flag[idx]; +#endif + } #ifndef NGTQ_DISTANCE_ANGLE - void set(size_t idx, double d) { flag[idx] = true; localDistanceLookup[idx] = d; } + void set(size_t idx, double d) { +#ifndef NGTQ_QBG + flag[idx] = true; +#endif + localDistanceLookup[idx] = d; + } double getDistance(size_t idx) { return localDistanceLookup[idx]; } #endif void initialize(size_t s) { @@ -362,7 +728,9 @@ class QuantizedObjectDistance { #else localDistanceLookup = new float[size]; #endif +#ifndef NGTQ_QBG flag.resize(size, false); +#endif } #ifdef NGTQ_DISTANCE_ANGLE LocalDistanceLookup *localDistanceLookup; @@ -370,7 +738,9 @@ class QuantizedObjectDistance { float *localDistanceLookup; #endif size_t size; +#ifndef NGTQ_QBG vector flag; +#endif }; class DistanceLookupTableUint8 { @@ -385,11 +755,13 @@ class QuantizedObjectDistance { } } void initialize(size_t numOfSubspaces, size_t localCodebookCentroidNo) { - size_t alignedNumOfSubvectors = ((numOfSubspaces - 1) / NGTQ_BATCH_SIZE + 1) * NGTQ_BATCH_SIZE; - size = alignedNumOfSubvectors * localCodebookCentroidNo; + size_t numOfAlignedSubvectors = ((numOfSubspaces - 1) / NGTQ_BATCH_SIZE + 1) * NGTQ_BATCH_SIZE; + size = numOfAlignedSubvectors * localCodebookCentroidNo; localDistanceLookup = new uint8_t[size]; - scales = new float[alignedNumOfSubvectors]; - offsets = new float[alignedNumOfSubvectors]; + scales = new float[numOfAlignedSubvectors]; + offsets = new float[numOfAlignedSubvectors]; + range512 = (numOfSubspaces >> 2) * step512; + range256 = (((numOfSubspaces - 1) >> 1) + 1) * step256; } uint8_t *localDistanceLookup; @@ -399,29 +771,37 @@ class QuantizedObjectDistance { float *scales; float *offsets; float totalOffset; + size_t range512; + size_t range256; + static constexpr size_t step512 = 32; + static constexpr size_t step256 = 16; }; QuantizedObjectDistance(){} virtual ~QuantizedObjectDistance() { delete[] localCentroids; + delete[] localCentroidsForSIMD; } virtual double operator()(NGT::Object &object, size_t objectID, void *localID) = 0; virtual double operator()(void *localID, DistanceLookupTable &distanceLUT) = 0; +#ifdef NGTQBG_MIN + virtual float operator()(void *inv, float *distances, size_t size, DistanceLookupTableUint8 &distanceLUT) = 0; +#else virtual void operator()(void *inv, float *distances, size_t size, DistanceLookupTableUint8 &distanceLUT) = 0; - +#endif virtual double operator()(NGT::Object &object, size_t objectID, void *localID, DistanceLookupTable &distanceLUT) = 0; template inline double getAngleDistanceUint8(NGT::Object &object, size_t objectID, T localID[]) { - assert(globalCodebook != 0); - NGT::PersistentObject &gcentroid = *globalCodebook->getObjectSpace().getRepository().get(objectID); - size_t sizeOfObject = globalCodebook->getObjectSpace().getByteSizeOfObject(); + assert(globalCodebookIndex != 0); + NGT::PersistentObject &gcentroid = *globalCodebookIndex->getObjectSpace().getRepository().get(objectID); + size_t sizeOfObject = globalCodebookIndex->getObjectSpace().getByteSizeOfObject(); size_t localDataSize = sizeOfObject / localDivisionNo / sizeof(uint8_t); #if defined(NGT_SHARED_MEMORY_ALLOCATOR) - unsigned char *gcptr = &gcentroid.at(0, globalCodebook->getObjectSpace().getRepository().allocator); + unsigned char *gcptr = &gcentroid.at(0, globalCodebookIndex->getObjectSpace().getRepository().allocator); #else unsigned char *gcptr = &gcentroid[0]; #endif @@ -431,9 +811,9 @@ class QuantizedObjectDistance { double sum = 0.0F; for (size_t li = 0; li < localDivisionNo; li++) { size_t idx = localCodebookNo == 1 ? 0 : li; - NGT::PersistentObject &lcentroid = *localCodebook[idx].getObjectSpace().getRepository().get(localID[li]); + NGT::PersistentObject &lcentroid = *localCodebookIndexes[idx].getObjectSpace().getRepository().get(localID[li]); #if defined(NGT_SHARED_MEMORY_ALLOCATOR) - float *lcptr = (float*)&lcentroid.at(0, localCodebook[idx].getObjectSpace().getRepository().allocator); + float *lcptr = (float*)&lcentroid.at(0, localCodebookIndexes[idx].getObjectSpace().getRepository().allocator); #else float *lcptr = (float*)&lcentroid[0]; #endif @@ -458,12 +838,12 @@ class QuantizedObjectDistance { #if defined(NGT_NO_AVX) template inline double getL2DistanceUint8(NGT::Object &object, size_t objectID, T localID[]) { - assert(globalCodebook != 0); - NGT::PersistentObject &gcentroid = *globalCodebook->getObjectSpace().getRepository().get(objectID); - size_t sizeOfObject = globalCodebook->getObjectSpace().getByteSizeOfObject(); + assert(globalCodebookIndex != 0); + NGT::PersistentObject &gcentroid = *globalCodebookIndex->getObjectSpace().getRepository().get(objectID); + size_t sizeOfObject = globalCodebookIndex->getObjectSpace().getByteSizeOfObject(); size_t localDataSize = sizeOfObject / localDivisionNo / sizeof(uint8_t); #if defined(NGT_SHARED_MEMORY_ALLOCATOR) - unsigned char *gcptr = &gcentroid.at(0, globalCodebook->getObjectSpace().getRepository().allocator); + unsigned char *gcptr = &gcentroid.at(0, globalCodebookIndex->getObjectSpace().getRepository().allocator); #else unsigned char *gcptr = &gcentroid[0]; #endif @@ -471,9 +851,9 @@ class QuantizedObjectDistance { double distance = 0.0; for (size_t li = 0; li < localDivisionNo; li++) { size_t idx = localCodebookNo == 1 ? 0 : li; - NGT::PersistentObject &lcentroid = *localCodebook[idx].getObjectSpace().getRepository().get(localID[li]); + NGT::PersistentObject &lcentroid = *localCodebookIndexes[idx].getObjectSpace().getRepository().get(localID[li]); #if defined(NGT_SHARED_MEMORY_ALLOCATOR) - float *lcptr = (float*)&lcentroid.at(0, localCodebook[idx].getObjectSpace().getRepository().allocator); + float *lcptr = (float*)&lcentroid.at(0, localCodebookIndexes[idx].getObjectSpace().getRepository().allocator); #else float *lcptr = (float*)&lcentroid[0]; #endif @@ -490,12 +870,12 @@ class QuantizedObjectDistance { #else template inline double getL2DistanceUint8(NGT::Object &object, size_t objectID, T localID[]) { - assert(globalCodebook != 0); - NGT::PersistentObject &gcentroid = *globalCodebook->getObjectSpace().getRepository().get(objectID); - size_t sizeOfObject = globalCodebook->getObjectSpace().getByteSizeOfObject(); + assert(globalCodebookIndex != 0); + NGT::PersistentObject &gcentroid = *globalCodebookIndex->getObjectSpace().getRepository().get(objectID); + size_t sizeOfObject = globalCodebookIndex->getObjectSpace().getByteSizeOfObject(); size_t localDataSize = sizeOfObject / localDivisionNo / sizeof(uint8_t); #if defined(NGT_SHARED_MEMORY_ALLOCATOR) - unsigned char *gcptr = &gcentroid.at(0, globalCodebook->getObjectSpace().getRepository().allocator); + unsigned char *gcptr = &gcentroid.at(0, globalCodebookIndex->getObjectSpace().getRepository().allocator); #else unsigned char *gcptr = &gcentroid[0]; #endif @@ -503,9 +883,9 @@ class QuantizedObjectDistance { double distance = 0.0; for (size_t li = 0; li < localDivisionNo; li++) { size_t idx = localCodebookNo == 1 ? 0 : li; - NGT::PersistentObject &lcentroid = *localCodebook[idx].getObjectSpace().getRepository().get(localID[li]); + NGT::PersistentObject &lcentroid = *localCodebookIndexes[idx].getObjectSpace().getRepository().get(localID[li]); #if defined(NGT_SHARED_MEMORY_ALLOCATOR) - float *lcptr = (float*)&lcentroid.at(0, localCodebook[idx].getObjectSpace().getRepository().allocator); + float *lcptr = (float*)&lcentroid.at(0, localCodebookIndexes[idx].getObjectSpace().getRepository().allocator); #else float *lcptr = (float*)&lcentroid[0]; #endif @@ -538,12 +918,12 @@ class QuantizedObjectDistance { template inline double getAngleDistanceFloat(NGT::Object &object, size_t objectID, T localID[]) { - assert(globalCodebook != 0); - NGT::PersistentObject &gcentroid = *globalCodebook->getObjectSpace().getRepository().get(objectID); - size_t sizeOfObject = globalCodebook->getObjectSpace().getByteSizeOfObject(); + assert(globalCodebookIndex != 0); + NGT::PersistentObject &gcentroid = *globalCodebookIndex->getObjectSpace().getRepository().get(objectID); + size_t sizeOfObject = globalCodebookIndex->getObjectSpace().getByteSizeOfObject(); size_t localDataSize = sizeOfObject / localDivisionNo / sizeof(float); #if defined(NGT_SHARED_MEMORY_ALLOCATOR) - float *gcptr = (float*)&gcentroid.at(0, globalCodebook->getObjectSpace().getRepository().allocator); + float *gcptr = (float*)&gcentroid.at(0, globalCodebookIndex->getObjectSpace().getRepository().allocator); #else float *gcptr = (float*)&gcentroid[0]; #endif @@ -553,9 +933,9 @@ class QuantizedObjectDistance { double sum = 0.0F; for (size_t li = 0; li < localDivisionNo; li++) { size_t idx = localCodebookNo == 1 ? 0 : li; - NGT::PersistentObject &lcentroid = *localCodebook[idx].getObjectSpace().getRepository().get(localID[li]); + NGT::PersistentObject &lcentroid = *localCodebookIndexes[idx].getObjectSpace().getRepository().get(localID[li]); #if defined(NGT_SHARED_MEMORY_ALLOCATOR) - float *lcptr = (float*)&lcentroid.at(0, localCodebook[idx].getObjectSpace().getRepository().allocator); + float *lcptr = (float*)&lcentroid.at(0, localCodebookIndexes[idx].getObjectSpace().getRepository().allocator); #else float *lcptr = (float*)&lcentroid[0]; #endif @@ -579,12 +959,12 @@ class QuantizedObjectDistance { template inline double getL2DistanceFloat(NGT::Object &object, size_t objectID, T localID[]) { - assert(globalCodebook != 0); - NGT::PersistentObject &gcentroid = *globalCodebook->getObjectSpace().getRepository().get(objectID); - size_t sizeOfObject = globalCodebook->getObjectSpace().getByteSizeOfObject(); + assert(globalCodebookIndex != 0); + NGT::PersistentObject &gcentroid = *globalCodebookIndex->getObjectSpace().getRepository().get(objectID); + size_t sizeOfObject = globalCodebookIndex->getObjectSpace().getByteSizeOfObject(); size_t localDataSize = sizeOfObject / localDivisionNo / sizeof(float); #if defined(NGT_SHARED_MEMORY_ALLOCATOR) - float *gcptr = (float*)&gcentroid.at(0, globalCodebook->getObjectSpace().getRepository().allocator); + float *gcptr = (float*)&gcentroid.at(0, globalCodebookIndex->getObjectSpace().getRepository().allocator); #else float *gcptr = (float*)&gcentroid[0]; #endif @@ -592,9 +972,9 @@ class QuantizedObjectDistance { double distance = 0.0; for (size_t li = 0; li < localDivisionNo; li++) { size_t idx = localCodebookNo == 1 ? 0 : li; - NGT::PersistentObject &lcentroid = *localCodebook[idx].getObjectSpace().getRepository().get(localID[li]); + NGT::PersistentObject &lcentroid = *localCodebookIndexes[idx].getObjectSpace().getRepository().get(localID[li]); #if defined(NGT_SHARED_MEMORY_ALLOCATOR) - float *lcptr = (float*)&lcentroid.at(0, localCodebook[idx].getObjectSpace().getRepository().allocator); + float *lcptr = (float*)&lcentroid.at(0, localCodebookIndexes[idx].getObjectSpace().getRepository().allocator); #else float *lcptr = (float*)&lcentroid[0]; #endif @@ -612,9 +992,9 @@ class QuantizedObjectDistance { #ifdef NGTQ_DISTANCE_ANGLE inline void createDistanceLookup(NGT::Object &object, size_t objectID, DistanceLookupTable &distanceLUT) { - assert(globalCodebook != 0); - NGT::Object &gcentroid = (NGT::Object &)*globalCodebook->getObjectSpace().getRepository().get(objectID); - size_t sizeOfObject = globalCodebook->getObjectSpace().getByteSizeOfObject(); + assert(globalCodebookIndex != 0); + NGT::Object &gcentroid = (NGT::Object &)*globalCodebookIndex->getObjectSpace().getRepository().get(objectID); + size_t sizeOfObject = globalCodebookIndex->getObjectSpace().getByteSizeOfObject(); size_t localDataSize = sizeOfObject / localDivisionNo / sizeof(float); float *optr = (float*)&((NGT::Object&)object)[0]; float *gcptr = (float*)&gcentroid[0]; @@ -623,7 +1003,7 @@ class QuantizedObjectDistance { for (size_t li = 0; li < localCodebookNo; li++, oft += localDataSize) { dlu++; for (size_t k = 1; k < localCodebookCentroidNo; k++) { - NGT::Object &lcentroid = (NGT::Object&)*localCodebook[li].getObjectSpace().getRepository().get(k); + NGT::Object &lcentroid = (NGT::Object&)*localCodebookIndexes[li].getObjectSpace().getRepository().get(k); float *lcptr = (float*)&lcentroid[0]; float *lcendptr = lcptr + localDataSize; float *toptr = optr + oft; @@ -645,9 +1025,9 @@ class QuantizedObjectDistance { } #else inline void createDistanceLookup(NGT::Object &object, size_t objectID, DistanceLookupTable &distanceLUT) { - assert(globalCodebook != 0); - NGT::Object &gcentroid = (NGT::Object &)*globalCodebook->getObjectSpace().getRepository().get(objectID); - size_t sizeOfObject = globalCodebook->getObjectSpace().getByteSizeOfObject(); + assert(globalCodebookIndex != 0); + NGT::Object &gcentroid = (NGT::Object &)*globalCodebookIndex->getObjectSpace().getRepository().get(objectID); + size_t sizeOfObject = globalCodebookIndex->getObjectSpace().getByteSizeOfObject(); createFloatL2DistanceLookup(&((NGT::Object&)object)[0], sizeOfObject, &gcentroid[0], distanceLUT.localDistanceLookup); @@ -682,110 +1062,293 @@ class QuantizedObjectDistance { + inline void createFloatL2DistanceLookup(void *object, size_t sizeOfObject, void *globalCentroid, DistanceLookupTableUint8 &distanceLUT) { + assert(localCodebookCentroidNoSIMD == 16); + size_t dim = sizeOfObject / sizeof(float); + size_t localDim = dim / localDivisionNo; +#if defined(NGT_AVX512) + __m512 flut[dim]; + __m512 *flutptr = &flut[0]; + __m512 mmin = _mm512_set1_ps(std::numeric_limits::max()); + __m512 mmax = _mm512_set1_ps(-std::numeric_limits::max()); + auto *lcptr = static_cast(&localCentroidsForSIMD[0]); + auto *optr = static_cast(object); + auto *optrend = optr + dim; +#ifndef NGTQG_ZERO_GLOBAL + auto *gcptr = static_cast(globalCentroid); +#endif + while (optr < optrend) { + auto *optrllast = optr + localDim; +#ifdef NGTQG_ZERO_GLOBAL + float rsv = *optr++; +#else + float rsv = *optr++ - *gcptr++; +#endif + __m512 vtmp = _mm512_sub_ps(_mm512_set1_ps(rsv), _mm512_loadu_ps(lcptr)); + __m512 v = _mm512_mul_ps(vtmp, vtmp); + lcptr += 16; + while (optr < optrllast) { +#ifdef NGTQG_ZERO_GLOBAL + rsv = *optr++; +#else + rsv = *optr++ - *gcptr++; +#endif + vtmp = _mm512_sub_ps(_mm512_set1_ps(rsv), _mm512_loadu_ps(lcptr)); + v = _mm512_add_ps(v, _mm512_mul_ps(vtmp, vtmp)); + lcptr += 16; + } + mmin = _mm512_min_ps(mmin, v); + mmax = _mm512_max_ps(mmax, v); + *flutptr = v; + flutptr++; + } + float min = _mm512_reduce_min_ps(mmin); + float max = _mm512_reduce_max_ps(mmax); + float offset = min; + float scale = (max - min) / 255.0; + + + distanceLUT.totalOffset = offset; + auto *blutptr = distanceLUT.localDistanceLookup; + flutptr = &flut[0]; + for (size_t li = 0; li < localCodebookNo; li++) { + __m512 v = _mm512_div_ps(_mm512_sub_ps(*flutptr, _mm512_set1_ps(offset)), _mm512_set1_ps(scale)); + __m512i b4v = _mm512_cvtps_epi32(_mm512_roundscale_ps(v, _MM_FROUND_TO_NEAREST_INT)); + __m128i b = _mm512_cvtepi32_epi8(b4v); + _mm_storeu_si128((__m128i_u*)blutptr, b); + //_mm512_storeu_si512(blutptr, b4v); + //_mm512_storeu_ps(blutptr, v); + flutptr++; + blutptr += 16; + distanceLUT.offsets[li] = offset; + distanceLUT.scales[li] = scale; + } + //std::cerr << "offset=" << distanceLUT.totalOffset << std::endl; +#elif defined(NGT_AVX2) + __m256 flut[dim * 2]; + __m256 *flutptr = &flut[0]; + __m256 mmin = _mm256_set1_ps(std::numeric_limits::max()); + __m256 mmax = _mm256_set1_ps(-std::numeric_limits::max()); + auto *lcptr = static_cast(&localCentroidsForSIMD[0]); + auto *optr = static_cast(object); + auto *optrend = optr + dim; +#ifndef NGTQG_ZERO_GLOBAL + auto *gcptr = static_cast(globalCentroid); +#endif + while (optr < optrend) { + auto *optrllast = optr + localDim; +#ifdef NGTQG_ZERO_GLOBAL + float rsv = *optr++; +#else + float rsv = *optr++ - *gcptr++; +#endif + __m256 vtmp = _mm256_sub_ps(_mm256_set1_ps(rsv), _mm256_loadu_ps(lcptr)); + __m256 vt = _mm256_mul_ps(vtmp, vtmp); + lcptr += 8; + vtmp = _mm256_sub_ps(_mm256_set1_ps(rsv), _mm256_loadu_ps(lcptr)); + __m256 vb = _mm256_mul_ps(vtmp, vtmp); + lcptr += 8; + while (optr < optrllast) { +#ifdef NGTQG_ZERO_GLOBAL + rsv = *optr++; +#else + rsv = *optr++ - *gcptr++; +#endif + vtmp = _mm256_sub_ps(_mm256_set1_ps(rsv), _mm256_loadu_ps(lcptr)); + vt = _mm256_add_ps(vt, _mm256_mul_ps(vtmp, vtmp)); + lcptr += 8; + vtmp = _mm256_sub_ps(_mm256_set1_ps(rsv), _mm256_loadu_ps(lcptr)); + vb = _mm256_add_ps(vb, _mm256_mul_ps(vtmp, vtmp)); + lcptr += 8; + } + mmin = _mm256_min_ps(mmin, vt); + mmax = _mm256_max_ps(mmax, vt); + *flutptr++ = vt; + mmin = _mm256_min_ps(mmin, vb); + mmax = _mm256_max_ps(mmax, vb); + *flutptr++ = vb; + } + float f[8]; + _mm256_storeu_ps(f, mmin); + float min = f[0]; + for (size_t i = 1; i < 8; i++) { + if (min > f[i]) min = f[i]; + } + _mm256_storeu_ps(f, mmax); + float max = f[0]; + for (size_t i = 1; i < 8; i++) { + if (max < f[i]) max = f[i]; + } + float offset = min; + float scale = (max - min) / 255.0; + + distanceLUT.totalOffset = offset; + //uint8_t blut[dim / 2 * localCodebookCentroidNoSIMD]; + auto *blutptr = distanceLUT.localDistanceLookup; + flutptr = &flut[0]; + for (size_t li = 0; li < localCodebookNo; li++) { + __m256 v = _mm256_div_ps(_mm256_sub_ps(*flutptr++, _mm256_set1_ps(offset)), _mm256_set1_ps(scale)); + __m256i b4v = _mm256_cvtps_epi32(_mm256_round_ps(v, _MM_FROUND_TO_NEAREST_INT)); + *blutptr++ = _mm256_extract_epi8(b4v, 0); + *blutptr++ = _mm256_extract_epi8(b4v, 4); + *blutptr++ = _mm256_extract_epi8(b4v, 8); + *blutptr++ = _mm256_extract_epi8(b4v, 12); + __m128i b = _mm256_extracti128_si256(b4v, 1); + *blutptr++ = _mm_extract_epi8(b, 0); + *blutptr++ = _mm_extract_epi8(b, 4); + *blutptr++ = _mm_extract_epi8(b, 8); + *blutptr++ = _mm_extract_epi8(b, 12); + v = _mm256_div_ps(_mm256_sub_ps(*flutptr++, _mm256_set1_ps(offset)), _mm256_set1_ps(scale)); + b4v = _mm256_cvtps_epi32(_mm256_round_ps(v, _MM_FROUND_TO_NEAREST_INT)); + *blutptr++ = _mm256_extract_epi8(b4v, 0); + *blutptr++ = _mm256_extract_epi8(b4v, 4); + *blutptr++ = _mm256_extract_epi8(b4v, 8); + *blutptr++ = _mm256_extract_epi8(b4v, 12); + b = _mm256_extracti128_si256(b4v, 1); + *blutptr++ = _mm_extract_epi8(b, 0); + *blutptr++ = _mm_extract_epi8(b, 4); + *blutptr++ = _mm_extract_epi8(b, 8); + *blutptr++ = _mm_extract_epi8(b, 12); + distanceLUT.offsets[li] = offset; + distanceLUT.scales[li] = scale; + } +#else + float flut[dim / 2 * localCodebookCentroidNoSIMD]; + auto *flutptr = &flut[0]; + auto min = std::numeric_limits::max(); + auto max = -std::numeric_limits::max(); + auto *lcptr = static_cast(&localCentroidsForSIMD[0]); + auto *optr = static_cast(object); +#ifndef NGTQG_ZERO_GLOBAL + auto *gcptr = static_cast(globalCentroid); +#endif + float sub[localCodebookCentroidNoSIMD]; + for (size_t d = 0; d < dim; d++) { +#ifdef NGTQG_ZERO_GLOBAL + float rsv = optr[d]; +#else + float rsv = optr[d] - *gcptr++; +#endif + for (size_t k = 0; k < localCodebookCentroidNoSIMD; k++) { + auto v = rsv - *lcptr++; + v *= v; + if (d % 2 == 0) { + sub[k] = v; + } else { + v += sub[k]; + if (v < min) { + min = v; + } else if (v > max) { + max = v; + } + *flutptr++ = v; + } + } + } + float offset = min; + float scale = (max - min) / 255.0; + + distanceLUT.totalOffset = offset; + auto *blutptr = distanceLUT.localDistanceLookup; + flutptr = &flut[0]; + for (size_t li = 0; li < localCodebookNo; li++) { + for (size_t k = 0; k < localCodebookCentroidNoSIMD; k++) { + int32_t tmp = std::round((*flutptr - offset) / scale); + assert(tmp >= 0 && tmp <= 255); + *blutptr++ = static_cast(tmp); + flutptr++; + } + distanceLUT.offsets[li] = offset; + distanceLUT.scales[li] = scale; + } + +#endif + + + + } + + + inline void createFloatL2DistanceLookup(void *object, size_t sizeOfObject, void *globalCentroid, float *lut) { size_t localDataSize = sizeOfObject / localDivisionNo / sizeof(float); - float *optr = static_cast(object); +#if !defined(NGTQG_ZERO_GLOBAL) float *gcptr = static_cast(globalCentroid); - +#endif size_t oft = 0; float *lcptr = static_cast(&localCentroids[0]); for (size_t li = 0; li < localCodebookNo; li++, oft += localDataSize) { - lut++; + *lut++ = 0; lcptr += localDataSize; for (size_t k = 1; k < localCodebookCentroidNo; k++) { float *lcendptr = lcptr + localDataSize; float *toptr = optr + oft; +#if !defined(NGTQG_ZERO_GLOBAL) float *tgcptr = gcptr + oft; +#endif float d = 0.0; while (lcptr != lcendptr) { +#if defined(NGTQG_ZERO_GLOBAL) + float sub = *toptr++ - *lcptr++; +#else float sub = *toptr++ - *tgcptr++ - *lcptr++; +#endif d += sub * sub; } *lut++ = d; } } - } + } + inline void createDistanceLookup(NGT::Object &object, size_t objectID, DistanceLookupTableUint8 &distanceLUT) { - assert(globalCodebook != 0); + void *objectPtr = &((NGT::Object&)object)[0]; + createDistanceLookup(objectPtr, objectID, distanceLUT); + } + inline void createDistanceLookup(void *objectPtr, size_t objectID, DistanceLookupTableUint8 &distanceLUT) { + assert(globalCodebookIndex != 0); size_t sizeOfObject = dimension * sizeOfType; - float dlutmp[distanceLUT.size]; #ifdef NGTQG_DOT_PRODUCT - createFloatDotProductLookup(&((NGT::Object&)object)[0], sizeOfObject, &gcentroid[0], dlutmp); + std::cerr << "Not implemented." << std::endl; + abort(); #else - createFloatL2DistanceLookup(&((NGT::Object&)object)[0], sizeOfObject, globalCentroid.data(), dlutmp); + assert(objectID < quantizationCodebook->size()); + createFloatL2DistanceLookup(objectPtr, sizeOfObject, quantizationCodebook->data(objectID), distanceLUT); #endif - { - uint8_t *cdlu = distanceLUT.localDistanceLookup; - float *dlu = dlutmp; - distanceLUT.totalOffset = 0.0; - float min = FLT_MAX; - float max = -FLT_MAX; - for (size_t li = 0; li < localCodebookNo; li++) { - for (size_t k = 1; k < localCodebookCentroidNo; k++) { - if (dlu[k] > max) { - max = dlu[k]; - } - if (dlu[k] < min) { - min = dlu[k]; - } - } - dlu += localCodebookCentroidNo; - } - float offset = min; - float scale = (max - min) / 255.0; - dlu = dlutmp; - for (size_t li = 0; li < localCodebookNo; li++) { - for (size_t k = 1; k < localCodebookCentroidNo; k++) { - int32_t tmp = std::round((dlu[k] - offset) / scale); - assert(tmp >= 0 && tmp <= 255); - *cdlu++ = static_cast(tmp); - } - dlu += localCodebookCentroidNo; - distanceLUT.offsets[li] = offset; - distanceLUT.scales[li] = scale; - distanceLUT.totalOffset += offset; - } - if ((localCodebookNo & 0x1) != 0) { - for (size_t k = 0; k < localCodebookCentroidNo; k++) { - *cdlu++ = 0; - } - distanceLUT.offsets[localCodebookNo] = 0.0; - distanceLUT.scales[localCodebookNo] = 0.0; - } - - } } - void set(NGT::Index *gcb, NGT::Index lcb[], size_t dn, size_t lcn, size_t sizeoftype, size_t dim) { - globalCodebook = gcb; - localCodebook = lcb; + void set(NGT::Index *gcb, NGT::Index lcb[], QuantizationCodebook *qcodebook, size_t dn, size_t lcn, + size_t sizeoftype, size_t dim, Rotation *r) { + globalCodebookIndex = gcb; + localCodebookIndexes = lcb; localDivisionNo = dn; dimension = dim; assert(dimension % localDivisionNo == 0); localDataSize = dimension / localDivisionNo; sizeOfType = sizeoftype; set(lcb, lcn); - if (globalCodebook->getObjectSpace().getRepository().size() == 2) { + if (globalCodebookIndex->getObjectSpace().getRepository().size() == 2) { NGT::ObjectID id = 1; try { - globalCodebook->getObjectSpace().getObject(id, globalCentroid); + globalCodebookIndex->getObjectSpace().getObject(id, globalCentroid); } catch (NGT::Exception &err) { std::cerr << "Cannot load the global centroid. id=" << id << std::endl; } } + quantizationCodebook = qcodebook; + float *lc = new float[localCodebookNo * localCodebookCentroidNo * localDataSize]; for (size_t li = 0; li < localCodebookNo; li++) { for (size_t k = 1; k < localCodebookCentroidNo; k++) { #if defined(NGT_SHARED_MEMORY_ALLOCATOR) - NGT::PersistentObject &lcentroid = *static_cast(localCodebook[li].getObjectSpace().getRepository().get(k)); + NGT::PersistentObject &lcentroid = *static_cast(localCodebookIndexes[li].getObjectSpace().getRepository().get(k)); memcpy(&lc[li * localCodebookCentroidNo * localDataSize + k * localDataSize], - &lcentroid.at(0, localCodebook[li].getObjectSpace().getRepository().allocator), localDataSize * sizeof(float)); + &lcentroid.at(0, localCodebookIndexes[li].getObjectSpace().getRepository().allocator), localDataSize * sizeof(float)); #else - NGT::Object &lcentroid = *static_cast(localCodebook[li].getObjectSpace().getRepository().get(k)); + NGT::Object &lcentroid = *static_cast(localCodebookIndexes[li].getObjectSpace().getRepository().get(k)); memcpy(&lc[li * localCodebookCentroidNo * localDataSize + k * localDataSize], &lcentroid[0], localDataSize * sizeof(float)); #endif } @@ -793,6 +1356,26 @@ class QuantizedObjectDistance { localCentroids = lc; + rotation = r; + + localCodebookCentroidNoSIMD = localCodebookCentroidNo == 0 ? 0 : localCodebookCentroidNo - 1; + lc = new float[dimension * localCodebookCentroidNoSIMD]; + for (size_t li = 0; li < localCodebookNo; li++) { + for (size_t k = 1; k < localCodebookCentroidNo; k++) { +#if defined(NGT_SHARED_MEMORY_ALLOCATOR) + NGT::PersistentObject &lcentroid = *static_cast(localCodebookIndexes[li].getObjectSpace().getRepository().get(k)); + float *subVector = reinterpret_cast(&lcentroid.at(0, localCodebookIndexes[li].getObjectSpace().getRepository().allocator)); +#else + NGT::Object &lcentroid = *static_cast(localCodebookIndexes[li].getObjectSpace().getRepository().get(k)); + float *subVector = reinterpret_cast(&lcentroid[0]); +#endif + for (size_t d = 0; d < localDataSize; d++) { + lc[(li * localDataSize + d) * localCodebookCentroidNoSIMD + (k - 1)] = subVector[d]; + } + } + } + + localCentroidsForSIMD = lc; } @@ -810,8 +1393,8 @@ class QuantizedObjectDistance { c.initialize(localCodebookNo, localCodebookCentroidNo); } - NGT::Index *globalCodebook; - NGT::Index *localCodebook; + NGT::Index *globalCodebookIndex; + NGT::Index *localCodebookIndexes; size_t localDivisionNo; size_t localCodebookNo; size_t localCodebookCentroidNo; @@ -820,11 +1403,14 @@ class QuantizedObjectDistance { size_t sizeOfType; size_t dimension; vector globalCentroid; - + QuantizationCodebook *quantizationCodebook; + float *localCentroids; + float *localCentroidsForSIMD; size_t localCodebookCentroidNoSIMD; + Rotation *rotation; }; template @@ -874,11 +1460,11 @@ class QuantizedObjectDistanceUint8 : public QuantizedObjectDistance { } inline double operator()(NGT::Object &object, size_t objectID, void *l, DistanceLookupTable &distanceLUT) { T *localID = static_cast(l); - NGT::PersistentObject &gcentroid = *globalCodebook->getObjectSpace().getRepository().get(objectID); - size_t sizeOfObject = globalCodebook->getObjectSpace().getByteSizeOfObject(); + NGT::PersistentObject &gcentroid = *globalCodebookIndex->getObjectSpace().getRepository().get(objectID); + size_t sizeOfObject = globalCodebookIndex->getObjectSpace().getByteSizeOfObject(); size_t localDataSize = sizeOfObject / localDivisionNo / sizeof(uint8_t); #if defined(NGT_SHARED_MEMORY_ALLOCATOR) - unsigned char *gcptr = &gcentroid.at(0, globalCodebook->getObjectSpace().getRepository().allocator); + unsigned char *gcptr = &gcentroid.at(0, globalCodebookIndex->getObjectSpace().getRepository().allocator); #else unsigned char *gcptr = &gcentroid[0]; #endif @@ -891,9 +1477,9 @@ class QuantizedObjectDistanceUint8 : public QuantizedObjectDistance { gcptr += localDataSize; } else { size_t idx = localCodebookNo == 1 ? 0 : li; - NGT::PersistentObject &lcentroid = *localCodebook[idx].getObjectSpace().getRepository().get(localID[li]); + NGT::PersistentObject &lcentroid = *localCodebookIndexes[idx].getObjectSpace().getRepository().get(localID[li]); #if defined(NGT_SHARED_MEMORY_ALLOCATOR) - float *lcptr = (float*)&lcentroid.at(0, localCodebook[idx].getObjectSpace().getRepository().allocator); + float *lcptr = (float*)&lcentroid.at(0, localCodebookIndexes[idx].getObjectSpace().getRepository().allocator); #else float *lcptr = (float*)&lcentroid[0]; #endif @@ -909,11 +1495,15 @@ class QuantizedObjectDistanceUint8 : public QuantizedObjectDistance { } return sqrt(distance); } +#ifdef NGTQBG_MIN + inline float operator()(void *inv, float *distances, size_t size, DistanceLookupTableUint8 &distanceLUT) { +#else inline void operator()(void *inv, float *distances, size_t size, DistanceLookupTableUint8 &distanceLUT) { +#endif cerr << "operator is not implemented" << endl; abort(); } -#endif // NGTQ_DISTANCE_ANGLE +#endif }; @@ -964,22 +1554,50 @@ class QuantizedObjectDistanceFloat : public QuantizedObjectDistance { } + + inline float horizontalMin(__m256 &a, __m256 &b, float noOfObjects) { + __m256 seq = _mm256_set_ps(8, 7, 6, 5, 4, 3, 2, 1); + __m256 mask = _mm256_sub_ps(_mm256_set1_ps(noOfObjects), seq); + mask = _mm256_blendv_ps(_mm256_set1_ps(0.0), _mm256_set1_ps(std::numeric_limits::max()), mask); + __m256 data = _mm256_max_ps(a, mask); + noOfObjects -= 8; + if (noOfObjects > 0.0) { + mask = _mm256_sub_ps(_mm256_set1_ps(noOfObjects), seq); + mask = _mm256_blendv_ps(_mm256_set1_ps(0.0), _mm256_set1_ps(std::numeric_limits::max()), mask); + b = _mm256_max_ps(b, mask); + data = _mm256_min_ps(data, b); + } + + data = _mm256_min_ps(data, (__m256)_mm256_permute4x64_epi64((__m256i)data, _MM_SHUFFLE(3, 2, 3, 2))); + data = _mm256_min_ps(data, (__m256)_mm256_srli_si256((__m256i)data, 8)); + data = _mm256_min_ps(data, (__m256)_mm256_srli_si256((__m256i)data, 4)); + //std::cerr << "ret=" << data[0] << std::endl; + return data[0]; + } + #if defined(NGTQG_AVX512) || defined(NGTQG_AVX2) - inline void operator()(void *inv, float *distances, size_t size, DistanceLookupTableUint8 &distanceLUT) { +#ifdef NGTQBG_MIN + inline float operator()(void *inv, float *distances, size_t noOfObjects, DistanceLookupTableUint8 &distanceLUT) { +#else + inline void operator()(void *inv, float *distances, size_t noOfObjects, DistanceLookupTableUint8 &distanceLUT) { +#endif uint8_t *localID = static_cast(inv); float *d = distances; - + float *lastd = distances + noOfObjects; +#ifdef NGTQBG_MIN + float min = std::numeric_limits::max(); +#endif #if defined(NGTQG_AVX512) - __m512i mask512x0F = _mm512_set1_epi16(0x000f); - __m512i mask512xF0 = _mm512_set1_epi16(0x00f0); - constexpr size_t step512 = 32; - const size_t range512 = (localDivisionNo >> 2) * step512; + const __m512i mask512x0F = _mm512_set1_epi16(0x000f); + const __m512i mask512xF0 = _mm512_set1_epi16(0x00f0); + const size_t range512 = distanceLUT.range512; + auto step512 = distanceLUT.step512; #endif const __m256i mask256x0F = _mm256_set1_epi16(0x000f); const __m256i mask256xF0 = _mm256_set1_epi16(0x00f0); - constexpr size_t step256 = 16; - const size_t range256 = (((localDivisionNo - 1) >> 1) + 1) * step256; - auto *last = localID + range256 / NGTQ_SIMD_BLOCK_SIZE * size; + const size_t range256 = distanceLUT.range256; + auto step256 = distanceLUT.step256; + auto *last = localID + range256 / NGTQ_SIMD_BLOCK_SIZE * noOfObjects; while (localID < last) { uint8_t *lut = distanceLUT.localDistanceLookup; auto *lastgroup256 = localID + range256; @@ -1011,20 +1629,15 @@ class QuantizedObjectDistanceFloat : public QuantizedObjectDistance { __m256i hi = _mm256_slli_epi16(_mm256_and_si256(packedobj, mask256xF0), 4); __m256i obj = _mm256_or_si256(lo, hi); __m256i vtmp = _mm256_shuffle_epi8(lookupTable, obj); - __attribute__((aligned(32))) uint8_t v[32]; - _mm256_storeu_si256((__m256i*)&v, vtmp); #if defined(NGTQG_AVX512) depu16 = _mm512_adds_epu16(depu16, _mm512_cvtepu8_epi16(vtmp)); #else - __attribute__((aligned(32))) uint16_t v16[16]; - _mm256_storeu_si256((__m256i*)&v16, depu16l); - _mm256_storeu_si256((__m256i*)&v16, depu16h); depu16l = _mm256_adds_epu16(depu16l, _mm256_cvtepu8_epi16(_mm256_extractf128_si256(vtmp, 0))); depu16h = _mm256_adds_epu16(depu16h, _mm256_cvtepu8_epi16(_mm256_extractf128_si256(vtmp, 1))); #endif lut += (localCodebookCentroidNo - 1) * 2; - localID += 16; + localID += step256; } #if defined(NGTQG_AVX512) @@ -1041,6 +1654,21 @@ class QuantizedObjectDistanceFloat : public QuantizedObjectDistance { #endif distance = _mm512_sqrt_ps(distance); _mm512_storeu_ps(d, distance); +#ifdef NGTQBG_MIN + { + float tmpmin; + int rest = 16 - (lastd - d); + if (rest > 0) { + __mmask16 mask = 0xffff; + mask >>= rest; + tmpmin = _mm512_mask_reduce_min_ps(mask, distance); + } else { + tmpmin = _mm512_reduce_min_ps(distance); + } + //std::cerr << "tmpmin=" << tmpmin << std::endl; + if (min > tmpmin) min = tmpmin; + } +#endif #else __m256i lol = _mm256_cvtepu16_epi32(_mm256_extractf128_si256(depu16l, 0)); __m256i loh = _mm256_cvtepu16_epi32(_mm256_extractf128_si256(depu16l, 1)); @@ -1048,9 +1676,6 @@ class QuantizedObjectDistanceFloat : public QuantizedObjectDistance { __m256i hih = _mm256_cvtepu16_epi32(_mm256_extractf128_si256(depu16h, 1)); __m256 distancel = _mm256_cvtepi32_ps(_mm256_add_epi32(lol, hil)); __m256 distanceh = _mm256_cvtepi32_ps(_mm256_add_epi32(loh, hih)); - __attribute__((aligned(32))) float v32[8]; - _mm256_storeu_ps((float*)&v32, distancel); - _mm256_storeu_ps((float*)&v32, distanceh); __m256 scalel = _mm256_broadcastss_ps(*reinterpret_cast<__m128*>(&distanceLUT.scales[0])); __m256 scaleh = _mm256_broadcastss_ps(*reinterpret_cast<__m128*>(&distanceLUT.scales[0])); distancel = _mm256_mul_ps(distancel, scalel); @@ -1067,25 +1692,34 @@ class QuantizedObjectDistanceFloat : public QuantizedObjectDistance { distanceh = _mm256_sqrt_ps(distanceh); _mm256_storeu_ps(d, distancel); _mm256_storeu_ps(d + 8, distanceh); +#ifdef NGTQBG_MIN + { + float tmpmin = horizontalMin(distancel, distanceh, lastd - d); + if (min > tmpmin) min = tmpmin; + } +#endif #endif d += 16; } +#ifdef NGTQBG_MIN + return min; +#endif } #else inline void operator()(void *inv, float *distances, size_t size, DistanceLookupTableUint8 &distanceLUT) { uint8_t *localID = static_cast(inv); - size_t alignedNumOfSubvectors = ((localDivisionNo - 1) / NGTQ_BATCH_SIZE + 1) * NGTQ_BATCH_SIZE; + size_t numOfAlignedSubvectors = ((localDivisionNo - 1) / NGTQ_BATCH_SIZE + 1) * NGTQ_BATCH_SIZE; size_t alignedSize = ((size - 1) / 2 + 1) * 2; uint32_t d[NGTQ_SIMD_BLOCK_SIZE]; size_t didx = 0; - size_t byteSize = alignedNumOfSubvectors * alignedSize / 2; + size_t byteSize = numOfAlignedSubvectors * alignedSize / 2; auto *last = localID + byteSize; while (localID < last) { uint8_t *lut = distanceLUT.localDistanceLookup; memset(d, 0, sizeof(uint32_t) * NGTQ_SIMD_BLOCK_SIZE); - for (size_t li = 0; li < alignedNumOfSubvectors; li++) { + for (size_t li = 0; li < numOfAlignedSubvectors; li++) { for (size_t i = 0; i < NGTQ_SIMD_BLOCK_SIZE; i++) { uint8_t obj = *localID; if (i % 2 == 0) { @@ -1112,9 +1746,9 @@ class QuantizedObjectDistanceFloat : public QuantizedObjectDistance { } inline double operator()(NGT::Object &object, size_t objectID, void *l, DistanceLookupTable &distanceLUT) { T *localID = static_cast(l); - NGT::PersistentObject &gcentroid = *globalCodebook->getObjectSpace().getRepository().get(objectID); + NGT::PersistentObject &gcentroid = *globalCodebookIndex->getObjectSpace().getRepository().get(objectID); #if defined(NGT_SHARED_MEMORY_ALLOCATOR) - float *gcptr = (float*)&gcentroid.at(0, globalCodebook->getObjectSpace().getRepository().allocator); + float *gcptr = (float*)&gcentroid.at(0, globalCodebookIndex->getObjectSpace().getRepository().allocator); #else float *gcptr = (float*)&gcentroid[0]; #endif @@ -1128,9 +1762,9 @@ class QuantizedObjectDistanceFloat : public QuantizedObjectDistance { gcptr += localDataSize; } else { size_t idx = li; - NGT::PersistentObject &lcentroid = *localCodebook[idx].getObjectSpace().getRepository().get(localID[li]); + NGT::PersistentObject &lcentroid = *localCodebookIndexes[idx].getObjectSpace().getRepository().get(localID[li]); #if defined(NGT_SHARED_MEMORY_ALLOCATOR) - float *lcptr = (float*)&lcentroid.at(0, localCodebook[idx].getObjectSpace().getRepository().allocator); + float *lcptr = (float*)&lcentroid.at(0, localCodebookIndexes[idx].getObjectSpace().getRepository().allocator); #else float *lcptr = (float*)&lcentroid[0]; #endif @@ -1169,36 +1803,42 @@ class QuantizedObjectDistanceFloat : public QuantizedObjectDistance { class Quantizer { public: - typedef ArrayFile ObjectList; +#ifdef NGTQ_STATIC_OBJECT_FILE + typedef StaticObjectFile ObjectList; +#else + typedef ObjectFile ObjectList; +#endif - Quantizer(DataType dt, size_t dim) { - property.dimension = dim; - property.dataType = dt; - switch (property.dataType) { - case DataTypeUint8: - property.dataSize = sizeof(uint8_t) * property.dimension; - break; - case DataTypeFloat: - property.dataSize = sizeof(float) * property.dimension; - break; - default: - cerr << "Quantizer constructor: Inner error. Invalid data type." << endl; - break; - } - } virtual ~Quantizer() { } virtual void create(const string &index, - NGT::Property &globalPropertySet, + NGT::Property &globalPropertySet, +#ifdef NGTQ_QBG + NGT::Property &localPropertySet, + std::vector *rotation = 0, + const string &objectFile = "") = 0; +#else NGT::Property &localPropertySet) = 0; +#endif +#ifndef NGTQ_QBG virtual void insert(vector > &objects) = 0; - virtual void insert(const string &line, vector > &objects, size_t id) = 0; virtual void insert(vector &object, vector > &objects, size_t id) = 0; +#endif + virtual void insertIntoObjectRepository(vector &object, size_t id) = 0; +#ifdef NGTQ_QBG + virtual void createIndex(size_t beginID, size_t endID) = 0; +#endif + virtual void setupInvertedIndex(std::vector> &quantizationCodebook, + std::vector &codebookIndex, + std::vector &objectIndex) = 0; +#ifndef NGTQ_QBG virtual void rebuildIndex() = 0; +#endif virtual void save() = 0; - virtual void open(const string &index, NGT::Property &globalProperty) = 0; - virtual void open(const string &index) = 0; + virtual void saveRotation(const std::vector &rotation) = 0; + virtual void open(const string &index, NGT::Property &globalProperty, bool readOnly) = 0; + virtual void open(const string &index, bool readOnly) = 0; virtual void close() = 0; virtual void closeCodebooks() = 0; #ifdef NGTQ_SHARED_INVERTED_INDEX @@ -1207,10 +1847,18 @@ class Quantizer { virtual void validate() = 0; +#ifdef NGTQ_QBG + virtual void extractInvertedIndexObject(InvertedIndexEntry &invertedIndexObjects, size_t id) = 0; virtual void extractInvertedIndexObject(InvertedIndexEntry &invertedIndexObjects) = 0; +#endif virtual void extractInvertedIndex(std::vector> &invertedIndex) = 0; + virtual void eraseInvertedIndexObject(size_t id) = 0; + virtual void eraseInvertedIndexObject() = 0; - virtual NGT::Distance getApproximateDistance(NGT::Object &query, uint32_t globalID, uint16_t *localID, QuantizedObjectDistance::DistanceLookupTable &distanceLUT) { abort(); } + virtual NGT::Distance getApproximateDistance(NGT::Object &query, uint32_t globalID, uint16_t *localID, QuantizedObjectDistance::DistanceLookupTable &distanceLUT) { + std::cerr << "getApproximateDistance() is not implemented." << std::endl; + abort(); + } virtual void search(NGT::Object *object, NGT::ObjectDistances &objs, size_t size, size_t approximateSearchSize, @@ -1229,21 +1877,20 @@ class Quantizer { virtual void info(ostream &os, char mode) = 0; - virtual NGT::Index & getLocalCodebook(size_t size) = 0; - virtual void verify() = 0; - virtual size_t getLocalCodebookSize(size_t size) = 0; - virtual size_t getInstanceSharedMemorySize(ostream &os, SharedMemoryAllocator::GetMemorySizeType t = SharedMemoryAllocator::GetTotalMemorySize) = 0; NGT::Object *allocateObject(string &line, const string &sep) { - return globalCodebook.allocateObject(line, " \t"); + return globalCodebookIndex.allocateObject(line, " \t"); } NGT::Object *allocateObject(vector &obj) { - return globalCodebook.allocateObject(obj); + return globalCodebookIndex.allocateObject(obj); + } + NGT::Object *allocateObject(vector &obj) { + return globalCodebookIndex.allocateObject(obj); } - void deleteObject(NGT::Object *object) { globalCodebook.deleteObject(object); } + void deleteObject(NGT::Object *object) { globalCodebookIndex.deleteObject(object); } void setThreadSize(size_t size) { property.threadSize = size; } void setGlobalRange(float r) { property.globalRange = r; } @@ -1253,11 +1900,14 @@ class Quantizer { void setDimension(size_t s) { property.dimension = s; } void setDistanceType(DistanceType t) { property.distanceType = t; } + NGT::Index &getLocalCodebook(size_t idx) { return localCodebookIndexes[idx]; } + size_t getLocalCodebookSize(size_t size) { return localCodebookIndexes[size].getObjectRepositorySize(); } + string getRootDirectory() { return rootDirectory; } size_t getSharedMemorySize(ostream &os, SharedMemoryAllocator::GetMemorySizeType t = SharedMemoryAllocator::GetTotalMemorySize) { os << "Global centroid:" << endl; - return globalCodebook.getSharedMemorySize(os, t) + getInstanceSharedMemorySize(os, t); + return globalCodebookIndex.getSharedMemorySize(os, t) + getInstanceSharedMemorySize(os, t); } virtual QuantizedObjectDistance &getQuantizedObjectDistance() = 0; @@ -1267,7 +1917,7 @@ class Quantizer { Property property; - NGT::Index globalCodebook; + NGT::Index globalCodebookIndex; size_t distanceComputationCount; @@ -1275,6 +1925,12 @@ class Quantizer { NGT::ObjectSpace::ObjectType objectType; size_t divisionNo; + std::vector localCodebookIndexes; + + QuantizationCodebook quantizationCodebook; + std::vector objectToBlobIndex; + Rotation rotation; + }; class QuantizedObjectProcessingStream { @@ -1294,16 +1950,21 @@ class QuantizedObjectProcessingStream { } void initialize(size_t divisionNo) { - alignedNumOfSubvectors = ((divisionNo - 1) / NGTQ_BATCH_SIZE + 1) * NGTQ_BATCH_SIZE; - alignedBlockSize = NGTQ_SIMD_BLOCK_SIZE * alignedNumOfSubvectors; + numOfAlignedSubvectors = ((divisionNo - 1) / NGTQ_BATCH_SIZE + 1) * NGTQ_BATCH_SIZE; + alignedBlockSize = NGTQ_SIMD_BLOCK_SIZE * numOfAlignedSubvectors; } + static size_t getNumOfAlignedObjects(size_t numOfObjects) { + return (((numOfObjects - 1) / NGTQ_SIMD_BLOCK_SIZE + 1) * NGTQ_SIMD_BLOCK_SIZE); + } + void setStreamSize(size_t numOfObjects) { - alignedNumOfObjects = (((numOfObjects - 1) / NGTQ_SIMD_BLOCK_SIZE + 1) * NGTQ_SIMD_BLOCK_SIZE); - streamSize = alignedNumOfObjects * alignedNumOfSubvectors; + numOfAlignedObjects = getNumOfAlignedObjects(numOfObjects); + streamSize = numOfAlignedObjects * numOfAlignedSubvectors; return; } +#ifdef NGTQ_QBG void arrangeQuantizedObject(size_t dataNo, size_t subvectorNo, uint8_t quantizedObject) { #if defined(NGT_SHARED_MEMORY_ALLOCATOR) abort(); @@ -1313,13 +1974,14 @@ class QuantizedObjectProcessingStream { stream[blkNo * alignedBlockSize + NGTQ_SIMD_BLOCK_SIZE * subvectorNo + oft] = quantizedObject; #endif } +#endif uint8_t* compressIntoUint4() { size_t idx = 0; size_t uint4StreamSize = streamSize / 2; uint8_t *uint4Objects = new uint8_t[uint4StreamSize](); while (idx < streamSize) { - for (size_t lidx = 0; lidx < alignedNumOfSubvectors; lidx++) { + for (size_t lidx = 0; lidx < numOfAlignedSubvectors; lidx++) { for (size_t bidx = 0; bidx < NGTQ_SIMD_BLOCK_SIZE; bidx++) { if (idx / 2 > uint4StreamSize) { std::stringstream msg; @@ -1355,48 +2017,70 @@ class QuantizedObjectProcessingStream { } uint8_t *stream; - size_t alignedNumOfSubvectors; + size_t numOfAlignedSubvectors; size_t alignedBlockSize; - size_t alignedNumOfObjects; + size_t numOfAlignedObjects ; size_t streamSize; }; class GenerateResidualObject { public: + GenerateResidualObject():globalCodebookIndex(0), objectList(0), quantizationCodebook(0) {} virtual ~GenerateResidualObject() {} +#ifdef NGTQ_QBG + +#ifdef NGTQ_VECTOR_OBJECT + virtual void operator()(std::vector &object, size_t centroidID, float *subspaceObject) = 0; +#else + virtual void operator()(NGT::Object &object, size_t centroidID, float *subspaceObject) = 0; +#endif + virtual void operator()(NGT::Object &object, size_t centroidID, + vector > > &localObjs) = 0; +#else virtual void operator()(size_t objectID, size_t centroidID, vector > > &localObjs) = 0; - +#endif void set(NGT::Index &gc, NGT::Index lc[], size_t dn, size_t lcn, - Quantizer::ObjectList *ol) { - globalCodebook = &(NGT::GraphAndTreeIndex&)gc.getIndex(); + Quantizer::ObjectList *ol, QuantizationCodebook *qc) { + globalCodebookIndex = &(NGT::GraphAndTreeIndex&)gc.getIndex(); divisionNo = dn; objectList = ol; set(lc, lcn); + quantizationCodebook = qc; } void set(NGT::Index lc[], size_t lcn) { - localCodebook.clear(); + localCodebookIndexes.clear(); localCodebookNo = lcn; for (size_t i = 0; i < localCodebookNo; ++i) { - localCodebook.push_back(&(NGT::GraphAndTreeIndex&)lc[i].getIndex()); + localCodebookIndexes.push_back(&(NGT::GraphAndTreeIndex&)lc[i].getIndex()); } } - NGT::GraphAndTreeIndex *globalCodebook; - vector localCodebook; + NGT::GraphAndTreeIndex *globalCodebookIndex; + vector localCodebookIndexes; size_t divisionNo; size_t localCodebookNo; Quantizer::ObjectList *objectList; + QuantizationCodebook *quantizationCodebook; }; class GenerateResidualObjectUint8 : public GenerateResidualObject { public: +#ifdef NGTQ_QBG +#ifdef NGTQ_VECTOR_OBJECT + void operator()(std::vector &object, size_t centroidID, float *subspaceObject) { abort(); } +#else + void operator()(NGT::Object &object, size_t centroidID, float *subspaceObject) { abort(); } +#endif + void operator()(NGT::Object &xobject, size_t centroidID, + vector > > &localObjs) { abort(); } +#else void operator()(size_t objectID, size_t centroidID, vector > > &localObjs) { - NGT::PersistentObject &globalCentroid = *globalCodebook->getObjectSpace().getRepository().get(centroidID); - NGT::Object object(&globalCodebook->getObjectSpace()); - objectList->get(objectID, object, &globalCodebook->getObjectSpace()); - size_t sizeOfObject = globalCodebook->getObjectSpace().getByteSizeOfObject(); + NGT::PersistentObject &globalCentroid = *globalCodebookIndex->getObjectSpace().getRepository().get(centroidID); + NGT::Object object(&globalCodebookIndex->getObjectSpace()); + objectList->get(objectID, object, &globalCodebookIndex->getObjectSpace()); + size_t sizeOfObject = globalCodebookIndex->getObjectSpace().getByteSizeOfObject(); size_t lsize = sizeOfObject / divisionNo; for (size_t di = 0; di < divisionNo; di++) { vector subObject; @@ -1404,26 +2088,83 @@ class GenerateResidualObjectUint8 : public GenerateResidualObject { for (size_t d = 0; d < lsize; d++) { #if defined(NGT_SHARED_MEMORY_ALLOCATOR) subObject[d] = (double)object[di * lsize + d] - - (double)globalCentroid.at(di * lsize + d, globalCodebook->getObjectSpace().getRepository().allocator); + (double)globalCentroid.at(di * lsize + d, globalCodebookIndex->getObjectSpace().getRepository().allocator); #else subObject[d] = (double)object[di * lsize + d] - (double)globalCentroid[di * lsize + d]; #endif } size_t idx = localCodebookNo == 1 ? 0 : di; - NGT::Object *localObj = localCodebook[idx]->allocateObject(subObject); + NGT::Object *localObj = localCodebookIndexes[idx]->allocateObject(subObject); localObjs[idx].push_back(pair(localObj, 0)); } } +#endif + }; +#ifdef NGTQ_QBG +class GenerateResidualObjectFloat : public GenerateResidualObject { +public: +#ifdef NGTQ_VECTOR_OBJECT + void operator()(std::vector &object, size_t centroidID, float *subspaceObject) { +#else + void operator()(NGT::Object &object, size_t centroidID, float *subspaceObject) { +#endif + size_t dimension = globalCodebookIndex->getObjectSpace().getPaddedDimension(); +#ifdef NGTQ_VECTOR_OBJECT + auto *vector = object.data(); +#else + auto *vector = static_cast(object.getPointer()); +#endif + for (size_t d = 0; d < dimension; d++) { +#ifdef NGTQG_ZERO_GLOBAL + subspaceObject[d] = vector[d]; +#else + if (centroidID == std::numeric_limits::max()) { + subspaceObject[d] = vector[d]; + } else { + subspaceObject[d] = vector[d] - quantizationCodebook->at(centroidID, d); + } +#endif + } + } + void operator()(NGT::Object &object, size_t centroidID, + vector>> &localObjs) { + size_t byteSizeOfObject = globalCodebookIndex->getObjectSpace().getByteSizeOfObject(); + size_t localByteSize = byteSizeOfObject / divisionNo; + size_t localDimension = localByteSize / sizeof(float); + for (size_t di = 0; di < divisionNo; di++) { +#ifndef NGTQG_ZERO_GLOBAL + vector subObject; + subObject.resize(localDimension); +#endif + float *subVector = static_cast(object.getPointer(di * localByteSize)); +#ifndef NGTQG_ZERO_GLOBAL +#if defined(NGT_SHARED_MEMORY_ALLOCATOR) + float *quantizationCodebookSubvector = quantizationCodebook->data(centroidID) + di * localDimension; +#else + float *quantizationCodebookSubvector = quantizationCodebook->data(centroidID) + di * localDimension; +#endif + for (size_t d = 0; d < localDimension; d++) { + subObject[d] = static_cast(subVector[d]) - static_cast(quantizationCodebookSubvector[d]); + } +#endif /// ///////////////////// + size_t idx = localCodebookNo == 1 ? 0 : di; + NGT::Object *localObj = localCodebookIndexes[idx]->allocateObject(subVector, localDimension); + localObjs[idx].push_back(pair(localObj, 0)); + } + } +}; + +#else class GenerateResidualObjectFloat : public GenerateResidualObject { public: void operator()(size_t objectID, size_t centroidID, vector > > &localObjs) { - NGT::PersistentObject &globalCentroid = *globalCodebook->getObjectSpace().getRepository().get(centroidID); - NGT::Object object(&globalCodebook->getObjectSpace()); - objectList->get(objectID, object, &globalCodebook->getObjectSpace()); - size_t byteSizeOfObject = globalCodebook->getObjectSpace().getByteSizeOfObject(); + NGT::PersistentObject &globalCentroid = *globalCodebookIndex->getObjectSpace().getRepository().get(centroidID); + NGT::Object object(&globalCodebookIndex->getObjectSpace()); + objectList->get(objectID, object, &globalCodebookIndex->getObjectSpace()); + size_t byteSizeOfObject = globalCodebookIndex->getObjectSpace().getByteSizeOfObject(); size_t localByteSize = byteSizeOfObject / divisionNo; size_t localDimension = localByteSize / sizeof(float); for (size_t di = 0; di < divisionNo; di++) { @@ -1432,7 +2173,7 @@ class GenerateResidualObjectFloat : public GenerateResidualObject { float *subVector = static_cast(object.getPointer(di * localByteSize)); #if defined(NGT_SHARED_MEMORY_ALLOCATOR) float *globalCentroidSubVector = static_cast(globalCentroid.getPointer(di * localByteSize, - globalCodebook->getObjectSpace().getRepository().allocator)); + globalCodebookIndex->getObjectSpace().getRepository().allocator)); #else float *globalCentroidSubVector = static_cast(globalCentroid.getPointer(di * localByteSize)); #endif @@ -1440,12 +2181,13 @@ class GenerateResidualObjectFloat : public GenerateResidualObject { subObject[d] = (double)subVector[d] - (double)globalCentroidSubVector[d]; } size_t idx = localCodebookNo == 1 ? 0 : di; - NGT::Object *localObj = localCodebook[idx]->allocateObject(subObject); + NGT::Object *localObj = localCodebookIndexes[idx]->allocateObject(subObject); localObjs[idx].push_back(pair(localObj, 0)); } } }; - +#endif + template class QuantizerInstance : public Quantizer { public: @@ -1453,22 +2195,24 @@ class QuantizerInstance : public Quantizer { typedef void (QuantizerInstance::*AggregateObjectsFunction)(NGT::ObjectDistance &, NGT::Object *, size_t size, NGT::ObjectSpace::ResultSet &, size_t); typedef InvertedIndexEntry IIEntry; - QuantizerInstance(DataType dataType, size_t dimension, size_t nCodebooks):Quantizer(dataType, dimension) { - if (nCodebooks < 1 || nCodebooks > 100000) { - stringstream msg; - msg << "Quantizer::Error. Invalid divion no. " << nCodebooks; - NGTThrowException(msg); - } - property.localDivisionNo = nCodebooks; + QuantizerInstance() { quantizedObjectDistance = 0; generateResidualObject = 0; + localCodebooks = 0; + silence = true; } virtual ~QuantizerInstance() { close(); } void createEmptyIndex(const string &index, - NGT::Property &globalProperty, - NGT::Property &localProperty) + NGT::Property &globalProperty, +#ifdef NGTQ_QBG + NGT::Property &localProperty, + std::vector *rotation, + const string &objectFile) +#else + NGT::Property &localProperty) +#endif { rootDirectory = index; NGT::Index::mkdir(rootDirectory); @@ -1513,53 +2257,115 @@ class QuantizerInstance : public Quantizer { #endif string fname = rootDirectory + "/obj"; if (property.dataSize == 0) { - NGTThrowException("Quantizer: data size of the object list is 0."); + std::stringstream msg; +#ifdef NGTQ_QBG + msg << "Quantizer: data size of the object is zero. " << property.dataSize << ":" << property.dimension + << ":" << property.dataType << ":" << property.genuineDataType; +#else + msg << "Quantizer: data size of the object is zero. " << property.dataSize << ":" << property.dimension + << ":" << property.dataType; +#endif + NGTThrowException(msg); + } +#ifdef NGTQ_STATIC_OBJECT_FILE + if (!objectList.create(fname, objectFile)) { + std::stringstream msg; + msg << "Quantizer::createEmptyIndex: cannot construct the object list. " << fname << ":" << objectFile; + NGTThrowException(msg); } + objectList.open(fname, property.dimension); +#ifdef MULTIPLE_OBJECT_LISTS + objectList.openMultipleStreams(omp_get_max_threads()); +#endif +#else objectList.create(fname, property.dataSize); - objectList.open(fname); - objectList.close(); - +#endif +#ifdef NGTQ_QBG + if (rotation != 0) { + saveRotation(*rotation); + } +#endif property.save(rootDirectory); } - void open(const string &index, NGT::Property &globalProperty) { - open(index); - globalCodebook.setProperty(globalProperty); + void saveRotation(const std::vector &rotation) { + Rotation r; + r = rotation; + ofstream ofs(rootDirectory + "/qr"); + r.serialize(ofs); } - void open(const string &index) { + void open(const string &index, NGT::Property &globalProperty, bool readOnly) { + open(index, readOnly); + globalCodebookIndex.setProperty(globalProperty); + } + + void open(const string &index, bool readOnly) { + NGT::StdOstreamRedirector redirector(silence); + redirector.begin(); rootDirectory = index; property.load(rootDirectory); string globalIndex = index + "/global"; - globalCodebook.open(globalIndex); + globalCodebookIndex.open(globalIndex, readOnly); + if ((globalCodebookIndex.getObjectRepositorySize() == 0) && readOnly) { + std::cerr << "open: Warning. global codebook is empty." << std::endl; + } size_t localCodebookNo = property.getLocalCodebookNo(); - localCodebook.resize(localCodebookNo); + localCodebookIndexes.resize(localCodebookNo); for (size_t i = 0; i < localCodebookNo; ++i) { stringstream localIndex; localIndex << index << "/local-" << i; - localCodebook[i].open(localIndex.str()); + localCodebookIndexes[i].open(localIndex.str()); } +#ifdef NGTQ_QBG + if (!readOnly) { +#else + { +#endif + #ifdef NGTQ_SHARED_INVERTED_INDEX - invertedIndex.open(index + "/ivt", 0); + invertedIndex.open(index + "/ivt", 0); #else - ifstream ifs(index + "/ivt"); - if (!ifs) { - cerr << "Cannot open " << index + "/ivt" << "." << endl; - return; + ifstream ifs(index + "/ivt"); + if (!ifs) { + cerr << "Cannot open " << index + "/ivt" << "." << endl; + return; + } + invertedIndex.deserialize(ifs); +#endif } - invertedIndex.deserialize(ifs); +#ifdef NGTQ_QBG + if (!objectList.open(index + "/obj", property.genuineDataType, property.distanceType, property.dimension)) { +#else + if (!objectList.open(index + "/obj", static_cast(property.dataType), property.distanceType, property.dimension)) { +#endif + stringstream msg; + msg << "NGTQ::Quantizer::open: cannot open the object file. " << index + "/obj" << std::endl; + std::cerr << "Ignore. " << msg.str() << std::endl; + } +#ifdef MULTIPLE_OBJECT_LISTS + objectList.openMultipleStreams(omp_get_max_threads()); #endif - objectList.open(index + "/obj"); NGT::Property globalProperty; - globalCodebook.getProperty(globalProperty); + globalCodebookIndex.getProperty(globalProperty); size_t sizeoftype = 0; +#ifdef NGT_HALF_FLOAT + if (globalProperty.objectType == NGT::Property::ObjectType::Float || + globalProperty.objectType == NGT::Property::ObjectType::Float16) { +#else if (globalProperty.objectType == NGT::Property::ObjectType::Float) { +#endif if (property.localIDByteSize == 4) { quantizedObjectDistance = new QuantizedObjectDistanceFloat; } else if (property.localIDByteSize == 2) { quantizedObjectDistance = new QuantizedObjectDistanceFloat; +#ifdef NGTQ_QBG + } else if (property.localIDByteSize == 1) { + quantizedObjectDistance = new QuantizedObjectDistanceFloat; +#endif } else { + std::cerr << "Invalid localIDByteSize : " << property.localIDByteSize << std::endl; abort(); } generateResidualObject = new GenerateResidualObjectFloat; @@ -1577,20 +2383,61 @@ class QuantizerInstance : public Quantizer { } else { cerr << "NGTQ::open: Fatal Inner Error: invalid object type. " << globalProperty.objectType << endl; cerr << " check NGT version consistency between the caller and the library." << endl; - assert(0); + abort(); } assert(quantizedObjectDistance != 0); - quantizedObjectDistance->set(&globalCodebook, localCodebook.data(), property.localDivisionNo, property.getLocalCodebookNo(), sizeoftype, property.dimension); - generateResidualObject->set(globalCodebook, localCodebook.data(), property.localDivisionNo, property.getLocalCodebookNo(), &objectList); + quantizedObjectDistance->set(&globalCodebookIndex, localCodebookIndexes.data(), &quantizationCodebook, + property.localDivisionNo, property.getLocalCodebookNo(), sizeoftype, + property.dimension, &rotation); + generateResidualObject->set(globalCodebookIndex, localCodebookIndexes.data(), property.localDivisionNo, property.getLocalCodebookNo(), &objectList, &quantizationCodebook); localIDByteSize = property.localIDByteSize; objectType = globalProperty.objectType; divisionNo = property.localDivisionNo; +#ifdef NGTQ_QBG + { + std::string streamName(rootDirectory + "/qr"); + ifstream ifs(streamName); + if (ifs) { + std::cerr << "loading rotation..." << std::endl; + rotation.deserialize(ifs); + } else { + std::cerr << "Warning. Not found the rotation file. " << streamName << std::endl; + } + } + { +#ifdef NGTQG_ROTATION + std::string rqcbName(rootDirectory + "/rqcb"); + ifstream irfs(rqcbName); + if (!irfs) { + std::cerr << "Warning. Not found the rqcb file. " << rqcbName << std::endl; + std::string qcbName(rootDirectory + "/qcb"); + ifstream ifs(qcbName); + if (!ifs) { + std::cerr << "Warning. Not found the qcb file. " << qcbName << std::endl; + } else { + std::cerr << "loading the global codebooks..." << std::endl; + quantizationCodebook.deserialize(ifs, readOnly); + std::cerr << "rotating the global codebooks... " << rotation.size() << std::endl; + quantizationCodebook.rotate(rotation); + } + } else { + std::cerr << "loading the rotated global codebooks..." << std::endl; + quantizationCodebook.deserialize(irfs, readOnly); + } +#else + std::string qcbName(rootDirectory + "/qcb"); + ifstream ifs(qcbName); + quantizationCodebook.deserialize(ifs, readOnly); +#endif + } +#endif + redirector.end(); } void save() { #ifndef NGT_SHARED_MEMORY_ALLOCATOR string global = rootDirectory + "/global"; - globalCodebook.saveIndex(global); + globalCodebookIndex.saveIndex(global); size_t localCodebookNo = property.getLocalCodebookNo(); for (size_t i = 0; i < localCodebookNo; ++i) { stringstream local; @@ -1598,20 +2445,30 @@ class QuantizerInstance : public Quantizer { try { NGT::Index::mkdir(local.str()); } catch (...) {} - localCodebook[i].saveIndex(local.str()); + localCodebookIndexes[i].saveIndex(local.str()); } #endif // NGT_SHARED_MEMORY_ALLOCATOR #ifndef NGTQ_SHARED_INVERTED_INDEX ofstream of(rootDirectory + "/ivt"); invertedIndex.serialize(of); +#endif +#ifdef NGTQ_QBG + { +#ifdef NGTQG_ROTATED_GLOBAL_CODEBOOKS + ofstream ofs(rootDirectory + "/rqcb"); +#else + ofstream ofs(rootDirectory + "/qcb"); +#endif + quantizationCodebook.serialize(ofs); + } #endif property.save(rootDirectory); } void closeCodebooks() { - globalCodebook.close(); - for (size_t i = 0; i < localCodebook.size(); ++i) { - localCodebook[i].close(); + globalCodebookIndex.close(); + for (size_t i = 0; i < localCodebookIndexes.size(); ++i) { + localCodebookIndexes[i].close(); } } @@ -1629,6 +2486,7 @@ class QuantizerInstance : public Quantizer { #ifndef NGTQ_SHARED_INVERTED_INDEX invertedIndex.deleteAll(); #endif + delete[] localCodebooks; } #ifdef NGTQ_SHARED_INVERTED_INDEX @@ -1649,7 +2507,8 @@ class QuantizerInstance : public Quantizer { if (id % 100000 == 0) { cerr << "Processed " << id << endl; } - IIEntry *entry = new(tmpInvertedIndex.getAllocator()) InvertedIndexEntry(localCodebook.size(), tmpInvertedIndex.getAllocator()); + auto codebookSize = property.localCentroidLimit; + IIEntry *entry = new(tmpInvertedIndex.getAllocator()) InvertedIndexEntry(codebookSize, tmpInvertedIndex.getAllocator()); size_t esize = (*invertedIndex.at(id)).size(); (*entry).reserve(esize, tmpInvertedIndex.getAllocator()); for (size_t i = 0; i < esize; ++i) { @@ -1734,13 +2593,17 @@ class QuantizerInstance : public Quantizer { } } +#ifdef NGTQ_VECTOR_OBJECT + void setGlobalCodeToInvertedEntry(NGT::Index::InsertionResult &id, pair, size_t> &object, vector &localData) { +#else void setGlobalCodeToInvertedEntry(NGT::Index::InsertionResult &id, pair &object, vector &localData) { +#endif size_t globalCentroidID = id.id; if (invertedIndex.isEmpty(globalCentroidID)) { #ifdef NGTQ_SHARED_INVERTED_INDEX - invertedIndex.put(globalCentroidID, new(invertedIndex.allocator) InvertedIndexEntry(localCodebook.size(), invertedIndex.allocator)); + invertedIndex.put(globalCentroidID, new(invertedIndex.allocator) InvertedIndexEntry(localCodebookIndexes.size(), invertedIndex.allocator)); #else - invertedIndex.put(globalCentroidID, new InvertedIndexEntry(localCodebook.size())); + invertedIndex.put(globalCentroidID, new InvertedIndexEntry(localCodebookIndexes.size())); #endif } assert(!invertedIndex.isEmpty(globalCentroidID)); @@ -1754,7 +2617,8 @@ class QuantizerInstance : public Quantizer { #else invertedIndexEntry.pushBack(object.second); #endif - if (property.centroidCreationMode == CentroidCreationModeStatic) { + if (property.centroidCreationMode == CentroidCreationModeStatic || + property.centroidCreationMode == CentroidCreationModeStaticLayer) { localData.push_back(LocalDatam(globalCentroidID, invertedIndexEntry.size() - 1)); } else { @@ -1765,8 +2629,11 @@ class QuantizerInstance : public Quantizer { } } else { if (property.centroidCreationMode != CentroidCreationModeDynamic) { - cerr << "Quantizer: Error! Although it is an original quantizer, object has been added to the global." << endl; - cerr << " Specify the size limitation of the global." << endl; + cerr << "Quantizer: Fatal error! Although it is a static global codebook, an object has been added to the global." << endl; + cerr << " The actual size of the global codebook=" << globalCodebookIndex.getObjectRepositorySize() - 1 + << ", The size of the global codebook in the property=" << property.globalCentroidLimit << std::endl; + cerr << " The both numbers above should be the same." << std::endl; + cerr << " Specify a proper size limitation for the global codebook?" << endl; assert(id.identical); abort(); } @@ -1783,6 +2650,10 @@ class QuantizerInstance : public Quantizer { invertedIndexEntry[0].setID(object.second); #endif } + if (property.quantizerType == QuantizerTypeQBG) { + localData.push_back(LocalDatam(globalCentroidID, + invertedIndexEntry.size() - 1)); + } } } @@ -1807,18 +2678,20 @@ class QuantizerInstance : public Quantizer { #endif } #ifdef NGT_SHARED_MEMORY_ALLOCATOR - localCodebook[0].deleteObject(localObjs[0][i].first); + localCodebookIndexes[0].deleteObject(localObjs[0][i].first); #else if (lids[i].identical) { - localCodebook[0].deleteObject(localObjs[0][i].first); + localCodebookIndexes[0].deleteObject(localObjs[0][i].first); } #endif } } +#ifndef NGTQ_QBG bool setMultipleLocalCodeToInvertedIndexEntry(vector &lcodebook, vector &localData, vector > > &localObjs) { size_t localCodebookNo = property.getLocalCodebookNo(); bool localCodebookFull = true; +#pragma omp parallel for for (size_t li = 0; li < localCodebookNo; ++li) { float lr = property.localRange; size_t localCentroidLimit = property.localCentroidLimit; @@ -1848,23 +2721,168 @@ class QuantizerInstance : public Quantizer { (*invertedIndex.at(localData[i].iiIdx))[localData[i].iiLocalIdx].localID[li] = id; #endif #ifdef NGT_SHARED_MEMORY_ALLOCATOR - localCodebook[li].deleteObject(localObjs[li][i].first); + localCodebookIndexes[li].deleteObject(localObjs[li][i].first); #else if (lids[i].identical) { - localCodebook[li].deleteObject(localObjs[li][i].first); + localCodebookIndexes[li].deleteObject(localObjs[li][i].first); } #endif } } return localCodebookFull; } - +#endif + +#ifdef NGTQ_QBG + bool setMultipleLocalCodeToInvertedIndexEntry(vector &lcodebook, + vector &localData, + float *subspaceObjects) { + size_t paddedDimension = globalCodebookIndex.getObjectSpace().getPaddedDimension(); + size_t localCodebookNo = property.getLocalCodebookNo(); + bool localCodebookFull = true; + for (size_t li = 0; li < localCodebookNo; ++li) { + float lr = property.localRange; + size_t localCentroidLimit = property.localCentroidLimit; + if (property.localCentroidCreationMode == CentroidCreationModeDynamicKmeans) { + localCentroidLimit *= property.localClusteringSampleCoefficient; + } + if (property.localCodebookState) { + lr = FLT_MAX; + localCentroidLimit = 0; + } else { + if (property.localCentroidCreationMode == CentroidCreationModeDynamicKmeans) { + lr = -1.0; + } + } + vector lids; + size_t localDimension = lcodebook[li]->getObjectSpace().getDimension(); + vector> localObjects(localData.size()); + for (size_t i = 0; i < localData.size(); i++) { + localObjects[i].first = lcodebook[li]->allocateObject(&subspaceObjects[i * paddedDimension + (li * localDimension)], localDimension); + localObjects[i].second = 0; + } + createIndex(*lcodebook[li], localCentroidLimit, localObjects, lids, lr); + if (lr != FLT_MAX) { + localCodebookFull = false; + } + assert(localData.size() == lids.size()); + for (size_t i = 0; i < localData.size(); i++) { + size_t id = lids[i].id; + assert(!property.localCodebookState || id <= ((1UL << (sizeof(LOCAL_ID_TYPE) * 8)) - 1)); +#ifdef NGTQ_SHARED_INVERTED_INDEX + (*invertedIndex.at(localData[i].iiIdx)).at(localData[i].iiLocalIdx, invertedIndex.allocator).localID[li] = id; +#else + (*invertedIndex.at(localData[i].iiIdx))[localData[i].iiLocalIdx].localID[li] = id; +#endif +#ifdef NGT_SHARED_MEMORY_ALLOCATOR + lcodebook[li]->deleteObject(localObjects[i].first); +#else + if (lids[i].identical) { + lcodebook[li]->deleteObject(localObjects[i].first); + } +#endif + } + } + return localCodebookFull; + } +#endif + + void constructLocalCodebooks() { + delete localCodebooks; + size_t localCodebookNo = property.getLocalCodebookNo(); + size_t codebookSize = localCodebookIndexes[0].getObjectSpace().getSize() - 1; + size_t paddedDimension = globalCodebookIndex.getObjectSpace().getPaddedDimension(); + size_t localDimension = localCodebookIndexes[0].getObjectSpace().getDimension(); + localCodebooks = new float[codebookSize * paddedDimension]; + size_t oft = 0; + for (size_t li = 0; li < localCodebookNo; li++) { + if (localCodebookIndexes[li].getObjectSpace().getSize() - 1 != codebookSize) { + std::cerr << "Fatal Error!" << std::endl; + abort(); + } + for (size_t cid = 1; cid <= codebookSize; cid++) { + std::vector v; + localCodebookIndexes[li].getObjectSpace().getObject(cid, v); + for (size_t ld = 0; ld < v.size(); ld++) { + localCodebooks[(cid - 1) * paddedDimension + oft + ld] = v[ld]; + } + } + oft += localDimension; + } + if (oft != globalCodebookIndex.getObjectSpace().getDimension()) { + std::cerr << "somethig wrong " << oft << ":" << globalCodebookIndex.getObjectSpace().getDimension() << std::endl; + abort(); + } + } + +#ifdef NGTQ_QBG + void setMultipleLocalCodeToInvertedIndexEntryFixed(vector &localData, + float *subspaceObjects) { + if (localData.empty()) { + return; + } + if (localCodebooks == 0) { + constructLocalCodebooks(); + } + size_t paddedDimension = globalCodebookIndex.getObjectSpace().getPaddedDimension(); + size_t localCodebookNo = property.getLocalCodebookNo(); + size_t codebookSize = property.localCentroidLimit; + if (property.dimension % property.localDivisionNo != 0) { + std::cerr << "setMultipleLocalCodeToInvertedIndexEntry!!!" << std::endl; + abort(); + } + size_t localDimension = property.dimension / property.localDivisionNo; + std::unique_ptr distance(new float[localData.size() * codebookSize * localCodebookNo]()); + std::vector> min(localData.size() * localCodebookNo, std::make_pair(std::numeric_limits::max(), std::numeric_limits::max())); + if (localCodebooks == 0) { + std::cerr << "Quantizer::setMultipleLocalCodeToInvertedEntry: FatalError!" << std::endl; + abort(); + } + { +#pragma omp parallel for + for (size_t idx = 0; idx < localData.size(); idx++) { + for (size_t cid = 0; cid < codebookSize; cid++) { + for (size_t svi = 0; svi < localCodebookNo; svi++) { + float localDistance = 0.0; + for (size_t ld = 0; ld < localDimension; ld++) { + size_t d = svi * localDimension + ld; + auto dist = subspaceObjects[idx * paddedDimension + d] - localCodebooks[cid * paddedDimension + d]; + dist *= dist; + distance[idx * codebookSize * localCodebookNo + cid * localCodebookNo + svi] += dist; + localDistance = distance[idx * codebookSize * localCodebookNo + cid * localCodebookNo + svi]; + } + if (localDistance < min[idx * localCodebookNo + svi].first) { + min[idx * localCodebookNo + svi].first = localDistance; + min[idx * localCodebookNo + svi].second = cid; + } + } + } + } + + } +#pragma omp parallel for + for (size_t li = 0; li < localCodebookNo; ++li) { + for (size_t i = 0; i < localData.size(); i++) { + size_t id = min[i * localCodebookNo + li].second + 1; + assert(!property.localCodebookState || id <= ((1UL << (sizeof(LOCAL_ID_TYPE) * 8)) - 1)); +#ifdef NGTQ_SHARED_INVERTED_INDEX + (*invertedIndex.at(localData[i].iiIdx)).at(localData[i].iiLocalIdx, invertedIndex.allocator).localID[li] = id; +#else + (*invertedIndex.at(localData[i].iiIdx))[localData[i].iiLocalIdx].localID[li] = id; +#endif + } + } + return; + } +#endif + void buildMultipleLocalCodebooks(NGT::Index *localCodebook, size_t localCodebookNo, size_t numberOfCentroids) { NGT::Clustering clustering; clustering.epsilonFrom = 0.10; clustering.epsilonTo = 0.50; clustering.epsilonStep = 0.05; clustering.maximumIteration = 20; + clustering.clusterSizeConstraint = false; for (size_t li = 0; li < localCodebookNo; ++li) { double diff = clustering.kmeansWithNGT(localCodebook[li], numberOfCentroids); if (diff > 0.0) { @@ -1874,6 +2892,8 @@ class QuantizerInstance : public Quantizer { } } + +#ifdef NGTQ_QBG void replaceInvertedIndexEntry(size_t localCodebookNo) { vector localData; for (size_t gidx = 1; gidx < invertedIndex.size(); gidx++) { @@ -1882,42 +2902,245 @@ class QuantizerInstance : public Quantizer { continue; } IIEntry &invertedIndexEntry = *invertedIndex.at(gidx); - for (size_t oi = property.centroidCreationMode == CentroidCreationModeStatic ? 0 : 1; + for (size_t oi = ((property.centroidCreationMode == CentroidCreationModeStatic || + property.centroidCreationMode == CentroidCreationModeStaticLayer) || + property.quantizerType == QuantizerTypeQBG) ? 0 : 1; oi < invertedIndexEntry.size(); oi++) { +#ifdef NGTQ_QBG + localData.push_back(LocalDatam(gidx, oi, invertedIndexEntry.subspaceID)); +#else localData.push_back(LocalDatam(gidx, oi)); +#endif } } - vector > > localObjs; - localObjs.resize(localCodebookNo); + float subspaceObjects[localData.size()][globalCodebookIndex.getObjectSpace().getPaddedDimension()]; for (size_t i = 0; i < localData.size(); i++) { IIEntry &invertedIndexEntry = *invertedIndex.at(localData[i].iiIdx); #ifdef NGTQ_SHARED_INVERTED_INDEX - (*generateResidualObject)(invertedIndexEntry.at(localData[i].iiLocalIdx, invertedIndex.allocator).id, - localData[i].iiIdx, // centroid:ID of global codebook - localObjs); +#ifdef NGTQ_QBG + std::cerr << "not implemented" << std::endl; + abort(); #else - (*generateResidualObject)(invertedIndexEntry[localData[i].iiLocalIdx].id, - localData[i].iiIdx, // centroid:ID of global codebook - localObjs); + NGT::Object object(&globalCodebookIndex.getObjectSpace()); + objectList.get(invertedIndexEntry[localData[i].iiLocalIdx].id, object, &globalCodebookIndex.getObjectSpace()); + (*generateResidualObject)(object, // object + invertedIndexEntry.subspaceID, + subspaceObjects[i]); // subspace objects +#endif +#else +#ifdef NGTQ_VECTOR_OBJECT + std::vector object; + objectList.get(invertedIndexEntry[localData[i].iiLocalIdx].id, object, &globalCodebookIndex.getObjectSpace()); +#else + NGT::Object object(&globalCodebookIndex.getObjectSpace()); +#endif + (*generateResidualObject)(object, // object + invertedIndexEntry.subspaceID, + subspaceObjects[i]); // subspace objects #endif } - vector lcodebook; - for (size_t i = 0; i < localCodebookNo; i++) { - lcodebook.push_back(&(NGT::GraphAndTreeIndex &)localCodebook[i].getIndex()); - } - setMultipleLocalCodeToInvertedIndexEntry(lcodebook, localData, localObjs); + setMultipleLocalCodeToInvertedIndexEntryFixed(localData, &subspaceObjects[0][0]); } +#endif +#ifndef NGTQ_QBG void insert(vector > &objects) { - NGT::GraphAndTreeIndex &gcodebook = (NGT::GraphAndTreeIndex &)globalCodebook.getIndex(); - vector lcodebook; + std::cerr << "insert() is not implemented." << std::endl; + abort(); + } +#endif + + void searchIndex(NGT::GraphAndTreeIndex &codebook, + size_t centroidLimit, +#ifdef NGTQ_VECTOR_OBJECT + const vector, size_t>> &objects, +#else + const vector> &objects, +#endif + vector &ids, + float &range, NGT::Index *gqindex) + { + if (quantizationCodebook.size() == 0) { + std::cerr << "Fatal error. quantizationCodebook is empty" << std::endl; + abort(); + } + ids.clear(); + ids.resize(objects.size()); + size_t foundCount = 0; + double foundRank = 0.0; +#pragma omp parallel for + for (size_t idx = 0; idx < objects.size(); idx++) { +#ifdef NGTQ_VECTOR_OBJECT + auto *object = globalCodebookIndex.allocateObject(objects[idx].first); + auto qid = quantizationCodebook.search(*object); + globalCodebookIndex.deleteObject(object); +#else + auto qid = quantizationCodebook.search(*objects[idx].first); +#endif + NGT::ObjectDistances result; + if (gqindex != 0) { + std::vector object(globalCodebookIndex.getObjectSpace().getDimension()); +#ifdef NGTQ_VECTOR_OBJECT + memcpy(object.data(), objects[idx].first.data(), sizeof(float) * object.size()); +#else + memcpy(object.data(), objects[idx].first->getPointer(), sizeof(float) * object.size()); +#endif +#define QID_WEIGHT 100 + object.push_back(qid * QID_WEIGHT); + NGT::SearchQuery sc(object);; + sc.setResults(&result); + sc.setSize(50); + sc.radius = FLT_MAX; + sc.setEpsilon(0.1); + gqindex->search(sc); + } else { +#ifdef NGTQ_VECTOR_OBJECT + auto *object = globalCodebookIndex.allocateObject(objects[idx].first); + NGT::SearchContainer sc(*object); +#else + NGT::SearchContainer sc(*objects[idx].first); +#endif + sc.setResults(&result); + sc.setSize(50); + sc.radius = FLT_MAX; + sc.setEpsilon(0.1); + globalCodebookIndex.search(sc); +#ifdef NGTQ_VECTOR_OBJECT + globalCodebookIndex.deleteObject(object); +#endif + } + int32_t eqi = -1; + for (size_t i = 0; i < result.size(); i++) { + auto &invertedIndexEntry = *invertedIndex.at(result[i].id); + if (invertedIndexEntry.subspaceID == qid) { +#pragma omp critical + { + foundCount++; + foundRank += i; + } + eqi = i; + break; + } + } + if (eqi < 0) { + eqi = 0; + } + ids[idx].id = result[eqi].id; + ids[idx].distance = result[eqi].distance; + ids[idx].identical = true; + } + return; + + } + +#ifdef NGTQ_VECTOR_OBJECT + void getBlobIDFromObjectToBlobIndex(const vector, size_t>> &objects, + vector &ids) +#else + void getBlobIDFromObjectToBlobIndex(const vector> &objects, + vector &ids) +#endif + { + ids.clear(); + ids.resize(objects.size()); +#ifdef GET_BLOG_EVAL + size_t identicalObjectCount = 0; +#endif + for (size_t idx = 0; idx < objects.size(); idx++) { + if (objects[idx].second - 1 >= objectToBlobIndex.size()) { + std::cerr << "Quantizer::insert: Fatal Error! Object ID is invalid. " + << idx << ":" << objects[idx].second - 1 << ":" << objectToBlobIndex.size() + << ":" << objects.size()<< std::endl; + abort(); + } + ids[idx].id = objectToBlobIndex[objects[idx].second - 1] + 1; + ids[idx].distance = 0.0; + ids[idx].identical = true; +#ifdef GET_BLOG_EVAL + { + NGT::ObjectDistances result; + NGT::SearchContainer sc(*objects[idx].first); + sc.setResults(&result); + sc.setSize(50); + sc.radius = FLT_MAX; + sc.setEpsilon(0.1); + globalCodebookIndex.search(sc); + //std::cerr << "insert:Eval: "; + if (result[0].id == ids[idx].id) { + identicalObjectCount++; + } else { + } + } +#endif + } +#ifdef GET_BLOG_EVAL + std::cerr << identicalObjectCount << "/" << objects.size() << std::endl; +#endif + return; + } + +#ifdef NGTQ_QBG + NGT::Index *buildGlobalCodebookWithQIDIndex() { +#if defined(NGT_SHARED_MEMORY_ALLOCATOR) + std::cerr << "buildGlobalCodebookWithQIDIndex: Not implemented." << std::endl; + abort(); +#else + NGT::Property property; + + property.dimension = globalCodebookIndex.getObjectSpace().getDimension() + 1; + property.distanceType = NGT::Index::Property::DistanceType::DistanceTypeL2; +#ifdef NGTQ_SHARED_INVERTED_INDEX + NGT::Index *index = new NGT::Index("dummy", property); + std::cerr << "Not implemented" << std::endl; + abort(); +#else + NGT::Index *index = new NGT::Index(property); +#endif + for (size_t id = 1; id < globalCodebookIndex.getObjectRepositorySize(); id++) { + std::vector object; + if (id % 10000 == 0) { + std::cerr << "# of processed objects=" << id << std::endl; + } + globalCodebookIndex.getObjectSpace().getObject(id, object); + object.push_back(invertedIndex.at(id)->subspaceID * QID_WEIGHT); + index->append(object); + } + index->createIndex(50); + return index; +#endif + } +#endif + +#ifdef NGTQ_QBG +#ifdef NGTQ_VECTOR_OBJECT + void insert(vector, size_t> > &objects, NGT::Index *gqindex) { +#else + void insert(vector > &objects, NGT::Index *gqindex) { +#endif +#ifdef NGTQ_SHARED_INVERTED_INDEX + std::cerr << "insert: Not implemented." << std::endl; + abort(); +#else + NGT::GraphAndTreeIndex &gcodebook = (NGT::GraphAndTreeIndex &)globalCodebookIndex.getIndex(); size_t localCodebookNo = property.getLocalCodebookNo(); + vector lcodebook; + lcodebook.reserve(localCodebookNo); for (size_t i = 0; i < localCodebookNo; i++) { - lcodebook.push_back(&(NGT::GraphAndTreeIndex &)localCodebook[i].getIndex()); + lcodebook.push_back(&static_cast(localCodebookIndexes[i].getIndex())); } float gr = property.globalRange; vector ids; - createIndex(gcodebook, property.globalCentroidLimit, objects, ids, gr); + if (property.centroidCreationMode == CentroidCreationModeStaticLayer || + property.centroidCreationMode == CentroidCreationModeStatic) { + if (objectToBlobIndex.empty()) { + searchIndex(gcodebook, property.globalCentroidLimit, objects, ids, gr, gqindex); + } else { + getBlobIDFromObjectToBlobIndex(objects, ids); + } + } else { + std::cerr << "Quantizer::InsertQBG: Warning! invalid centroidCreationMode. " << property.centroidCreationMode << std::endl; + abort(); + } #ifdef NGTQ_SHARED_INVERTED_INDEX if (invertedIndex.getAllocatedSize() <= invertedIndex.size() + objects.size()) { invertedIndex.reserve(invertedIndex.getAllocatedSize() * 2); @@ -1929,30 +3152,69 @@ class QuantizerInstance : public Quantizer { for (size_t i = 0; i < ids.size(); i++) { setGlobalCodeToInvertedEntry(ids[i], objects[i], localData); } - vector > > localObjs; - localObjs.resize(property.getLocalCodebookNo()); + float subspaceObjects[localData.size()][globalCodebookIndex.getObjectSpace().getPaddedDimension()]; +#pragma omp parallel for for (size_t i = 0; i < localData.size(); i++) { IIEntry &invertedIndexEntry = *invertedIndex.at(localData[i].iiIdx); #ifdef NGTQ_SHARED_INVERTED_INDEX +#ifdef NGTQ_QBG + std::cerr << "Not implemented" << std::endl; + abort(); +#else (*generateResidualObject)(invertedIndexEntry.at(localData[i].iiLocalIdx, invertedIndex.allocator).id, localData[i].iiIdx, // centroid:ID of global codebook localObjs); +#endif +#else +#ifdef NGTQ_QBG + +#ifdef NGTQG_ROTATED_GLOBAL_CODEBOOKS + assert(!rotation.empty()); + if (!rotation.empty()) { +#ifdef NGTQ_VECTOR_OBJECT + rotation.mul(objects[i].first.data()); +#else + rotation.mul(static_cast(objects[i].first->getPointer())); +#endif + } +#endif +#ifdef NGTQ_VECTOR_OBJECT + (*generateResidualObject)(objects[i].first, // object + invertedIndexEntry.subspaceID, + subspaceObjects[i]); // subspace objects +#else + (*generateResidualObject)(*objects[i].first, // object + invertedIndexEntry.subspaceID, + subspaceObjects[i]); // subspace objects +#endif +#ifndef NGTQG_ROTATED_GLOBAL_CODEBOOKS + rotation.mul(subspaceObjects[i]); +#endif #else (*generateResidualObject)(invertedIndexEntry[localData[i].iiLocalIdx].id, localData[i].iiIdx, // centroid:ID of global codebook localObjs); #endif - } +#endif + + } + if (property.singleLocalCodebook) { // single local codebook - setSingleLocalCodeToInvertedIndexEntry(lcodebook, localData, localObjs); + std::cerr << "insert: Fatal Error. single local codebook isn't available." << std::endl; + abort(); } else { // multiple local codebooks - bool localCodebookFull = setMultipleLocalCodeToInvertedIndexEntry(lcodebook, localData, localObjs); - if ((!property.localCodebookState) && localCodebookFull) { + bool localCodebookFull = true; + if (property.localCodebookState) { + setMultipleLocalCodeToInvertedIndexEntryFixed(localData, &subspaceObjects[0][0]); + } else { + localCodebookFull = setMultipleLocalCodeToInvertedIndexEntry(lcodebook, localData, &subspaceObjects[0][0]); + } + if ((!property.localCodebookState) && localCodebookFull) { if (property.localCentroidCreationMode == CentroidCreationModeDynamicKmeans) { - buildMultipleLocalCodebooks(localCodebook.data(), localCodebookNo, property.localCentroidLimit); - (*generateResidualObject).set(localCodebook.data(), localCodebookNo); + buildMultipleLocalCodebooks(localCodebookIndexes.data(), localCodebookNo, property.localCentroidLimit); + (*generateResidualObject).set(localCodebookIndexes.data(), localCodebookNo); property.localCodebookState = true; localCodebookFull = false; replaceInvertedIndexEntry(localCodebookNo); @@ -1962,79 +3224,176 @@ class QuantizerInstance : public Quantizer { } } } +#pragma omp parallel for for (size_t i = 0; i < objects.size(); i++) { #ifdef NGT_SHARED_MEMORY_ALLOCATOR - globalCodebook.deleteObject(objects[i].first); + globalCodebookIndex.deleteObject(objects[i].first); #else +#ifndef NGTQ_VECTOR_OBJECT if (ids[i].identical == true) { - globalCodebook.deleteObject(objects[i].first); + globalCodebookIndex.deleteObject(objects[i].first); } +#endif #endif } objects.clear(); +#endif } +#endif - void insert(const string &line, vector > &objects, size_t count) { + +#ifndef NGTQ_QBG + void insert(vector &objvector, vector > &objects, size_t count) { size_t id = count; if (count == 0) { id = objectList.size(); id = id == 0 ? 1 : id; } - NGT::Object *object = globalCodebook.allocateObject(line, " \t"); - objectList.put(id, *object, &globalCodebook.getObjectSpace()); + + NGT::Object *object = globalCodebookIndex.allocateObject(objvector); + objectList.put(id, *object, &globalCodebookIndex.getObjectSpace()); + objects.push_back(pair(object, id)); + if (objects.size() >= property.batchSize) { - insert(objects); // batch insert + insert(objects); // batch insert } } +#endif - void insert(vector &objvector, vector > &objects, size_t count) { + void insertIntoObjectRepository(vector &objvector, size_t count) { size_t id = count; if (count == 0) { id = objectList.size(); id = id == 0 ? 1 : id; } + NGT::Object *object = globalCodebookIndex.allocateObject(objvector); + std::vector vs = globalCodebookIndex.getObjectSpace().getObject(*object); + objectList.put(id, *object, &globalCodebookIndex.getObjectSpace()); + globalCodebookIndex.deleteObject(object); + } - NGT::Object *object = globalCodebook.allocateObject(objvector); - objectList.put(id, *object, &globalCodebook.getObjectSpace()); +#ifdef NGTQ_QBG + void createIndex(size_t beginID = 1, size_t endID = 0) { + NGT::Index *gqindex = 0; + if (property.centroidCreationMode == CentroidCreationModeStaticLayer) { + gqindex = buildGlobalCodebookWithQIDIndex(); + } +#ifdef NGTQ_VECTOR_OBJECT + vector, size_t>> objects; +#else + vector> objects; +#endif + objects.reserve(property.batchSize); + if (endID == 0) { + endID = objectList.size() - 1; + } + NGT::Timer timer; + timer.start(); + for (size_t id = beginID; id <= endID; id++) { + if (id % 1000000 == 0) { + timer.stop(); + std::cerr << "# of processed objects=" << id << " Time=" << timer << ", vm size=" + << NGT::Common::getProcessVmSizeStr() << std::endl; + timer.restart(); + } +#ifdef NGTQ_VECTOR_OBJECT + std::vector object; + if (!objectList.get(id, object, &globalCodebookIndex.getObjectSpace())) { + std::cerr << "Cannot get object. ID=" << id << std::endl; + continue; + } +#else + NGT::Object *object = globalCodebookIndex.getObjectSpace().allocateObject(); + objectList.get(id, *object, &globalCodebookIndex.getObjectSpace()); +#endif +#ifdef NGTQ_VECTOR_OBJECT + objects.push_back(pair, size_t>(object, id)); +#else + objects.push_back(pair(object, id)); +#endif + if (objects.size() >= property.batchSize) { + insert(objects, gqindex); // batch insert + } + } + if (objects.size() > 0) { + insert(objects, gqindex); // batch insert + } + delete gqindex; + } +#endif - objects.push_back(pair(object, id)); + void setupInvertedIndex(std::vector> &qCodebook, + std::vector &codebookIndex, + std::vector &objectIndex) { +#if !defined(NGTQ_QBG) + std::cerr << "setupInvertedIndex: Not implemented." << std::endl; + abort(); +#else + if (globalCodebookIndex.getObjectRepositorySize() != codebookIndex.size() + 1) { + std::cerr << "Error? " << globalCodebookIndex.getObjectRepositorySize() << ":" << codebookIndex.size() + 1 << std::endl; + } + if (!invertedIndex.empty()) { + stringstream msg; + msg << "Fatal Error! inverted index is not empty. " << invertedIndex.size(); + NGTThrowException(msg); + } + invertedIndex.reserve(codebookIndex.size() + 1); + std::cerr << "codebook Index size=" << codebookIndex.size()<< std::endl; + for (size_t idx = 0; idx < codebookIndex.size(); idx++) { + auto gid = idx + 1; +#ifdef NGTQ_SHARED_INVERTED_INDEX + invertedIndex.put(gid, new(invertedIndex.allocator) InvertedIndexEntry(localCodebookIndexes.size(), invertedIndex.allocator)); +#else + invertedIndex.put(gid, new InvertedIndexEntry(localCodebookIndexes.size())); +#endif + invertedIndex.at(gid)->subspaceID = codebookIndex[idx]; + } + + quantizationCodebook.setPaddedDimension(globalCodebookIndex.getObjectSpace().getPaddedDimension()); + quantizationCodebook = qCodebook; +#ifdef NGTQG_ROTATED_GLOBAL_CODEBOOKS + quantizationCodebook.rotate(rotation); +#endif - if (objects.size() >= property.batchSize) { - insert(objects); // batch insert + objectToBlobIndex = std::move(objectIndex); + objectIndex.clear(); + std::vector invertedIndexCount(codebookIndex.size()); + for (size_t idx = 0; idx < objectToBlobIndex.size(); idx++) { + invertedIndexCount[objectToBlobIndex[idx]]++; + } + for (size_t idx = 0; idx < codebookIndex.size(); idx++) { + auto gid = idx + 1; +#ifdef NGTQ_SHARED_INVERTED_INDEX + IIEntry &invertedIndexEntry = *invertedIndex.at(gid, invertedIndex.allocator); +#else + IIEntry &invertedIndexEntry = *invertedIndex.at(gid); +#endif + invertedIndexEntry.reserve(invertedIndexCount[idx]); } +#endif } + +#ifndef NGTQ_QBG void rebuildIndex() { - vector > objects; - size_t objectCount = objectList.size(); - size_t count = 0; - for (size_t idx = 1; idx < objectCount; idx++) { - count++; - if (count % 100000 == 0) { - cerr << "Processed " << count; - cerr << endl; - } - NGT::Object *object = globalCodebook.getObjectSpace().allocateObject(); - objectList.get(idx, *object, &globalCodebook.getObjectSpace()); - objects.push_back(pair(object, idx)); - if (objects.size() >= property.batchSize) { - insert(objects); - } - } - if (objects.size() >= 0) { - insert(objects); - } + abort(); } +#endif void create(const string &index, - NGT::Property &globalProperty, + NGT::Property &globalProperty, +#ifdef NGTQ_QBG + NGT::Property &localProperty, + std::vector *rotation = 0, + const string &objectFile = "" +#else NGT::Property &localProperty +#endif ) { - if (property.localCentroidLimit > ((1UL << (sizeof(LOCAL_ID_TYPE) * 8)) - 1)) { stringstream msg; - msg << "Quantizer::Error. Local centroid limit is too large. " << property.localCentroidLimit << " It must be less than " << (1UL << (sizeof(LOCAL_ID_TYPE) * 8)); + msg << "Quantizer::Error. Local centroid limit " << property.localCentroidLimit << " is too large. It must be less than " << (1UL << (sizeof(LOCAL_ID_TYPE) * 8)); NGTThrowException(msg); } @@ -2073,13 +3432,15 @@ class QuantizerInstance : public Quantizer { lp.edgeSizeForSearch = 40; lp.objectType = NGT::Index::Property::ObjectType::Float; - gp.dimension = property.dimension; if (gp.dimension == 0) { stringstream msg; msg << "NGTQ::Quantizer::create: specified dimension is zero!"; NGTThrowException(msg); } + if (property.localDivisionNo == 0) { + NGTThrowException("NGTQ::Quantizer::create: # of subvectors is zero"); + } if (property.localDivisionNo != 1 && property.dimension % property.localDivisionNo != 0) { stringstream msg; msg << "NGTQ::Quantizer::create: dimension and localDivisionNo are not proper. " @@ -2092,6 +3453,9 @@ class QuantizerInstance : public Quantizer { case DataTypeFloat: gp.objectType = NGT::Index::Property::ObjectType::Float; break; + case DataTypeFloat16: + gp.objectType = NGT::Index::Property::ObjectType::Float16; + break; case DataTypeUint8: gp.objectType = NGT::Index::Property::ObjectType::Uint8; break; @@ -2104,11 +3468,11 @@ class QuantizerInstance : public Quantizer { } switch (property.distanceType) { - case DistanceTypeL1: + case DistanceType::DistanceTypeL1: gp.distanceType = NGT::Index::Property::DistanceType::DistanceTypeL1; lp.distanceType = NGT::Index::Property::DistanceType::DistanceTypeL1; break; - case DistanceTypeL2: + case DistanceType::DistanceTypeL2: #ifdef NGTQ_DISTANCE_ANGLE { stringstream msg; @@ -2119,11 +3483,11 @@ class QuantizerInstance : public Quantizer { gp.distanceType = NGT::Index::Property::DistanceType::DistanceTypeL2; lp.distanceType = NGT::Index::Property::DistanceType::DistanceTypeL2; break; - case DistanceTypeHamming: + case DistanceType::DistanceTypeHamming: gp.distanceType = NGT::Index::Property::DistanceType::DistanceTypeHamming; lp.distanceType = NGT::Index::Property::DistanceType::DistanceTypeHamming; break; - case DistanceTypeAngle: + case DistanceType::DistanceTypeAngle: #ifndef NGTQ_DISTANCE_ANGLE { stringstream msg; @@ -2134,15 +3498,15 @@ class QuantizerInstance : public Quantizer { gp.distanceType = NGT::Index::Property::DistanceType::DistanceTypeAngle; lp.distanceType = NGT::Index::Property::DistanceType::DistanceTypeAngle; break; - case DistanceTypeNormalizedCosine: + case DistanceType::DistanceTypeNormalizedCosine: gp.distanceType = NGT::Index::Property::DistanceType::DistanceTypeNormalizedCosine; lp.distanceType = NGT::Index::Property::DistanceType::DistanceTypeL2; break; - case DistanceTypeCosine: + case DistanceType::DistanceTypeCosine: gp.distanceType = NGT::Index::Property::DistanceType::DistanceTypeCosine; lp.distanceType = NGT::Index::Property::DistanceType::DistanceTypeL2; break; - case DistanceTypeNormalizedL2: + case DistanceType::DistanceTypeNormalizedL2: gp.distanceType = NGT::Index::Property::DistanceType::DistanceTypeNormalizedL2; lp.distanceType = NGT::Index::Property::DistanceType::DistanceTypeL2; break; @@ -2154,32 +3518,29 @@ class QuantizerInstance : public Quantizer { } } +#ifdef NGTQ_QBG + createEmptyIndex(index, gp, lp, rotation, objectFile); +#else createEmptyIndex(index, gp, lp); +#endif } void validate() { - size_t gcbSize = globalCodebook.getObjectRepositorySize(); - cerr << "global codebook size=" << gcbSize << endl; +#ifndef NGTQ_QBG + size_t gcbSize = globalCodebookIndex.getObjectRepositorySize(); for (size_t gidx = 1; gidx < 4 && gidx < gcbSize; gidx++) { if (invertedIndex[gidx] == 0) { cerr << "something wrong" << endl; exit(1); } - cerr << gidx << " inverted index size=" << (*invertedIndex[gidx]).size() << endl; if ((*invertedIndex[gidx]).size() == 0) { cerr << "something wrong" << endl; continue; } - NGT::PersistentObject &gcentroid = *globalCodebook.getObjectSpace().getRepository().get(gidx); + NGT::PersistentObject &gcentroid = *globalCodebookIndex.getObjectSpace().getRepository().get(gidx); vector gco; - globalCodebook.getObjectSpace().getRepository().extractObject(&gcentroid, gco); - cerr << "global centroid object(" << gco.size() << ")="; - for (size_t i = 0; i < gco.size(); i++) { - cerr << gco[i] << " "; - } - cerr << endl; - + globalCodebookIndex.getObjectSpace().getRepository().extractObject(&gcentroid, gco); { #ifdef NGTQ_SHARED_INVERTED_INDEX InvertedIndexObject &invertedIndexEntry = (*invertedIndex[gidx]).at(0, invertedIndex.allocator); @@ -2191,10 +3552,9 @@ class QuantizerInstance : public Quantizer { exit(1); } } - NGT::Object *gcentroidFromList = globalCodebook.getObjectSpace().getRepository().allocateObject(); - objectList.get(gidx, *gcentroidFromList, &globalCodebook.getObjectSpace()); + NGT::Object *gcentroidFromList = globalCodebookIndex.getObjectSpace().getRepository().allocateObject(); vector gcolist; - globalCodebook.getObjectSpace().getRepository().extractObject(gcentroidFromList, gcolist); + globalCodebookIndex.getObjectSpace().getRepository().extractObject(gcentroidFromList, gcolist); if (gco != gcolist) { cerr << "Fatal error! centroid in NGT is different from object list in NGTQ" << endl; exit(1); @@ -2209,9 +3569,9 @@ class QuantizerInstance : public Quantizer { elements.push_back(invertedIndexEntry.id); cerr << " object ID=" << invertedIndexEntry.id; { - NGT::Object *o = globalCodebook.getObjectSpace().getRepository().allocateObject(); - objectList.get(invertedIndexEntry.id, *o, &globalCodebook.getObjectSpace()); - NGT::Distance distance = globalCodebook.getObjectSpace().getComparator()(*gcentroidFromList, *o); + NGT::Object *o = globalCodebookIndex.getObjectSpace().getRepository().allocateObject(); + objectList.get(invertedIndexEntry.id, *o, &globalCodebookIndex.getObjectSpace()); + NGT::Distance distance = globalCodebookIndex.getObjectSpace().getComparator()(*gcentroidFromList, *o); cerr << ":distance=" << distance; } cerr << ":local codebook IDs="; @@ -2221,7 +3581,8 @@ class QuantizerInstance : public Quantizer { cerr << endl; for (size_t li = 0; li < property.localDivisionNo; li++) { if (invertedIndexEntry.localID[li] == 0) { - if (property.centroidCreationMode != CentroidCreationModeStatic) { + if (property.centroidCreationMode != CentroidCreationModeStatic && + property.centroidCreationMode != CentroidCreationModeStaticLayer) { if (iidx == 0) { break; } @@ -2248,65 +3609,46 @@ class QuantizerInstance : public Quantizer { cerr << "x "; ngid.push_back(objects[resulti].id); NGT::ObjectDistances result; - NGT::Object *o = globalCodebook.getObjectSpace().getRepository().allocateObject(); - objectList.get(ngid.back(), *o, &globalCodebook.getObjectSpace()); - NGT::GraphAndTreeIndex &graphIndex = (NGT::GraphAndTreeIndex &)globalCodebook.getIndex(); + NGT::Object *o = globalCodebookIndex.getObjectSpace().getRepository().allocateObject(); + objectList.get(ngid.back(), *o, &globalCodebookIndex.getObjectSpace()); + NGT::GraphAndTreeIndex &graphIndex = (NGT::GraphAndTreeIndex &)globalCodebookIndex.getIndex(); graphIndex.searchForNNGInsertion(*o, result); if (result[0].distance > objects[resulti].distance) { cerr << " Strange! "; cerr << result[0].distance << ":" << objects[resulti].distance << " "; } - globalCodebook.getObjectSpace().getRepository().deleteObject(o); + globalCodebookIndex.getObjectSpace().getRepository().deleteObject(o); } cerr << " search object " << resulti << " ID=" << objects[resulti].id << " distance=" << objects[resulti].distance << endl; } } - globalCodebook.getObjectSpace().getRepository().deleteObject(gcentroidFromList); + globalCodebookIndex.getObjectSpace().getRepository().deleteObject(gcentroidFromList); } +#endif // NGTQ_QBG } void searchGlobalCodebook(NGT::Object *query, size_t size, NGT::ObjectDistances &objects, size_t &approximateSearchSize, size_t codebookSearchSize, double epsilon) { - +#ifdef NGTQ_TRACE + std::cerr << "searchGlobalCodebook codebookSearchSize=" << codebookSearchSize << std::endl; +#endif NGT::SearchContainer sc(*query); sc.setResults(&objects); sc.size = codebookSearchSize; sc.radius = FLT_MAX; sc.explorationCoefficient = epsilon + 1.0; if (epsilon >= FLT_MAX) { - globalCodebook.linearSearch(sc); + globalCodebookIndex.linearSearch(sc); } else { - globalCodebook.search(sc); + globalCodebookIndex.search(sc); } } inline void aggregateObjectsWithExactDistance(NGT::ObjectDistance &globalCentroid, NGT::Object *query, size_t size, NGT::ObjectSpace::ResultSet &results, size_t approximateSearchSize) { - NGT::ObjectSpace &objectSpace = globalCodebook.getObjectSpace(); - for (size_t j = 0; j < invertedIndex[globalCentroid.id]->size() && results.size() < approximateSearchSize; j++) { -#ifdef NGTQ_SHARED_INVERTED_INDEX - InvertedIndexObject &invertedIndexEntry = (*invertedIndex[globalCentroid.id]).at(j, invertedIndex.allocator); -#else - InvertedIndexObject &invertedIndexEntry = (*invertedIndex[globalCentroid.id])[j]; -#endif - double distance; - if (invertedIndexEntry.localID[0] == 0) { - distance = globalCentroid.distance; - } else { - NGT::Object o(&objectSpace); - objectList.get(invertedIndexEntry.id, (NGT::Object&)o, &objectSpace); - distance = objectSpace.getComparator()(*query, (NGT::Object&)o); - } - - NGT::ObjectDistance obj; - obj.id = invertedIndexEntry.id; - obj.distance = distance; - assert(obj.id > 0); - results.push(obj); - - } + abort(); } inline void aggregateObjectsWithLookupTable(NGT::ObjectDistance &globalCentroid, NGT::Object *query, size_t size, NGT::ObjectSpace::ResultSet &results, size_t approximateSearchSize) { @@ -2336,6 +3678,66 @@ class QuantizerInstance : public Quantizer { } } + void eraseInvertedIndexObject(size_t id) { + invertedIndex.erase(id); + } + void eraseInvertedIndexObject() { + for (size_t id = 0; id < invertedIndex.size(); id++) { + try { + invertedIndex.erase(id); + } catch(...) {} + } + } +#ifdef NGTQ_QBG + void extractInvertedIndexObject(InvertedIndexEntry &invertedIndexObjects, size_t gid) { +#ifdef NGTQ_SHARED_INVERTED_INDEX + std::cerr << "InvertedIndex: Not implemented." << std::endl; + abort(); +#else + if (gid >= invertedIndex.size()) { + std::stringstream msg; + msg << "Quantizer::extractInvertedIndexObject: Fatal error! Invalid gid. " << invertedIndex.size() << ":" << gid; + NGTThrowException(msg); + } + if (invertedIndex[gid] == 0) { +#ifdef NGTQ_SHARED_INVERTED_INDEX + std::cerr << "Not implemented" << std::endl; +#else + invertedIndexObjects.clear(); +#endif + return; + } + invertedIndexObjects.subspaceID = invertedIndex[gid]->subspaceID; + invertedIndexObjects.resize(invertedIndex[gid]->size()); + for (size_t idx = 0; idx < invertedIndex[gid]->size(); idx++) { +#ifdef NGTQ_SHARED_INVERTED_INDEX + NGTQ::InvertedIndexObject &entry = (*invertedIndex[gid]).at(idx, invertedIndex.allocator); + invertedIndexObjects.at(idx, invertedIndex.allocator).id = entry.id; + if (sizeof(entry.localID[0]) > sizeof(invertedIndexObjects.at(idx, invertedIndex.allocator).localID[0])) { + std::cerr << "you should change the object ID type." << std::endl; + abort(); + } +#else + NGTQ::InvertedIndexObject &entry = (*invertedIndex[gid])[idx]; + invertedIndexObjects[idx].id = entry.id; + if (sizeof(entry.localID[0]) > sizeof(invertedIndexObjects[idx].localID[0])) { + std::cerr << "you should change the object ID type." << std::endl; + abort(); + } +#endif + for (size_t i = 0; i < localCodebookIndexes.size(); i++) { +#ifdef NGTQ_SHARED_INVERTED_INDEX + invertedIndexObjects.at(idx,invertedIndex.allocator).localID[i] = entry.localID[i]; +#else + invertedIndexObjects[idx].localID[i] = entry.localID[i]; +#endif + } + } +#endif + } +#endif + +#ifdef NGTQ_QBG void extractInvertedIndexObject(InvertedIndexEntry &invertedIndexObjects) { #ifdef NGTQ_SHARED_INVERTED_INDEX std::cerr << "not implemented" << std::endl; @@ -2362,10 +3764,11 @@ class QuantizerInstance : public Quantizer { for (size_t idx = 0; idx < invertedIndex[gid]->size(); idx++) { #ifdef NGTQ_SHARED_INVERTED_INDEX NGTQ::InvertedIndexObject &entry = (*invertedIndex[gid]).at(idx, invertedIndex.allocator); + invertedIndexObjects.at(entry.id, invertedIndex.allocator).id = entry.id; #else NGTQ::InvertedIndexObject &entry = (*invertedIndex[gid])[idx]; -#endif invertedIndexObjects[entry.id].id = entry.id; +#endif if (sizeof(entry.localID[0]) > sizeof(invertedIndexObjects[entry.id].localID[0])) { std::cerr << "you should change the object ID type." << std::endl; abort(); @@ -2378,6 +3781,7 @@ class QuantizerInstance : public Quantizer { } #endif } +#endif void extractInvertedIndex(std::vector> &ii) { ii.resize(invertedIndex.size()); @@ -2477,7 +3881,8 @@ class QuantizerInstance : public Quantizer { } void refineDistance(NGT::Object *query, NGT::ObjectDistances &results) { - NGT::ObjectSpace &objectSpace = globalCodebook.getObjectSpace(); +#ifndef NGTQ_QBG + NGT::ObjectSpace &objectSpace = globalCodebookIndex.getObjectSpace(); for (auto i = results.begin(); i != results.end(); ++i) { NGT::ObjectDistance &result = *i; NGT::Object o(&objectSpace); @@ -2486,6 +3891,7 @@ class QuantizerInstance : public Quantizer { result.distance = distance; } std::sort(results.begin(), results.end()); +#endif } void search(NGT::Object *query, NGT::ObjectDistances &objs, @@ -2494,7 +3900,7 @@ class QuantizerInstance : public Quantizer { AggregationMode aggregationMode, double epsilon = FLT_MAX) { size_t approximateSearchSize = size * expansion; - size_t codebookSearchSize = approximateSearchSize / (objectList.size() / globalCodebook.getObjectRepositorySize()) + 1; + size_t codebookSearchSize = approximateSearchSize / (objectList.size() / globalCodebookIndex.getObjectRepositorySize()) + 1; search(query, objs, size, approximateSearchSize, codebookSearchSize, aggregationMode, epsilon); } @@ -2570,13 +3976,17 @@ class QuantizerInstance : public Quantizer { double calculateQuantizationError() { - NGT::ObjectSpace &objectSpace = globalCodebook.getObjectSpace(); +#ifdef NGTQ_QBG + std::cerr << "calculateQuantizationError: Not implemented." << std::endl; + return 0.0; +#else + NGT::ObjectSpace &objectSpace = globalCodebookIndex.getObjectSpace(); double distance = 0.0; double globalDistance = 0.0; size_t count = 0; for (size_t gi = 0; gi < invertedIndex.size(); gi++) { if (invertedIndex[gi] != 0) { - NGT::PersistentObject &gcentroid = *globalCodebook.getObjectSpace().getRepository().get(gi); + NGT::PersistentObject &gcentroid = *globalCodebookIndex.getObjectSpace().getRepository().get(gi); for (size_t li = 0; li < invertedIndex[gi]->size(); li++) { #ifdef NGTQ_SHARED_INVERTED_INDEX size_t id = invertedIndex[gi]->at(li, invertedIndex.allocator).id; @@ -2592,7 +4002,7 @@ class QuantizerInstance : public Quantizer { #endif distance += d; count++; - NGT::Distance gd = globalCodebook.getObjectSpace().getComparator()(object, gcentroid); + NGT::Distance gd = globalCodebookIndex.getObjectSpace().getComparator()(object, gcentroid); globalDistance += gd; } } @@ -2601,6 +4011,7 @@ class QuantizerInstance : public Quantizer { globalDistance /= count; std::cerr << distance << ":" << globalDistance << std::endl; return distance; +#endif } void info(ostream &os, char mode) { @@ -2627,22 +4038,20 @@ class QuantizerInstance : public Quantizer { } } - NGT::Index &getLocalCodebook(size_t idx) { return localCodebook[idx]; } - void verify() { cerr << "sizeof(LOCAL_ID_TYPE)=" << sizeof(LOCAL_ID_TYPE) << endl; size_t objcount = objectList.size(); cerr << "Object count=" << objcount << endl; - size_t gcount = globalCodebook.getObjectRepositorySize(); + size_t gcount = globalCodebookIndex.getObjectRepositorySize(); cerr << "Global codebook size=" << gcount << endl; - size_t lcount = localCodebook[0].getObjectRepositorySize(); + size_t lcount = localCodebookIndexes[0].getObjectRepositorySize(); cerr << "Local codebook size=" << lcount << endl; lcount *= 1.1; cerr << "Inverted index size=" << invertedIndex.size() << endl; cerr << "Started verifying global codebook..." << endl; vector status; - globalCodebook.verify(status); + globalCodebookIndex.verify(status); cerr << "Started verifing the inverted index." << endl; size_t errorCount = 0; @@ -2687,7 +4096,6 @@ class QuantizerInstance : public Quantizer { } } - size_t getLocalCodebookSize(size_t size) { return localCodebook[size].getObjectRepositorySize(); } size_t getInstanceSharedMemorySize(ostream &os, SharedMemoryAllocator::GetMemorySizeType t = SharedMemoryAllocator::GetTotalMemorySize) { #ifdef NGTQ_SHARED_INVERTED_INDEX @@ -2697,8 +4105,8 @@ class QuantizerInstance : public Quantizer { #endif os << "inverted=" << size << endl; os << "Local centroid:" << endl; - for (size_t di = 0; di < localCodebook.size(); di++) { - size += localCodebook[di].getSharedMemorySize(os, t); + for (size_t di = 0; di < localCodebookIndexes.size(); di++) { + size += localCodebookIndexes[di].getSharedMemorySize(os, t); } return size; } @@ -2716,30 +4124,32 @@ class QuantizerInstance : public Quantizer { #endif QuantizedObjectDistance *quantizedObjectDistance; GenerateResidualObject *generateResidualObject; - std::vector localCodebook; - + float *localCodebooks; + bool silence; }; class Quantization { public: static Quantizer *generate(Property &property) { - DataType dataType = property.dataType; - size_t dimension = property.dimension; - size_t divisionNo = property.getLocalCodebookNo(); - size_t localIDByteSize = property.localIDByteSize; - Quantizer *quantizer = 0; + size_t localIDByteSize = property.localIDByteSize; + Quantizer *quantizer = 0; if (property.centroidCreationMode == CentroidCreationModeNone) { NGTThrowException("Centroid creation mode is not specified"); } else { if (localIDByteSize == 4) { - quantizer = new QuantizerInstance(dataType, dimension, divisionNo); + quantizer = new QuantizerInstance; } else if (localIDByteSize == 2) { - quantizer = new QuantizerInstance(dataType, dimension, divisionNo); + quantizer = new QuantizerInstance; +#ifdef NGTQ_QBG + } else if (localIDByteSize == 1) { + quantizer = new QuantizerInstance; +#endif } else { std::stringstream msg; msg << "Not support the specified size of local ID. " << localIDByteSize; NGTThrowException(msg); } + } return quantizer; @@ -2749,23 +4159,44 @@ class Quantization { class Index { public: Index():quantizer(0) {} - Index(const string& index, bool rdOnly = false):quantizer(0) { open(index, rdOnly); } + Index(const string& index, bool rdOnly = false):quantizer(0) { + open(index, rdOnly); + } ~Index() { close(); } static void create(const string &index, Property &property, NGT::Property &globalProperty, +#ifdef NGTQ_QBG + NGT::Property &localProperty, + std::vector *rotation = 0, + const std::string &objectFile = "") { +#else NGT::Property &localProperty) { +#endif if (property.dimension == 0) { NGTThrowException("NGTQ::create: Error. The dimension is zero."); } - property.setupLocalIDByteSize(); - NGTQ::Quantizer *quantizer = - NGTQ::Quantization::generate(property); + property.setup(property); + NGTQ::Quantizer *quantizer = NGTQ::Quantization::generate(property); try { - quantizer->property.setup(property); +#ifdef NGTQ_QBG + if (property.dimension == 0) { + property.dimension = property.genuineDimension; + } + if (property.dimension % 4 != 0) { + property.dimension = ((property.dimension - 1) / 4 + 1) * 4; + } + quantizer->property = property; + quantizer->create(index, globalProperty, localProperty, rotation, objectFile); +#else + quantizer->property = property; quantizer->create(index, globalProperty, localProperty); +#endif + if (property.dimension == 0) { + NGTThrowException("Quantizer: Dimension is zero."); + } } catch(NGT::Exception &err) { delete quantizer; throw err; @@ -2789,47 +4220,7 @@ class Quantization { } #endif - static void append(const string &indexName, // index file - const string &data, // data file - size_t dataSize = 0 // data size - ) { - NGTQ::Index index(indexName); - istream *is; - if (data == "-") { - is = &cin; - } else { - ifstream *ifs = new ifstream; - ifs->ifstream::open(data); - if (!(*ifs)) { - cerr << "Cannot open the specified file. " << data << endl; - return; - } - is = ifs; - } - string line; - vector > objects; - size_t count = 0; - // extract objects from the file and insert them to the object list. - while(getline(*is, line)) { - count++; - index.insert(line, objects, 0); - if (count % 10000 == 0) { - cerr << "Processed " << count; - cerr << endl; - } - } - if (objects.size() > 0) { - index.insert(objects); - } - cerr << "end of insertion. " << count << endl; - if (data != "-") { - delete is; - } - - index.save(); - index.close(); - } - +#ifndef NGTQ_QBG static void rebuild(const string &indexName, const string &rebuiltIndexName ) { @@ -2856,17 +4247,19 @@ class Quantization { } } +#endif void open(const string &index, bool readOnly = false) { close(); NGT::Property globalProperty; globalProperty.clear(); globalProperty.edgeSizeForSearch = 40; - quantizer = getQuantizer(index, globalProperty); - if (readOnly) { + quantizer = getQuantizer(index, globalProperty, readOnly); + if ((quantizer->property.quantizerType == NGTQ::QuantizerTypeQG) && readOnly) { quantizer->closeCodebooks(); } - } + + } void save() { getQuantizer().save(); @@ -2878,17 +4271,43 @@ class Quantization { quantizer = 0; } } - void insert(string &line, vector > &objects, size_t id) { - getQuantizer().insert(line, objects, id); - } +#ifndef NGTQ_QBG void insert(vector > &objects) { - getQuantizer().insert(objects); + std::cerr << "Not implemented." << std::endl; + abort(); + } +#endif + + void insertIntoObjectRepository(std::vector object) { + getQuantizer().insertIntoObjectRepository(object, 0); + } +#ifdef NGTQ_QBG + void createIndex(size_t beginID = 1, size_t endID = 0) { + getQuantizer().createIndex(beginID, endID); + } + + void createIndex(std::vector> &quantizationCodebook, + std::vector &codebookIndex, + std::vector &objectIndex, + size_t beginID = 1, size_t endID = 0) { + setupInvertedIndex(quantizationCodebook, codebookIndex, objectIndex); + createIndex(beginID, endID); + } +#endif + + void setupInvertedIndex(std::vector> &quantizationCodebook, + std::vector &codebookIndex, + std::vector &objectIndex) { + getQuantizer().setupInvertedIndex(quantizationCodebook, codebookIndex, objectIndex); } + +#ifndef NGTQ_QBG void rebuildIndex() { getQuantizer().rebuildIndex(); } +#endif NGT::Object *allocateObject(string &line, const string &sep, size_t dimension) { return getQuantizer().allocateObject(line, sep); @@ -2898,6 +4317,10 @@ class Quantization { return getQuantizer().allocateObject(obj); } + NGT::Object *allocateObject(vector &obj) { + return getQuantizer().allocateObject(obj); + } + void deleteObject(NGT::Object *object) { getQuantizer().deleteObject(object); } void search(NGT::Object *object, NGT::ObjectDistances &objs, @@ -2927,7 +4350,7 @@ class Quantization { return *quantizer; } - size_t getGlobalCodebookSize() { return quantizer->globalCodebook.getObjectRepositorySize(); } + size_t getGlobalCodebookSize() { return quantizer->globalCodebookIndex.getObjectRepositorySize(); } size_t getLocalCodebookSize(size_t idx) { return quantizer->getLocalCodebookSize(idx); } size_t getSharedMemorySize(ostream &os, SharedMemoryAllocator::GetMemorySizeType t = SharedMemoryAllocator::GetTotalMemorySize) { @@ -2935,14 +4358,7 @@ class Quantization { } protected: - - static NGTQ::Quantizer *getQuantizer(const string &index) { - NGT::Property globalProperty; - globalProperty.clear(); - return getQuantizer(index, globalProperty); - } - - static NGTQ::Quantizer *getQuantizer(const string &index, NGT::Property &globalProperty) { + static NGTQ::Quantizer *getQuantizer(const string &index, NGT::Property &globalProperty, bool readOnly) { NGTQ::Property property; try { property.load(index); @@ -2956,7 +4372,7 @@ class Quantization { NGTThrowException("NGTQ::Index: Cannot get quantizer."); } try { - quantizer->open(index, globalProperty); + quantizer->open(index, globalProperty, property.quantizerType == NGTQ::QuantizerTypeQBG ? readOnly : false); } catch(NGT::Exception &err) { delete quantizer; throw err; @@ -2965,6 +4381,8 @@ class Quantization { } NGTQ::Quantizer *quantizer; + + bool silence; }; } // namespace NGTQ diff --git a/lib/NGT/ObjectSpace.h b/lib/NGT/ObjectSpace.h index 5784beb..5254ab1 100644 --- a/lib/NGT/ObjectSpace.h +++ b/lib/NGT/ObjectSpace.h @@ -191,7 +191,8 @@ namespace NGT { typedef std::priority_queue, std::less > ResultSet; - ObjectSpace(size_t d):dimension(d), distanceType(DistanceTypeNone), comparator(0), normalization(false) {} + ObjectSpace(size_t d):dimension(d), distanceType(DistanceTypeNone), comparator(0), normalization(false), + prefetchOffset(-1), prefetchSize(-1) {} virtual ~ObjectSpace() { if (comparator != 0) { delete comparator; } } #ifdef NGT_SHARED_MEMORY_ALLOCATOR @@ -268,22 +269,24 @@ namespace NGT { data[i] = static_cast(data[i]) / sum; } } - uint32_t getPrefetchOffset() { return prefetchOffset; } - uint32_t setPrefetchOffset(size_t offset) { - if (offset == 0) { - prefetchOffset = floor(300.0 / (static_cast(getPaddedDimension()) + 30.0) + 1.0); - } else { + int32_t getPrefetchOffset() { return prefetchOffset; } + int32_t setPrefetchOffset(int offset) { + if (offset > 0) { prefetchOffset = offset; } + if (prefetchOffset <= 0) { + prefetchOffset = floor(300.0 / (static_cast(getPaddedDimension()) + 30.0) + 1.0); + } return prefetchOffset; } - uint32_t getPrefetchSize() { return prefetchSize; } - uint32_t setPrefetchSize(size_t size) { - if (size == 0) { - prefetchSize = getByteSizeOfObject(); - } else { + int32_t getPrefetchSize() { return prefetchSize; } + int32_t setPrefetchSize(int size) { + if (size > 0) { prefetchSize = size; } + if (prefetchSize <= 0) { + prefetchSize = getByteSizeOfObject(); + } return prefetchSize; } protected: @@ -291,8 +294,8 @@ namespace NGT { DistanceType distanceType; Comparator *comparator; bool normalization; - uint32_t prefetchOffset; - uint32_t prefetchSize; + int32_t prefetchOffset; + int32_t prefetchSize; }; class BaseObject { @@ -369,7 +372,9 @@ namespace NGT { class Object : public BaseObject { public: Object(NGT::ObjectSpace *os = 0):vector(0) { - assert(os != 0); + if (os == 0) { + return; + } size_t s = os->getByteSizeOfObject(); construct(s); } @@ -379,6 +384,9 @@ namespace NGT { construct(s); } + void attach(void *ptr) { vector = static_cast(ptr); } + void detach() { vector = 0; } + void copy(Object &o, size_t s) { assert(vector != 0); for (size_t i = 0; i < s; i++) { diff --git a/lib/NGT/PrimitiveComparator.h b/lib/NGT/PrimitiveComparator.h index 49136d5..a2d3ae4 100644 --- a/lib/NGT/PrimitiveComparator.h +++ b/lib/NGT/PrimitiveComparator.h @@ -251,7 +251,6 @@ namespace NGT { __attribute__((aligned(32))) float f[4]; _mm_store_ps(f, sum128); - double s = f[0] + f[1] + f[2] + f[3]; return sqrt(s); } diff --git a/lib/NGT/Thread.h b/lib/NGT/Thread.h index 943d506..d583835 100644 --- a/lib/NGT/Thread.h +++ b/lib/NGT/Thread.h @@ -31,8 +31,8 @@ void * evaluate_responce(void *); class ThreadTerminationException : public Exception { public: - ThreadTerminationException(const std::string &file, size_t line, std::stringstream &m) { set(file, line, m.str()); } - ThreadTerminationException(const std::string &file, size_t line, const std::string &m) { set(file, line, m); } + ThreadTerminationException(const std::string &file, const std::string &function, size_t line, std::stringstream &m) { set(file, function, line, m.str()); } + ThreadTerminationException(const std::string &file, const std::string &function, size_t line, const std::string &m) { set(file, function, line, m); } }; class ThreadInfo; diff --git a/lib/NGT/defines.h.in b/lib/NGT/defines.h.in index 9375d6f..e936106 100644 --- a/lib/NGT/defines.h.in +++ b/lib/NGT/defines.h.in @@ -22,6 +22,9 @@ #cmakedefine NGT_AVX_DISABLED // not use avx to compare #cmakedefine NGT_LARGE_DATASET // more than 10M objects #cmakedefine NGT_DISTANCE_COMPUTATION_COUNT // count # of distance computations +#cmakedefine NGT_QBG_DISABLED +#cmakedefine NGTQG_ZERO_GLOBAL +#cmakedefine NGTQG_NO_ROTATION // End of cmake defines ////////////////////////////////////////////////////////////////////////// diff --git a/python/README-ngtpy.md b/python/README-ngtpy.md index 314eab0..994823d 100644 --- a/python/README-ngtpy.md +++ b/python/README-ngtpy.md @@ -107,7 +107,7 @@ Get the number of the registered objects. int get_num_of_objects() **Returns** -The the number of the registered objects. +The number of the registered objects. ### search diff --git a/python/src/ngtpy.cpp b/python/src/ngtpy.cpp index 767617b..5159cec 100644 --- a/python/src/ngtpy.cpp +++ b/python/src/ngtpy.cpp @@ -17,7 +17,12 @@ #include "NGT/Index.h" #include "NGT/GraphOptimizer.h" #include "NGT/version_defs.h" +#include "NGT/NGTQ/Quantizer.h" +#ifdef NGTQ_QBG +#include "NGT/NGTQ/QuantizedBlobGraph.h" +#else #include "NGT/NGTQ/QuantizedGraph.h" +#endif #include #include @@ -293,7 +298,7 @@ class Index : public NGT::Index { int numOfExploredEdges, // # of explored edges for search size_t batchSize) // batch size to search at the same time { - bool unlog = NGT::Index::redirector.enabled; + bool unlog = NGT::Index::redirect; NGT::GraphReconstructor::refineANNG(*this, unlog, epsilon, accuracy, numOfEdges, numOfExploredEdges, batchSize); } @@ -507,6 +512,466 @@ class QuantizedIndex : public NGTQG::Index { }; +class BatchResults { +public: + BatchResults() {} + size_t getSize() { return size; } + void convert() { + if (results.size() == 0) { + return; + } + resultList.clear(); + resultList.resize(results.size()); + for (size_t idx = 0; idx < size; idx++) { + if (resultList[idx].size() != results[idx].size() && results[idx].size() != 0) { + resultList[idx].clear(); + resultList[idx].resize(results[idx].size()); + size_t rank = results[idx].size(); + resultList[idx].resize(rank); + rank--; + while (!results[idx].empty()) { + resultList[idx][rank] = results[idx].top(); + results[idx].pop(); + rank--; + } + } + } + results.clear(); + } + py::object get(size_t idx) { + convert(); + if (idx >= size) { + py::list result; + return result; + } + py::list result; + for (auto ri = resultList[idx].begin(); ri != resultList[idx].end(); ++ri) { + result.append(py::make_tuple((*ri).id - 1, (*ri).distance)); + } + return result; + } + py::array_t getIDs() { + convert(); + if (size == 0 || resultList[0].size() == 0) { + NGTThrowException("ngtpy::BatchResults::get: empty."); + } + size_t nobjects = resultList[0].size(); + py::array_t r({size, nobjects}); + auto wr = r.mutable_unchecked<2>(); + for (size_t idx = 0; idx < size; idx++) { + if (resultList[idx].size() != nobjects) { + NGTThrowException("ngtpy::BatchResults::get: not knn results."); + } + for (auto ri = resultList[idx].begin(); ri != resultList[idx].end(); ++ri) { + wr(idx, std::distance(resultList[idx].begin(), ri)) = (*ri).id - 1; + } + } + return r; + } + + py::array_t getIndexedIDs() { + convert(); + size_t count = 0; + for (size_t idx = 0; idx < size; idx++) { + count += resultList[idx].size(); + } + py::array_t results(count); + auto wresults = results.mutable_unchecked<1>(); + size_t pos = 0; + for (size_t idx = 0; idx < size; idx++) { + for (auto ri = resultList[idx].begin(); ri != resultList[idx].end(); ++ri) { + wresults(pos++) = (*ri).id - 1; + } + } + return results; + } + + py::array_t getIndexedDistances() { + convert(); + size_t count = 0; + for (size_t idx = 0; idx < size; idx++) { + count += resultList[idx].size(); + } + py::array_t results(count); + auto wresults = results.mutable_unchecked<1>(); + size_t pos = 0; + for (size_t idx = 0; idx < size; idx++) { + for (auto ri = resultList[idx].begin(); ri != resultList[idx].end(); ++ri) { + wresults(pos++) = (*ri).distance; + } + } + return results; + } + + py::array_t getIndex() { + convert(); + py::array_t results(size + 1); + auto wresults = results.mutable_unchecked<1>(); + size_t count = 0; + wresults(0) = 0; + for (size_t idx = 0; idx < size; idx++) { + count += resultList[idx].size(); + wresults(idx + 1) = count; + } + return results; + } + + std::vector results; + std::vector resultList; + size_t size; +}; + +class QuantizedBlobIndex : public QBG::Index { +public: + QuantizedBlobIndex( + const std::string path, // ngt index path. + size_t maxNoOfEdges, // the maximum number of quantized graph edges. + bool zeroBasedNumbering, // object ID numbering. + bool treeDisabled, // not use the tree index. + bool logDisabled // stderr log is disabled. + ):QBG::Index(path, true) { + zeroNumbering = zeroBasedNumbering; + numOfDistanceComputations = 0; + treeIndex = !treeDisabled; + withDistance = true;; + defaultNumOfSearchObjects = 20; + defaultEpsilon = 0.02; + defaultBlobEpsilon = 0.0; + defaultResultExpansion = 3.0; + defaultEdgeSize = -2; + defaultExplorationSize = 200; + defaultExactResultExpansion = 0.0; + defaultNumOfProbes = 0; + } + +py::array_t batchSearchTmp( + py::array_t queries, + size_t size + ) { + const py::buffer_info &qinfo = queries.request(); + const std::vector &qshape = qinfo.shape; + auto nOfQueries = qshape[0]; + auto dimension = qshape[1]; + auto *queryPtr = static_cast(qinfo.ptr); + + size = size > 0 ? size : defaultNumOfSearchObjects; + + py::array_t results({nOfQueries, static_cast(size)}); + auto wresults = results.mutable_unchecked<2>(); + +#pragma omp parallel for schedule(dynamic) + for (int idx = 0; idx < nOfQueries; idx++) { + NGT::Object queryObject(dimension * sizeof(float)); + float *qptr = queryPtr + idx * dimension; + memcpy(queryObject.getPointer(), qptr, dimension * sizeof(float)); + QBG::SearchContainer sc(queryObject); + sc.setSize(size); + sc.setEpsilon(defaultEpsilon); + sc.setBlobEpsilon(defaultBlobEpsilon); + sc.setEdgeSize(defaultEdgeSize); + sc.setGraphExplorationSize(defaultExplorationSize); + QBG::Index::searchBlobGraph(sc); + NGT::ResultPriorityQueue &r = sc.getWorkingResult(); + if (r.size() != size) { + std::cerr << "result size is invalid? " << r.size() << ":" << size << std::endl; + } + size_t rank = size; + while (!r.empty()) { + rank--; + wresults(idx, rank) = r.top().id - 1; + r.pop(); + } + } + return results; + } + + void batchApproximateSearchWithoutGraph( + py::array_t queries, + BatchResults &results, + size_t size + ) { + const py::buffer_info &qinfo = queries.request(); + const std::vector &qshape = qinfo.shape; + auto nOfQueries = qshape[0]; + size_t dimension = qshape[1]; + size_t psedoDimension = QBG::Index::getQuantizer().globalCodebookIndex.getObjectSpace().getPaddedDimension(); + auto *queryPtr = static_cast(qinfo.ptr); + + size = size > 0 ? size : defaultNumOfSearchObjects; + + results.results.clear(); + results.resultList.clear(); + results.results.resize(nOfQueries); + +#pragma omp parallel for schedule(dynamic) + for (int idx = 0; idx < nOfQueries; idx++) { + NGT::Object queryObject(psedoDimension * sizeof(float)); + float *qptr = queryPtr + idx * dimension; + if (psedoDimension > dimension) { + memset(queryObject.getPointer(), 0, psedoDimension * sizeof(float)); + } + memcpy(queryObject.getPointer(), qptr, dimension * sizeof(float)); + QBG::SearchContainer sc(queryObject); + sc.setSize(size); + sc.setEpsilon(defaultEpsilon); + sc.setEdgeSize(defaultEdgeSize); + sc.setNumOfProbes(defaultNumOfProbes); + QBG::Index::searchBlobNaively(sc); + results.results[idx] = std::move(sc.getWorkingResult()); + } + results.size = results.results.size(); + return; + } + + void batchApproximateSearch( + py::array_t queries, + BatchResults &results, + size_t size + ) { + const py::buffer_info &qinfo = queries.request(); + const std::vector &qshape = qinfo.shape; + auto nOfQueries = qshape[0]; + size_t dimension = qshape[1]; + size_t psedoDimension = QBG::Index::getQuantizer().globalCodebookIndex.getObjectSpace().getPaddedDimension(); + auto *queryPtr = static_cast(qinfo.ptr); + + size = size > 0 ? size : defaultNumOfSearchObjects; + + results.results.clear(); + results.resultList.clear(); + results.results.resize(nOfQueries); + +#pragma omp parallel for schedule(dynamic) + for (int idx = 0; idx < nOfQueries; idx++) { + float *qptr = queryPtr + idx * dimension; + vector query(psedoDimension, 0); + memcpy(query.data(), qptr, dimension * sizeof(float)); + QBG::SearchContainer sc; + sc.setObjectVector(query); + sc.setSize(size); + sc.setEpsilon(defaultEpsilon); + sc.setBlobEpsilon(defaultBlobEpsilon); + sc.setEdgeSize(defaultEdgeSize); + sc.setGraphExplorationSize(defaultExplorationSize); + QBG::Index::searchBlobGraph(sc); + results.results[idx] = std::move(sc.getWorkingResult()); + //QBG::Index::getQuantizer().globalCodebookIndex.deleteObject(queryObject); + } + results.size = results.results.size(); + return; + } + + void batchExactSearch( + py::array_t queries, + BatchResults &results, + size_t size + ) { + const py::buffer_info &qinfo = queries.request(); + const std::vector &qshape = qinfo.shape; + auto nOfQueries = qshape[0]; + size_t dimension = qshape[1]; + size_t psedoDimension = QBG::Index::getQuantizer().globalCodebookIndex.getObjectSpace().getPaddedDimension(); + auto *queryPtr = static_cast(qinfo.ptr); + + size = size > 0 ? size : defaultNumOfSearchObjects; + + results.results.clear(); + results.resultList.clear(); + results.resultList.resize(nOfQueries); + +#pragma omp parallel for schedule(dynamic) + for (int idx = 0; idx < nOfQueries; idx++) { + NGT::Object queryObject(psedoDimension * sizeof(float)); + float *qptr = queryPtr + idx * dimension; + if (psedoDimension > dimension) { + memset(queryObject.getPointer(), 0, psedoDimension * sizeof(float)); + } + memcpy(queryObject.getPointer(), qptr, dimension * sizeof(float)); + NGT::ObjectDistances rs; + QBG::SearchContainer sc(queryObject); + sc.setResults(&rs); + sc.setSize(static_cast(size) * defaultExactResultExpansion); + sc.setExactResultSize(size); + sc.setEpsilon(defaultEpsilon); + sc.setBlobEpsilon(defaultBlobEpsilon); + sc.setEdgeSize(defaultEdgeSize); + sc.setGraphExplorationSize(defaultExplorationSize); + QBG::Index::searchBlobGraph(sc); + results.resultList[idx] = std::move(rs); + } + results.size = results.resultList.size(); + return; + } + + void batchSearch( + py::array_t queries, + BatchResults &results, + size_t size + ) { + if (defaultNumOfProbes == 0) { + if (defaultExactResultExpansion > 1.0) { + batchExactSearch(queries, results, size); + } else { + batchApproximateSearch(queries, results, size); + } + } else { + batchApproximateSearchWithoutGraph(queries, results, size); + } + return; + } + + void batchRangeSearch( + py::array_t queries, + BatchResults &results, + float radius + ) { + const py::buffer_info &qinfo = queries.request(); + const std::vector &qshape = qinfo.shape; + auto nOfQueries = qshape[0]; + size_t dimension = qshape[1]; + size_t psedoDimension = QBG::Index::getQuantizer().globalCodebookIndex.getObjectSpace().getPaddedDimension(); + auto *queryPtr = static_cast(qinfo.ptr); + radius = radius >= 0 ? radius : defaultRadius; + radius = sqrt(radius); + + results.results.clear(); + results.resultList.clear(); + results.results.resize(nOfQueries); + +#pragma omp parallel for schedule(dynamic) + for (int idx = 0; idx < nOfQueries; idx++) { + NGT::Object queryObject(dimension * sizeof(float)); + float *qptr = queryPtr + idx * dimension; + if (psedoDimension > dimension) { + memset(queryObject.getPointer(), 0, psedoDimension * sizeof(float)); + } + memcpy(queryObject.getPointer(), qptr, dimension * sizeof(float)); + QBG::SearchContainer sc(queryObject); + sc.setSize(std::numeric_limits::max()); + sc.setRadius(radius); + sc.setEpsilon(defaultEpsilon); + sc.setBlobEpsilon(defaultBlobEpsilon); + sc.setEdgeSize(defaultEdgeSize); + sc.setGraphExplorationSize(defaultExplorationSize); + QBG::Index::searchBlobGraph(sc); + results.results[idx] = std::move(sc.getWorkingResult()); + } + results.size = results.results.size(); + return; + } + + py::object search( + py::object query, + size_t size, // the number of resultant objects + float epsilon, // search parameter epsilon. the adequate range is from 0.0 to 0.05. + float resultExpansion, // the number of inner resultant objects + int edgeSize // the number of used edges for each node during the exploration of the graph. + ) { + py::array_t qobject(query); + py::buffer_info qinfo = qobject.request(); + std::vector qvector(static_cast(qinfo.ptr), static_cast(qinfo.ptr) + qinfo.size); + try { + NGT::Object *qobject = QBG::Index::getQuantizer().globalCodebookIndex.allocateObject(qvector); + QBG::SearchContainer sc(*qobject); + size = size > 0 ? size : defaultNumOfSearchObjects; + epsilon = epsilon > -1.0 ? epsilon : defaultEpsilon; + sc.setSize(size); // the number of resulting objects. + sc.setEpsilon(epsilon); // set exploration coefficient. + sc.setBlobEpsilon(defaultBlobEpsilon); + std::cerr << "ngtpy search" << std::endl; + QBG::Index::searchBlobGraph(sc); + + QBG::Index::getQuantizer().globalCodebookIndex.deleteObject(qobject); + numOfDistanceComputations += sc.distanceComputationCount; + + if (!withDistance) { + std::cerr << "without distances" << std::endl; + NGT::ResultPriorityQueue &r = sc.getWorkingResult(); + std::cerr << "size=" << r.size() << std::endl; + py::array_t ids(r.size()); + py::buffer_info idsinfo = ids.request(); + int *endptr = reinterpret_cast(idsinfo.ptr); + int *ptr = endptr + (r.size() - 1); + if (zeroNumbering) { + while (ptr >= endptr) { + *ptr-- = r.top().id - 1; + r.pop(); + } + } else { + while (ptr >= endptr) { + *ptr-- = r.top().id; + r.pop(); + } + } + + return ids; + } + std::cerr << "with distances" << std::endl; + py::list results; + NGT::ObjectDistances r; + r.moveFrom(sc.getWorkingResult()); + std::cerr << "size=" << r.size() << std::endl; + if (zeroNumbering) { + for (auto ri = r.begin(); ri != r.end(); ++ri) { + results.append(py::make_tuple((*ri).id - 1, (*ri).distance)); + std::cerr << "ID(1)=" << (*ri).id << std::endl; + } + } else { + for (auto ri = r.begin(); ri != r.end(); ++ri) { + results.append(py::make_tuple((*ri).id, (*ri).distance)); + std::cerr << "ID(2)=" << (*ri).id << std::endl; + } + } + return results; + } catch (NGT::Exception &e) { + std::cerr << e.what() << std::endl; + if (!withDistance) { + return py::array_t(); + } else { + return py::list(); + } + } + } + + void setWithDistance(bool v) { withDistance = v; } + + void set( + size_t numOfSearchObjects, // the number of resultant objects + float epsilon, // search parameter epsilon. the adequate range is from 0.0 to 0.05. + float blobEpsilon, // search parameter blob epsilon. the recommended value is 0.0. + float resultExpansion, // the number of inner resultant objects + float radius, + int edgeSize, + int explorationSize, // the maximum number of nodes that are searched + float exactResultExpansion, + int numOfProbes + ) { + defaultNumOfSearchObjects = numOfSearchObjects > 0 ? numOfSearchObjects : defaultNumOfSearchObjects; + defaultEpsilon = epsilon > -1.0 ? epsilon : defaultEpsilon; + defaultBlobEpsilon = blobEpsilon > -1.0 ? blobEpsilon : defaultBlobEpsilon; + defaultResultExpansion = resultExpansion >= 0.0 ? resultExpansion : defaultResultExpansion; + defaultEdgeSize = edgeSize >= -2 ? edgeSize : defaultEdgeSize; + defaultExplorationSize = explorationSize > 0 ? explorationSize : defaultExplorationSize; + defaultRadius = radius >= 0.0 ? radius : defaultRadius; + defaultExactResultExpansion = exactResultExpansion > 0.0 ? exactResultExpansion : defaultExactResultExpansion; + defaultNumOfProbes = numOfProbes > 0 ? numOfProbes : defaultNumOfProbes; + } + + bool zeroNumbering; // for object ID numbering. zero-based or one-based numbering. + size_t numOfDistanceComputations; + bool treeIndex; + bool withDistance; + size_t defaultNumOfSearchObjects; // k + float defaultEpsilon; + float defaultBlobEpsilon; + float defaultResultExpansion; + int64_t defaultEdgeSize; + size_t defaultExplorationSize; + float defaultRadius; + float defaultExactResultExpansion; + size_t defaultNumOfProbes; +}; + PYBIND11_MODULE(ngtpy, m) { m.doc() = "ngt python"; @@ -650,4 +1115,53 @@ PYBIND11_MODULE(ngtpy, m) { py::arg("epsilon") = -FLT_MAX, py::arg("result_expansion") = -FLT_MAX, py::arg("edge_size") = INT_MIN); + + + py::class_(m, "QuantizedBlobIndex") + .def(py::init(), + py::arg("path"), + py::arg("max_no_of_edges") = 128, + py::arg("zero_based_numbering") = true, + py::arg("tree_disabled") = false, + py::arg("log_disabled") = false) + .def("batchSearchTmp", &::QuantizedBlobIndex::batchSearchTmp, + py::arg("query"), + py::arg("size") = 0) + .def("batchSearch", &::QuantizedBlobIndex::batchSearch, + py::arg("query"), + py::arg("results"), + py::arg("size") = 0) + .def("batchRangeSearch", &::QuantizedBlobIndex::batchRangeSearch, + py::arg("query"), + py::arg("results"), + py::arg("radius") = -FLT_MAX) + .def("search", &::QuantizedBlobIndex::search, + py::arg("query"), + py::arg("size") = 0, + py::arg("epsilon") = -FLT_MAX, + py::arg("result_expansion") = -FLT_MAX, + py::arg("edge_size") = INT_MIN) + .def("set_with_distance", &::QuantizedBlobIndex::setWithDistance, + py::arg("boolean") = true) + .def("set", &::QuantizedBlobIndex::set, + py::arg("num_of_search_objects") = 0, + py::arg("epsilon") = -FLT_MAX, + py::arg("blob_epsilon") = -FLT_MAX, + py::arg("result_expansion") = -FLT_MAX, + py::arg("radius") = -FLT_MAX, + py::arg("edge_size") = INT_MIN, + py::arg("exploration_size") = 0, + py::arg("exact_result_expansion") = 0.0, + py::arg("num_of_probes") = INT_MIN); + + py::class_(m, "BatchResults") + .def(py::init<>()) + .def("get", &::BatchResults::get, + py::arg("position")) + .def("getIDs", &::BatchResults::getIDs) + .def("getIndexedIDs", &::BatchResults::getIndexedIDs) + .def("getIndexedDistances", &::BatchResults::getIndexedDistances) + .def("getIndex", &::BatchResults::getIndex) + .def("getSize", &::BatchResults::getSize); + } diff --git a/samples/jaccard-sparse/jaccard-sparse.cpp b/samples/jaccard-sparse/jaccard-sparse.cpp index f8ac096..e1286dd 100644 --- a/samples/jaccard-sparse/jaccard-sparse.cpp +++ b/samples/jaccard-sparse/jaccard-sparse.cpp @@ -75,7 +75,6 @@ append(NGT::Args &args) std::cerr << "jaccard-sparse: Empty line or invalid value. " << count << ":" << line << std::endl; continue; } - NGT::ObjectID id = index.append(index.makeSparseObject(object)); } if (data != "-") { delete ifs; diff --git a/tests/ann-benchmarks-results/fashion-mnist-784-euclidean.png b/tests/ann-benchmarks-results/fashion-mnist-784-euclidean.png index 3f08b70..a4ef511 100755 Binary files a/tests/ann-benchmarks-results/fashion-mnist-784-euclidean.png and b/tests/ann-benchmarks-results/fashion-mnist-784-euclidean.png differ diff --git a/tests/ann-benchmarks-results/gist-960-euclidean.png b/tests/ann-benchmarks-results/gist-960-euclidean.png index b0970ab..4decec4 100755 Binary files a/tests/ann-benchmarks-results/gist-960-euclidean.png and b/tests/ann-benchmarks-results/gist-960-euclidean.png differ diff --git a/tests/ann-benchmarks-results/glove-100-angular.png b/tests/ann-benchmarks-results/glove-100-angular.png index 85952a2..f38f548 100755 Binary files a/tests/ann-benchmarks-results/glove-100-angular.png and b/tests/ann-benchmarks-results/glove-100-angular.png differ diff --git a/tests/ann-benchmarks-results/glove-25-angular.png b/tests/ann-benchmarks-results/glove-25-angular.png index 4a52037..2d86c16 100755 Binary files a/tests/ann-benchmarks-results/glove-25-angular.png and b/tests/ann-benchmarks-results/glove-25-angular.png differ diff --git a/tests/ann-benchmarks-results/nytimes-256-angular.png b/tests/ann-benchmarks-results/nytimes-256-angular.png index e326e62..2c4784b 100755 Binary files a/tests/ann-benchmarks-results/nytimes-256-angular.png and b/tests/ann-benchmarks-results/nytimes-256-angular.png differ diff --git a/tests/ann-benchmarks-results/sift-128-euclidean.png b/tests/ann-benchmarks-results/sift-128-euclidean.png index e70dd79..d1f8924 100755 Binary files a/tests/ann-benchmarks-results/sift-128-euclidean.png and b/tests/ann-benchmarks-results/sift-128-euclidean.png differ