diff --git a/.clang-format b/.clang-format deleted file mode 100644 index 452fcea..0000000 --- a/.clang-format +++ /dev/null @@ -1,63 +0,0 @@ ---- -BasedOnStyle: LLVM -AccessModifierOffset: '0' -AlignConsecutiveMacros: 'false' -AlignConsecutiveAssignments: 'false' -AlignConsecutiveDeclarations: 'false' -AlignEscapedNewlines: Left -AlignTrailingComments: 'true' -AllowAllArgumentsOnNextLine: 'true' -AllowAllConstructorInitializersOnNextLine: 'false' -AllowAllParametersOfDeclarationOnNextLine: 'true' -AllowShortBlocksOnASingleLine: 'false' -AllowShortCaseLabelsOnASingleLine: 'false' -AllowShortFunctionsOnASingleLine: None -AllowShortIfStatementsOnASingleLine: Never -AllowShortLambdasOnASingleLine: None -AllowShortLoopsOnASingleLine: 'false' -AlwaysBreakAfterReturnType: None -AlwaysBreakBeforeMultilineStrings: 'false' -AlwaysBreakTemplateDeclarations: 'Yes' -BinPackArguments: 'false' -BinPackParameters: 'false' -BreakBeforeBraces: Attach -BreakBeforeTernaryOperators: 'false' -BreakConstructorInitializers: AfterColon -BreakInheritanceList: AfterColon -ColumnLimit: '0' -CompactNamespaces: 'false' -ConstructorInitializerAllOnOneLineOrOnePerLine: 'false' -Cpp11BracedListStyle: 'true' -FixNamespaceComments: 'false' -IncludeBlocks: Regroup -IndentCaseLabels: 'true' -IndentPPDirectives: BeforeHash -IndentWidth: '4' -IndentWrappedFunctionNames: 'false' -JavaScriptQuotes: Double -KeepEmptyLinesAtTheStartOfBlocks: 'false' -Language: Cpp -MaxEmptyLinesToKeep: '1' -NamespaceIndentation: All -PointerAlignment: Middle -SortIncludes: 'true' -SpaceAfterCStyleCast: 'false' -SpaceAfterLogicalNot: 'false' -SpaceAfterTemplateKeyword: 'false' -SpaceBeforeAssignmentOperators: 'true' -SpaceBeforeCpp11BracedList: 'false' -SpaceBeforeCtorInitializerColon: 'false' -SpaceBeforeInheritanceColon: 'false' -SpaceBeforeParens: Never -SpaceBeforeRangeBasedForLoopColon: 'false' -SpaceInEmptyParentheses: 'false' -SpacesInAngles: 'false' -SpacesInCStyleCastParentheses: 'false' -SpacesInContainerLiterals: 'false' -SpacesInParentheses: 'false' -SpacesInSquareBrackets: 'false' -Standard: Cpp11 -TabWidth: '4' -UseTab: Never - -... diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..ee18514 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,23 @@ +name: Build +on: [push, pull_request] + +jobs: + build: + if: github.event_name == 'push' && !startsWith(github.event.ref, 'refs/tags') + runs-on: ubuntu-20.04 + strategy: + matrix: + python-version: [3.6, 3.7, 3.8, 3.9, pypy3] + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install Ubuntu packages + run: >- + sudo apt install libidn2-dev; + - name: Test + run: >- + python setup.py test diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..8baba3e --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,23 @@ +name: Deploy +on: + release: + types: [published] + +jobs: + deploy: + runs-on: ubuntu-20.04 + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Build a source tarball + run: >- + python -m pip install --user --upgrade setuptools; + python setup.py sdist; + - name: Publish distribution 📦 to PyPI + uses: pypa/gh-action-pypi-publish@master + with: + password: ${{ secrets.pypi_password }} diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml deleted file mode 100644 index 23df2bd..0000000 --- a/.github/workflows/pythonpackage.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Build -on: [push, pull_request] -jobs: - build: - runs-on: ubuntu-20.04 - strategy: - max-parallel: 4 - matrix: - python-version: [3.6, 3.7, 3.8, 3.9, pypy3] - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Test module - run: >- - python setup.py test - deploy: - needs: build - runs-on: ubuntu-20.04 - if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Build a source tarball - run: >- - python -m pip install --user --upgrade setuptools; - python setup.py sdist; - - name: Publish distribution 📦 to PyPI - uses: pypa/gh-action-pypi-publish@master - with: - password: ${{ secrets.pypi_password }} diff --git a/README.md b/README.md index a439a45..3901b90 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,8 @@ - [About The Project](#about-the-project) - [Built With](#built-with) - [Performance](#performance) + - [500MB File](#500mb-file) + - [6000MB File](#6000mb-file) - [Prerequisites](#prerequisites) - [Installation](#installation) - [Usage](#usage) @@ -29,7 +31,10 @@ PySubstringSearch is a library intended for searching over an index file for substring patterns. The library is written in C++ to achieve speed and efficiency. The library also uses [Msufsort](https://github.com/michaelmaniscalco/msufsort) suffix array construction library for string indexing. The created index consists of the original text and a 32bit suffix array structs. The library relies on a proprietary container protocol to hold the original text along with the index in chunks of 512mb to evade the limitation of the Suffix Array Construction implementation. -The module implements two methods, search_sequential & search_parallel. search_sequential searches through the inner chunks one by one where search_parallel searches concurrently. When dealing with big indices, bigger than 1gb for example, search_parallel would function faster. I advice to check them both with the resulted index to find which one fits better. +The module implements multiple methods. +- `search` - search concurrently for a substring existed in different entries within the index file. As the index file getting bigger with multiple inner chunks, the concurrency effect increases. +- `count_entries` - return the number of entries in the index file consisting of the substring. +- `count_occurrences` - return the number of occurrences of the substring in all the entries. If the substring exists multiple times in the same entry, each occurrence will be counted. ### Built With @@ -39,20 +44,21 @@ The module implements two methods, search_sequential & search_parallel. search_s ### Performance -| Library | Text Size | Function | Time | #Results | Improvement Factor | -| ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | -| [ripgrepy](https://pypi.org/project/ripgrepy/) | 500mb | Ripgrepy('text_one', '500mb').run().as_string.split('\n') | 127 ms ± 694 µs per loop | 12553 | 1.0x | -| [PySubstringSearch](https://github.com/Intsights/PySubstringSearch) | 500mb | reader.search_sequential('text_one') | 2.48 ms ± 53.4 µs per loop | 12553 | 51.2x | -| [PySubstringSearch](https://github.com/Intsights/PySubstringSearch) | 500mb | reader.search_parallel('text_one') | 3.78 ms ± 350 µs per loop | 12553 | 33.6x | -| [ripgrepy](https://pypi.org/project/ripgrepy/) | 500mb | Ripgrepy('text_two', '500mb').run().as_string.split('\n') | 127 ms ± 623 µs per loop | 769 | 1.0x | -| [PySubstringSearch](https://github.com/Intsights/PySubstringSearch) | 500mb | reader.search_sequential('text_two') | 156 µs ± 916 ns per loop | 769 | 814.0x | -| [PySubstringSearch](https://github.com/Intsights/PySubstringSearch) | 500mb | reader.search_parallel('text_two') | 251 µs ± 80.2 µs per loop | 769 | 506.0x | -| [ripgrepy](https://pypi.org/project/ripgrepy/) | 6gb | Ripgrepy('text_one', '6gb').run().as_string.split('\n') | 1.38 s ± 3.82 ms | 206884 | 1.0x | -| [PySubstringSearch](https://github.com/Intsights/PySubstringSearch) | 6gb | reader.search_sequential('text_one') | 93.7 ms ± 2.16 ms per loop | 206884 | 15.3x | -| [PySubstringSearch](https://github.com/Intsights/PySubstringSearch) | 6gb | reader.search_parallel('text_one') | 34.3 ms ± 321 µs per loop | 206884 | 40.5x | -| [ripgrepy](https://pypi.org/project/ripgrepy/) | 6gb | Ripgrepy('text_two', '6gb').run().as_string.split('\n') | 1.61 s ± 37.2 ms per loop | 6921 | 1.0x | -| [PySubstringSearch](https://github.com/Intsights/PySubstringSearch) | 6gb | reader.search_sequential('text_two') | 2.22 ms ± 79.3 µs per loop | 6921 | 725.2x | -| [PySubstringSearch](https://github.com/Intsights/PySubstringSearch) | 6gb | reader.search_parallel('text_two') | 1.38 ms ± 26 µs per loop | 6921 | 1166.6x | +#### 500MB File +| Library | Function | Time | #Results | Improvement Factor | +| ------------- | ------------- | ------------- | ------------- | ------------- | +| [ripgrepy](https://pypi.org/project/ripgrepy/) | Ripgrepy('text_one', '500mb').run().as_string.split('\n') | 148ms | 2367 | 1.0x | +| [PySubstringSearch](https://github.com/Intsights/PySubstringSearch) | reader.search('text_one') | 1.28ms | 2367 | 115.6x | +| [ripgrepy](https://pypi.org/project/ripgrepy/) | Ripgrepy('text_two', '500mb').run().as_string.split('\n') | 116ms | 159 | 1.0x | +| [PySubstringSearch](https://github.com/Intsights/PySubstringSearch) | reader.search('text_two') | 228µs | 159 | 508.7x | + +#### 6000MB File +| Library | Function | Time | #Results | Improvement Factor | +| ------------- | ------------- | ------------- | ------------- | ------------- | +| [ripgrepy](https://pypi.org/project/ripgrepy/) | Ripgrepy('text_one', '6000mb').run().as_string.split('\n') | 2.4s | 59538 | 1.0x | +| [PySubstringSearch](https://github.com/Intsights/PySubstringSearch) | reader.search('text_one') | 15.4ms | 59538 | 155.8x | +| [ripgrepy](https://pypi.org/project/ripgrepy/) | Ripgrepy('text_two', '6000mb').run().as_string.split('\n') | 1.5s | 7266 | 1.0x | +| [PySubstringSearch](https://github.com/Intsights/PySubstringSearch) | reader.search('text_two') | 1.97ms | 7266 | 761.4x | ### Prerequisites @@ -104,21 +110,23 @@ reader = pysubstringsearch.Reader( index_file_path='output.idx', ) -# lookup for a substring sequentially -reader.search_sequential('short') +# lookup for a substring +reader.search('short') >>> ['some short string'] -# lookup for a substring sequentially -reader.search_sequential('string') +# lookup for a substring +reader.search('string') >>> ['some short string', 'another but now a longer string'] -# lookup for a substring concurrently -reader.search_parallel('short') ->>> ['some short string'] +# count the number of occurrences +# ['some short string', 'another string now, but a longer string'] +reader.count_occurences('string') +>>> 3 -# lookup for a substring concurrently -reader.search_parallel('string') ->>> ['some short string', 'another but now a longer string'] +# count the number of entries +# ['some short string', 'another string now, but a longer string'] +reader.count_occurences('string') +>>> 2 ``` diff --git a/setup.py b/setup.py index 718aa07..9988250 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name='PySubstringSearch', - version='0.3.1', + version='0.4.0', author='Gal Ben David', author_email='gal@intsights.com', url='https://github.com/Intsights/PySubstringSearch', diff --git a/src/pysubstringsearch.cpp b/src/pysubstringsearch.cpp index b5065f8..5d59413 100644 --- a/src/pysubstringsearch.cpp +++ b/src/pysubstringsearch.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include "msufsort.hpp" @@ -191,137 +192,224 @@ class Reader { ~Reader() {} - std::vector search_parallel( + std::vector search( const std::string & substring ) { - std::vector results; - std::vector>> futures; + std::vector entries; + std::vector> futures; for (std::uint32_t file_index = 0; file_index < this->files.size(); ++file_index) { auto future = std::async( - &Reader::search_specific_file, + &Reader::search_entries_in_file, this, + std::ref(entries), substring, file_index ); - if (this->files.size() == 1) { - return future.get(); - } else { - futures.push_back(std::move(future)); - } + futures.push_back(std::move(future)); } for (auto & future : futures) { - auto result = future.get(); - results.insert(results.end(), result.begin(), result.end()); + future.wait(); } - return results; + return entries; } - std::vector search_sequential( + std::uint32_t count_occurrences( const std::string & substring ) { - std::vector results; - std::unordered_set results_indices; + auto number_of_occurrences = 0; + std::vector>>> futures; for (std::uint32_t file_index = 0; file_index < this->files.size(); ++file_index) { - auto result = this->search_specific_file( + auto future = std::async( + &Reader::get_substring_positions, + this, substring, file_index ); - if (this->files.size() == 1) { - return result; - } else { - results.insert(results.end(), result.begin(), result.end()); + futures.push_back(std::move(future)); + } + + for (auto & future : futures) { + auto result = future.get(); + if (result.has_value()) { + auto [first_text_index, last_text_index] = result.value(); + auto number_of_text_indices = ((last_text_index - first_text_index) / 4) + 1; + number_of_occurrences += number_of_text_indices; } } - return results; + return number_of_occurrences; + } + + std::uint32_t count_entries( + const std::string & substring + ) { + auto number_of_entries = 0; + std::vector> futures; + + for (std::uint32_t file_index = 0; file_index < this->files.size(); ++file_index) { + auto future = std::async( + &Reader::count_entries_in_file, + this, + substring, + file_index + ); + futures.push_back(std::move(future)); + } + + for (auto & future : futures) { + auto number_of_entries_in_file = future.get(); + number_of_entries += number_of_entries_in_file; + } + + return number_of_entries; } - inline std::vector search_specific_file( + inline std::optional> get_substring_positions( const std::string & substring, std::uint32_t file_index ) { - std::vector results; - std::unordered_set results_indices; - const auto & [suffix_array_file_stream, text_vector] = this->files[file_index]; - std::int32_t index; + // The index file first 4 bytes are the size of the index file + // in a uint32_t format, hence starting from the 5th byte. std::uint64_t left_anchor = 4; std::uint64_t right_anchor = suffix_array_file_stream->size; - std::uint64_t first_suffix_array_index = std::numeric_limits::max(); + std::optional first_suffix_array_index = std::nullopt; + std::optional last_suffix_array_index = std::nullopt; while (left_anchor <= right_anchor) { std::uint64_t middle_anchor = left_anchor + ((right_anchor - left_anchor) / 4 / 2 * 4); + std::int32_t text_index; suffix_array_file_stream->seekg(middle_anchor); - suffix_array_file_stream->read((char *)&index, sizeof(index)); - - std::string_view suffix(&text_vector[index], substring.size()); + suffix_array_file_stream->read((char *)&text_index, sizeof(text_index)); - auto distance = std::memcmp( - substring.c_str(), - suffix.data(), - substring.size() - ); + auto distance = std::memcmp(substring.c_str(), &text_vector[text_index], substring.size()); if (distance < 0) { right_anchor = middle_anchor - 4; } else if (distance > 0) { left_anchor = middle_anchor + 4; } else { first_suffix_array_index = middle_anchor; + if (!last_suffix_array_index.has_value()) { + last_suffix_array_index = middle_anchor; + } right_anchor = middle_anchor - 4; } } - if (first_suffix_array_index == std::numeric_limits::max()) { - return results; + if (!first_suffix_array_index.has_value()) { + return std::nullopt; } - std::uint64_t current_index_position = first_suffix_array_index; - while (current_index_position < suffix_array_file_stream->size) { - std::vector suffixes_vector(100, -1); - suffix_array_file_stream->seekg(current_index_position); - suffix_array_file_stream->read((char *)suffixes_vector.data(), sizeof(std::int32_t) * 100); + left_anchor = last_suffix_array_index.value(); + right_anchor = suffix_array_file_stream->size; + while (left_anchor <= right_anchor) { + std::uint64_t middle_anchor = left_anchor + ((right_anchor - left_anchor) / 4 / 2 * 4); - bool finished = false; - for (std::int32_t suffix_index : suffixes_vector) { - if (suffix_index == -1) { - break; - } - auto distance = std::memcmp( - substring.c_str(), - &text_vector[suffix_index], - substring.size() - ); - if (distance != 0) { - finished = true; + std::int32_t text_index; + suffix_array_file_stream->seekg(middle_anchor); + suffix_array_file_stream->read((char *)&text_index, sizeof(text_index)); + + auto distance = std::memcmp(substring.c_str(), &text_vector[text_index], substring.size()); + if (distance < 0) { + right_anchor = middle_anchor - 4; + } else if (distance > 0) { + left_anchor = middle_anchor + 4; + } else { + last_suffix_array_index = middle_anchor; + left_anchor = middle_anchor + 4; + } + } + + return std::make_tuple( + first_suffix_array_index.value(), + last_suffix_array_index.value() + ); + } + + inline void search_entries_in_file( + std::vector & entries, + const std::string & substring, + std::uint32_t file_index + ) { + auto substring_positions = this->get_substring_positions( + substring, + file_index + ); + if (!substring_positions.has_value()) { + return; + } + auto [first_text_index, last_text_index] = substring_positions.value(); + + auto number_of_text_indices = ((last_text_index - first_text_index) / 4) + 1; + std::vector text_indices(number_of_text_indices); + + const auto & [suffix_array_file_stream, text_vector] = this->files[file_index]; + suffix_array_file_stream->seekg(first_text_index); + suffix_array_file_stream->read( + (char *)text_indices.data(), + sizeof(std::int32_t) * number_of_text_indices + ); + + std::unordered_set current_entries(number_of_text_indices); + for (std::int32_t text_index : text_indices) { + std::int32_t entry_start = text_index; + for (; entry_start > 0; entry_start -= 1) { + if (text_vector[entry_start - 1] == '\0') { break; } + } + current_entries.emplace(&text_vector[entry_start]); + } - for (; suffix_index > 0; suffix_index -= 1) { - if (text_vector[suffix_index - 1] == '\0') { - break; - } - } - const auto & [iterator, inserted] = results_indices.emplace(suffix_index); - if (inserted == true) { - results.push_back(std::string(&text_vector[suffix_index])); + this->entries_lock.lock(); + entries.insert(entries.end(), current_entries.begin(), current_entries.end()); + this->entries_lock.unlock(); + } + + inline std::size_t count_entries_in_file( + const std::string & substring, + std::uint32_t file_index + ) { + auto substring_positions = this->get_substring_positions( + substring, + file_index + ); + if (!substring_positions.has_value()) { + return 0; + } + auto [first_text_index, last_text_index] = substring_positions.value(); + + auto number_of_text_indices = ((last_text_index - first_text_index) / 4) + 1; + std::vector text_indices(number_of_text_indices); + + const auto & [suffix_array_file_stream, text_vector] = this->files[file_index]; + suffix_array_file_stream->seekg(first_text_index); + suffix_array_file_stream->read( + (char *)text_indices.data(), + sizeof(std::int32_t) * number_of_text_indices + ); + + std::unordered_set entries_start_indices(number_of_text_indices); + for (std::int32_t text_index : text_indices) { + std::int32_t entry_start = text_index; + for (; entry_start > 0; entry_start -= 1) { + if (text_vector[entry_start - 1] == '\0') { + break; } } - if (finished) { - break; - } - - current_index_position += sizeof(std::int32_t) * 100; + entries_start_indices.emplace(entry_start); } - return results; + return entries_start_indices.size(); } std::vector, std::vector>> files; + std::mutex entries_lock; }; PYBIND11_MODULE(pysubstringsearch, m) { @@ -332,15 +420,21 @@ PYBIND11_MODULE(pysubstringsearch, m) { pybind11::arg("index_file_path") ) .def( - "search_parallel", - &Reader::search_parallel, - "search over an index file for a substring", + "search", + &Reader::search, + "search over an index file for a substring, returns list of distinct entries", + pybind11::arg("substring") + ) + .def( + "count_occurrences", + &Reader::count_occurrences, + "search over an index file for a substring, returns the number of occurrences", pybind11::arg("substring") ) .def( - "search_sequential", - &Reader::search_sequential, - "search over an index file for a substring", + "count_entries", + &Reader::count_entries, + "search over an index file for a substring, returns the number of entries", pybind11::arg("substring") ); pybind11::class_(m, "Writer") diff --git a/tests/test_pysubstringsearch.py b/tests/test_pysubstringsearch.py index 1aea87d..dee2942 100644 --- a/tests/test_pysubstringsearch.py +++ b/tests/test_pysubstringsearch.py @@ -27,16 +27,31 @@ def assert_substring_search( index_file_path=f'{tmp_directory}/output.idx', ) self.assertCountEqual( - first=reader.search_parallel( + first=reader.search( substring=substring, ), second=expected_results, ) - self.assertCountEqual( - first=reader.search_sequential( + + number_of_entries = 0 + for entry in strings: + if substring in entry: + number_of_entries += 1 + self.assertEqual( + first=reader.count_entries( substring=substring, ), - second=expected_results, + second=number_of_entries, + ) + + total_occurrences = 0 + for entry in strings: + total_occurrences += entry.count(substring) + self.assertEqual( + first=reader.count_occurrences( + substring=substring, + ), + second=total_occurrences, ) def test_file_not_found( @@ -146,29 +161,56 @@ def test_unicode( self, ): strings = [ - '诶比西', + 'رجعوني عنيك لأيامي اللي راحوا', + 'علموني أندم على الماضي وجراحه', + 'اللي شفته قبل ما تشوفك عنيه', + 'عمر ضايع يحسبوه إزاي عليّ', + 'انت عمري اللي ابتدي بنورك صباحه', + 'قد ايه من عمري قبلك راح وعدّى', + 'يا حبيبي قد ايه من عمري راح', + 'ولا شاف القلب قبلك فرحة واحدة', + 'ولا داق في الدنيا غير طعم الجراح', + 'ابتديت دلوقت بس أحب عمري', + 'ابتديت دلوقت اخاف لا العمر يجري', + 'كل فرحه اشتاقها من قبلك خيالي', + 'التقاها في نور عنيك قلبي وفكري', + 'يا حياة قلبي يا أغلى من حياتي', + 'ليه ما قابلتش هواك يا حبيبي بدري', + 'اللي شفته قبل ما تشوفك عنيه', + 'عمر ضايع يحسبوه إزاي عليّ', + 'انت عمري اللي ابتدي بنورك صباحه', + 'الليالي الحلوه والشوق والمحبة', + 'من زمان والقلب شايلهم عشانك', + 'دوق معايا الحب دوق حبه بحبه', + 'من حنان قلبي اللي طال شوقه لحنانك', + 'هات عنيك تسرح في دنيتهم عنيه', + 'هات ايديك ترتاح للمستهم ايديه', ] self.assert_substring_search( strings=strings, - substring='诶', + substring='زمان', expected_results=[ - '诶比西', + 'من زمان والقلب شايلهم عشانك', ], ) self.assert_substring_search( strings=strings, - substring='诶比', + substring='في', expected_results=[ - '诶比西', + 'هات عنيك تسرح في دنيتهم عنيه', + 'التقاها في نور عنيك قلبي وفكري', + 'ولا داق في الدنيا غير طعم الجراح', ], ) self.assert_substring_search( strings=strings, - substring='比诶', - expected_results=[], + substring='حنان', + expected_results=[ + 'من حنان قلبي اللي طال شوقه لحنانك', + ], ) self.assert_substring_search(