From f985fa66b8d35657fc5c80d2834e1fbe20378abe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= Date: Sun, 1 Nov 2026 21:20:20 +0100 Subject: [PATCH 1/3] Add hash_word_ngrams() and hash_char_ngrams() --- dlib/statistics.h | 1 + dlib/statistics/feature_hashing.h | 121 ++++++++++++++++ dlib/statistics/feature_hashing_abstract.h | 70 ++++++++++ dlib/test/CMakeLists.txt | 1 + dlib/test/feature_hashing.cpp | 152 +++++++++++++++++++++ dlib/test/makefile | 1 + docs/docs/ml.xml | 29 ++++ docs/docs/release_notes.xml | 1 + docs/docs/term_index.xml | 2 + 9 files changed, 378 insertions(+) create mode 100644 dlib/statistics/feature_hashing.h create mode 100644 dlib/statistics/feature_hashing_abstract.h create mode 100644 dlib/test/feature_hashing.cpp diff --git a/dlib/statistics.h b/dlib/statistics.h index 0558d27601..83cde2a6ca 100644 --- a/dlib/statistics.h +++ b/dlib/statistics.h @@ -13,6 +13,7 @@ #include "statistics/vector_normalizer_frobmetric.h" #include "statistics/lda.h" #include "statistics/tfidf.h" +#include "statistics/feature_hashing.h" #endif // DLIB_STATISTICs_H_ diff --git a/dlib/statistics/feature_hashing.h b/dlib/statistics/feature_hashing.h new file mode 100644 index 0000000000..0221688ee0 --- /dev/null +++ b/dlib/statistics/feature_hashing.h @@ -0,0 +1,121 @@ +// Copyright (C) 2026 Davis E. King (davis@dlib.net) +// License: Boost Software License See LICENSE.txt for the full license. +#ifndef DLIB_FEATURE_HASHING_Hh_ +#define DLIB_FEATURE_HASHING_Hh_ + +#include "feature_hashing_abstract.h" +#include "../algs.h" +#include "../general_hash/murmur_hash3.h" +#include "../sparse_vector.h" +#include +#include +#include +#include + +namespace dlib +{ + +// ---------------------------------------------------------------------------------------- + + namespace impl + { + template + void add_ngram ( + const char* data, + std::size_t size, + unsigned long num_buckets, + sample_type& counts + ) + { + using index_type = typename sample_type::value_type::first_type; + const auto bucket = murmur_hash3(data, static_cast(size), 0) % num_buckets; + counts.emplace_back(static_cast(bucket), 1); + } + } + +// ---------------------------------------------------------------------------------------- + + template < + typename sample_type = std::vector>, + typename string_type + > + sample_type hash_word_ngrams ( + const std::vector& tokens, + unsigned long min_n, + unsigned long max_n, + unsigned long num_buckets + ) + { + DLIB_ASSERT(1 <= min_n && min_n <= max_n && num_buckets > 0, + "\t sample_type hash_word_ngrams()" + << "\n\t Invalid arguments were given to this function." + << "\n\t min_n: " << min_n + << "\n\t max_n: " << max_n + << "\n\t num_buckets: " << num_buckets + ); + sample_type counts; + std::string ngram; + for (unsigned long n = min_n; n <= std::min(max_n, tokens.size()); ++n) + { + for (unsigned long i = 0; i + n <= tokens.size(); ++i) + { + ngram.assign(tokens[i].data(), tokens[i].size()); + for (unsigned long j = i + 1; j < i + n; ++j) + { + ngram += ' '; + ngram.append(tokens[j].data(), tokens[j].size()); + } + impl::add_ngram(ngram.data(), ngram.size(), num_buckets, counts); + } + } + make_sparse_vector_inplace(counts); + return counts; + } + +// ---------------------------------------------------------------------------------------- + + template < + typename sample_type = std::vector>, + typename string_type + > + sample_type hash_char_ngrams ( + const std::vector& tokens, + unsigned long min_n, + unsigned long max_n, + unsigned long num_buckets + ) + { + DLIB_ASSERT(1 <= min_n && min_n <= max_n && num_buckets > 0, + "\t sample_type hash_char_ngrams()" + << "\n\t Invalid arguments were given to this function." + << "\n\t min_n: " << min_n + << "\n\t max_n: " << max_n + << "\n\t num_buckets: " << num_buckets + ); + sample_type counts; + std::string padded; + for (const auto& token : tokens) + { + padded.assign(1, ' '); + padded.append(token.data(), token.size()); + padded += ' '; + if (padded.size() < min_n) + { + impl::add_ngram(padded.data(), padded.size(), num_buckets, counts); + continue; + } + for (unsigned long n = min_n; n <= std::min(max_n, padded.size()); ++n) + { + for (unsigned long i = 0; i + n <= padded.size(); ++i) + impl::add_ngram(padded.data() + i, n, num_buckets, counts); + } + } + make_sparse_vector_inplace(counts); + return counts; + } + +// ---------------------------------------------------------------------------------------- + +} + +#endif // DLIB_FEATURE_HASHING_Hh_ diff --git a/dlib/statistics/feature_hashing_abstract.h b/dlib/statistics/feature_hashing_abstract.h new file mode 100644 index 0000000000..549fc04c68 --- /dev/null +++ b/dlib/statistics/feature_hashing_abstract.h @@ -0,0 +1,70 @@ +// Copyright (C) 2026 Davis E. King (davis@dlib.net) +// License: Boost Software License See LICENSE.txt for the full license. +#undef DLIB_FEATURE_HASHING_ABSTRACT_Hh_ +#ifdef DLIB_FEATURE_HASHING_ABSTRACT_Hh_ + +#include "../general_hash/murmur_hash3_abstract.h" +#include +#include + +namespace dlib +{ + +// ---------------------------------------------------------------------------------------- + + template < + typename sample_type = std::vector>, + typename string_type + > + sample_type hash_word_ngrams ( + const std::vector& tokens, + unsigned long min_n, + unsigned long max_n, + unsigned long num_buckets + ); + /*! + requires + - string_type has data() and size(), e.g. std::string or std::string_view. + - sample_type is a std::vector> where I can hold + num_buckets-1. + - 1 <= min_n <= max_n + - num_buckets > 0 + ensures + - Counts the word n-grams of a document, hashed into num_buckets buckets, + without needing a vocabulary. The result is suitable for + tfidf_transformer or a sparse linear trainer. + - An n-gram is n adjacent tokens joined by a single space, for each n in + [min_n, max_n]. + - The n-gram s goes to bucket murmur_hash3(s.data(), s.size(), 0) % num_buckets. + - Returns a sorted sparse vector: (b,c) means c n-grams went to bucket b. + !*/ + +// ---------------------------------------------------------------------------------------- + + template < + typename sample_type = std::vector>, + typename string_type + > + sample_type hash_char_ngrams ( + const std::vector& tokens, + unsigned long min_n, + unsigned long max_n, + unsigned long num_buckets + ); + /*! + requires + - The same as hash_word_ngrams(). + ensures + - Counts the character n-grams within each token, hashed and returned + as hash_word_ngrams() does. + - Each token is padded with a space on both sides. Its n-grams are the + runs of n consecutive bytes of the padded token, for each n in + [min_n, max_n] no longer than it. A padded token shorter than min_n is + one n-gram, whole. This matches scikit-learn's analyzer='char_wb'. + !*/ + +// ---------------------------------------------------------------------------------------- + +} + +#endif // DLIB_FEATURE_HASHING_ABSTRACT_Hh_ diff --git a/dlib/test/CMakeLists.txt b/dlib/test/CMakeLists.txt index 879743469c..bcf17847e7 100644 --- a/dlib/test/CMakeLists.txt +++ b/dlib/test/CMakeLists.txt @@ -55,6 +55,7 @@ add_executable(${target_name} main.cpp tester.cpp entropy_encoder_model.cpp example_args.cpp face.cpp + feature_hashing.cpp fft.cpp fhog.cpp filtering.cpp diff --git a/dlib/test/feature_hashing.cpp b/dlib/test/feature_hashing.cpp new file mode 100644 index 0000000000..538a6bc645 --- /dev/null +++ b/dlib/test/feature_hashing.cpp @@ -0,0 +1,152 @@ +// Copyright (C) 2026 Davis E. King (davis@dlib.net) +// License: Boost Software License See LICENSE.txt for the full license. + +#include +#include +#include "tester.h" +#include +#include +#include +#include + +namespace +{ + using namespace test; + using namespace dlib; + using namespace std; + dlib::logger dlog("test.feature_hashing"); + + using sparse_type = std::vector>; + + unsigned long bucket (const std::string& s, unsigned long num_buckets) + { + return murmur_hash3(s.data(), s.size(), 0) % num_buckets; + } + + // The expected counts, from the n-grams written out by hand. + sparse_type expected (const std::vector& ngrams, unsigned long num_buckets) + { + std::map counts; + for (const auto& s : ngrams) + counts[bucket(s, num_buckets)] += 1; + return sparse_type(counts.begin(), counts.end()); + } + +// ---------------------------------------------------------------------------------------- + + void test_word_ngrams() + { + const unsigned long num_buckets = 1 << 20; + const std::vector tokens = {"a", "b", "a", "b"}; + DLIB_TEST(hash_word_ngrams(tokens, 1, 2, num_buckets) == + expected({"a", "b", "a", "b", "a b", "b a", "a b"}, num_buckets)); + DLIB_TEST(hash_word_ngrams(tokens, 3, 3, num_buckets) == + expected({"a b a", "b a b"}, num_buckets)); + DLIB_TEST(hash_word_ngrams(tokens, 5, 6, num_buckets).empty()); + DLIB_TEST(hash_word_ngrams(std::vector(), 1, 2, num_buckets).empty()); + } + +// ---------------------------------------------------------------------------------------- + + void test_char_ngrams() + { + const unsigned long num_buckets = 1 << 20; + + // " ab " has two 3-grams, is itself the only 4-gram, and is too short for 5. + DLIB_TEST(hash_char_ngrams(std::vector{"ab"}, 3, 5, num_buckets) == + expected({" ab", "ab ", " ab "}, num_buckets)); + + // " a " is shorter than min_n, so it counts once, whole. + DLIB_TEST(hash_char_ngrams(std::vector{"a"}, 4, 5, num_buckets) == + expected({" a "}, num_buckets)); + + // N-grams stay inside their token. + DLIB_TEST(hash_char_ngrams(std::vector{"abc", "de"}, 3, 3, num_buckets) == + expected({" ab", "abc", "bc ", " de", "de "}, num_buckets)); + + DLIB_TEST(hash_char_ngrams(std::vector{"abcd"}, 2, 3, num_buckets) == + expected({" a", "ab", "bc", "cd", "d ", " ab", "abc", "bcd", "cd "}, num_buckets)); + + DLIB_TEST(hash_char_ngrams(std::vector(), 3, 5, num_buckets).empty()); + } + +// ---------------------------------------------------------------------------------------- + + void test_collisions() + { + // With one bucket, every n-gram lands in bucket 0. + DLIB_TEST(hash_word_ngrams(std::vector{"a", "b", "c"}, 1, 2, 1) == + sparse_type({{0, 5}})); + DLIB_TEST(hash_char_ngrams(std::vector{"ab"}, 3, 5, 1) == + sparse_type({{0, 3}})); + } + +// ---------------------------------------------------------------------------------------- + + void test_types() + { + const unsigned long num_buckets = 1000; + const std::vector words = {"to", "be", "or", "not", "to", "be"}; + const std::vector views(words.begin(), words.end()); + + DLIB_TEST(hash_word_ngrams(words, 1, 2, num_buckets) == + hash_word_ngrams(views, 1, 2, num_buckets)); + const auto from_strings = hash_char_ngrams(words, 2, 4, num_buckets); + DLIB_TEST(from_strings == hash_char_ngrams(views, 2, 4, num_buckets)); + + using integer_type = std::vector>; + const auto integer_counts = hash_char_ngrams(words, 2, 4, num_buckets); + DLIB_TEST(integer_counts.size() == from_strings.size()); + for (unsigned long i = 0; i < integer_counts.size(); ++i) + { + DLIB_TEST(integer_counts[i].first == from_strings[i].first); + DLIB_TEST(integer_counts[i].second == from_strings[i].second); + } + } + +// ---------------------------------------------------------------------------------------- + + void test_with_tfidf() + { + // The intended use: hashed counts in, TF-IDF weighted vectors out. + const std::vector> documents = { + {"the", "cat", "sat"}, + {"the", "dog", "sat"}, + {"the", "cat", "ran"} + }; + std::vector counts; + for (const auto& document : documents) + counts.push_back(hash_word_ngrams(document, 1, 1, 1 << 20)); + + tfidf_transformer tfidf; + tfidf.train(counts); + DLIB_TEST(tfidf.vocabulary_size() == 5); + DLIB_TEST(std::abs(length(tfidf(counts[0])) - 1) < 1e-12); + } + +// ---------------------------------------------------------------------------------------- + + class feature_hashing_tester : public tester + { + public: + feature_hashing_tester ( + ) : + tester ( + "test_feature_hashing", + "Run tests on hash_word_ngrams() and hash_char_ngrams().", + 0 + ) + {} + + void perform_test ( + ) + { + test_word_ngrams(); + test_char_ngrams(); + test_collisions(); + test_types(); + test_with_tfidf(); + } + } a; + +} diff --git a/dlib/test/makefile b/dlib/test/makefile index f4a7bad1c9..b72d89b4e6 100644 --- a/dlib/test/makefile +++ b/dlib/test/makefile @@ -61,6 +61,7 @@ SRC += empirical_kernel_map.cpp SRC += entropy_coder.cpp SRC += entropy_encoder_model.cpp SRC += face.cpp +SRC += feature_hashing.cpp SRC += fft.cpp SRC += fhog.cpp SRC += filtering.cpp diff --git a/docs/docs/ml.xml b/docs/docs/ml.xml index 2c7d2bde78..b4afd8bec8 100644 --- a/docs/docs/ml.xml +++ b/docs/docs/ml.xml @@ -604,6 +604,8 @@ Davis E. King. + + + + hash_word_ngrams + dlib/statistics.h + dlib/statistics/feature_hashing_abstract.h + + This function counts the word n-grams of a document, hashed into a fixed + number of buckets, giving a sparse vector for a + tfidf_transformer or a sparse linear trainer. + + + + + + + + hash_char_ngrams + dlib/statistics.h + dlib/statistics/feature_hashing_abstract.h + + This function is hash_word_ngrams for the + character n-grams within each token. + + + + diff --git a/docs/docs/release_notes.xml b/docs/docs/release_notes.xml index 10228a0e11..6f863137eb 100644 --- a/docs/docs/release_notes.xml +++ b/docs/docs/release_notes.xml @@ -16,6 +16,7 @@ New Features and Improvements: - Added tfidf_transformer, which learns TF-IDF weights for sparse vectors of term counts (issue #3176). - Added normalize() and normalize_inplace() for sparse vectors. + - Added hash_word_ngrams() and hash_char_ngrams() for feature hashing of text. - Deep learning tooling: - Added an Adaptive Computation Time layer. (PR #3111) diff --git a/docs/docs/term_index.xml b/docs/docs/term_index.xml index 958d362e1b..3209ceb86c 100644 --- a/docs/docs/term_index.xml +++ b/docs/docs/term_index.xml @@ -670,6 +670,8 @@ + + From a8e609745467cddceeb6bdaeef829067ca2060b4 Mon Sep 17 00:00:00 2001 From: Davis King Date: Mon, 2 Nov 2026 22:56:13 -0500 Subject: [PATCH 2/3] Clarify feature hashing contracts and guard n-gram lengths --- dlib/statistics/feature_hashing.h | 19 +++++++++++++++++++ dlib/statistics/feature_hashing_abstract.h | 19 +++++++++++++++---- dlib/test/feature_hashing.cpp | 6 ++++++ 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/dlib/statistics/feature_hashing.h b/dlib/statistics/feature_hashing.h index 0221688ee0..7940677f69 100644 --- a/dlib/statistics/feature_hashing.h +++ b/dlib/statistics/feature_hashing.h @@ -5,10 +5,13 @@ #include "feature_hashing_abstract.h" #include "../algs.h" +#include "../error.h" #include "../general_hash/murmur_hash3.h" #include "../sparse_vector.h" #include +#include #include +#include #include #include @@ -26,11 +29,27 @@ namespace dlib unsigned long num_buckets, sample_type& counts ) + /*! + requires + - data points to size bytes. + - num_buckets > 0 + - sample_type satisfies the requirements of hash_word_ngrams(). + ensures + - Appends (murmur_hash3(data,size,0) % num_buckets, 1) to counts. + throws + - dlib::error if size exceeds the length supported by murmur_hash3(). + !*/ { using index_type = typename sample_type::value_type::first_type; + static_assert(std::is_integral_v && std::is_unsigned_v && + !std::is_same_v, "N-gram indices must be unsigned integers."); + DLIB_ASSERT(num_buckets > 0 && num_buckets-1 <= std::numeric_limits::max()); + if (size > static_cast(std::numeric_limits::max())) + throw error("N-gram length exceeds the length supported by murmur_hash3()."); const auto bucket = murmur_hash3(data, static_cast(size), 0) % num_buckets; counts.emplace_back(static_cast(bucket), 1); } + } // ---------------------------------------------------------------------------------------- diff --git a/dlib/statistics/feature_hashing_abstract.h b/dlib/statistics/feature_hashing_abstract.h index 549fc04c68..5bb025c953 100644 --- a/dlib/statistics/feature_hashing_abstract.h +++ b/dlib/statistics/feature_hashing_abstract.h @@ -4,6 +4,7 @@ #ifdef DLIB_FEATURE_HASHING_ABSTRACT_Hh_ #include "../general_hash/murmur_hash3_abstract.h" +#include "../error.h" #include #include @@ -24,9 +25,12 @@ namespace dlib ); /*! requires - - string_type has data() and size(), e.g. std::string or std::string_view. - - sample_type is a std::vector> where I can hold - num_buckets-1. + - string_type provides data() and size() giving a const char* and the + number of bytes in the token, e.g. std::string or std::string_view. + - sample_type is a std::vector> where I is an unsigned + integral type and V is an integral or floating point type. Neither is bool. + - I can represent num_buckets-1. + - V can represent the number of n-grams assigned to each bucket. - 1 <= min_n <= max_n - num_buckets > 0 ensures @@ -37,6 +41,9 @@ namespace dlib [min_n, max_n]. - The n-gram s goes to bucket murmur_hash3(s.data(), s.size(), 0) % num_buckets. - Returns a sorted sparse vector: (b,c) means c n-grams went to bucket b. + throws + - dlib::error if an n-gram exceeds the length supported by murmur_hash3() + (std::numeric_limits::max() bytes). !*/ // ---------------------------------------------------------------------------------------- @@ -60,7 +67,11 @@ namespace dlib - Each token is padded with a space on both sides. Its n-grams are the runs of n consecutive bytes of the padded token, for each n in [min_n, max_n] no longer than it. A padded token shorter than min_n is - one n-gram, whole. This matches scikit-learn's analyzer='char_wb'. + one n-gram, whole. This uses the boundary padding and short-token rule + of scikit-learn's analyzer='char_wb', applied to bytes. For UTF-8 text, + n measures bytes and a window may split a multi-byte character. + throws + - The same as hash_word_ngrams(). !*/ // ---------------------------------------------------------------------------------------- diff --git a/dlib/test/feature_hashing.cpp b/dlib/test/feature_hashing.cpp index 538a6bc645..5da296c4ae 100644 --- a/dlib/test/feature_hashing.cpp +++ b/dlib/test/feature_hashing.cpp @@ -68,6 +68,12 @@ namespace expected({" a", "ab", "bc", "cd", "d ", " ab", "abc", "bcd", "cd "}, num_buckets)); DLIB_TEST(hash_char_ngrams(std::vector(), 3, 5, num_buckets).empty()); + + // UTF-8 e-acute has two bytes, so its padded form has two byte trigrams. + DLIB_TEST(hash_char_ngrams(std::vector{"\xc3\xa9"}, 3, 3, num_buckets) == + expected({" \xc3\xa9", "\xc3\xa9 "}, num_buckets)); + DLIB_TEST(hash_char_ngrams(std::vector{"\xc3\xa9"}, 1, 1, num_buckets) == + expected({" ", "\xc3", "\xa9", " "}, num_buckets)); } // ---------------------------------------------------------------------------------------- From 110cc332d60b34c26fdd6a7c87d3e1c5dff18174 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= Date: Tue, 3 Nov 2026 09:16:51 +0100 Subject: [PATCH 3/3] Check the bucket range once per call --- dlib/statistics/feature_hashing.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dlib/statistics/feature_hashing.h b/dlib/statistics/feature_hashing.h index 7940677f69..17c9feae84 100644 --- a/dlib/statistics/feature_hashing.h +++ b/dlib/statistics/feature_hashing.h @@ -43,13 +43,11 @@ namespace dlib using index_type = typename sample_type::value_type::first_type; static_assert(std::is_integral_v && std::is_unsigned_v && !std::is_same_v, "N-gram indices must be unsigned integers."); - DLIB_ASSERT(num_buckets > 0 && num_buckets-1 <= std::numeric_limits::max()); if (size > static_cast(std::numeric_limits::max())) throw error("N-gram length exceeds the length supported by murmur_hash3()."); const auto bucket = murmur_hash3(data, static_cast(size), 0) % num_buckets; counts.emplace_back(static_cast(bucket), 1); } - } // ---------------------------------------------------------------------------------------- @@ -65,7 +63,8 @@ namespace dlib unsigned long num_buckets ) { - DLIB_ASSERT(1 <= min_n && min_n <= max_n && num_buckets > 0, + DLIB_ASSERT(1 <= min_n && min_n <= max_n && num_buckets > 0 && + num_buckets-1 <= std::numeric_limits::max(), "\t sample_type hash_word_ngrams()" << "\n\t Invalid arguments were given to this function." << "\n\t min_n: " << min_n @@ -104,7 +103,8 @@ namespace dlib unsigned long num_buckets ) { - DLIB_ASSERT(1 <= min_n && min_n <= max_n && num_buckets > 0, + DLIB_ASSERT(1 <= min_n && min_n <= max_n && num_buckets > 0 && + num_buckets-1 <= std::numeric_limits::max(), "\t sample_type hash_char_ngrams()" << "\n\t Invalid arguments were given to this function." << "\n\t min_n: " << min_n