Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions dlib/statistics.h
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
#include "statistics/vector_normalizer_frobmetric.h"
#include "statistics/lda.h"
#include "statistics/tfidf.h"
#include "statistics/feature_hashing.h"

#endif // DLIB_STATISTICs_H_

Expand Down
140 changes: 140 additions & 0 deletions dlib/statistics/feature_hashing.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
// Copyright (C) 2026 Davis E. King (davis@dlib.net)
// License: Boost Software License See LICENSE.txt for the full license.
#ifndef DLIB_FEATURE_HASHING_Hh_
#define DLIB_FEATURE_HASHING_Hh_

#include "feature_hashing_abstract.h"
#include "../algs.h"
#include "../error.h"
#include "../general_hash/murmur_hash3.h"
#include "../sparse_vector.h"
#include <algorithm>
#include <limits>
#include <string>
#include <type_traits>
#include <utility>
#include <vector>

namespace dlib
{

// ----------------------------------------------------------------------------------------

namespace impl
{
template <typename sample_type>
void add_ngram (
const char* data,
std::size_t size,
unsigned long num_buckets,
sample_type& counts
)
/*!
requires
- data points to size bytes.
- num_buckets > 0
- sample_type satisfies the requirements of hash_word_ngrams().
ensures
- Appends (murmur_hash3(data,size,0) % num_buckets, 1) to counts.
throws
- dlib::error if size exceeds the length supported by murmur_hash3().
!*/
{
using index_type = typename sample_type::value_type::first_type;
static_assert(std::is_integral_v<index_type> && std::is_unsigned_v<index_type> &&
!std::is_same_v<index_type,bool>, "N-gram indices must be unsigned integers.");
if (size > static_cast<std::size_t>(std::numeric_limits<int>::max()))
throw error("N-gram length exceeds the length supported by murmur_hash3().");
const auto bucket = murmur_hash3(data, static_cast<int>(size), 0) % num_buckets;
counts.emplace_back(static_cast<index_type>(bucket), 1);
}
}

// ----------------------------------------------------------------------------------------

template <
typename sample_type = std::vector<std::pair<unsigned long,double>>,
typename string_type
>
sample_type hash_word_ngrams (
const std::vector<string_type>& tokens,
unsigned long min_n,
unsigned long max_n,
unsigned long num_buckets
)
{
DLIB_ASSERT(1 <= min_n && min_n <= max_n && num_buckets > 0 &&
num_buckets-1 <= std::numeric_limits<typename sample_type::value_type::first_type>::max(),
"\t sample_type hash_word_ngrams()"
<< "\n\t Invalid arguments were given to this function."
<< "\n\t min_n: " << min_n
<< "\n\t max_n: " << max_n
<< "\n\t num_buckets: " << num_buckets
);
sample_type counts;
std::string ngram;
for (unsigned long n = min_n; n <= std::min<unsigned long>(max_n, tokens.size()); ++n)
{
for (unsigned long i = 0; i + n <= tokens.size(); ++i)
{
ngram.assign(tokens[i].data(), tokens[i].size());
for (unsigned long j = i + 1; j < i + n; ++j)
{
ngram += ' ';
ngram.append(tokens[j].data(), tokens[j].size());
}
impl::add_ngram(ngram.data(), ngram.size(), num_buckets, counts);
}
}
make_sparse_vector_inplace(counts);
return counts;
}

// ----------------------------------------------------------------------------------------

template <
typename sample_type = std::vector<std::pair<unsigned long,double>>,
typename string_type
>
sample_type hash_char_ngrams (
const std::vector<string_type>& tokens,
unsigned long min_n,
unsigned long max_n,
unsigned long num_buckets
)
{
DLIB_ASSERT(1 <= min_n && min_n <= max_n && num_buckets > 0 &&
num_buckets-1 <= std::numeric_limits<typename sample_type::value_type::first_type>::max(),
"\t sample_type hash_char_ngrams()"
<< "\n\t Invalid arguments were given to this function."
<< "\n\t min_n: " << min_n
<< "\n\t max_n: " << max_n
<< "\n\t num_buckets: " << num_buckets
);
sample_type counts;
std::string padded;
for (const auto& token : tokens)
{
padded.assign(1, ' ');
padded.append(token.data(), token.size());
padded += ' ';
if (padded.size() < min_n)
{
impl::add_ngram(padded.data(), padded.size(), num_buckets, counts);
continue;
}
for (unsigned long n = min_n; n <= std::min<unsigned long>(max_n, padded.size()); ++n)
{
for (unsigned long i = 0; i + n <= padded.size(); ++i)
impl::add_ngram(padded.data() + i, n, num_buckets, counts);
}
}
make_sparse_vector_inplace(counts);
return counts;
}

// ----------------------------------------------------------------------------------------

}

#endif // DLIB_FEATURE_HASHING_Hh_
81 changes: 81 additions & 0 deletions dlib/statistics/feature_hashing_abstract.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
// Copyright (C) 2026 Davis E. King (davis@dlib.net)
// License: Boost Software License See LICENSE.txt for the full license.
#undef DLIB_FEATURE_HASHING_ABSTRACT_Hh_
#ifdef DLIB_FEATURE_HASHING_ABSTRACT_Hh_

#include "../general_hash/murmur_hash3_abstract.h"
#include "../error.h"
#include <utility>
#include <vector>

namespace dlib
{

// ----------------------------------------------------------------------------------------

template <
typename sample_type = std::vector<std::pair<unsigned long,double>>,
typename string_type
>
sample_type hash_word_ngrams (
const std::vector<string_type>& tokens,
unsigned long min_n,
unsigned long max_n,
unsigned long num_buckets
);
/*!
requires
- string_type provides data() and size() giving a const char* and the
number of bytes in the token, e.g. std::string or std::string_view.
- sample_type is a std::vector<std::pair<I,V>> where I is an unsigned
integral type and V is an integral or floating point type. Neither is bool.
- I can represent num_buckets-1.
- V can represent the number of n-grams assigned to each bucket.
- 1 <= min_n <= max_n
- num_buckets > 0
ensures
- Counts the word n-grams of a document, hashed into num_buckets buckets,
without needing a vocabulary. The result is suitable for
tfidf_transformer or a sparse linear trainer.
- An n-gram is n adjacent tokens joined by a single space, for each n in
[min_n, max_n].
- The n-gram s goes to bucket murmur_hash3(s.data(), s.size(), 0) % num_buckets.
- Returns a sorted sparse vector: (b,c) means c n-grams went to bucket b.
throws
- dlib::error if an n-gram exceeds the length supported by murmur_hash3()
(std::numeric_limits<int>::max() bytes).
!*/

// ----------------------------------------------------------------------------------------

template <
typename sample_type = std::vector<std::pair<unsigned long,double>>,
typename string_type
>
sample_type hash_char_ngrams (
const std::vector<string_type>& tokens,
unsigned long min_n,
unsigned long max_n,
unsigned long num_buckets
);
/*!
requires
- The same as hash_word_ngrams().
ensures
- Counts the character n-grams within each token, hashed and returned
as hash_word_ngrams() does.
- Each token is padded with a space on both sides. Its n-grams are the
runs of n consecutive bytes of the padded token, for each n in
[min_n, max_n] no longer than it. A padded token shorter than min_n is
one n-gram, whole. This uses the boundary padding and short-token rule
of scikit-learn's analyzer='char_wb', applied to bytes. For UTF-8 text,
n measures bytes and a window may split a multi-byte character.
throws
- The same as hash_word_ngrams().
!*/

// ----------------------------------------------------------------------------------------

}

#endif // DLIB_FEATURE_HASHING_ABSTRACT_Hh_
1 change: 1 addition & 0 deletions dlib/test/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ add_executable(${target_name} main.cpp tester.cpp
entropy_encoder_model.cpp
example_args.cpp
face.cpp
feature_hashing.cpp
fft.cpp
fhog.cpp
filtering.cpp
Expand Down
158 changes: 158 additions & 0 deletions dlib/test/feature_hashing.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
// Copyright (C) 2026 Davis E. King (davis@dlib.net)
// License: Boost Software License See LICENSE.txt for the full license.

#include <dlib/statistics.h>
#include <dlib/sparse_vector.h>
#include "tester.h"
#include <map>
#include <string>
#include <string_view>
#include <vector>

namespace
{
using namespace test;
using namespace dlib;
using namespace std;
dlib::logger dlog("test.feature_hashing");

using sparse_type = std::vector<std::pair<unsigned long,double>>;

unsigned long bucket (const std::string& s, unsigned long num_buckets)
{
return murmur_hash3(s.data(), s.size(), 0) % num_buckets;
}

// The expected counts, from the n-grams written out by hand.
sparse_type expected (const std::vector<std::string>& ngrams, unsigned long num_buckets)
{
std::map<unsigned long,double> counts;
for (const auto& s : ngrams)
counts[bucket(s, num_buckets)] += 1;
return sparse_type(counts.begin(), counts.end());
}

// ----------------------------------------------------------------------------------------

void test_word_ngrams()
{
const unsigned long num_buckets = 1 << 20;
const std::vector<std::string> tokens = {"a", "b", "a", "b"};
DLIB_TEST(hash_word_ngrams(tokens, 1, 2, num_buckets) ==
expected({"a", "b", "a", "b", "a b", "b a", "a b"}, num_buckets));
DLIB_TEST(hash_word_ngrams(tokens, 3, 3, num_buckets) ==
expected({"a b a", "b a b"}, num_buckets));
DLIB_TEST(hash_word_ngrams(tokens, 5, 6, num_buckets).empty());
DLIB_TEST(hash_word_ngrams(std::vector<std::string>(), 1, 2, num_buckets).empty());
}

// ----------------------------------------------------------------------------------------

void test_char_ngrams()
{
const unsigned long num_buckets = 1 << 20;

// " ab " has two 3-grams, is itself the only 4-gram, and is too short for 5.
DLIB_TEST(hash_char_ngrams(std::vector<std::string>{"ab"}, 3, 5, num_buckets) ==
expected({" ab", "ab ", " ab "}, num_buckets));

// " a " is shorter than min_n, so it counts once, whole.
DLIB_TEST(hash_char_ngrams(std::vector<std::string>{"a"}, 4, 5, num_buckets) ==
expected({" a "}, num_buckets));

// N-grams stay inside their token.
DLIB_TEST(hash_char_ngrams(std::vector<std::string>{"abc", "de"}, 3, 3, num_buckets) ==
expected({" ab", "abc", "bc ", " de", "de "}, num_buckets));

DLIB_TEST(hash_char_ngrams(std::vector<std::string>{"abcd"}, 2, 3, num_buckets) ==
expected({" a", "ab", "bc", "cd", "d ", " ab", "abc", "bcd", "cd "}, num_buckets));

DLIB_TEST(hash_char_ngrams(std::vector<std::string>(), 3, 5, num_buckets).empty());

// UTF-8 e-acute has two bytes, so its padded form has two byte trigrams.
DLIB_TEST(hash_char_ngrams(std::vector<std::string>{"\xc3\xa9"}, 3, 3, num_buckets) ==
expected({" \xc3\xa9", "\xc3\xa9 "}, num_buckets));
DLIB_TEST(hash_char_ngrams(std::vector<std::string>{"\xc3\xa9"}, 1, 1, num_buckets) ==
expected({" ", "\xc3", "\xa9", " "}, num_buckets));
}

// ----------------------------------------------------------------------------------------

void test_collisions()
{
// With one bucket, every n-gram lands in bucket 0.
DLIB_TEST(hash_word_ngrams(std::vector<std::string>{"a", "b", "c"}, 1, 2, 1) ==
sparse_type({{0, 5}}));
DLIB_TEST(hash_char_ngrams(std::vector<std::string>{"ab"}, 3, 5, 1) ==
sparse_type({{0, 3}}));
}

// ----------------------------------------------------------------------------------------

void test_types()
{
const unsigned long num_buckets = 1000;
const std::vector<std::string> words = {"to", "be", "or", "not", "to", "be"};
const std::vector<std::string_view> views(words.begin(), words.end());

DLIB_TEST(hash_word_ngrams(words, 1, 2, num_buckets) ==
hash_word_ngrams(views, 1, 2, num_buckets));
const auto from_strings = hash_char_ngrams(words, 2, 4, num_buckets);
DLIB_TEST(from_strings == hash_char_ngrams(views, 2, 4, num_buckets));

using integer_type = std::vector<std::pair<unsigned int,unsigned int>>;
const auto integer_counts = hash_char_ngrams<integer_type>(words, 2, 4, num_buckets);
DLIB_TEST(integer_counts.size() == from_strings.size());
for (unsigned long i = 0; i < integer_counts.size(); ++i)
{
DLIB_TEST(integer_counts[i].first == from_strings[i].first);
DLIB_TEST(integer_counts[i].second == from_strings[i].second);
}
}

// ----------------------------------------------------------------------------------------

void test_with_tfidf()
{
// The intended use: hashed counts in, TF-IDF weighted vectors out.
const std::vector<std::vector<std::string>> documents = {
{"the", "cat", "sat"},
{"the", "dog", "sat"},
{"the", "cat", "ran"}
};
std::vector<sparse_type> counts;
for (const auto& document : documents)
counts.push_back(hash_word_ngrams(document, 1, 1, 1 << 20));

tfidf_transformer<sparse_type> tfidf;
tfidf.train(counts);
DLIB_TEST(tfidf.vocabulary_size() == 5);
DLIB_TEST(std::abs(length(tfidf(counts[0])) - 1) < 1e-12);
}

// ----------------------------------------------------------------------------------------

class feature_hashing_tester : public tester
{
public:
feature_hashing_tester (
) :
tester (
"test_feature_hashing",
"Run tests on hash_word_ngrams() and hash_char_ngrams().",
0
)
{}

void perform_test (
)
{
test_word_ngrams();
test_char_ngrams();
test_collisions();
test_types();
test_with_tfidf();
}
} a;

}
Loading
Loading