/*
* Copyright (C) 2017 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "feature-processor.h"
#include "model-executor.h"
#include "tensor-view.h"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
namespace libtextclassifier2 {
namespace {
using testing::ElementsAreArray;
using testing::FloatEq;
using testing::Matcher;
flatbuffers::DetachedBuffer PackFeatureProcessorOptions(
const FeatureProcessorOptionsT& options) {
flatbuffers::FlatBufferBuilder builder;
builder.Finish(CreateFeatureProcessorOptions(builder, &options));
return builder.Release();
}
template <typename T>
std::vector<T> Subvector(const std::vector<T>& vector, int start, int end) {
return std::vector<T>(vector.begin() + start, vector.begin() + end);
}
Matcher<std::vector<float>> ElementsAreFloat(const std::vector<float>& values) {
std::vector<Matcher<float>> matchers;
for (const float value : values) {
matchers.push_back(FloatEq(value));
}
return ElementsAreArray(matchers);
}
class TestingFeatureProcessor : public FeatureProcessor {
public:
using FeatureProcessor::CountIgnoredSpanBoundaryCodepoints;
using FeatureProcessor::FeatureProcessor;
using FeatureProcessor::ICUTokenize;
using FeatureProcessor::IsCodepointInRanges;
using FeatureProcessor::SpanToLabel;
using FeatureProcessor::StripTokensFromOtherLines;
using FeatureProcessor::supported_codepoint_ranges_;
using FeatureProcessor::SupportedCodepointsRatio;
};
// EmbeddingExecutor that always returns features based on
class FakeEmbeddingExecutor : public EmbeddingExecutor {
public:
bool AddEmbedding(const TensorView<int>& sparse_features, float* dest,
int dest_size) const override {
TC_CHECK_GE(dest_size, 4);
EXPECT_EQ(sparse_features.size(), 1);
dest[0] = sparse_features.data()[0];
dest[1] = sparse_features.data()[0];
dest[2] = -sparse_features.data()[0];
dest[3] = -sparse_features.data()[0];
return true;
}
private:
std::vector<float> storage_;
};
TEST(FeatureProcessorTest, SplitTokensOnSelectionBoundariesMiddle) {
std::vector<Token> tokens{Token("Hělló", 0, 5),
Token("fěěbař@google.com", 6, 23),
Token("heře!", 24, 29)};
internal::SplitTokensOnSelectionBoundaries({9, 12}, &tokens);
// clang-format off
EXPECT_THAT(tokens, ElementsAreArray(
{Token("Hělló", 0, 5),
Token("fěě", 6, 9),
Token("bař", 9, 12),
Token("@google.com", 12, 23),
Token("heře!", 24, 29)}));
// clang-format on
}
TEST(FeatureProcessorTest, SplitTokensOnSelectionBoundariesBegin) {
std::vector<Token> tokens{Token("Hělló", 0, 5),
Token("fěěbař@google.com", 6, 23),
Token("heře!", 24, 29)};
internal::SplitTokensOnSelectionBoundaries({6, 12}, &tokens);
// clang-format off
EXPECT_THAT(tokens, ElementsAreArray(
{Token("Hělló", 0, 5),
Token("fěěbař", 6, 12),
Token("@google.com", 12, 23),
Token("heře!", 24, 29)}));
// clang-format on
}
TEST(FeatureProcessorTest, SplitTokensOnSelectionBoundariesEnd) {
std::vector<Token> tokens{Token("Hělló", 0, 5),
Token("fěěbař@google.com", 6, 23),
Token("heře!", 24, 29)};
internal::SplitTokensOnSelectionBoundaries({9, 23}, &tokens);
// clang-format off
EXPECT_THAT(tokens, ElementsAreArray(
{Token("Hělló", 0, 5),
Token("fěě", 6, 9),
Token("bař@google.com", 9, 23),
Token("heře!", 24, 29)}));
// clang-format on
}
TEST(FeatureProcessorTest, SplitTokensOnSelectionBoundariesWhole) {
std::vector<Token> tokens{Token("Hělló", 0, 5),
Token("fěěbař@google.com", 6, 23),
Token("heře!", 24, 29)};
internal::SplitTokensOnSelectionBoundaries({6, 23}, &tokens);
// clang-format off
EXPECT_THAT(tokens, ElementsAreArray(
{Token("Hělló", 0, 5),
Token("fěěbař@google.com", 6, 23),
Token("heře!", 24, 29)}));
// clang-format on
}
TEST(FeatureProcessorTest, SplitTokensOnSelectionBoundariesCrossToken) {
std::vector<Token> tokens{Token("Hělló", 0, 5),
Token("fěěbař@google.com", 6, 23),
Token("heře!", 24, 29)};
internal::SplitTokensOnSelectionBoundaries({2, 9}, &tokens);
// clang-format off
EXPECT_THAT(tokens, ElementsAreArray(
{Token("Hě", 0, 2),
Token("lló", 2, 5),
Token("fěě", 6, 9),
Token("bař@google.com", 9, 23),
Token("heře!", 24, 29)}));
// clang-format on
}
TEST(FeatureProcessorTest, KeepLineWithClickFirst) {
CREATE_UNILIB_FOR_TESTING;
FeatureProcessorOptionsT options;
options.only_use_line_with_click = true;
flatbuffers::DetachedBuffer options_fb = PackFeatureProcessorOptions(options);
TestingFeatureProcessor feature_processor(
flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()),
&unilib);
const std::string context = "Fiřst Lině\nSěcond Lině\nThiřd Lině";
const CodepointSpan span = {0, 5};
// clang-format off
std::vector<Token> tokens = {Token("Fiřst", 0, 5),
Token("Lině", 6, 10),
Token("Sěcond", 11, 17),
Token("Lině", 18, 22),
Token("Thiřd", 23, 28),
Token("Lině", 29, 33)};
// clang-format on
// Keeps the first line.
feature_processor.StripTokensFromOtherLines(context, span, &tokens);
EXPECT_THAT(tokens,
ElementsAreArray({Token("Fiřst", 0, 5), Token("Lině", 6, 10)}));
}
TEST(FeatureProcessorTest, KeepLineWithClickSecond) {
CREATE_UNILIB_FOR_TESTING;
FeatureProcessorOptionsT options;
options.only_use_line_with_click = true;
flatbuffers::DetachedBuffer options_fb = PackFeatureProcessorOptions(options);
TestingFeatureProcessor feature_processor(
flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()),
&unilib);
const std::string context = "Fiřst Lině\nSěcond Lině\nThiřd Lině";
const CodepointSpan span = {18, 22};
// clang-format off
std::vector<Token> tokens = {Token("Fiřst", 0, 5),
Token("Lině", 6, 10),
Token("Sěcond", 11, 17),
Token("Lině", 18, 22),
Token("Thiřd", 23, 28),
Token("Lině", 29, 33)};
// clang-format on
// Keeps the first line.
feature_processor.StripTokensFromOtherLines(context, span, &tokens);
EXPECT_THAT(tokens, ElementsAreArray(
{Token("Sěcond", 11, 17), Token("Lině", 18, 22)}));
}
TEST(FeatureProcessorTest, KeepLineWithClickThird) {
CREATE_UNILIB_FOR_TESTING;
FeatureProcessorOptionsT options;
options.only_use_line_with_click = true;
flatbuffers::DetachedBuffer options_fb = PackFeatureProcessorOptions(options);
TestingFeatureProcessor feature_processor(
flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()),
&unilib);
const std::string context = "Fiřst Lině\nSěcond Lině\nThiřd Lině";
const CodepointSpan span = {24, 33};
// clang-format off
std::vector<Token> tokens = {Token("Fiřst", 0, 5),
Token("Lině", 6, 10),
Token("Sěcond", 11, 17),
Token("Lině", 18, 22),
Token("Thiřd", 23, 28),
Token("Lině", 29, 33)};
// clang-format on
// Keeps the first line.
feature_processor.StripTokensFromOtherLines(context, span, &tokens);
EXPECT_THAT(tokens, ElementsAreArray(
{Token("Thiřd", 23, 28), Token("Lině", 29, 33)}));
}
TEST(FeatureProcessorTest, KeepLineWithClickSecondWithPipe) {
CREATE_UNILIB_FOR_TESTING;
FeatureProcessorOptionsT options;
options.only_use_line_with_click = true;
flatbuffers::DetachedBuffer options_fb = PackFeatureProcessorOptions(options);
TestingFeatureProcessor feature_processor(
flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()),
&unilib);
const std::string context = "Fiřst Lině|Sěcond Lině\nThiřd Lině";
const CodepointSpan span = {18, 22};
// clang-format off
std::vector<Token> tokens = {Token("Fiřst", 0, 5),
Token("Lině", 6, 10),
Token("Sěcond", 11, 17),
Token("Lině", 18, 22),
Token("Thiřd", 23, 28),
Token("Lině", 29, 33)};
// clang-format on
// Keeps the first line.
feature_processor.StripTokensFromOtherLines(context, span, &tokens);
EXPECT_THAT(tokens, ElementsAreArray(
{Token("Sěcond", 11, 17), Token("Lině", 18, 22)}));
}
TEST(FeatureProcessorTest, KeepLineWithCrosslineClick) {
CREATE_UNILIB_FOR_TESTING;
FeatureProcessorOptionsT options;
options.only_use_line_with_click = true;
flatbuffers::DetachedBuffer options_fb = PackFeatureProcessorOptions(options);
TestingFeatureProcessor feature_processor(
flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()),
&unilib);
const std::string context = "Fiřst Lině\nSěcond Lině\nThiřd Lině";
const CodepointSpan span = {5, 23};
// clang-format off
std::vector<Token> tokens = {Token("Fiřst", 0, 5),
Token("Lině", 6, 10),
Token("Sěcond", 18, 23),
Token("Lině", 19, 23),
Token("Thiřd", 23, 28),
Token("Lině", 29, 33)};
// clang-format on
// Keeps the first line.
feature_processor.StripTokensFromOtherLines(context, span, &tokens);
EXPECT_THAT(tokens, ElementsAreArray(
{Token("Fiřst", 0, 5), Token("Lině", 6, 10),
Token("Sěcond", 18, 23), Token("Lině", 19, 23),
Token("Thiřd", 23, 28), Token("Lině", 29, 33)}));
}
TEST(FeatureProcessorTest, SpanToLabel) {
CREATE_UNILIB_FOR_TESTING;
FeatureProcessorOptionsT options;
options.context_size = 1;
options.max_selection_span = 1;
options.snap_label_span_boundaries_to_containing_tokens = false;
options.tokenization_codepoint_config.emplace_back(
new TokenizationCodepointRangeT());
auto& config = options.tokenization_codepoint_config.back();
config->start = 32;
config->end = 33;
config->role = TokenizationCodepointRange_::Role_WHITESPACE_SEPARATOR;
flatbuffers::DetachedBuffer options_fb = PackFeatureProcessorOptions(options);
TestingFeatureProcessor feature_processor(
flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()),
&unilib);
std::vector<Token> tokens = feature_processor.Tokenize("one, two, three");
ASSERT_EQ(3, tokens.size());
int label;
ASSERT_TRUE(feature_processor.SpanToLabel({5, 8}, tokens, &label));
EXPECT_EQ(kInvalidLabel, label);
ASSERT_TRUE(feature_processor.SpanToLabel({5, 9}, tokens, &label));
EXPECT_NE(kInvalidLabel, label);
TokenSpan token_span;
feature_processor.LabelToTokenSpan(label, &token_span);
EXPECT_EQ(0, token_span.first);
EXPECT_EQ(0, token_span.second);
// Reconfigure with snapping enabled.
options.snap_label_span_boundaries_to_containing_tokens = true;
flatbuffers::DetachedBuffer options2_fb =
PackFeatureProcessorOptions(options);
TestingFeatureProcessor feature_processor2(
flatbuffers::GetRoot<FeatureProcessorOptions>(options2_fb.data()),
&unilib);
int label2;
ASSERT_TRUE(feature_processor2.SpanToLabel({5, 8}, tokens, &label2));
EXPECT_EQ(label, label2);
ASSERT_TRUE(feature_processor2.SpanToLabel({6, 9}, tokens, &label2));
EXPECT_EQ(label, label2);
ASSERT_TRUE(feature_processor2.SpanToLabel({5, 9}, tokens, &label2));
EXPECT_EQ(label, label2);
// Cross a token boundary.
ASSERT_TRUE(feature_processor2.SpanToLabel({4, 9}, tokens, &label2));
EXPECT_EQ(kInvalidLabel, label2);
ASSERT_TRUE(feature_processor2.SpanToLabel({5, 10}, tokens, &label2));
EXPECT_EQ(kInvalidLabel, label2);
// Multiple tokens.
options.context_size = 2;
options.max_selection_span = 2;
flatbuffers::DetachedBuffer options3_fb =
PackFeatureProcessorOptions(options);
TestingFeatureProcessor feature_processor3(
flatbuffers::GetRoot<FeatureProcessorOptions>(options3_fb.data()),
&unilib);
tokens = feature_processor3.Tokenize("zero, one, two, three, four");
ASSERT_TRUE(feature_processor3.SpanToLabel({6, 15}, tokens, &label2));
EXPECT_NE(kInvalidLabel, label2);
feature_processor3.LabelToTokenSpan(label2, &token_span);
EXPECT_EQ(1, token_span.first);
EXPECT_EQ(0, token_span.second);
int label3;
ASSERT_TRUE(feature_processor3.SpanToLabel({6, 14}, tokens, &label3));
EXPECT_EQ(label2, label3);
ASSERT_TRUE(feature_processor3.SpanToLabel({6, 13}, tokens, &label3));
EXPECT_EQ(label2, label3);
ASSERT_TRUE(feature_processor3.SpanToLabel({7, 13}, tokens, &label3));
EXPECT_EQ(label2, label3);
}
TEST(FeatureProcessorTest, SpanToLabelIgnoresPunctuation) {
CREATE_UNILIB_FOR_TESTING;
FeatureProcessorOptionsT options;
options.context_size = 1;
options.max_selection_span = 1;
options.snap_label_span_boundaries_to_containing_tokens = false;
options.tokenization_codepoint_config.emplace_back(
new TokenizationCodepointRangeT());
auto& config = options.tokenization_codepoint_config.back();
config->start = 32;
config->end = 33;
config->role = TokenizationCodepointRange_::Role_WHITESPACE_SEPARATOR;
flatbuffers::DetachedBuffer options_fb = PackFeatureProcessorOptions(options);
TestingFeatureProcessor feature_processor(
flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()),
&unilib);
std::vector<Token> tokens = feature_processor.Tokenize("one, two, three");
ASSERT_EQ(3, tokens.size());
int label;
ASSERT_TRUE(feature_processor.SpanToLabel({5, 8}, tokens, &label));
EXPECT_EQ(kInvalidLabel, label);
ASSERT_TRUE(feature_processor.SpanToLabel({5, 9}, tokens, &label));
EXPECT_NE(kInvalidLabel, label);
TokenSpan token_span;
feature_processor.LabelToTokenSpan(label, &token_span);
EXPECT_EQ(0, token_span.first);
EXPECT_EQ(0, token_span.second);
// Reconfigure with snapping enabled.
options.snap_label_span_boundaries_to_containing_tokens = true;
flatbuffers::DetachedBuffer options2_fb =
PackFeatureProcessorOptions(options);
TestingFeatureProcessor feature_processor2(
flatbuffers::GetRoot<FeatureProcessorOptions>(options2_fb.data()),
&unilib);
int label2;
ASSERT_TRUE(feature_processor2.SpanToLabel({5, 8}, tokens, &label2));
EXPECT_EQ(label, label2);
ASSERT_TRUE(feature_processor2.SpanToLabel({6, 9}, tokens, &label2));
EXPECT_EQ(label, label2);
ASSERT_TRUE(feature_processor2.SpanToLabel({5, 9}, tokens, &label2));
EXPECT_EQ(label, label2);
// Cross a token boundary.
ASSERT_TRUE(feature_processor2.SpanToLabel({4, 9}, tokens, &label2));
EXPECT_EQ(kInvalidLabel, label2);
ASSERT_TRUE(feature_processor2.SpanToLabel({5, 10}, tokens, &label2));
EXPECT_EQ(kInvalidLabel, label2);
// Multiple tokens.
options.context_size = 2;
options.max_selection_span = 2;
flatbuffers::DetachedBuffer options3_fb =
PackFeatureProcessorOptions(options);
TestingFeatureProcessor feature_processor3(
flatbuffers::GetRoot<FeatureProcessorOptions>(options3_fb.data()),
&unilib);
tokens = feature_processor3.Tokenize("zero, one, two, three, four");
ASSERT_TRUE(feature_processor3.SpanToLabel({6, 15}, tokens, &label2));
EXPECT_NE(kInvalidLabel, label2);
feature_processor3.LabelToTokenSpan(label2, &token_span);
EXPECT_EQ(1, token_span.first);
EXPECT_EQ(0, token_span.second);
int label3;
ASSERT_TRUE(feature_processor3.SpanToLabel({6, 14}, tokens, &label3));
EXPECT_EQ(label2, label3);
ASSERT_TRUE(feature_processor3.SpanToLabel({6, 13}, tokens, &label3));
EXPECT_EQ(label2, label3);
ASSERT_TRUE(feature_processor3.SpanToLabel({7, 13}, tokens, &label3));
EXPECT_EQ(label2, label3);
}
TEST(FeatureProcessorTest, CenterTokenFromClick) {
int token_index;
// Exactly aligned indices.
token_index = internal::CenterTokenFromClick(
{6, 11},
{Token("Hělló", 0, 5), Token("world", 6, 11), Token("heře!", 12, 17)});
EXPECT_EQ(token_index, 1);
// Click is contained in a token.
token_index = internal::CenterTokenFromClick(
{13, 17},
{Token("Hělló", 0, 5), Token("world", 6, 11), Token("heře!", 12, 17)});
EXPECT_EQ(token_index, 2);
// Click spans two tokens.
token_index = internal::CenterTokenFromClick(
{6, 17},
{Token("Hělló", 0, 5), Token("world", 6, 11), Token("heře!", 12, 17)});
EXPECT_EQ(token_index, kInvalidIndex);
}
TEST(FeatureProcessorTest, CenterTokenFromMiddleOfSelection) {
int token_index;
// Selection of length 3. Exactly aligned indices.
token_index = internal::CenterTokenFromMiddleOfSelection(
{7, 27},
{Token("Token1", 0, 6), Token("Token2", 7, 13), Token("Token3", 14, 20),
Token("Token4", 21, 27), Token("Token5", 28, 34)});
EXPECT_EQ(token_index, 2);
// Selection of length 1 token. Exactly aligned indices.
token_index = internal::CenterTokenFromMiddleOfSelection(
{21, 27},
{Token("Token1", 0, 6), Token("Token2", 7, 13), Token("Token3", 14, 20),
Token("Token4", 21, 27), Token("Token5", 28, 34)});
EXPECT_EQ(token_index, 3);
// Selection marks sub-token range, with no tokens in it.
token_index = internal::CenterTokenFromMiddleOfSelection(
{29, 33},
{Token("Token1", 0, 6), Token("Token2", 7, 13), Token("Token3", 14, 20),
Token("Token4", 21, 27), Token("Token5", 28, 34)});
EXPECT_EQ(token_index, kInvalidIndex);
// Selection of length 2. Sub-token indices.
token_index = internal::CenterTokenFromMiddleOfSelection(
{3, 25},
{Token("Token1", 0, 6), Token("Token2", 7, 13), Token("Token3", 14, 20),
Token("Token4", 21, 27), Token("Token5", 28, 34)});
EXPECT_EQ(token_index, 1);
// Selection of length 1. Sub-token indices.
token_index = internal::CenterTokenFromMiddleOfSelection(
{22, 34},
{Token("Token1", 0, 6), Token("Token2", 7, 13), Token("Token3", 14, 20),
Token("Token4", 21, 27), Token("Token5", 28, 34)});
EXPECT_EQ(token_index, 4);
// Some invalid ones.
token_index = internal::CenterTokenFromMiddleOfSelection({7, 27}, {});
EXPECT_EQ(token_index, -1);
}
TEST(FeatureProcessorTest, SupportedCodepointsRatio) {
FeatureProcessorOptionsT options;
options.context_size = 2;
options.max_selection_span = 2;
options.snap_label_span_boundaries_to_containing_tokens = false;
options.feature_version = 2;
options.embedding_size = 4;
options.bounds_sensitive_features.reset(
new FeatureProcessorOptions_::BoundsSensitiveFeaturesT());
options.bounds_sensitive_features->enabled = true;
options.bounds_sensitive_features->num_tokens_before = 5;
options.bounds_sensitive_features->num_tokens_inside_left = 3;
options.bounds_sensitive_features->num_tokens_inside_right = 3;
options.bounds_sensitive_features->num_tokens_after = 5;
options.bounds_sensitive_features->include_inside_bag = true;
options.bounds_sensitive_features->include_inside_length = true;
options.tokenization_codepoint_config.emplace_back(
new TokenizationCodepointRangeT());
auto& config = options.tokenization_codepoint_config.back();
config->start = 32;
config->end = 33;
config->role = TokenizationCodepointRange_::Role_WHITESPACE_SEPARATOR;
{
options.supported_codepoint_ranges.emplace_back(
new FeatureProcessorOptions_::CodepointRangeT());
auto& range = options.supported_codepoint_ranges.back();
range->start = 0;
range->end = 128;
}
{
options.supported_codepoint_ranges.emplace_back(
new FeatureProcessorOptions_::CodepointRangeT());
auto& range = options.supported_codepoint_ranges.back();
range->start = 10000;
range->end = 10001;
}
{
options.supported_codepoint_ranges.emplace_back(
new FeatureProcessorOptions_::CodepointRangeT());
auto& range = options.supported_codepoint_ranges.back();
range->start = 20000;
range->end = 30000;
}
flatbuffers::DetachedBuffer options_fb = PackFeatureProcessorOptions(options);
CREATE_UNILIB_FOR_TESTING;
TestingFeatureProcessor feature_processor(
flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()),
&unilib);
EXPECT_THAT(feature_processor.SupportedCodepointsRatio(
{0, 3}, feature_processor.Tokenize("aaa bbb ccc")),
FloatEq(1.0));
EXPECT_THAT(feature_processor.SupportedCodepointsRatio(
{0, 3}, feature_processor.Tokenize("aaa bbb ěěě")),
FloatEq(2.0 / 3));
EXPECT_THAT(feature_processor.SupportedCodepointsRatio(
{0, 3}, feature_processor.Tokenize("ěěě řřř ěěě")),
FloatEq(0.0));
EXPECT_FALSE(feature_processor.IsCodepointInRanges(
-1, feature_processor.supported_codepoint_ranges_));
EXPECT_TRUE(feature_processor.IsCodepointInRanges(
0, feature_processor.supported_codepoint_ranges_));
EXPECT_TRUE(feature_processor.IsCodepointInRanges(
10, feature_processor.supported_codepoint_ranges_));
EXPECT_TRUE(feature_processor.IsCodepointInRanges(
127, feature_processor.supported_codepoint_ranges_));
EXPECT_FALSE(feature_processor.IsCodepointInRanges(
128, feature_processor.supported_codepoint_ranges_));
EXPECT_FALSE(feature_processor.IsCodepointInRanges(
9999, feature_processor.supported_codepoint_ranges_));
EXPECT_TRUE(feature_processor.IsCodepointInRanges(
10000, feature_processor.supported_codepoint_ranges_));
EXPECT_FALSE(feature_processor.IsCodepointInRanges(
10001, feature_processor.supported_codepoint_ranges_));
EXPECT_TRUE(feature_processor.IsCodepointInRanges(
25000, feature_processor.supported_codepoint_ranges_));
const std::vector<Token> tokens = {Token("ěěě", 0, 3), Token("řřř", 4, 7),
Token("eee", 8, 11)};
options.min_supported_codepoint_ratio = 0.0;
flatbuffers::DetachedBuffer options2_fb =
PackFeatureProcessorOptions(options);
TestingFeatureProcessor feature_processor2(
flatbuffers::GetRoot<FeatureProcessorOptions>(options2_fb.data()),
&unilib);
EXPECT_TRUE(feature_processor2.HasEnoughSupportedCodepoints(
tokens, /*token_span=*/{0, 3}));
options.min_supported_codepoint_ratio = 0.2;
flatbuffers::DetachedBuffer options3_fb =
PackFeatureProcessorOptions(options);
TestingFeatureProcessor feature_processor3(
flatbuffers::GetRoot<FeatureProcessorOptions>(options3_fb.data()),
&unilib);
EXPECT_TRUE(feature_processor3.HasEnoughSupportedCodepoints(
tokens, /*token_span=*/{0, 3}));
options.min_supported_codepoint_ratio = 0.5;
flatbuffers::DetachedBuffer options4_fb =
PackFeatureProcessorOptions(options);
TestingFeatureProcessor feature_processor4(
flatbuffers::GetRoot<FeatureProcessorOptions>(options4_fb.data()),
&unilib);
EXPECT_FALSE(feature_processor4.HasEnoughSupportedCodepoints(
tokens, /*token_span=*/{0, 3}));
}
TEST(FeatureProcessorTest, InSpanFeature) {
FeatureProcessorOptionsT options;
options.context_size = 2;
options.max_selection_span = 2;
options.snap_label_span_boundaries_to_containing_tokens = false;
options.feature_version = 2;
options.embedding_size = 4;
options.extract_selection_mask_feature = true;
flatbuffers::DetachedBuffer options_fb = PackFeatureProcessorOptions(options);
CREATE_UNILIB_FOR_TESTING;
TestingFeatureProcessor feature_processor(
flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()),
&unilib);
std::unique_ptr<CachedFeatures> cached_features;
FakeEmbeddingExecutor embedding_executor;
const std::vector<Token> tokens = {Token("aaa", 0, 3), Token("bbb", 4, 7),
Token("ccc", 8, 11), Token("ddd", 12, 15)};
EXPECT_TRUE(feature_processor.ExtractFeatures(
tokens, /*token_span=*/{0, 4},
/*selection_span_for_feature=*/{4, 11}, &embedding_executor,
/*embedding_cache=*/nullptr, /*feature_vector_size=*/5,
&cached_features));
std::vector<float> features;
cached_features->AppendClickContextFeaturesForClick(1, &features);
ASSERT_EQ(features.size(), 25);
EXPECT_THAT(features[4], FloatEq(0.0));
EXPECT_THAT(features[9], FloatEq(0.0));
EXPECT_THAT(features[14], FloatEq(1.0));
EXPECT_THAT(features[19], FloatEq(1.0));
EXPECT_THAT(features[24], FloatEq(0.0));
}
TEST(FeatureProcessorTest, EmbeddingCache) {
FeatureProcessorOptionsT options;
options.context_size = 2;
options.max_selection_span = 2;
options.snap_label_span_boundaries_to_containing_tokens = false;
options.feature_version = 2;
options.embedding_size = 4;
options.bounds_sensitive_features.reset(
new FeatureProcessorOptions_::BoundsSensitiveFeaturesT());
options.bounds_sensitive_features->enabled = true;
options.bounds_sensitive_features->num_tokens_before = 3;
options.bounds_sensitive_features->num_tokens_inside_left = 2;
options.bounds_sensitive_features->num_tokens_inside_right = 2;
options.bounds_sensitive_features->num_tokens_after = 3;
flatbuffers::DetachedBuffer options_fb = PackFeatureProcessorOptions(options);
CREATE_UNILIB_FOR_TESTING;
TestingFeatureProcessor feature_processor(
flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()),
&unilib);
std::unique_ptr<CachedFeatures> cached_features;
FakeEmbeddingExecutor embedding_executor;
const std::vector<Token> tokens = {
Token("aaa", 0, 3), Token("bbb", 4, 7), Token("ccc", 8, 11),
Token("ddd", 12, 15), Token("eee", 16, 19), Token("fff", 20, 23)};
// We pre-populate the cache with dummy embeddings, to make sure they are
// used when populating the features vector.
const std::vector<float> cached_padding_features = {10.0, -10.0, 10.0, -10.0};
const std::vector<float> cached_features1 = {1.0, 2.0, 3.0, 4.0};
const std::vector<float> cached_features2 = {5.0, 6.0, 7.0, 8.0};
FeatureProcessor::EmbeddingCache embedding_cache = {
{{kInvalidIndex, kInvalidIndex}, cached_padding_features},
{{4, 7}, cached_features1},
{{12, 15}, cached_features2},
};
EXPECT_TRUE(feature_processor.ExtractFeatures(
tokens, /*token_span=*/{0, 6},
/*selection_span_for_feature=*/{kInvalidIndex, kInvalidIndex},
&embedding_executor, &embedding_cache, /*feature_vector_size=*/4,
&cached_features));
std::vector<float> features;
cached_features->AppendBoundsSensitiveFeaturesForSpan({2, 4}, &features);
ASSERT_EQ(features.size(), 40);
// Check that the dummy embeddings were used.
EXPECT_THAT(Subvector(features, 0, 4),
ElementsAreFloat(cached_padding_features));
EXPECT_THAT(Subvector(features, 8, 12), ElementsAreFloat(cached_features1));
EXPECT_THAT(Subvector(features, 16, 20), ElementsAreFloat(cached_features2));
EXPECT_THAT(Subvector(features, 24, 28), ElementsAreFloat(cached_features2));
EXPECT_THAT(Subvector(features, 36, 40),
ElementsAreFloat(cached_padding_features));
// Check that the real embeddings were cached.
EXPECT_EQ(embedding_cache.size(), 7);
EXPECT_THAT(Subvector(features, 4, 8),
ElementsAreFloat(embedding_cache.at({0, 3})));
EXPECT_THAT(Subvector(features, 12, 16),
ElementsAreFloat(embedding_cache.at({8, 11})));
EXPECT_THAT(Subvector(features, 20, 24),
ElementsAreFloat(embedding_cache.at({8, 11})));
EXPECT_THAT(Subvector(features, 28, 32),
ElementsAreFloat(embedding_cache.at({16, 19})));
EXPECT_THAT(Subvector(features, 32, 36),
ElementsAreFloat(embedding_cache.at({20, 23})));
}
TEST(FeatureProcessorTest, StripUnusedTokensWithNoRelativeClick) {
std::vector<Token> tokens_orig{
Token("0", 0, 0), Token("1", 0, 0), Token("2", 0, 0), Token("3", 0, 0),
Token("4", 0, 0), Token("5", 0, 0), Token("6", 0, 0), Token("7", 0, 0),
Token("8", 0, 0), Token("9", 0, 0), Token("10", 0, 0), Token("11", 0, 0),
Token("12", 0, 0)};
std::vector<Token> tokens;
int click_index;
// Try to click first token and see if it gets padded from left.
tokens = tokens_orig;
click_index = 0;
internal::StripOrPadTokens({0, 0}, 2, &tokens, &click_index);
// clang-format off
EXPECT_EQ(tokens, std::vector<Token>({Token(),
Token(),
Token("0", 0, 0),
Token("1", 0, 0),
Token("2", 0, 0)}));
// clang-format on
EXPECT_EQ(click_index, 2);
// When we click the second token nothing should get padded.
tokens = tokens_orig;
click_index = 2;
internal::StripOrPadTokens({0, 0}, 2, &tokens, &click_index);
// clang-format off
EXPECT_EQ(tokens, std::vector<Token>({Token("0", 0, 0),
Token("1", 0, 0),
Token("2", 0, 0),
Token("3", 0, 0),
Token("4", 0, 0)}));
// clang-format on
EXPECT_EQ(click_index, 2);
// When we click the last token tokens should get padded from the right.
tokens = tokens_orig;
click_index = 12;
internal::StripOrPadTokens({0, 0}, 2, &tokens, &click_index);
// clang-format off
EXPECT_EQ(tokens, std::vector<Token>({Token("10", 0, 0),
Token("11", 0, 0),
Token("12", 0, 0),
Token(),
Token()}));
// clang-format on
EXPECT_EQ(click_index, 2);
}
TEST(FeatureProcessorTest, StripUnusedTokensWithRelativeClick) {
std::vector<Token> tokens_orig{
Token("0", 0, 0), Token("1", 0, 0), Token("2", 0, 0), Token("3", 0, 0),
Token("4", 0, 0), Token("5", 0, 0), Token("6", 0, 0), Token("7", 0, 0),
Token("8", 0, 0), Token("9", 0, 0), Token("10", 0, 0), Token("11", 0, 0),
Token("12", 0, 0)};
std::vector<Token> tokens;
int click_index;
// Try to click first token and see if it gets padded from left to maximum
// context_size.
tokens = tokens_orig;
click_index = 0;
internal::StripOrPadTokens({2, 3}, 2, &tokens, &click_index);
// clang-format off
EXPECT_EQ(tokens, std::vector<Token>({Token(),
Token(),
Token("0", 0, 0),
Token("1", 0, 0),
Token("2", 0, 0),
Token("3", 0, 0),
Token("4", 0, 0),
Token("5", 0, 0)}));
// clang-format on
EXPECT_EQ(click_index, 2);
// Clicking to the middle with enough context should not produce any padding.
tokens = tokens_orig;
click_index = 6;
internal::StripOrPadTokens({3, 1}, 2, &tokens, &click_index);
// clang-format off
EXPECT_EQ(tokens, std::vector<Token>({Token("1", 0, 0),
Token("2", 0, 0),
Token("3", 0, 0),
Token("4", 0, 0),
Token("5", 0, 0),
Token("6", 0, 0),
Token("7", 0, 0),
Token("8", 0, 0),
Token("9", 0, 0)}));
// clang-format on
EXPECT_EQ(click_index, 5);
// Clicking at the end should pad right to maximum context_size.
tokens = tokens_orig;
click_index = 11;
internal::StripOrPadTokens({3, 1}, 2, &tokens, &click_index);
// clang-format off
EXPECT_EQ(tokens, std::vector<Token>({Token("6", 0, 0),
Token("7", 0, 0),
Token("8", 0, 0),
Token("9", 0, 0),
Token("10", 0, 0),
Token("11", 0, 0),
Token("12", 0, 0),
Token(),
Token()}));
// clang-format on
EXPECT_EQ(click_index, 5);
}
TEST(FeatureProcessorTest, InternalTokenizeOnScriptChange) {
CREATE_UNILIB_FOR_TESTING;
FeatureProcessorOptionsT options;
options.tokenization_codepoint_config.emplace_back(
new TokenizationCodepointRangeT());
{
auto& config = options.tokenization_codepoint_config.back();
config->start = 0;
config->end = 256;
config->role = TokenizationCodepointRange_::Role_DEFAULT_ROLE;
config->script_id = 1;
}
options.tokenize_on_script_change = false;
flatbuffers::DetachedBuffer options_fb = PackFeatureProcessorOptions(options);
TestingFeatureProcessor feature_processor(
flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()),
&unilib);
EXPECT_EQ(feature_processor.Tokenize("앨라배마123웹사이트"),
std::vector<Token>({Token("앨라배마123웹사이트", 0, 11)}));
options.tokenize_on_script_change = true;
flatbuffers::DetachedBuffer options_fb2 =
PackFeatureProcessorOptions(options);
TestingFeatureProcessor feature_processor2(
flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb2.data()),
&unilib);
EXPECT_EQ(feature_processor2.Tokenize("앨라배마123웹사이트"),
std::vector<Token>({Token("앨라배마", 0, 4), Token("123", 4, 7),
Token("웹사이트", 7, 11)}));
}
#ifdef LIBTEXTCLASSIFIER_TEST_ICU
TEST(FeatureProcessorTest, ICUTokenize) {
FeatureProcessorOptionsT options;
options.tokenization_type = FeatureProcessorOptions_::TokenizationType_ICU;
flatbuffers::DetachedBuffer options_fb = PackFeatureProcessorOptions(options);
TestingFeatureProcessor feature_processor(
flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()));
std::vector<Token> tokens = feature_processor.Tokenize("พระบาทสมเด็จพระปรมิ");
ASSERT_EQ(tokens,
// clang-format off
std::vector<Token>({Token("พระบาท", 0, 6),
Token("สมเด็จ", 6, 12),
Token("พระ", 12, 15),
Token("ปร", 15, 17),
Token("มิ", 17, 19)}));
// clang-format on
}
#endif
#ifdef LIBTEXTCLASSIFIER_TEST_ICU
TEST(FeatureProcessorTest, ICUTokenizeWithWhitespaces) {
FeatureProcessorOptionsT options;
options.tokenization_type = FeatureProcessorOptions_::TokenizationType_ICU;
options.icu_preserve_whitespace_tokens = true;
flatbuffers::DetachedBuffer options_fb = PackFeatureProcessorOptions(options);
TestingFeatureProcessor feature_processor(
flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()));
std::vector<Token> tokens =
feature_processor.Tokenize("พระบาท สมเด็จ พระ ปร มิ");
ASSERT_EQ(tokens,
// clang-format off
std::vector<Token>({Token("พระบาท", 0, 6),
Token(" ", 6, 7),
Token("สมเด็จ", 7, 13),
Token(" ", 13, 14),
Token("พระ", 14, 17),
Token(" ", 17, 18),
Token("ปร", 18, 20),
Token(" ", 20, 21),
Token("มิ", 21, 23)}));
// clang-format on
}
#endif
#ifdef LIBTEXTCLASSIFIER_TEST_ICU
TEST(FeatureProcessorTest, MixedTokenize) {
FeatureProcessorOptionsT options;
options.tokenization_type = FeatureProcessorOptions_::TokenizationType_MIXED;
options.tokenization_codepoint_config.emplace_back(
new TokenizationCodepointRangeT());
auto& config = options.tokenization_codepoint_config.back();
config->start = 32;
config->end = 33;
config->role = TokenizationCodepointRange_::Role_WHITESPACE_SEPARATOR;
{
options.internal_tokenizer_codepoint_ranges.emplace_back(
new FeatureProcessorOptions_::CodepointRangeT());
auto& range = options.internal_tokenizer_codepoint_ranges.back();
range->start = 0;
range->end = 128;
}
{
options.internal_tokenizer_codepoint_ranges.emplace_back(
new FeatureProcessorOptions_::CodepointRangeT());
auto& range = options.internal_tokenizer_codepoint_ranges.back();
range->start = 128;
range->end = 256;
}
{
options.internal_tokenizer_codepoint_ranges.emplace_back(
new FeatureProcessorOptions_::CodepointRangeT());
auto& range = options.internal_tokenizer_codepoint_ranges.back();
range->start = 256;
range->end = 384;
}
{
options.internal_tokenizer_codepoint_ranges.emplace_back(
new FeatureProcessorOptions_::CodepointRangeT());
auto& range = options.internal_tokenizer_codepoint_ranges.back();
range->start = 384;
range->end = 592;
}
flatbuffers::DetachedBuffer options_fb = PackFeatureProcessorOptions(options);
TestingFeatureProcessor feature_processor(
flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()));
std::vector<Token> tokens = feature_processor.Tokenize(
"こんにちはJapanese-ląnguagę text 世界 http://www.google.com/");
ASSERT_EQ(tokens,
// clang-format off
std::vector<Token>({Token("こんにちは", 0, 5),
Token("Japanese-ląnguagę", 5, 22),
Token("text", 23, 27),
Token("世界", 28, 30),
Token("http://www.google.com/", 31, 53)}));
// clang-format on
}
#endif
TEST(FeatureProcessorTest, IgnoredSpanBoundaryCodepoints) {
CREATE_UNILIB_FOR_TESTING;
FeatureProcessorOptionsT options;
options.ignored_span_boundary_codepoints.push_back('.');
options.ignored_span_boundary_codepoints.push_back(',');
options.ignored_span_boundary_codepoints.push_back('[');
options.ignored_span_boundary_codepoints.push_back(']');
flatbuffers::DetachedBuffer options_fb = PackFeatureProcessorOptions(options);
TestingFeatureProcessor feature_processor(
flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()),
&unilib);
const std::string text1_utf8 = "ěščř";
const UnicodeText text1 = UTF8ToUnicodeText(text1_utf8, /*do_copy=*/false);
EXPECT_EQ(feature_processor.CountIgnoredSpanBoundaryCodepoints(
text1.begin(), text1.end(),
/*count_from_beginning=*/true),
0);
EXPECT_EQ(feature_processor.CountIgnoredSpanBoundaryCodepoints(
text1.begin(), text1.end(),
/*count_from_beginning=*/false),
0);
const std::string text2_utf8 = ".,abčd";
const UnicodeText text2 = UTF8ToUnicodeText(text2_utf8, /*do_copy=*/false);
EXPECT_EQ(feature_processor.CountIgnoredSpanBoundaryCodepoints(
text2.begin(), text2.end(),
/*count_from_beginning=*/true),
2);
EXPECT_EQ(feature_processor.CountIgnoredSpanBoundaryCodepoints(
text2.begin(), text2.end(),
/*count_from_beginning=*/false),
0);
const std::string text3_utf8 = ".,abčd[]";
const UnicodeText text3 = UTF8ToUnicodeText(text3_utf8, /*do_copy=*/false);
EXPECT_EQ(feature_processor.CountIgnoredSpanBoundaryCodepoints(
text3.begin(), text3.end(),
/*count_from_beginning=*/true),
2);
EXPECT_EQ(feature_processor.CountIgnoredSpanBoundaryCodepoints(
text3.begin(), text3.end(),
/*count_from_beginning=*/false),
2);
const std::string text4_utf8 = "[abčd]";
const UnicodeText text4 = UTF8ToUnicodeText(text4_utf8, /*do_copy=*/false);
EXPECT_EQ(feature_processor.CountIgnoredSpanBoundaryCodepoints(
text4.begin(), text4.end(),
/*count_from_beginning=*/true),
1);
EXPECT_EQ(feature_processor.CountIgnoredSpanBoundaryCodepoints(
text4.begin(), text4.end(),
/*count_from_beginning=*/false),
1);
const std::string text5_utf8 = "";
const UnicodeText text5 = UTF8ToUnicodeText(text5_utf8, /*do_copy=*/false);
EXPECT_EQ(feature_processor.CountIgnoredSpanBoundaryCodepoints(
text5.begin(), text5.end(),
/*count_from_beginning=*/true),
0);
EXPECT_EQ(feature_processor.CountIgnoredSpanBoundaryCodepoints(
text5.begin(), text5.end(),
/*count_from_beginning=*/false),
0);
const std::string text6_utf8 = "012345ěščř";
const UnicodeText text6 = UTF8ToUnicodeText(text6_utf8, /*do_copy=*/false);
UnicodeText::const_iterator text6_begin = text6.begin();
std::advance(text6_begin, 6);
EXPECT_EQ(feature_processor.CountIgnoredSpanBoundaryCodepoints(
text6_begin, text6.end(),
/*count_from_beginning=*/true),
0);
EXPECT_EQ(feature_processor.CountIgnoredSpanBoundaryCodepoints(
text6_begin, text6.end(),
/*count_from_beginning=*/false),
0);
const std::string text7_utf8 = "012345.,ěščř";
const UnicodeText text7 = UTF8ToUnicodeText(text7_utf8, /*do_copy=*/false);
UnicodeText::const_iterator text7_begin = text7.begin();
std::advance(text7_begin, 6);
EXPECT_EQ(feature_processor.CountIgnoredSpanBoundaryCodepoints(
text7_begin, text7.end(),
/*count_from_beginning=*/true),
2);
UnicodeText::const_iterator text7_end = text7.begin();
std::advance(text7_end, 8);
EXPECT_EQ(feature_processor.CountIgnoredSpanBoundaryCodepoints(
text7.begin(), text7_end,
/*count_from_beginning=*/false),
2);
// Test not stripping.
EXPECT_EQ(feature_processor.StripBoundaryCodepoints(
"Hello [[[Wořld]] or not?", {0, 24}),
std::make_pair(0, 24));
// Test basic stripping.
EXPECT_EQ(feature_processor.StripBoundaryCodepoints(
"Hello [[[Wořld]] or not?", {6, 16}),
std::make_pair(9, 14));
// Test stripping when everything is stripped.
EXPECT_EQ(
feature_processor.StripBoundaryCodepoints("Hello [[[]] or not?", {6, 11}),
std::make_pair(6, 6));
// Test stripping empty string.
EXPECT_EQ(feature_processor.StripBoundaryCodepoints("", {0, 0}),
std::make_pair(0, 0));
}
TEST(FeatureProcessorTest, CodepointSpanToTokenSpan) {
const std::vector<Token> tokens{Token("Hělló", 0, 5),
Token("fěěbař@google.com", 6, 23),
Token("heře!", 24, 29)};
// Spans matching the tokens exactly.
EXPECT_EQ(TokenSpan(0, 1), CodepointSpanToTokenSpan(tokens, {0, 5}));
EXPECT_EQ(TokenSpan(1, 2), CodepointSpanToTokenSpan(tokens, {6, 23}));
EXPECT_EQ(TokenSpan(2, 3), CodepointSpanToTokenSpan(tokens, {24, 29}));
EXPECT_EQ(TokenSpan(0, 2), CodepointSpanToTokenSpan(tokens, {0, 23}));
EXPECT_EQ(TokenSpan(1, 3), CodepointSpanToTokenSpan(tokens, {6, 29}));
EXPECT_EQ(TokenSpan(0, 3), CodepointSpanToTokenSpan(tokens, {0, 29}));
// Snapping to containing tokens has no effect.
EXPECT_EQ(TokenSpan(0, 1), CodepointSpanToTokenSpan(tokens, {0, 5}, true));
EXPECT_EQ(TokenSpan(1, 2), CodepointSpanToTokenSpan(tokens, {6, 23}, true));
EXPECT_EQ(TokenSpan(2, 3), CodepointSpanToTokenSpan(tokens, {24, 29}, true));
EXPECT_EQ(TokenSpan(0, 2), CodepointSpanToTokenSpan(tokens, {0, 23}, true));
EXPECT_EQ(TokenSpan(1, 3), CodepointSpanToTokenSpan(tokens, {6, 29}, true));
EXPECT_EQ(TokenSpan(0, 3), CodepointSpanToTokenSpan(tokens, {0, 29}, true));
// Span boundaries inside tokens.
EXPECT_EQ(TokenSpan(1, 2), CodepointSpanToTokenSpan(tokens, {1, 28}));
EXPECT_EQ(TokenSpan(0, 3), CodepointSpanToTokenSpan(tokens, {1, 28}, true));
// Tokens adjacent to the span, but not overlapping.
EXPECT_EQ(TokenSpan(1, 2), CodepointSpanToTokenSpan(tokens, {5, 24}));
EXPECT_EQ(TokenSpan(1, 2), CodepointSpanToTokenSpan(tokens, {5, 24}, true));
}
} // namespace
} // namespace libtextclassifier2