// Copyright 2016 Google Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/mutator.h" #include <algorithm> #include <set> #include <string> #include <tuple> #include <utility> #include <vector> #include "port/gtest.h" #include "src/binary_format.h" #include "src/mutator_test_proto2.pb.h" #include "src/mutator_test_proto3.pb.h" #include "src/text_format.h" namespace protobuf_mutator { using protobuf::util::MessageDifferencer; using testing::TestWithParam; using testing::ValuesIn; const char kMessages[] = R"( required_msg {} optional_msg {} repeated_msg {} repeated_msg {required_sint32: 56} repeated_msg {} repeated_msg { required_msg {} optional_msg {} repeated_msg {} repeated_msg { required_int32: 67 } repeated_msg {} } )"; const char kMessagesProto3[] = R"( optional_msg {} repeated_msg {} repeated_msg {optional_sint32: 56} repeated_msg {} repeated_msg { optional_msg {} repeated_msg {} repeated_msg { optional_int32: 67 } repeated_msg {} } )"; const char kRequiredFields[] = R"( required_double: 1.26685288449177e-313 required_float: 5.9808638e-39 required_int32: 67 required_int64: 5285068 required_uint32: 14486213 required_uint64: 520229415 required_sint32: 56 required_sint64: -6057486163525532641 required_fixed32: 8812173 required_fixed64: 273731277756 required_sfixed32: 43142 required_sfixed64: 132 required_bool: false required_string: "qwert" required_bytes: "asdf" )"; const char kOptionalFields[] = R"( optional_double: 1.93177850152856e-314 optional_float: 4.7397519e-41 optional_int32: 40020 optional_int64: 10 optional_uint32: 40 optional_uint64: 159 optional_sint32: 44015 optional_sint64: 17493625000076 optional_fixed32: 193 optional_fixed64: 8542688694448488723 optional_sfixed32: 4926 optional_sfixed64: 60 optional_bool: true optional_string: "QWERT" optional_bytes: "ASDF" optional_enum: ENUM_5 )"; const char kRepeatedFields[] = R"( repeated_double: 1.93177850152856e-314 repeated_double: 1.26685288449177e-313 repeated_float: 4.7397519e-41 repeated_float: 5.9808638e-39 repeated_int32: 40020 repeated_int32: 67 repeated_int64: 10 repeated_int64: 5285068 repeated_uint32: 40 repeated_uint32: 14486213 repeated_uint64: 159 repeated_uint64: 520229415 repeated_sint32: 44015 repeated_sint32: 56 repeated_sint64: 17493625000076 repeated_sint64: -6057486163525532641 repeated_fixed32: 193 repeated_fixed32: 8812173 repeated_fixed64: 8542688694448488723 repeated_fixed64: 273731277756 repeated_sfixed32: 4926 repeated_sfixed32: 43142 repeated_sfixed64: 60 repeated_sfixed64: 132 repeated_bool: false repeated_bool: true repeated_string: "QWERT" repeated_string: "qwert" repeated_bytes: "ASDF" repeated_bytes: "asdf" repeated_enum: ENUM_5 repeated_enum: ENUM_4 )"; const char kRequiredNestedFields[] = R"( required_int32: 123 optional_msg { required_double: 1.26685288449177e-313 required_float: 5.9808638e-39 required_int32: 67 required_int64: 5285068 required_uint32: 14486213 required_uint64: 520229415 required_sint32: 56 required_sint64: -6057486163525532641 required_fixed32: 8812173 required_fixed64: 273731277756 required_sfixed32: 43142 required_sfixed64: 132 required_bool: false required_string: "qwert" required_bytes: "asdf" } )"; const char kOptionalNestedFields[] = R"( optional_int32: 123 optional_msg { optional_double: 1.93177850152856e-314 optional_float: 4.7397519e-41 optional_int32: 40020 optional_int64: 10 optional_uint32: 40 optional_uint64: 159 optional_sint32: 44015 optional_sint64: 17493625000076 optional_fixed32: 193 optional_fixed64: 8542688694448488723 optional_sfixed32: 4926 optional_sfixed64: 60 optional_bool: true optional_string: "QWERT" optional_bytes: "ASDF" optional_enum: ENUM_5 } )"; const char kRepeatedNestedFields[] = R"( optional_int32: 123 optional_msg { repeated_double: 1.93177850152856e-314 repeated_double: 1.26685288449177e-313 repeated_float: 4.7397519e-41 repeated_float: 5.9808638e-39 repeated_int32: 40020 repeated_int32: 67 repeated_int64: 10 repeated_int64: 5285068 repeated_uint32: 40 repeated_uint32: 14486213 repeated_uint64: 159 repeated_uint64: 520229415 repeated_sint32: 44015 repeated_sint32: 56 repeated_sint64: 17493625000076 repeated_sint64: -6057486163525532641 repeated_fixed32: 193 repeated_fixed32: 8812173 repeated_fixed64: 8542688694448488723 repeated_fixed64: 273731277756 repeated_sfixed32: 4926 repeated_sfixed32: 43142 repeated_sfixed64: 60 repeated_sfixed64: 132 repeated_bool: false repeated_bool: true repeated_string: "QWERT" repeated_string: "qwert" repeated_bytes: "ASDF" repeated_bytes: "asdf" repeated_enum: ENUM_5 repeated_enum: ENUM_4 } )"; class TestMutator : public Mutator { public: explicit TestMutator(bool keep_initialized) : Mutator(&random_), random_(17) { keep_initialized_ = keep_initialized; } // Avoids dedup logic for some tests. void NoDeDupCrossOver(const protobuf::Message& message1, protobuf::Message* message2) { CrossOverImpl(message1, message2); } private: RandomEngine random_; }; class ReducedTestMutator : public TestMutator { public: ReducedTestMutator() : TestMutator(false) { for (float i = 1000; i > 0.1; i /= 7) { values_.push_back(i); values_.push_back(-i); } values_.push_back(-1.0); values_.push_back(0.0); values_.push_back(1.0); } protected: int32_t MutateInt32(int32_t value) override { return GetRandomValue(); } int64_t MutateInt64(int64_t value) override { return GetRandomValue(); } uint32_t MutateUInt32(uint32_t value) override { return fabs(GetRandomValue()); } uint64_t MutateUInt64(uint64_t value) override { return fabs(GetRandomValue()); } float MutateFloat(float value) override { return GetRandomValue(); } double MutateDouble(double value) override { return GetRandomValue(); } std::string MutateString(const std::string& value, size_t size_increase_hint) override { return strings_[std::uniform_int_distribution<uint8_t>( 0, strings_.size() - 1)(*random())]; } private: float GetRandomValue() { return values_[std::uniform_int_distribution<uint8_t>( 0, values_.size() - 1)(*random())]; } std::vector<float> values_; std::vector<std::string> strings_ = { "", "\001", "\000", "a", "b", "ab", }; }; std::vector<std::string> Split(const std::string& str) { std::istringstream iss(str); std::vector<std::string> result; for (std::string line; std::getline(iss, line, '\n');) result.push_back(line); return result; } using TestParams = std::tuple<const protobuf::Message*, const char*, size_t>; template <class T> std::vector<TestParams> GetFieldTestParams( const std::vector<const char*>& tests) { std::vector<TestParams> results; for (auto t : tests) { auto lines = Split(t); for (size_t i = 0; i != lines.size(); ++i) { if (lines[i].find(':') != std::string::npos) results.push_back(std::make_tuple(&T::default_instance(), t, i)); } } return results; } template <class T> std::vector<TestParams> GetMessageTestParams( const std::vector<const char*>& tests) { std::vector<TestParams> results; for (auto t : tests) { auto lines = Split(t); for (size_t i = 0; i != lines.size(); ++i) { if (lines[i].find("{}") != std::string::npos) results.push_back(std::make_tuple(&T::default_instance(), t, i)); } } return results; } bool Mutate(const protobuf::Message& from, const protobuf::Message& to) { EXPECT_FALSE(MessageDifferencer::Equals(from, to)); ReducedTestMutator mutator; std::unique_ptr<protobuf::Message> message(from.New()); for (int j = 0; j < 1000000; ++j) { message->CopyFrom(from); mutator.Mutate(message.get(), 1000); if (MessageDifferencer::Equals(*message, to)) return true; } ADD_FAILURE() << "Failed to get from:\n" << SaveMessageAsText(from) << "\nto:\n" << SaveMessageAsText(to); return false; } class MutatorTest : public TestWithParam<TestParams> { protected: void SetUp() override { m1_.reset(std::get<0>(GetParam())->New()); m2_.reset(std::get<0>(GetParam())->New()); text_ = std::get<1>(GetParam()); line_ = std::get<2>(GetParam()); } void LoadMessage(protobuf::Message* message) { EXPECT_TRUE(ParseTextMessage(text_, message)); } bool LoadWithoutLine(protobuf::Message* message) { std::ostringstream oss; auto lines = Split(text_); for (size_t i = 0; i != lines.size(); ++i) { if (i != line_) oss << lines[i] << '\n'; } return ParseTextMessage(oss.str(), message); } bool LoadWithChangedLine(protobuf::Message* message, int value) { auto lines = Split(text_); std::ostringstream oss; for (size_t i = 0; i != lines.size(); ++i) { if (i != line_) { oss << lines[i] << '\n'; } else { std::string s = lines[i]; s.resize(s.find(':') + 2); if (lines[i].back() == '\"') { // strings s += value ? "\"\\" + std::to_string(value) + "\"" : "\"\""; } else if (lines[i].back() == 'e') { // bools s += value ? "true" : "false"; } else { s += std::to_string(value); } oss << s << '\n'; } } return ParseTextMessage(oss.str(), message); } std::string text_; size_t line_; std::unique_ptr<protobuf::Message> m1_; std::unique_ptr<protobuf::Message> m2_; }; // These tests are irrelevant for Proto3 as it has no required fields and // insertion/deletion. class MutatorFieldInsDelTest : public MutatorTest {}; INSTANTIATE_TEST_CASE_P(Proto2, MutatorFieldInsDelTest, ValuesIn(GetFieldTestParams<Msg>( {kRequiredFields, kOptionalFields, kRepeatedFields, kRequiredNestedFields, kOptionalNestedFields, kRepeatedNestedFields}))); TEST_P(MutatorFieldInsDelTest, DeleteField) { LoadMessage(m1_.get()); LoadWithoutLine(m2_.get()); EXPECT_TRUE(Mutate(*m1_, *m2_)); } TEST_P(MutatorFieldInsDelTest, InsertField) { LoadWithoutLine(m1_.get()); LoadWithChangedLine(m2_.get(), 0); EXPECT_TRUE(Mutate(*m1_, *m2_)); } class MutatorFieldTest : public MutatorTest { public: template <class Msg> void TestCopyField(); }; INSTANTIATE_TEST_CASE_P(Proto2, MutatorFieldTest, ValuesIn(GetFieldTestParams<Msg>( {kRequiredFields, kOptionalFields, kRepeatedFields, kRequiredNestedFields, kOptionalNestedFields, kRepeatedNestedFields}))); INSTANTIATE_TEST_CASE_P(Proto3, MutatorFieldTest, ValuesIn(GetFieldTestParams<Msg3>( {kOptionalFields, kRepeatedFields, kOptionalNestedFields, kRepeatedNestedFields}))); TEST_P(MutatorFieldTest, Initialized) { LoadWithoutLine(m1_.get()); TestMutator mutator(true); mutator.Mutate(m1_.get(), 1000); EXPECT_TRUE(m1_->IsInitialized()); } TEST_P(MutatorFieldTest, ChangeField) { LoadWithChangedLine(m1_.get(), 0); LoadWithChangedLine(m2_.get(), 1); EXPECT_TRUE(Mutate(*m1_, *m2_)); EXPECT_TRUE(Mutate(*m2_, *m1_)); } template <class Msg> void MutatorFieldTest::TestCopyField() { LoadWithChangedLine(m1_.get(), 7); LoadWithChangedLine(m2_.get(), 0); Msg from; from.add_repeated_msg()->CopyFrom(*m1_); from.add_repeated_msg()->CopyFrom(*m2_); Msg to; to.add_repeated_msg()->CopyFrom(*m1_); to.add_repeated_msg()->CopyFrom(*m1_); EXPECT_TRUE(Mutate(from, to)); to.Clear(); to.add_repeated_msg()->CopyFrom(*m2_); to.add_repeated_msg()->CopyFrom(*m2_); EXPECT_TRUE(Mutate(from, to)); } TEST_P(MutatorFieldTest, CopyField) { if (m1_->GetDescriptor() == Msg::descriptor()) TestCopyField<Msg>(); else TestCopyField<Msg3>(); } class MutatorSingleFieldTest : public MutatorTest {}; INSTANTIATE_TEST_CASE_P(Proto2, MutatorSingleFieldTest, ValuesIn(GetFieldTestParams<Msg>({ kRequiredFields, kOptionalFields, kRequiredNestedFields, kOptionalNestedFields, }))); INSTANTIATE_TEST_CASE_P(Proto3, MutatorSingleFieldTest, ValuesIn(GetFieldTestParams<Msg3>({ kOptionalFields, kOptionalNestedFields, }))); TEST_P(MutatorSingleFieldTest, CrossOver) { LoadWithoutLine(m1_.get()); LoadMessage(m2_.get()); EXPECT_FALSE(MessageDifferencer::Equals(*m1_, *m2_)); TestMutator mutator(false); int match_m1_ = 0; int match_m2_ = 0; int iterations = 1000; std::unique_ptr<protobuf::Message> message(m1_->New()); for (int j = 0; j < iterations; ++j) { message->CopyFrom(*m1_); mutator.NoDeDupCrossOver(*m2_, message.get()); if (MessageDifferencer::Equals(*message, *m2_)) ++match_m2_; if (MessageDifferencer::Equals(*message, *m1_)) ++match_m1_; } EXPECT_LT(iterations * .4, match_m1_); EXPECT_GE(iterations * .6, match_m1_); EXPECT_LT(iterations * .4, match_m2_); EXPECT_GE(iterations * .6, match_m2_); } template <typename T> class MutatorTypedTest : public ::testing::Test { public: using Message = T; }; using MutatorTypedTestTypes = testing::Types<Msg, Msg3>; TYPED_TEST_CASE(MutatorTypedTest, MutatorTypedTestTypes); TYPED_TEST(MutatorTypedTest, CrossOverRepeated) { typename TestFixture::Message m1; m1.add_repeated_int32(1); m1.add_repeated_int32(2); m1.add_repeated_int32(3); typename TestFixture::Message m2; m2.add_repeated_int32(4); m2.add_repeated_int32(5); m2.add_repeated_int32(6); int iterations = 10000; std::set<std::set<int>> sets; TestMutator mutator(false); for (int j = 0; j < iterations; ++j) { typename TestFixture::Message message; message.CopyFrom(m1); mutator.NoDeDupCrossOver(m2, &message); sets.insert( {message.repeated_int32().begin(), message.repeated_int32().end()}); } EXPECT_EQ(1u << 6, sets.size()); } TYPED_TEST(MutatorTypedTest, CrossOverRepeatedMessages) { typename TestFixture::Message m1; auto* rm1 = m1.add_repeated_msg(); rm1->add_repeated_int32(1); rm1->add_repeated_int32(2); typename TestFixture::Message m2; auto* rm2 = m2.add_repeated_msg(); rm2->add_repeated_int32(3); rm2->add_repeated_int32(4); rm2->add_repeated_int32(5); rm2->add_repeated_int32(6); int iterations = 10000; std::set<std::set<int>> sets; TestMutator mutator(false); for (int j = 0; j < iterations; ++j) { typename TestFixture::Message message; message.CopyFrom(m1); mutator.NoDeDupCrossOver(m2, &message); for (const auto& msg : message.repeated_msg()) sets.insert({msg.repeated_int32().begin(), msg.repeated_int32().end()}); } EXPECT_EQ(1u << 6, sets.size()); } TYPED_TEST(MutatorTypedTest, FailedMutations) { TestMutator mutator(false); size_t crossovers = 0; for (int i = 0; i < 10000; ++i) { typename TestFixture::Message messages[2]; typename TestFixture::Message tmp; for (int j = 0; j < 20; ++j) { for (auto& m : messages) { tmp.CopyFrom(m); mutator.Mutate(&m, 1000); // Mutate must not produce the same result. EXPECT_FALSE(MessageDifferencer::Equals(m, tmp)); } } tmp.CopyFrom(messages[1]); mutator.CrossOver(messages[0], &tmp); if (MessageDifferencer::Equals(tmp, messages[1]) || MessageDifferencer::Equals(tmp, messages[0])) ++crossovers; } // CrossOver may fail but very rare. EXPECT_LT(crossovers, 100u); } TYPED_TEST(MutatorTypedTest, Serialization) { TestMutator mutator(false); for (int i = 0; i < 10000; ++i) { typename TestFixture::Message message; for (int j = 0; j < 5; ++j) { mutator.Mutate(&message, 1000); typename TestFixture::Message parsed; EXPECT_TRUE(ParseTextMessage(SaveMessageAsText(message), &parsed)); EXPECT_TRUE(MessageDifferencer::Equals(parsed, message)); EXPECT_TRUE(ParseBinaryMessage(SaveMessageAsBinary(message), &parsed)); EXPECT_TRUE(MessageDifferencer::Equals(parsed, message)); } } } class MutatorMessagesTest : public MutatorTest {}; INSTANTIATE_TEST_CASE_P(Proto2, MutatorMessagesTest, ValuesIn(GetMessageTestParams<Msg>({kMessages}))); INSTANTIATE_TEST_CASE_P( Proto3, MutatorMessagesTest, ValuesIn(GetMessageTestParams<Msg3>({kMessagesProto3}))); TEST_P(MutatorMessagesTest, DeletedMessage) { LoadMessage(m1_.get()); LoadWithoutLine(m2_.get()); EXPECT_TRUE(Mutate(*m1_, *m2_)); } TEST_P(MutatorMessagesTest, InsertMessage) { LoadWithoutLine(m1_.get()); LoadMessage(m2_.get()); EXPECT_TRUE(Mutate(*m1_, *m2_)); } // TODO(vitalybuka): Special tests for oneof. TEST(MutatorMessagesTest, UsageExample) { SmallMessage message; TestMutator mutator(false); // Test that we can generate all variation of the message. std::set<std::string> mutations; for (int j = 0; j < 1000; ++j) { mutator.Mutate(&message, 1000); std::string str = SaveMessageAsText(message); mutations.insert(str); } // 3 states for boolean and 5 for enum, including missing fields. EXPECT_EQ(3u * 5u, mutations.size()); } TEST(MutatorMessagesTest, EmptyMessage) { EmptyMessage message; TestMutator mutator(false); for (int j = 0; j < 10000; ++j) mutator.Mutate(&message, 1000); } TEST(MutatorMessagesTest, Regressions) { RegressionMessage message; TestMutator mutator(false); for (int j = 0; j < 10000; ++j) mutator.Mutate(&message, 1000); } } // namespace protobuf_mutator