// Copyright (C) 2015 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "tokenizer.h"
#include <errno.h>
#include <gtest/gtest.h>
#include <string>
namespace init {
#define SETUP_TEST(test_data) \
std::string data(test_data); \
Tokenizer tokenizer(data); \
ASSERT_EQ(Tokenizer::TOK_START, tokenizer.current().type)
#define ASSERT_TEXT_TOKEN(test_text) \
ASSERT_TRUE(tokenizer.Next()); \
ASSERT_EQ(test_text, tokenizer.current().text); \
ASSERT_EQ(Tokenizer::TOK_TEXT, tokenizer.current().type)
#define ASSERT_NEWLINE_TOKEN() \
ASSERT_TRUE(tokenizer.Next()); \
ASSERT_EQ(Tokenizer::TOK_NEWLINE, tokenizer.current().type)
TEST(Tokenizer, Empty) {
SETUP_TEST("");
ASSERT_FALSE(tokenizer.Next());
}
TEST(Tokenizer, Simple) {
SETUP_TEST("test");
ASSERT_TEXT_TOKEN("test");
ASSERT_FALSE(tokenizer.Next());
}
TEST(Tokenizer, LeadingWhiteSpace) {
SETUP_TEST(" \t \r test");
ASSERT_TEXT_TOKEN("test");
ASSERT_FALSE(tokenizer.Next());
}
TEST(Tokenizer, TrailingWhiteSpace) {
SETUP_TEST("test \t \r ");
ASSERT_TEXT_TOKEN("test");
ASSERT_FALSE(tokenizer.Next());
}
TEST(Tokenizer, WhiteSpace) {
SETUP_TEST(" \t \r test \t \r ");
ASSERT_TEXT_TOKEN("test");
ASSERT_FALSE(tokenizer.Next());
}
TEST(Tokenizer, TwoTokens) {
SETUP_TEST(" foo bar ");
ASSERT_TEXT_TOKEN("foo");
ASSERT_TEXT_TOKEN("bar");
ASSERT_FALSE(tokenizer.Next());
}
TEST(Tokenizer, MultiToken) {
SETUP_TEST("one two three four five");
ASSERT_TEXT_TOKEN("one");
ASSERT_TEXT_TOKEN("two");
ASSERT_TEXT_TOKEN("three");
ASSERT_TEXT_TOKEN("four");
ASSERT_TEXT_TOKEN("five");
ASSERT_FALSE(tokenizer.Next());
}
TEST(Tokenizer, NewLine) {
SETUP_TEST("\n");
ASSERT_NEWLINE_TOKEN();
ASSERT_FALSE(tokenizer.Next());
}
TEST(Tokenizer, TextNewLine) {
SETUP_TEST("test\n");
ASSERT_TEXT_TOKEN("test");
ASSERT_NEWLINE_TOKEN();
ASSERT_FALSE(tokenizer.Next());
}
TEST(Tokenizer, MultiTextNewLine) {
SETUP_TEST("one\ntwo\nthree\n");
ASSERT_TEXT_TOKEN("one");
ASSERT_NEWLINE_TOKEN();
ASSERT_TEXT_TOKEN("two");
ASSERT_NEWLINE_TOKEN();
ASSERT_TEXT_TOKEN("three");
ASSERT_NEWLINE_TOKEN();
ASSERT_FALSE(tokenizer.Next());
}
TEST(Tokenizer, MultiTextNewLineNoLineEnding) {
SETUP_TEST("one\ntwo\nthree");
ASSERT_TEXT_TOKEN("one");
ASSERT_NEWLINE_TOKEN();
ASSERT_TEXT_TOKEN("two");
ASSERT_NEWLINE_TOKEN();
ASSERT_TEXT_TOKEN("three");
ASSERT_FALSE(tokenizer.Next());
}
TEST(Tokenizer, Comment) {
SETUP_TEST("#test");
ASSERT_FALSE(tokenizer.Next());
}
TEST(Tokenizer, CommentWhiteSpace) {
SETUP_TEST(" \t \r #test \t \r ");
ASSERT_FALSE(tokenizer.Next());
}
TEST(Tokenizer, CommentNewLine) {
SETUP_TEST(" #test \n");
ASSERT_NEWLINE_TOKEN();
ASSERT_FALSE(tokenizer.Next());
}
TEST(Tokenizer, CommentTwoNewLine) {
SETUP_TEST(" #test \n#test");
ASSERT_NEWLINE_TOKEN();
ASSERT_FALSE(tokenizer.Next());
}
TEST(Tokenizer, CommentWithText) {
SETUP_TEST("foo bar #test");
ASSERT_TEXT_TOKEN("foo");
ASSERT_TEXT_TOKEN("bar");
ASSERT_FALSE(tokenizer.Next());
}
TEST(Tokenizer, CommentWithTextNoSpace) {
SETUP_TEST("foo bar#test");
ASSERT_TEXT_TOKEN("foo");
ASSERT_TEXT_TOKEN("bar");
ASSERT_FALSE(tokenizer.Next());
}
TEST(Tokenizer, CommentWithTextLineFeed) {
SETUP_TEST("foo bar #test\n");
ASSERT_TEXT_TOKEN("foo");
ASSERT_TEXT_TOKEN("bar");
ASSERT_NEWLINE_TOKEN();
ASSERT_FALSE(tokenizer.Next());
}
TEST(Tokenizer, CommentWithMultiTextLineFeed) {
SETUP_TEST("#blah\nfoo bar #test\n#blah");
ASSERT_NEWLINE_TOKEN();
ASSERT_TEXT_TOKEN("foo");
ASSERT_TEXT_TOKEN("bar");
ASSERT_NEWLINE_TOKEN();
ASSERT_FALSE(tokenizer.Next());
}
TEST(Tokenizer, SimpleEscaped) {
SETUP_TEST("fo\\no bar");
ASSERT_TEXT_TOKEN("fo\\no");
ASSERT_TEXT_TOKEN("bar");
ASSERT_FALSE(tokenizer.Next());
}
TEST(Tokenizer, EscapedLineContNoLineFeed) {
SETUP_TEST("fo\\no bar \\ hello");
ASSERT_TEXT_TOKEN("fo\\no");
ASSERT_TEXT_TOKEN("bar");
ASSERT_FALSE(tokenizer.Next());
}
TEST(Tokenizer, EscapedLineContLineFeed) {
SETUP_TEST("fo\\no bar \\ hello\n");
ASSERT_TEXT_TOKEN("fo\\no");
ASSERT_TEXT_TOKEN("bar");
ASSERT_FALSE(tokenizer.Next());
}
TEST(Tokenizer, EscapedLineCont) {
SETUP_TEST("fo\\no bar \\\ntest");
ASSERT_TEXT_TOKEN("fo\\no");
ASSERT_TEXT_TOKEN("bar");
ASSERT_TEXT_TOKEN("test");
ASSERT_FALSE(tokenizer.Next());
}
TEST(Tokenizer, EscapedLineContWithBadChars) {
SETUP_TEST("fo\\no bar \\bad bad bad\ntest");
ASSERT_TEXT_TOKEN("fo\\no");
ASSERT_TEXT_TOKEN("bar");
ASSERT_TEXT_TOKEN("test");
ASSERT_FALSE(tokenizer.Next());
}
TEST(Tokenizer, SimpleQuotes) {
SETUP_TEST("foo \"single token\" bar");
ASSERT_TEXT_TOKEN("foo");
ASSERT_TEXT_TOKEN("single token");
ASSERT_TEXT_TOKEN("bar");
ASSERT_FALSE(tokenizer.Next());
}
TEST(Tokenizer, BadQuotes) {
SETUP_TEST("foo \"single token");
ASSERT_TEXT_TOKEN("foo");
ASSERT_TEXT_TOKEN("single token");
ASSERT_FALSE(tokenizer.Next());
}
} // namespace init