/*
* Copyright (C) 2008 Apple Inc. All Rights Reserved.
* Copyright (C) 2010 Google, Inc. All Rights Reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef HTMLTokenizer_h
#define HTMLTokenizer_h
#include "SegmentedString.h"
#include <wtf/Noncopyable.h>
#include <wtf/PassOwnPtr.h>
#include <wtf/Vector.h>
#include <wtf/text/AtomicString.h>
namespace WebCore {
class Element;
class Frame;
class HTMLToken;
class HTMLTokenizer {
WTF_MAKE_NONCOPYABLE(HTMLTokenizer); WTF_MAKE_FAST_ALLOCATED;
public:
enum State {
DataState,
CharacterReferenceInDataState,
RCDATAState,
CharacterReferenceInRCDATAState,
RAWTEXTState,
ScriptDataState,
PLAINTEXTState,
TagOpenState,
EndTagOpenState,
TagNameState,
RCDATALessThanSignState,
RCDATAEndTagOpenState,
RCDATAEndTagNameState,
RAWTEXTLessThanSignState,
RAWTEXTEndTagOpenState,
RAWTEXTEndTagNameState,
ScriptDataLessThanSignState,
ScriptDataEndTagOpenState,
ScriptDataEndTagNameState,
ScriptDataEscapeStartState,
ScriptDataEscapeStartDashState,
ScriptDataEscapedState,
ScriptDataEscapedDashState,
ScriptDataEscapedDashDashState,
ScriptDataEscapedLessThanSignState,
ScriptDataEscapedEndTagOpenState,
ScriptDataEscapedEndTagNameState,
ScriptDataDoubleEscapeStartState,
ScriptDataDoubleEscapedState,
ScriptDataDoubleEscapedDashState,
ScriptDataDoubleEscapedDashDashState,
ScriptDataDoubleEscapedLessThanSignState,
ScriptDataDoubleEscapeEndState,
BeforeAttributeNameState,
AttributeNameState,
AfterAttributeNameState,
BeforeAttributeValueState,
AttributeValueDoubleQuotedState,
AttributeValueSingleQuotedState,
AttributeValueUnquotedState,
CharacterReferenceInAttributeValueState,
AfterAttributeValueQuotedState,
SelfClosingStartTagState,
BogusCommentState,
// The ContinueBogusCommentState is not in the HTML5 spec, but we use
// it internally to keep track of whether we've started the bogus
// comment token yet.
ContinueBogusCommentState,
MarkupDeclarationOpenState,
CommentStartState,
CommentStartDashState,
CommentState,
CommentEndDashState,
CommentEndState,
CommentEndBangState,
DOCTYPEState,
BeforeDOCTYPENameState,
DOCTYPENameState,
AfterDOCTYPENameState,
AfterDOCTYPEPublicKeywordState,
BeforeDOCTYPEPublicIdentifierState,
DOCTYPEPublicIdentifierDoubleQuotedState,
DOCTYPEPublicIdentifierSingleQuotedState,
AfterDOCTYPEPublicIdentifierState,
BetweenDOCTYPEPublicAndSystemIdentifiersState,
AfterDOCTYPESystemKeywordState,
BeforeDOCTYPESystemIdentifierState,
DOCTYPESystemIdentifierDoubleQuotedState,
DOCTYPESystemIdentifierSingleQuotedState,
AfterDOCTYPESystemIdentifierState,
BogusDOCTYPEState,
CDATASectionState,
// These CDATA states are not in the HTML5 spec, but we use them internally.
CDATASectionRightSquareBracketState,
CDATASectionDoubleRightSquareBracketState,
};
static PassOwnPtr<HTMLTokenizer> create(bool usePreHTML5ParserQuirks) { return adoptPtr(new HTMLTokenizer(usePreHTML5ParserQuirks)); }
~HTMLTokenizer();
void reset();
// This function returns true if it emits a token. Otherwise, callers
// must provide the same (in progress) token on the next call (unless
// they call reset() first).
bool nextToken(SegmentedString&, HTMLToken&);
int lineNumber() const { return m_lineNumber; }
int columnNumber() const { return 1; } // Matches LegacyHTMLDocumentParser.h behavior.
State state() const { return m_state; }
void setState(State state) { m_state = state; }
// Updates the tokenizer's state according to the given tag name. This is
// an approximation of how the tree builder would update the tokenizer's
// state. This method is useful for approximating HTML tokenization. To
// get exactly the correct tokenization, you need the real tree builder.
//
// The main failures in the approximation are as follows:
//
// * The first set of character tokens emitted for a <pre> element might
// contain an extra leading newline.
// * The replacement of U+0000 with U+FFFD will not be sensitive to the
// tree builder's insertion mode.
// * CDATA sections in foreign content will be tokenized as bogus comments
// instead of as character tokens.
//
void updateStateFor(const AtomicString& tagName, Frame*);
// Hack to skip leading newline in <pre>/<listing> for authoring ease.
// http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#parsing-main-inbody
void setSkipLeadingNewLineForListing(bool value) { m_skipLeadingNewLineForListing = value; }
bool forceNullCharacterReplacement() const { return m_forceNullCharacterReplacement; }
void setForceNullCharacterReplacement(bool value) { m_forceNullCharacterReplacement = value; }
bool shouldAllowCDATA() const { return m_shouldAllowCDATA; }
void setShouldAllowCDATA(bool value) { m_shouldAllowCDATA = value; }
bool shouldSkipNullCharacters() const
{
return !m_forceNullCharacterReplacement
&& (m_state == DataState
|| m_state == RCDATAState
|| m_state == RAWTEXTState
|| m_state == PLAINTEXTState);
}
private:
// http://www.whatwg.org/specs/web-apps/current-work/#preprocessing-the-input-stream
class InputStreamPreprocessor {
WTF_MAKE_NONCOPYABLE(InputStreamPreprocessor);
public:
InputStreamPreprocessor(HTMLTokenizer* tokenizer)
: m_tokenizer(tokenizer)
, m_nextInputCharacter('\0')
, m_skipNextNewLine(false)
{
}
UChar nextInputCharacter() const { return m_nextInputCharacter; }
// Returns whether we succeeded in peeking at the next character.
// The only way we can fail to peek is if there are no more
// characters in |source| (after collapsing \r\n, etc).
ALWAYS_INLINE bool peek(SegmentedString& source, int& lineNumber)
{
PeekAgain:
m_nextInputCharacter = *source;
// Every branch in this function is expensive, so we have a
// fast-reject branch for characters that don't require special
// handling. Please run the parser benchmark whenever you touch
// this function. It's very hot.
static const UChar specialCharacterMask = '\n' | '\r' | '\0';
if (m_nextInputCharacter & ~specialCharacterMask) {
m_skipNextNewLine = false;
return true;
}
if (m_nextInputCharacter == '\n' && m_skipNextNewLine) {
m_skipNextNewLine = false;
source.advancePastNewline(lineNumber);
if (source.isEmpty())
return false;
m_nextInputCharacter = *source;
}
if (m_nextInputCharacter == '\r') {
m_nextInputCharacter = '\n';
m_skipNextNewLine = true;
} else {
m_skipNextNewLine = false;
// FIXME: The spec indicates that the surrogate pair range as well as
// a number of specific character values are parse errors and should be replaced
// by the replacement character. We suspect this is a problem with the spec as doing
// that filtering breaks surrogate pair handling and causes us not to match Minefield.
if (m_nextInputCharacter == '\0' && !shouldTreatNullAsEndOfFileMarker(source)) {
if (m_tokenizer->shouldSkipNullCharacters()) {
source.advancePastNonNewline();
if (source.isEmpty())
return false;
goto PeekAgain;
}
m_nextInputCharacter = 0xFFFD;
}
}
return true;
}
// Returns whether there are more characters in |source| after advancing.
bool advance(SegmentedString& source, int& lineNumber)
{
source.advance(lineNumber);
if (source.isEmpty())
return false;
return peek(source, lineNumber);
}
static const UChar endOfFileMarker;
private:
bool shouldTreatNullAsEndOfFileMarker(SegmentedString& source) const
{
return source.isClosed() && source.length() == 1;
}
HTMLTokenizer* m_tokenizer;
// http://www.whatwg.org/specs/web-apps/current-work/#next-input-character
UChar m_nextInputCharacter;
bool m_skipNextNewLine;
};
HTMLTokenizer(bool usePreHTML5ParserQuirks);
inline bool processEntity(SegmentedString&);
inline void parseError();
inline void bufferCharacter(UChar);
inline void bufferCodePoint(unsigned);
inline bool emitAndResumeIn(SegmentedString&, State);
inline bool emitAndReconsumeIn(SegmentedString&, State);
inline bool emitEndOfFile(SegmentedString&);
inline bool flushEmitAndResumeIn(SegmentedString&, State);
// Return whether we need to emit a character token before dealing with
// the buffered end tag.
inline bool flushBufferedEndTag(SegmentedString&);
inline bool temporaryBufferIs(const String&);
// Sometimes we speculatively consume input characters and we don't
// know whether they represent end tags or RCDATA, etc. These
// functions help manage these state.
inline void addToPossibleEndTag(UChar cc);
inline void saveEndTagNameIfNeeded();
inline bool isAppropriateEndTag();
inline bool haveBufferedCharacterToken();
State m_state;
Vector<UChar, 32> m_appropriateEndTagName;
// m_token is owned by the caller. If nextToken is not on the stack,
// this member might be pointing to unallocated memory.
HTMLToken* m_token;
int m_lineNumber;
bool m_skipLeadingNewLineForListing;
bool m_forceNullCharacterReplacement;
bool m_shouldAllowCDATA;
// http://www.whatwg.org/specs/web-apps/current-work/#temporary-buffer
Vector<UChar, 32> m_temporaryBuffer;
// We occationally want to emit both a character token and an end tag
// token (e.g., when lexing script). We buffer the name of the end tag
// token here so we remember it next time we re-enter the tokenizer.
Vector<UChar, 32> m_bufferedEndTagName;
// http://www.whatwg.org/specs/web-apps/current-work/#additional-allowed-character
UChar m_additionalAllowedCharacter;
// http://www.whatwg.org/specs/web-apps/current-work/#preprocessing-the-input-stream
InputStreamPreprocessor m_inputStreamPreprocessor;
bool m_usePreHTML5ParserQuirks;
};
}
#endif