/*
Copyright (C) 1999 Lars Knoll (knoll@mpi-hd.mpg.de)
Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 Apple Inc. All rights reserved.
Copyright (C) 2005, 2006, 2007 Alexey Proskuryakov (ap@nypop.com)
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Library General Public
License as published by the Free Software Foundation; either
version 2 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Library General Public License for more details.
You should have received a copy of the GNU Library General Public License
along with this library; see the file COPYING.LIB. If not, write to
the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
Boston, MA 02110-1301, USA.
*/
#include "config.h"
#include "TextResourceDecoder.h"
#include "DOMImplementation.h"
#include "HTMLNames.h"
#include "TextCodec.h"
#include "TextEncoding.h"
#include "TextEncodingDetector.h"
#include "TextEncodingRegistry.h"
#include <wtf/ASCIICType.h>
#include <wtf/StringExtras.h>
using namespace WTF;
namespace WebCore {
using namespace HTMLNames;
// You might think we should put these find functions elsewhere, perhaps with the
// similar functions that operate on UChar, but arguably only the decoder has
// a reason to process strings of char rather than UChar.
static int find(const char* subject, size_t subjectLength, const char* target)
{
size_t targetLength = strlen(target);
if (targetLength > subjectLength)
return -1;
for (size_t i = 0; i <= subjectLength - targetLength; ++i) {
bool match = true;
for (size_t j = 0; j < targetLength; ++j) {
if (subject[i + j] != target[j]) {
match = false;
break;
}
}
if (match)
return i;
}
return -1;
}
static int findIgnoringCase(const char* subject, size_t subjectLength, const char* target)
{
size_t targetLength = strlen(target);
if (targetLength > subjectLength)
return -1;
#ifndef NDEBUG
for (size_t i = 0; i < targetLength; ++i)
ASSERT(isASCIILower(target[i]));
#endif
for (size_t i = 0; i <= subjectLength - targetLength; ++i) {
bool match = true;
for (size_t j = 0; j < targetLength; ++j) {
if (toASCIILower(subject[i + j]) != target[j]) {
match = false;
break;
}
}
if (match)
return i;
}
return -1;
}
static TextEncoding findTextEncoding(const char* encodingName, int length)
{
Vector<char, 64> buffer(length + 1);
memcpy(buffer.data(), encodingName, length);
buffer[length] = '\0';
return buffer.data();
}
class KanjiCode {
public:
enum Type { ASCII, JIS, EUC, SJIS, UTF16, UTF8 };
static enum Type judge(const char* str, int length);
static const int ESC = 0x1b;
static const unsigned char sjisMap[256];
static int ISkanji(int code)
{
if (code >= 0x100)
return 0;
return sjisMap[code & 0xff] & 1;
}
static int ISkana(int code)
{
if (code >= 0x100)
return 0;
return sjisMap[code & 0xff] & 2;
}
};
const unsigned char KanjiCode::sjisMap[256] = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0
};
/*
* EUC-JP is
* [0xa1 - 0xfe][0xa1 - 0xfe]
* 0x8e[0xa1 - 0xfe](SS2)
* 0x8f[0xa1 - 0xfe][0xa1 - 0xfe](SS3)
*
* Shift_Jis is
* [0x81 - 0x9f, 0xe0 - 0xef(0xfe?)][0x40 - 0x7e, 0x80 - 0xfc]
*
* Shift_Jis Hankaku Kana is
* [0xa1 - 0xdf]
*/
/*
* KanjiCode::judge() is based on judge_jcode() from jvim
* http://hp.vector.co.jp/authors/VA003457/vim/
*
* Special Thanks to Kenichi Tsuchida
*/
enum KanjiCode::Type KanjiCode::judge(const char* str, int size)
{
enum Type code;
int i;
int bfr = false; /* Kana Moji */
int bfk = 0; /* EUC Kana */
int sjis = 0;
int euc = 0;
const unsigned char* ptr = reinterpret_cast<const unsigned char*>(str);
code = ASCII;
i = 0;
while (i < size) {
if (ptr[i] == ESC && (size - i >= 3)) {
if ((ptr[i + 1] == '$' && ptr[i + 2] == 'B')
|| (ptr[i + 1] == '(' && ptr[i + 2] == 'B')) {
code = JIS;
goto breakBreak;
} else if ((ptr[i + 1] == '$' && ptr[i + 2] == '@')
|| (ptr[i + 1] == '(' && ptr[i + 2] == 'J')) {
code = JIS;
goto breakBreak;
} else if (ptr[i + 1] == '(' && ptr[i + 2] == 'I') {
code = JIS;
i += 3;
} else if (ptr[i + 1] == ')' && ptr[i + 2] == 'I') {
code = JIS;
i += 3;
} else {
i++;
}
bfr = false;
bfk = 0;
} else {
if (ptr[i] < 0x20) {
bfr = false;
bfk = 0;
/* ?? check kudokuten ?? && ?? hiragana ?? */
if ((i >= 2) && (ptr[i - 2] == 0x81)
&& (0x41 <= ptr[i - 1] && ptr[i - 1] <= 0x49)) {
code = SJIS;
sjis += 100; /* kudokuten */
} else if ((i >= 2) && (ptr[i - 2] == 0xa1)
&& (0xa2 <= ptr[i - 1] && ptr[i - 1] <= 0xaa)) {
code = EUC;
euc += 100; /* kudokuten */
} else if ((i >= 2) && (ptr[i - 2] == 0x82) && (0xa0 <= ptr[i - 1])) {
sjis += 40; /* hiragana */
} else if ((i >= 2) && (ptr[i - 2] == 0xa4) && (0xa0 <= ptr[i - 1])) {
euc += 40; /* hiragana */
}
} else {
/* ?? check hiragana or katana ?? */
if ((size - i > 1) && (ptr[i] == 0x82) && (0xa0 <= ptr[i + 1])) {
sjis++; /* hiragana */
} else if ((size - i > 1) && (ptr[i] == 0x83)
&& (0x40 <= ptr[i + 1] && ptr[i + 1] <= 0x9f)) {
sjis++; /* katakana */
} else if ((size - i > 1) && (ptr[i] == 0xa4) && (0xa0 <= ptr[i + 1])) {
euc++; /* hiragana */
} else if ((size - i > 1) && (ptr[i] == 0xa5) && (0xa0 <= ptr[i + 1])) {
euc++; /* katakana */
}
if (bfr) {
if ((i >= 1) && (0x40 <= ptr[i] && ptr[i] <= 0xa0) && ISkanji(ptr[i - 1])) {
code = SJIS;
goto breakBreak;
} else if ((i >= 1) && (0x81 <= ptr[i - 1] && ptr[i - 1] <= 0x9f) && ((0x40 <= ptr[i] && ptr[i] < 0x7e) || (0x7e < ptr[i] && ptr[i] <= 0xfc))) {
code = SJIS;
goto breakBreak;
} else if ((i >= 1) && (0xfd <= ptr[i] && ptr[i] <= 0xfe) && (0xa1 <= ptr[i - 1] && ptr[i - 1] <= 0xfe)) {
code = EUC;
goto breakBreak;
} else if ((i >= 1) && (0xfd <= ptr[i - 1] && ptr[i - 1] <= 0xfe) && (0xa1 <= ptr[i] && ptr[i] <= 0xfe)) {
code = EUC;
goto breakBreak;
} else if ((i >= 1) && (ptr[i] < 0xa0 || 0xdf < ptr[i]) && (0x8e == ptr[i - 1])) {
code = SJIS;
goto breakBreak;
} else if (ptr[i] <= 0x7f) {
code = SJIS;
goto breakBreak;
} else {
if (0xa1 <= ptr[i] && ptr[i] <= 0xa6) {
euc++; /* sjis hankaku kana kigo */
} else if (0xa1 <= ptr[i] && ptr[i] <= 0xdf) {
; /* sjis hankaku kana */
} else if (0xa1 <= ptr[i] && ptr[i] <= 0xfe) {
euc++;
} else if (0x8e == ptr[i]) {
euc++;
} else if (0x20 <= ptr[i] && ptr[i] <= 0x7f) {
sjis++;
}
bfr = false;
bfk = 0;
}
} else if (0x8e == ptr[i]) {
if (size - i <= 1) {
;
} else if (0xa1 <= ptr[i + 1] && ptr[i + 1] <= 0xdf) {
/* EUC KANA or SJIS KANJI */
if (bfk == 1) {
euc += 100;
}
bfk++;
i++;
} else {
/* SJIS only */
code = SJIS;
goto breakBreak;
}
} else if (0x81 <= ptr[i] && ptr[i] <= 0x9f) {
/* SJIS only */
code = SJIS;
if ((size - i >= 1)
&& ((0x40 <= ptr[i + 1] && ptr[i + 1] <= 0x7e)
|| (0x80 <= ptr[i + 1] && ptr[i + 1] <= 0xfc))) {
goto breakBreak;
}
} else if (0xfd <= ptr[i] && ptr[i] <= 0xfe) {
/* EUC only */
code = EUC;
if ((size - i >= 1)
&& (0xa1 <= ptr[i + 1] && ptr[i + 1] <= 0xfe)) {
goto breakBreak;
}
} else if (ptr[i] <= 0x7f) {
;
} else {
bfr = true;
bfk = 0;
}
}
i++;
}
}
if (code == ASCII) {
if (sjis > euc) {
code = SJIS;
} else if (sjis < euc) {
code = EUC;
}
}
breakBreak:
return (code);
}
TextResourceDecoder::ContentType TextResourceDecoder::determineContentType(const String& mimeType)
{
if (equalIgnoringCase(mimeType, "text/css"))
return CSS;
if (equalIgnoringCase(mimeType, "text/html"))
return HTML;
if (DOMImplementation::isXMLMIMEType(mimeType))
return XML;
return PlainText;
}
const TextEncoding& TextResourceDecoder::defaultEncoding(ContentType contentType, const TextEncoding& specifiedDefaultEncoding)
{
// Despite 8.5 "Text/xml with Omitted Charset" of RFC 3023, we assume UTF-8 instead of US-ASCII
// for text/xml. This matches Firefox.
if (contentType == XML)
return UTF8Encoding();
if (!specifiedDefaultEncoding.isValid())
return Latin1Encoding();
return specifiedDefaultEncoding;
}
TextResourceDecoder::TextResourceDecoder(const String& mimeType, const TextEncoding& specifiedDefaultEncoding, bool usesEncodingDetector)
: m_contentType(determineContentType(mimeType))
, m_encoding(defaultEncoding(m_contentType, specifiedDefaultEncoding))
, m_source(DefaultEncoding)
, m_hintEncoding(0)
, m_checkedForBOM(false)
, m_checkedForCSSCharset(false)
, m_checkedForHeadCharset(false)
, m_useLenientXMLDecoding(false)
, m_sawError(false)
, m_usesEncodingDetector(usesEncodingDetector)
{
}
TextResourceDecoder::~TextResourceDecoder()
{
}
void TextResourceDecoder::setEncoding(const TextEncoding& encoding, EncodingSource source)
{
// In case the encoding didn't exist, we keep the old one (helps some sites specifying invalid encodings).
if (!encoding.isValid())
return;
// When encoding comes from meta tag (i.e. it cannot be XML files sent via XHR),
// treat x-user-defined as windows-1252 (bug 18270)
if (source == EncodingFromMetaTag && strcasecmp(encoding.name(), "x-user-defined") == 0)
m_encoding = "windows-1252";
else if (source == EncodingFromMetaTag || source == EncodingFromXMLHeader || source == EncodingFromCSSCharset)
m_encoding = encoding.closestByteBasedEquivalent();
else
m_encoding = encoding;
m_codec.clear();
m_source = source;
}
// Returns the position of the encoding string.
static int findXMLEncoding(const char* str, int len, int& encodingLength)
{
int pos = find(str, len, "encoding");
if (pos == -1)
return -1;
pos += 8;
// Skip spaces and stray control characters.
while (pos < len && str[pos] <= ' ')
++pos;
// Skip equals sign.
if (pos >= len || str[pos] != '=')
return -1;
++pos;
// Skip spaces and stray control characters.
while (pos < len && str[pos] <= ' ')
++pos;
// Skip quotation mark.
if (pos >= len)
return - 1;
char quoteMark = str[pos];
if (quoteMark != '"' && quoteMark != '\'')
return -1;
++pos;
// Find the trailing quotation mark.
int end = pos;
while (end < len && str[end] != quoteMark)
++end;
if (end >= len)
return -1;
encodingLength = end - pos;
return pos;
}
// true if there is more to parse
static inline bool skipWhitespace(const char*& pos, const char* dataEnd)
{
while (pos < dataEnd && (*pos == '\t' || *pos == ' '))
++pos;
return pos != dataEnd;
}
size_t TextResourceDecoder::checkForBOM(const char* data, size_t len)
{
// Check for UTF-16/32 or UTF-8 BOM mark at the beginning, which is a sure sign of a Unicode encoding.
// We let it override even a user-chosen encoding.
ASSERT(!m_checkedForBOM);
size_t lengthOfBOM = 0;
size_t bufferLength = m_buffer.size();
size_t buf1Len = bufferLength;
size_t buf2Len = len;
const unsigned char* buf1 = reinterpret_cast<const unsigned char*>(m_buffer.data());
const unsigned char* buf2 = reinterpret_cast<const unsigned char*>(data);
unsigned char c1 = buf1Len ? (--buf1Len, *buf1++) : buf2Len ? (--buf2Len, *buf2++) : 0;
unsigned char c2 = buf1Len ? (--buf1Len, *buf1++) : buf2Len ? (--buf2Len, *buf2++) : 0;
unsigned char c3 = buf1Len ? (--buf1Len, *buf1++) : buf2Len ? (--buf2Len, *buf2++) : 0;
unsigned char c4 = buf2Len ? (--buf2Len, *buf2++) : 0;
// Check for the BOM.
if (c1 == 0xFF && c2 == 0xFE) {
if (c3 != 0 || c4 != 0) {
setEncoding(UTF16LittleEndianEncoding(), AutoDetectedEncoding);
lengthOfBOM = 2;
} else {
setEncoding(UTF32LittleEndianEncoding(), AutoDetectedEncoding);
lengthOfBOM = 4;
}
} else if (c1 == 0xEF && c2 == 0xBB && c3 == 0xBF) {
setEncoding(UTF8Encoding(), AutoDetectedEncoding);
lengthOfBOM = 3;
} else if (c1 == 0xFE && c2 == 0xFF) {
setEncoding(UTF16BigEndianEncoding(), AutoDetectedEncoding);
lengthOfBOM = 2;
} else if (c1 == 0 && c2 == 0 && c3 == 0xFE && c4 == 0xFF) {
setEncoding(UTF32BigEndianEncoding(), AutoDetectedEncoding);
lengthOfBOM = 4;
}
if (lengthOfBOM || bufferLength + len >= 4)
m_checkedForBOM = true;
return lengthOfBOM;
}
bool TextResourceDecoder::checkForCSSCharset(const char* data, size_t len, bool& movedDataToBuffer)
{
if (m_source != DefaultEncoding && m_source != EncodingFromParentFrame) {
m_checkedForCSSCharset = true;
return true;
}
size_t oldSize = m_buffer.size();
m_buffer.grow(oldSize + len);
memcpy(m_buffer.data() + oldSize, data, len);
movedDataToBuffer = true;
if (m_buffer.size() > 8) { // strlen("@charset") == 8
const char* dataStart = m_buffer.data();
const char* dataEnd = dataStart + m_buffer.size();
if (dataStart[0] == '@' && dataStart[1] == 'c' && dataStart[2] == 'h' && dataStart[3] == 'a' && dataStart[4] == 'r' &&
dataStart[5] == 's' && dataStart[6] == 'e' && dataStart[7] == 't') {
dataStart += 8;
const char* pos = dataStart;
if (!skipWhitespace(pos, dataEnd))
return false;
if (*pos == '"' || *pos == '\'') {
char quotationMark = *pos;
++pos;
dataStart = pos;
while (pos < dataEnd && *pos != quotationMark)
++pos;
if (pos == dataEnd)
return false;
int encodingNameLength = pos - dataStart + 1;
++pos;
if (!skipWhitespace(pos, dataEnd))
return false;
if (*pos == ';')
setEncoding(findTextEncoding(dataStart, encodingNameLength), EncodingFromCSSCharset);
}
}
m_checkedForCSSCharset = true;
return true;
}
return false;
}
// Other browsers allow comments in the head section, so we need to also.
// It's important not to look for tags inside the comments.
static inline void skipComment(const char*& ptr, const char* pEnd)
{
const char* p = ptr;
if (p == pEnd)
return;
// Allow <!-->; other browsers do.
if (*p == '>') {
p++;
} else {
while (p + 2 < pEnd) {
if (*p == '-') {
// This is the real end of comment, "-->".
if (p[1] == '-' && p[2] == '>') {
p += 3;
break;
}
// This is the incorrect end of comment that other browsers allow, "--!>".
if (p + 3 < pEnd && p[1] == '-' && p[2] == '!' && p[3] == '>') {
p += 4;
break;
}
}
p++;
}
}
ptr = p;
}
const int bytesToCheckUnconditionally = 1024; // That many input bytes will be checked for meta charset even if <head> section is over.
bool TextResourceDecoder::checkForHeadCharset(const char* data, size_t len, bool& movedDataToBuffer)
{
if (m_source != DefaultEncoding && m_source != EncodingFromParentFrame) {
m_checkedForHeadCharset = true;
return true;
}
// This is not completely efficient, since the function might go
// through the HTML head several times.
size_t oldSize = m_buffer.size();
m_buffer.grow(oldSize + len);
memcpy(m_buffer.data() + oldSize, data, len);
movedDataToBuffer = true;
const char* ptr = m_buffer.data();
const char* pEnd = ptr + m_buffer.size();
// Is there enough data available to check for XML declaration?
if (m_buffer.size() < 8)
return false;
// Handle XML declaration, which can have encoding in it. This encoding is honored even for HTML documents.
// It is an error for an XML declaration not to be at the start of an XML document, and it is ignored in HTML documents in such case.
if (ptr[0] == '<' && ptr[1] == '?' && ptr[2] == 'x' && ptr[3] == 'm' && ptr[4] == 'l') {
const char* xmlDeclarationEnd = ptr;
while (xmlDeclarationEnd != pEnd && *xmlDeclarationEnd != '>')
++xmlDeclarationEnd;
if (xmlDeclarationEnd == pEnd)
return false;
// No need for +1, because we have an extra "?" to lose at the end of XML declaration.
int len;
int pos = findXMLEncoding(ptr, xmlDeclarationEnd - ptr, len);
if (pos != -1)
setEncoding(findTextEncoding(ptr + pos, len), EncodingFromXMLHeader);
// continue looking for a charset - it may be specified in an HTTP-Equiv meta
} else if (ptr[0] == '<' && ptr[1] == 0 && ptr[2] == '?' && ptr[3] == 0 && ptr[4] == 'x' && ptr[5] == 0) {
setEncoding(UTF16LittleEndianEncoding(), AutoDetectedEncoding);
return true;
} else if (ptr[0] == 0 && ptr[1] == '<' && ptr[2] == 0 && ptr[3] == '?' && ptr[4] == 0 && ptr[5] == 'x') {
setEncoding(UTF16BigEndianEncoding(), AutoDetectedEncoding);
return true;
} else if (ptr[0] == '<' && ptr[1] == 0 && ptr[2] == 0 && ptr[3] == 0 && ptr[4] == '?' && ptr[5] == 0 && ptr[6] == 0 && ptr[7] == 0) {
setEncoding(UTF32LittleEndianEncoding(), AutoDetectedEncoding);
return true;
} else if (ptr[0] == 0 && ptr[1] == 0 && ptr[2] == 0 && ptr[3] == '<' && ptr[4] == 0 && ptr[5] == 0 && ptr[6] == 0 && ptr[7] == '?') {
setEncoding(UTF32BigEndianEncoding(), AutoDetectedEncoding);
return true;
}
// we still don't have an encoding, and are in the head
// the following tags are allowed in <head>:
// SCRIPT|STYLE|META|LINK|OBJECT|TITLE|BASE
// We stop scanning when a tag that is not permitted in <head>
// is seen, rather when </head> is seen, because that more closely
// matches behavior in other browsers; more details in
// <http://bugs.webkit.org/show_bug.cgi?id=3590>.
// Additionally, we ignore things that looks like tags in <title>, <script> and <noscript>; see
// <http://bugs.webkit.org/show_bug.cgi?id=4560>, <http://bugs.webkit.org/show_bug.cgi?id=12165>
// and <http://bugs.webkit.org/show_bug.cgi?id=12389>.
// Since many sites have charset declarations after <body> or other tags that are disallowed in <head>,
// we don't bail out until we've checked at least bytesToCheckUnconditionally bytes of input.
AtomicStringImpl* enclosingTagName = 0;
bool inHeadSection = true; // Becomes false when </head> or any tag not allowed in head is encountered.
// the HTTP-EQUIV meta has no effect on XHTML
if (m_contentType == XML)
return true;
while (ptr + 3 < pEnd) { // +3 guarantees that "<!--" fits in the buffer - and certainly we aren't going to lose any "charset" that way.
if (*ptr == '<') {
bool end = false;
ptr++;
// Handle comments.
if (ptr[0] == '!' && ptr[1] == '-' && ptr[2] == '-') {
ptr += 3;
skipComment(ptr, pEnd);
if (ptr - m_buffer.data() >= bytesToCheckUnconditionally && !inHeadSection) {
// Some pages that test bandwidth from within the browser do it by having
// huge comments and measuring the time they take to load. Repeatedly scanning
// these comments can take a lot of CPU time.
m_checkedForHeadCharset = true;
return true;
}
continue;
}
if (*ptr == '/') {
++ptr;
end = true;
}
// Grab the tag name, but mostly ignore namespaces.
bool sawNamespace = false;
char tagBuffer[20];
int len = 0;
while (len < 19) {
if (ptr == pEnd)
return false;
char c = *ptr;
if (c == ':') {
len = 0;
sawNamespace = true;
ptr++;
continue;
}
if ((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9'))
;
else if (c >= 'A' && c <= 'Z')
c += 'a' - 'A';
else
break;
tagBuffer[len++] = c;
ptr++;
}
tagBuffer[len] = 0;
AtomicString tag(tagBuffer);
if (enclosingTagName) {
if (end && tag.impl() == enclosingTagName)
enclosingTagName = 0;
} else {
if (tag == titleTag)
enclosingTagName = titleTag.localName().impl();
else if (tag == scriptTag)
enclosingTagName = scriptTag.localName().impl();
else if (tag == noscriptTag)
enclosingTagName = noscriptTag.localName().impl();
}
// Find where the opening tag ends.
const char* tagContentStart = ptr;
if (!end) {
while (ptr != pEnd && *ptr != '>') {
if (*ptr == '\'' || *ptr == '"') {
char quoteMark = *ptr;
++ptr;
while (ptr != pEnd && *ptr != quoteMark)
++ptr;
if (ptr == pEnd)
return false;
}
++ptr;
}
if (ptr == pEnd)
return false;
++ptr;
}
if (!end && tag == metaTag && !sawNamespace) {
const char* str = tagContentStart;
int length = ptr - tagContentStart;
int pos = 0;
while (pos < length) {
int charsetPos = findIgnoringCase(str + pos, length - pos, "charset");
if (charsetPos == -1)
break;
pos += charsetPos + 7;
// skip whitespace
while (pos < length && str[pos] <= ' ')
pos++;
if (pos == length)
break;
if (str[pos++] != '=')
continue;
while ((pos < length) &&
(str[pos] <= ' ' || str[pos] == '=' || str[pos] == '"' || str[pos] == '\''))
pos++;
// end ?
if (pos == length)
break;
int end = pos;
while (end < length &&
str[end] != ' ' && str[end] != '"' && str[end] != '\'' &&
str[end] != ';' && str[end] != '>')
end++;
setEncoding(findTextEncoding(str + pos, end - pos), EncodingFromMetaTag);
if (m_source == EncodingFromMetaTag)
return true;
if (end >= length || str[end] == '/' || str[end] == '>')
break;
pos = end + 1;
}
} else {
if (!enclosingTagName && tag != scriptTag && tag != noscriptTag && tag != styleTag
&& tag != linkTag && tag != metaTag && tag != objectTag && tag != titleTag && tag != baseTag
&& (end || tag != htmlTag) && (end || tag != headTag) && isASCIIAlpha(tagBuffer[0])) {
inHeadSection = false;
}
if (ptr - m_buffer.data() >= bytesToCheckUnconditionally && !inHeadSection) {
m_checkedForHeadCharset = true;
return true;
}
}
} else
++ptr;
}
return false;
}
void TextResourceDecoder::detectJapaneseEncoding(const char* data, size_t len)
{
switch (KanjiCode::judge(data, len)) {
case KanjiCode::JIS:
setEncoding("ISO-2022-JP", AutoDetectedEncoding);
break;
case KanjiCode::EUC:
setEncoding("EUC-JP", AutoDetectedEncoding);
break;
case KanjiCode::SJIS:
setEncoding("Shift_JIS", AutoDetectedEncoding);
break;
case KanjiCode::ASCII:
case KanjiCode::UTF16:
case KanjiCode::UTF8:
break;
}
}
// We use the encoding detector in two cases:
// 1. Encoding detector is turned ON and no other encoding source is
// available (that is, it's DefaultEncoding).
// 2. Encoding detector is turned ON and the encoding is set to
// the encoding of the parent frame, which is also auto-detected.
// Note that condition #2 is NOT satisfied unless parent-child frame
// relationship is compliant to the same-origin policy. If they're from
// different domains, |m_source| would not be set to EncodingFromParentFrame
// in the first place.
bool TextResourceDecoder::shouldAutoDetect() const
{
// Just checking m_hintEncoding suffices here because it's only set
// in setHintEncoding when the source is AutoDetectedEncoding.
return m_usesEncodingDetector
&& (m_source == DefaultEncoding || (m_source == EncodingFromParentFrame && m_hintEncoding));
}
String TextResourceDecoder::decode(const char* data, size_t len)
{
size_t lengthOfBOM = 0;
if (!m_checkedForBOM)
lengthOfBOM = checkForBOM(data, len);
bool movedDataToBuffer = false;
if (m_contentType == CSS && !m_checkedForCSSCharset)
if (!checkForCSSCharset(data, len, movedDataToBuffer))
return "";
if ((m_contentType == HTML || m_contentType == XML) && !m_checkedForHeadCharset) // HTML and XML
if (!checkForHeadCharset(data, len, movedDataToBuffer))
return "";
// FIXME: It is wrong to change the encoding downstream after we have already done some decoding.
if (shouldAutoDetect()) {
if (m_encoding.isJapanese())
detectJapaneseEncoding(data, len); // FIXME: We should use detectTextEncoding() for all languages.
else {
TextEncoding detectedEncoding;
if (detectTextEncoding(data, len, m_hintEncoding, &detectedEncoding))
setEncoding(detectedEncoding, AutoDetectedEncoding);
}
}
ASSERT(m_encoding.isValid());
if (!m_codec)
m_codec.set(newTextCodec(m_encoding).release());
if (m_buffer.isEmpty())
return m_codec->decode(data + lengthOfBOM, len - lengthOfBOM, false, m_contentType == XML, m_sawError);
if (!movedDataToBuffer) {
size_t oldSize = m_buffer.size();
m_buffer.grow(oldSize + len);
memcpy(m_buffer.data() + oldSize, data, len);
}
String result = m_codec->decode(m_buffer.data() + lengthOfBOM, m_buffer.size() - lengthOfBOM, false, m_contentType == XML && !m_useLenientXMLDecoding, m_sawError);
m_buffer.clear();
return result;
}
String TextResourceDecoder::flush()
{
// If we can not identify the encoding even after a document is completely
// loaded, we need to detect the encoding if other conditions for
// autodetection is satisfied.
if (m_buffer.size() && shouldAutoDetect()
&& ((!m_checkedForHeadCharset && (m_contentType == HTML || m_contentType == XML)) || (!m_checkedForCSSCharset && (m_contentType == CSS)))) {
TextEncoding detectedEncoding;
if (detectTextEncoding(m_buffer.data(), m_buffer.size(),
m_hintEncoding, &detectedEncoding))
setEncoding(detectedEncoding, AutoDetectedEncoding);
}
if (!m_codec)
m_codec.set(newTextCodec(m_encoding).release());
String result = m_codec->decode(m_buffer.data(), m_buffer.size(), true, m_contentType == XML && !m_useLenientXMLDecoding, m_sawError);
m_buffer.clear();
m_codec.clear();
m_checkedForBOM = false; // Skip BOM again when re-decoding.
return result;
}
}