void CXML_Parser::SkipLiterals(const CFX_ByteStringC& str) {
m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex;
if (IsEOF()) {
return;
}
int32_t i = 0, iLen = str.GetLength();
do {
while (m_dwIndex < m_dwBufferSize) {
if (str.GetAt(i) != m_pBuffer[m_dwIndex++]) {
i = 0;
} else {
i++;
if (i == iLen) {
break;
}
}
}
m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex;
if (i == iLen) {
return;
}
if (m_dwIndex < m_dwBufferSize || IsEOF()) {
break;
}
} while (ReadNextBlock());
while (!m_pDataAcc->IsEOF()) {
ReadNextBlock();
m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwBufferSize;
}
m_dwIndex = m_dwBufferSize;
}
void CXML_Parser::GetName(CFX_ByteString& space, CFX_ByteString& name) {
m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex;
if (IsEOF()) {
return;
}
CFX_ByteTextBuf buf;
uint8_t ch;
do {
while (m_dwIndex < m_dwBufferSize) {
ch = m_pBuffer[m_dwIndex];
if (ch == ':') {
space = buf.AsStringC();
buf.Clear();
} else if (g_FXCRT_XML_IsNameChar(ch)) {
buf.AppendChar(ch);
} else {
break;
}
m_dwIndex++;
}
m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex;
if (m_dwIndex < m_dwBufferSize || IsEOF()) {
break;
}
} while (ReadNextBlock());
name = buf.AsStringC();
}
void DataProvider::OnOpenFile(Readers::iterator reader_iter, bool result) {
if (!result) {
DeleteReader(reader_iter);
return;
}
Reader* reader = reader_iter->get();
reader->state = ReaderState::kIdle;
consumer_->OnDataConsumerFileOpened(reader->file_id, reader->file_path);
ReadNextBlock();
}
void CXML_Parser::GetTagName(CFX_ByteString& space,
CFX_ByteString& name,
bool& bEndTag,
bool bStartTag) {
m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex;
if (IsEOF()) {
return;
}
bEndTag = false;
uint8_t ch;
int32_t iState = bStartTag ? 1 : 0;
do {
while (m_dwIndex < m_dwBufferSize) {
ch = m_pBuffer[m_dwIndex];
switch (iState) {
case 0:
m_dwIndex++;
if (ch != '<') {
break;
}
iState = 1;
break;
case 1:
if (ch == '?') {
m_dwIndex++;
SkipLiterals("?>");
iState = 0;
break;
} else if (ch == '!') {
m_dwIndex++;
SkipLiterals("-->");
iState = 0;
break;
}
if (ch == '/') {
m_dwIndex++;
GetName(space, name);
bEndTag = true;
} else {
GetName(space, name);
bEndTag = false;
}
return;
}
}
m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex;
if (m_dwIndex < m_dwBufferSize || IsEOF()) {
break;
}
} while (ReadNextBlock());
}
void CXML_Parser::GetTagName(CFX_ByteString &space, CFX_ByteString &name, FX_BOOL &bEndTag, FX_BOOL bStartTag)
{
m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex;
if (IsEOF()) {
return;
}
bEndTag = FALSE;
FX_BYTE ch;
FX_INT32 iState = bStartTag ? 1 : 0;
do {
while (m_dwIndex < m_dwBufferSize) {
ch = m_pBuffer[m_dwIndex];
switch (iState) {
case 0:
m_dwIndex ++;
if (ch != '<') {
break;
}
iState = 1;
break;
case 1:
if (ch == '?') {
m_dwIndex ++;
SkipLiterals(FX_BSTRC("?>"));
iState = 0;
break;
} else if (ch == '!') {
m_dwIndex ++;
SkipLiterals(FX_BSTRC("-->"));
iState = 0;
break;
}
if (ch == '/') {
m_dwIndex ++;
GetName(space, name);
bEndTag = TRUE;
} else {
GetName(space, name);
bEndTag = FALSE;
}
return;
}
}
m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex;
if (m_dwIndex < m_dwBufferSize || IsEOF()) {
break;
}
} while (ReadNextBlock());
}
void CXML_Parser::SkipWhiteSpaces() {
m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex;
if (IsEOF()) {
return;
}
do {
while (m_dwIndex < m_dwBufferSize &&
g_FXCRT_XML_IsWhiteSpace(m_pBuffer[m_dwIndex])) {
m_dwIndex++;
}
m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex;
if (m_dwIndex < m_dwBufferSize || IsEOF()) {
break;
}
} while (ReadNextBlock());
}
void DataProvider::OnReadFile(Readers::iterator reader_iter,
IOBuffer* io_buffer,
FileReaderResult read_result) {
Reader* reader = reader_iter->get();
io_buffer->is_busy = false;
if (read_result.status != FileReaderStatus::kSuccess) {
reader->state = ReaderState::kClosing;
reader->file_reader->Close(
std::bind(&DataProvider::OnCloseFile, this, reader_iter));
} else {
reader->state = ReaderState::kIdle;
reader->read_offset += read_result.size;
consumer_->OnDataConsumerBlockRead(reader->file_id, reader->file_path,
io_buffer->buffer.data(),
read_result.size);
}
ReadNextBlock();
}
void CXML_Parser::GetAttrValue(CFX_WideString &value)
{
m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex;
if (IsEOF()) {
return;
}
CFX_UTF8Decoder decoder;
uint8_t mark = 0, ch = 0;
do {
while (m_dwIndex < m_dwBufferSize) {
ch = m_pBuffer[m_dwIndex];
if (mark == 0) {
if (ch != '\'' && ch != '"') {
return;
}
mark = ch;
m_dwIndex ++;
ch = 0;
continue;
}
m_dwIndex ++;
if (ch == mark) {
break;
}
if (ch == '&') {
decoder.AppendChar(GetCharRef());
if (IsEOF()) {
value = decoder.GetResult();
return;
}
} else {
decoder.Input(ch);
}
}
m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex;
if (ch == mark || m_dwIndex < m_dwBufferSize || IsEOF()) {
break;
}
} while (ReadNextBlock());
value = decoder.GetResult();
}
size_t Lz4StreamReader::Read(void* buffer, size_t length)
{
uint32_t out = 0; // Bytes returned to caller
#ifdef _WIN64
if(length > UINT32_MAX) throw Exception(E_INVALIDARG);
#endif
if(length == 0) return 0; // Nothing to do
// Read uncompressed data into the output buffer until either
// the specified amount has been read or the stream ends
while(length > 0) {
// Take the smaller of what we have and what we still need
uint32_t next = std::min(m_blockremain, static_cast<uint32_t>(length));
if(next) {
// The buffer pointer can be NULL to just skip over data
if(buffer) memcpy(buffer, m_blockcurrent, next);
// Move the block data pointer
m_blockcurrent += next;
m_blockremain -= next;
// Move the output buffer pointer
if(buffer) buffer = reinterpret_cast<uint8_t*>(buffer) + next;
length -= next;
out += next; // Wrote this many output bytes
}
// If there is no more block data, decompress the next block; if
// there is no more data in the stream, we're done
if((m_blockremain == 0) && (ReadNextBlock() == 0)) break;
}
m_position += out; // Increment the current stream position
return out; // Return number of bytes written
}
CXML_Element* CXML_Parser::ParseElement(CXML_Element* pParent, bool bStartTag) {
m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex;
if (IsEOF()) {
return nullptr;
}
CFX_ByteString tag_name, tag_space;
bool bEndTag;
GetTagName(tag_space, tag_name, bEndTag, bStartTag);
if (tag_name.IsEmpty() || bEndTag) {
return nullptr;
}
CXML_Element* pElement = new CXML_Element;
pElement->m_pParent = pParent;
pElement->SetTag(tag_space.AsStringC(), tag_name.AsStringC());
do {
CFX_ByteString attr_space, attr_name;
while (m_dwIndex < m_dwBufferSize) {
SkipWhiteSpaces();
if (IsEOF()) {
break;
}
if (!g_FXCRT_XML_IsNameIntro(m_pBuffer[m_dwIndex])) {
break;
}
GetName(attr_space, attr_name);
SkipWhiteSpaces();
if (IsEOF()) {
break;
}
if (m_pBuffer[m_dwIndex] != '=') {
break;
}
m_dwIndex++;
SkipWhiteSpaces();
if (IsEOF()) {
break;
}
CFX_WideString attr_value;
GetAttrValue(attr_value);
pElement->m_AttrMap.SetAt(attr_space, attr_name, attr_value);
}
m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex;
if (m_dwIndex < m_dwBufferSize || IsEOF()) {
break;
}
} while (ReadNextBlock());
SkipWhiteSpaces();
if (IsEOF()) {
return pElement;
}
uint8_t ch = m_pBuffer[m_dwIndex++];
if (ch == '/') {
m_dwIndex++;
m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex;
return pElement;
}
if (ch != '>') {
m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex;
delete pElement;
return nullptr;
}
SkipWhiteSpaces();
if (IsEOF()) {
return pElement;
}
CFX_UTF8Decoder decoder;
CFX_WideTextBuf content;
bool bCDATA = false;
int32_t iState = 0;
do {
while (m_dwIndex < m_dwBufferSize) {
ch = m_pBuffer[m_dwIndex++];
switch (iState) {
case 0:
if (ch == '<') {
iState = 1;
} else if (ch == '&') {
decoder.ClearStatus();
decoder.AppendChar(GetCharRef());
} else {
decoder.Input(ch);
}
break;
case 1:
if (ch == '!') {
iState = 2;
} else if (ch == '?') {
SkipLiterals("?>");
SkipWhiteSpaces();
iState = 0;
} else if (ch == '/') {
CFX_ByteString space, name;
GetName(space, name);
SkipWhiteSpaces();
m_dwIndex++;
iState = 10;
} else {
content << decoder.GetResult();
CFX_WideString dataStr = content.MakeString();
if (!bCDATA && !m_bSaveSpaceChars) {
dataStr.TrimRight(L" \t\r\n");
}
InsertContentSegment(bCDATA, dataStr.AsStringC(), pElement);
content.Clear();
decoder.Clear();
bCDATA = false;
iState = 0;
m_dwIndex--;
CXML_Element* pSubElement = ParseElement(pElement, true);
if (!pSubElement) {
break;
}
pSubElement->m_pParent = pElement;
pElement->m_Children.push_back(
{CXML_Element::Element, pSubElement});
SkipWhiteSpaces();
}
break;
case 2:
if (ch == '[') {
SkipLiterals("]]>");
} else if (ch == '-') {
m_dwIndex++;
SkipLiterals("-->");
} else {
SkipLiterals(">");
}
decoder.Clear();
SkipWhiteSpaces();
iState = 0;
break;
}
if (iState == 10) {
break;
}
}
m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex;
if (iState == 10 || m_dwIndex < m_dwBufferSize || IsEOF()) {
break;
}
} while (ReadNextBlock());
content << decoder.GetResult();
CFX_WideString dataStr = content.MakeString();
if (!m_bSaveSpaceChars) {
dataStr.TrimRight(L" \t\r\n");
}
InsertContentSegment(bCDATA, dataStr.AsStringC(), pElement);
content.Clear();
decoder.Clear();
bCDATA = false;
return pElement;
}
uint32_t CXML_Parser::GetCharRef() {
m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex;
if (IsEOF()) {
return 0;
}
uint8_t ch;
int32_t iState = 0;
CFX_ByteTextBuf buf;
uint32_t code = 0;
do {
while (m_dwIndex < m_dwBufferSize) {
ch = m_pBuffer[m_dwIndex];
switch (iState) {
case 0:
if (ch == '#') {
m_dwIndex++;
iState = 2;
break;
}
iState = 1;
case 1:
m_dwIndex++;
if (ch == ';') {
CFX_ByteStringC ref = buf.AsStringC();
if (ref == "gt") {
code = '>';
} else if (ref == "lt") {
code = '<';
} else if (ref == "amp") {
code = '&';
} else if (ref == "apos") {
code = '\'';
} else if (ref == "quot") {
code = '"';
}
iState = 10;
break;
}
buf.AppendByte(ch);
break;
case 2:
if (ch == 'x') {
m_dwIndex++;
iState = 4;
break;
}
iState = 3;
case 3:
m_dwIndex++;
if (ch == ';') {
iState = 10;
break;
}
if (g_FXCRT_XML_IsDigital(ch))
code = code * 10 + FXSYS_toDecimalDigit(static_cast<FX_WCHAR>(ch));
break;
case 4:
m_dwIndex++;
if (ch == ';') {
iState = 10;
break;
}
uint8_t nHex =
g_FXCRT_XML_ByteTypes[ch] & FXCRTM_XML_CHARTYPE_HexChar;
if (nHex) {
if (nHex == FXCRTM_XML_CHARTYPE_HexDigital) {
code =
(code << 4) + FXSYS_toDecimalDigit(static_cast<FX_WCHAR>(ch));
} else if (nHex == FXCRTM_XML_CHARTYPE_HexLowerLetter) {
code = (code << 4) + ch - 87;
} else {
code = (code << 4) + ch - 55;
}
}
break;
}
if (iState == 10) {
break;
}
}
m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex;
if (iState == 10 || m_dwIndex < m_dwBufferSize || IsEOF()) {
break;
}
} while (ReadNextBlock());
return code;
}
bool CXML_Parser::Init(bool bOwndedStream) {
m_bOwnedStream = bOwndedStream;
m_nOffset = 0;
return ReadNextBlock();
}
FX_BOOL CXML_Parser::Init(FX_BOOL bOwndedStream)
{
m_bOwnedStream = bOwndedStream;
m_nOffset = 0;
return ReadNextBlock();
}
CXML_Element* CXML_Parser::ParseElement(CXML_Element* pParent, FX_BOOL bStartTag)
{
m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex;
if (IsEOF()) {
return NULL;
}
CFX_ByteString tag_name, tag_space;
FX_BOOL bEndTag;
GetTagName(tag_space, tag_name, bEndTag, bStartTag);
if (tag_name.IsEmpty() || bEndTag) {
return NULL;
}
CXML_Element* pElement;
pElement = FX_NEW CXML_Element;
if (pElement) {
pElement->m_pParent = pParent;
pElement->SetTag(tag_space, tag_name);
}
if (!pElement) {
return NULL;
}
do {
CFX_ByteString attr_space, attr_name;
while (m_dwIndex < m_dwBufferSize) {
SkipWhiteSpaces();
if (IsEOF()) {
break;
}
if (!g_FXCRT_XML_IsNameIntro(m_pBuffer[m_dwIndex])) {
break;
}
GetName(attr_space, attr_name);
SkipWhiteSpaces();
if (IsEOF()) {
break;
}
if (m_pBuffer[m_dwIndex] != '=') {
break;
}
m_dwIndex ++;
SkipWhiteSpaces();
if (IsEOF()) {
break;
}
CFX_WideString attr_value;
GetAttrValue(attr_value);
pElement->m_AttrMap.SetAt(attr_space, attr_name, attr_value);
}
m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex;
if (m_dwIndex < m_dwBufferSize || IsEOF()) {
break;
}
} while (ReadNextBlock());
SkipWhiteSpaces();
if (IsEOF()) {
return pElement;
}
FX_BYTE ch = m_pBuffer[m_dwIndex ++];
if (ch == '/') {
m_dwIndex ++;
m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex;
return pElement;
}
if (ch != '>') {
m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex;
delete pElement;
return NULL;
}
SkipWhiteSpaces();
if (IsEOF()) {
return pElement;
}
CFX_UTF8Decoder decoder;
CFX_WideTextBuf content;
FX_BOOL bCDATA = FALSE;
FX_INT32 iState = 0;
do {
while (m_dwIndex < m_dwBufferSize) {
ch = m_pBuffer[m_dwIndex ++];
switch (iState) {
case 0:
if (ch == '<') {
iState = 1;
} else if (ch == '&') {
decoder.ClearStatus();
decoder.AppendChar(GetCharRef());
} else {
decoder.Input(ch);
}
break;
case 1:
if (ch == '!') {
iState = 2;
} else if (ch == '?') {
SkipLiterals(FX_BSTRC("?>"));
SkipWhiteSpaces();
iState = 0;
} else if (ch == '/') {
CFX_ByteString space, name;
GetName(space, name);
SkipWhiteSpaces();
m_dwIndex ++;
iState = 10;
} else {
content << decoder.GetResult();
CFX_WideString dataStr = content.GetWideString();
if (!bCDATA && !m_bSaveSpaceChars) {
dataStr.TrimRight((FX_LPCWSTR)L" \t\r\n");
}
InsertContentSegment(bCDATA, dataStr, pElement);
content.Clear();
decoder.Clear();
bCDATA = FALSE;
iState = 0;
m_dwIndex --;
CXML_Element* pSubElement = ParseElement(pElement, TRUE);
if (pSubElement == NULL) {
break;
}
pSubElement->m_pParent = pElement;
pElement->m_Children.Add((FX_LPVOID)CXML_Element::Element);
pElement->m_Children.Add(pSubElement);
SkipWhiteSpaces();
}
break;
case 2:
if (ch == '[') {
SkipLiterals(FX_BSTRC("]]>"));
} else if (ch == '-') {
m_dwIndex ++;
SkipLiterals(FX_BSTRC("-->"));
} else {
SkipLiterals(FX_BSTRC(">"));
}
decoder.Clear();
SkipWhiteSpaces();
iState = 0;
break;
}
if (iState == 10) {
break;
}
}
m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex;
if (iState == 10 || m_dwIndex < m_dwBufferSize || IsEOF()) {
break;
}
} while (ReadNextBlock());
content << decoder.GetResult();
CFX_WideString dataStr = content.GetWideString();
if (!m_bSaveSpaceChars) {
dataStr.TrimRight((FX_LPCWSTR)L" \t\r\n");
}
InsertContentSegment(bCDATA, dataStr, pElement);
content.Clear();
decoder.Clear();
bCDATA = FALSE;
return pElement;
}
FX_DWORD CXML_Parser::GetCharRef()
{
m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex;
if (IsEOF()) {
return 0;
}
FX_BYTE ch;
FX_INT32 iState = 0;
CFX_ByteTextBuf buf;
FX_DWORD code = 0;
do {
while (m_dwIndex < m_dwBufferSize) {
ch = m_pBuffer[m_dwIndex];
switch (iState) {
case 0:
if (ch == '#') {
m_dwIndex ++;
iState = 2;
break;
}
iState = 1;
case 1:
m_dwIndex ++;
if (ch == ';') {
CFX_ByteStringC ref = buf.GetByteString();
if (ref == FX_BSTRC("gt")) {
code = '>';
} else if (ref == FX_BSTRC("lt")) {
code = '<';
} else if (ref == FX_BSTRC("amp")) {
code = '&';
} else if (ref == FX_BSTRC("apos")) {
code = '\'';
} else if (ref == FX_BSTRC("quot")) {
code = '"';
}
iState = 10;
break;
}
buf.AppendByte(ch);
break;
case 2:
if (ch == 'x') {
m_dwIndex ++;
iState = 4;
break;
}
iState = 3;
case 3:
m_dwIndex ++;
if (ch == ';') {
iState = 10;
break;
}
if (g_FXCRT_XML_IsDigital(ch)) {
code = code * 10 + ch - '0';
}
break;
case 4:
m_dwIndex ++;
if (ch == ';') {
iState = 10;
break;
}
FX_BYTE nHex = g_FXCRT_XML_ByteTypes[ch] & FXCRTM_XML_CHARTYPE_HexChar;
if (nHex) {
if (nHex == FXCRTM_XML_CHARTYPE_HexDigital) {
code = (code << 4) + ch - '0';
} else if (nHex == FXCRTM_XML_CHARTYPE_HexLowerLetter) {
code = (code << 4) + ch - 87;
} else {
code = (code << 4) + ch - 55;
}
}
break;
}
if (iState == 10) {
break;
}
}
m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex;
if (iState == 10 || m_dwIndex < m_dwBufferSize || IsEOF()) {
break;
}
} while (ReadNextBlock());
return code;
}
void DataProvider::Resume() {
if (!is_suspended_)
return;
is_suspended_ = false;
ReadNextBlock();
}
