void CXML_Parser::SkipLiterals(const CFX_ByteStringC& str) { m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex; if (IsEOF()) { return; } int32_t i = 0, iLen = str.GetLength(); do { while (m_dwIndex < m_dwBufferSize) { if (str.GetAt(i) != m_pBuffer[m_dwIndex++]) { i = 0; } else { i++; if (i == iLen) { break; } } } m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex; if (i == iLen) { return; } if (m_dwIndex < m_dwBufferSize || IsEOF()) { break; } } while (ReadNextBlock()); while (!m_pDataAcc->IsEOF()) { ReadNextBlock(); m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwBufferSize; } m_dwIndex = m_dwBufferSize; }
void CXML_Parser::GetName(CFX_ByteString& space, CFX_ByteString& name) { m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex; if (IsEOF()) { return; } CFX_ByteTextBuf buf; uint8_t ch; do { while (m_dwIndex < m_dwBufferSize) { ch = m_pBuffer[m_dwIndex]; if (ch == ':') { space = buf.AsStringC(); buf.Clear(); } else if (g_FXCRT_XML_IsNameChar(ch)) { buf.AppendChar(ch); } else { break; } m_dwIndex++; } m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex; if (m_dwIndex < m_dwBufferSize || IsEOF()) { break; } } while (ReadNextBlock()); name = buf.AsStringC(); }
void DataProvider::OnOpenFile(Readers::iterator reader_iter, bool result) { if (!result) { DeleteReader(reader_iter); return; } Reader* reader = reader_iter->get(); reader->state = ReaderState::kIdle; consumer_->OnDataConsumerFileOpened(reader->file_id, reader->file_path); ReadNextBlock(); }
void CXML_Parser::GetTagName(CFX_ByteString& space, CFX_ByteString& name, bool& bEndTag, bool bStartTag) { m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex; if (IsEOF()) { return; } bEndTag = false; uint8_t ch; int32_t iState = bStartTag ? 1 : 0; do { while (m_dwIndex < m_dwBufferSize) { ch = m_pBuffer[m_dwIndex]; switch (iState) { case 0: m_dwIndex++; if (ch != '<') { break; } iState = 1; break; case 1: if (ch == '?') { m_dwIndex++; SkipLiterals("?>"); iState = 0; break; } else if (ch == '!') { m_dwIndex++; SkipLiterals("-->"); iState = 0; break; } if (ch == '/') { m_dwIndex++; GetName(space, name); bEndTag = true; } else { GetName(space, name); bEndTag = false; } return; } } m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex; if (m_dwIndex < m_dwBufferSize || IsEOF()) { break; } } while (ReadNextBlock()); }
void CXML_Parser::GetTagName(CFX_ByteString &space, CFX_ByteString &name, FX_BOOL &bEndTag, FX_BOOL bStartTag) { m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex; if (IsEOF()) { return; } bEndTag = FALSE; FX_BYTE ch; FX_INT32 iState = bStartTag ? 1 : 0; do { while (m_dwIndex < m_dwBufferSize) { ch = m_pBuffer[m_dwIndex]; switch (iState) { case 0: m_dwIndex ++; if (ch != '<') { break; } iState = 1; break; case 1: if (ch == '?') { m_dwIndex ++; SkipLiterals(FX_BSTRC("?>")); iState = 0; break; } else if (ch == '!') { m_dwIndex ++; SkipLiterals(FX_BSTRC("-->")); iState = 0; break; } if (ch == '/') { m_dwIndex ++; GetName(space, name); bEndTag = TRUE; } else { GetName(space, name); bEndTag = FALSE; } return; } } m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex; if (m_dwIndex < m_dwBufferSize || IsEOF()) { break; } } while (ReadNextBlock()); }
void CXML_Parser::SkipWhiteSpaces() { m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex; if (IsEOF()) { return; } do { while (m_dwIndex < m_dwBufferSize && g_FXCRT_XML_IsWhiteSpace(m_pBuffer[m_dwIndex])) { m_dwIndex++; } m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex; if (m_dwIndex < m_dwBufferSize || IsEOF()) { break; } } while (ReadNextBlock()); }
void DataProvider::OnReadFile(Readers::iterator reader_iter, IOBuffer* io_buffer, FileReaderResult read_result) { Reader* reader = reader_iter->get(); io_buffer->is_busy = false; if (read_result.status != FileReaderStatus::kSuccess) { reader->state = ReaderState::kClosing; reader->file_reader->Close( std::bind(&DataProvider::OnCloseFile, this, reader_iter)); } else { reader->state = ReaderState::kIdle; reader->read_offset += read_result.size; consumer_->OnDataConsumerBlockRead(reader->file_id, reader->file_path, io_buffer->buffer.data(), read_result.size); } ReadNextBlock(); }
void CXML_Parser::GetAttrValue(CFX_WideString &value) { m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex; if (IsEOF()) { return; } CFX_UTF8Decoder decoder; uint8_t mark = 0, ch = 0; do { while (m_dwIndex < m_dwBufferSize) { ch = m_pBuffer[m_dwIndex]; if (mark == 0) { if (ch != '\'' && ch != '"') { return; } mark = ch; m_dwIndex ++; ch = 0; continue; } m_dwIndex ++; if (ch == mark) { break; } if (ch == '&') { decoder.AppendChar(GetCharRef()); if (IsEOF()) { value = decoder.GetResult(); return; } } else { decoder.Input(ch); } } m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex; if (ch == mark || m_dwIndex < m_dwBufferSize || IsEOF()) { break; } } while (ReadNextBlock()); value = decoder.GetResult(); }
size_t Lz4StreamReader::Read(void* buffer, size_t length) { uint32_t out = 0; // Bytes returned to caller #ifdef _WIN64 if(length > UINT32_MAX) throw Exception(E_INVALIDARG); #endif if(length == 0) return 0; // Nothing to do // Read uncompressed data into the output buffer until either // the specified amount has been read or the stream ends while(length > 0) { // Take the smaller of what we have and what we still need uint32_t next = std::min(m_blockremain, static_cast<uint32_t>(length)); if(next) { // The buffer pointer can be NULL to just skip over data if(buffer) memcpy(buffer, m_blockcurrent, next); // Move the block data pointer m_blockcurrent += next; m_blockremain -= next; // Move the output buffer pointer if(buffer) buffer = reinterpret_cast<uint8_t*>(buffer) + next; length -= next; out += next; // Wrote this many output bytes } // If there is no more block data, decompress the next block; if // there is no more data in the stream, we're done if((m_blockremain == 0) && (ReadNextBlock() == 0)) break; } m_position += out; // Increment the current stream position return out; // Return number of bytes written }
CXML_Element* CXML_Parser::ParseElement(CXML_Element* pParent, bool bStartTag) { m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex; if (IsEOF()) { return nullptr; } CFX_ByteString tag_name, tag_space; bool bEndTag; GetTagName(tag_space, tag_name, bEndTag, bStartTag); if (tag_name.IsEmpty() || bEndTag) { return nullptr; } CXML_Element* pElement = new CXML_Element; pElement->m_pParent = pParent; pElement->SetTag(tag_space.AsStringC(), tag_name.AsStringC()); do { CFX_ByteString attr_space, attr_name; while (m_dwIndex < m_dwBufferSize) { SkipWhiteSpaces(); if (IsEOF()) { break; } if (!g_FXCRT_XML_IsNameIntro(m_pBuffer[m_dwIndex])) { break; } GetName(attr_space, attr_name); SkipWhiteSpaces(); if (IsEOF()) { break; } if (m_pBuffer[m_dwIndex] != '=') { break; } m_dwIndex++; SkipWhiteSpaces(); if (IsEOF()) { break; } CFX_WideString attr_value; GetAttrValue(attr_value); pElement->m_AttrMap.SetAt(attr_space, attr_name, attr_value); } m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex; if (m_dwIndex < m_dwBufferSize || IsEOF()) { break; } } while (ReadNextBlock()); SkipWhiteSpaces(); if (IsEOF()) { return pElement; } uint8_t ch = m_pBuffer[m_dwIndex++]; if (ch == '/') { m_dwIndex++; m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex; return pElement; } if (ch != '>') { m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex; delete pElement; return nullptr; } SkipWhiteSpaces(); if (IsEOF()) { return pElement; } CFX_UTF8Decoder decoder; CFX_WideTextBuf content; bool bCDATA = false; int32_t iState = 0; do { while (m_dwIndex < m_dwBufferSize) { ch = m_pBuffer[m_dwIndex++]; switch (iState) { case 0: if (ch == '<') { iState = 1; } else if (ch == '&') { decoder.ClearStatus(); decoder.AppendChar(GetCharRef()); } else { decoder.Input(ch); } break; case 1: if (ch == '!') { iState = 2; } else if (ch == '?') { SkipLiterals("?>"); SkipWhiteSpaces(); iState = 0; } else if (ch == '/') { CFX_ByteString space, name; GetName(space, name); SkipWhiteSpaces(); m_dwIndex++; iState = 10; } else { content << decoder.GetResult(); CFX_WideString dataStr = content.MakeString(); if (!bCDATA && !m_bSaveSpaceChars) { dataStr.TrimRight(L" \t\r\n"); } InsertContentSegment(bCDATA, dataStr.AsStringC(), pElement); content.Clear(); decoder.Clear(); bCDATA = false; iState = 0; m_dwIndex--; CXML_Element* pSubElement = ParseElement(pElement, true); if (!pSubElement) { break; } pSubElement->m_pParent = pElement; pElement->m_Children.push_back( {CXML_Element::Element, pSubElement}); SkipWhiteSpaces(); } break; case 2: if (ch == '[') { SkipLiterals("]]>"); } else if (ch == '-') { m_dwIndex++; SkipLiterals("-->"); } else { SkipLiterals(">"); } decoder.Clear(); SkipWhiteSpaces(); iState = 0; break; } if (iState == 10) { break; } } m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex; if (iState == 10 || m_dwIndex < m_dwBufferSize || IsEOF()) { break; } } while (ReadNextBlock()); content << decoder.GetResult(); CFX_WideString dataStr = content.MakeString(); if (!m_bSaveSpaceChars) { dataStr.TrimRight(L" \t\r\n"); } InsertContentSegment(bCDATA, dataStr.AsStringC(), pElement); content.Clear(); decoder.Clear(); bCDATA = false; return pElement; }
uint32_t CXML_Parser::GetCharRef() { m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex; if (IsEOF()) { return 0; } uint8_t ch; int32_t iState = 0; CFX_ByteTextBuf buf; uint32_t code = 0; do { while (m_dwIndex < m_dwBufferSize) { ch = m_pBuffer[m_dwIndex]; switch (iState) { case 0: if (ch == '#') { m_dwIndex++; iState = 2; break; } iState = 1; case 1: m_dwIndex++; if (ch == ';') { CFX_ByteStringC ref = buf.AsStringC(); if (ref == "gt") { code = '>'; } else if (ref == "lt") { code = '<'; } else if (ref == "amp") { code = '&'; } else if (ref == "apos") { code = '\''; } else if (ref == "quot") { code = '"'; } iState = 10; break; } buf.AppendByte(ch); break; case 2: if (ch == 'x') { m_dwIndex++; iState = 4; break; } iState = 3; case 3: m_dwIndex++; if (ch == ';') { iState = 10; break; } if (g_FXCRT_XML_IsDigital(ch)) code = code * 10 + FXSYS_toDecimalDigit(static_cast<FX_WCHAR>(ch)); break; case 4: m_dwIndex++; if (ch == ';') { iState = 10; break; } uint8_t nHex = g_FXCRT_XML_ByteTypes[ch] & FXCRTM_XML_CHARTYPE_HexChar; if (nHex) { if (nHex == FXCRTM_XML_CHARTYPE_HexDigital) { code = (code << 4) + FXSYS_toDecimalDigit(static_cast<FX_WCHAR>(ch)); } else if (nHex == FXCRTM_XML_CHARTYPE_HexLowerLetter) { code = (code << 4) + ch - 87; } else { code = (code << 4) + ch - 55; } } break; } if (iState == 10) { break; } } m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex; if (iState == 10 || m_dwIndex < m_dwBufferSize || IsEOF()) { break; } } while (ReadNextBlock()); return code; }
bool CXML_Parser::Init(bool bOwndedStream) { m_bOwnedStream = bOwndedStream; m_nOffset = 0; return ReadNextBlock(); }
FX_BOOL CXML_Parser::Init(FX_BOOL bOwndedStream) { m_bOwnedStream = bOwndedStream; m_nOffset = 0; return ReadNextBlock(); }
CXML_Element* CXML_Parser::ParseElement(CXML_Element* pParent, FX_BOOL bStartTag) { m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex; if (IsEOF()) { return NULL; } CFX_ByteString tag_name, tag_space; FX_BOOL bEndTag; GetTagName(tag_space, tag_name, bEndTag, bStartTag); if (tag_name.IsEmpty() || bEndTag) { return NULL; } CXML_Element* pElement; pElement = FX_NEW CXML_Element; if (pElement) { pElement->m_pParent = pParent; pElement->SetTag(tag_space, tag_name); } if (!pElement) { return NULL; } do { CFX_ByteString attr_space, attr_name; while (m_dwIndex < m_dwBufferSize) { SkipWhiteSpaces(); if (IsEOF()) { break; } if (!g_FXCRT_XML_IsNameIntro(m_pBuffer[m_dwIndex])) { break; } GetName(attr_space, attr_name); SkipWhiteSpaces(); if (IsEOF()) { break; } if (m_pBuffer[m_dwIndex] != '=') { break; } m_dwIndex ++; SkipWhiteSpaces(); if (IsEOF()) { break; } CFX_WideString attr_value; GetAttrValue(attr_value); pElement->m_AttrMap.SetAt(attr_space, attr_name, attr_value); } m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex; if (m_dwIndex < m_dwBufferSize || IsEOF()) { break; } } while (ReadNextBlock()); SkipWhiteSpaces(); if (IsEOF()) { return pElement; } FX_BYTE ch = m_pBuffer[m_dwIndex ++]; if (ch == '/') { m_dwIndex ++; m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex; return pElement; } if (ch != '>') { m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex; delete pElement; return NULL; } SkipWhiteSpaces(); if (IsEOF()) { return pElement; } CFX_UTF8Decoder decoder; CFX_WideTextBuf content; FX_BOOL bCDATA = FALSE; FX_INT32 iState = 0; do { while (m_dwIndex < m_dwBufferSize) { ch = m_pBuffer[m_dwIndex ++]; switch (iState) { case 0: if (ch == '<') { iState = 1; } else if (ch == '&') { decoder.ClearStatus(); decoder.AppendChar(GetCharRef()); } else { decoder.Input(ch); } break; case 1: if (ch == '!') { iState = 2; } else if (ch == '?') { SkipLiterals(FX_BSTRC("?>")); SkipWhiteSpaces(); iState = 0; } else if (ch == '/') { CFX_ByteString space, name; GetName(space, name); SkipWhiteSpaces(); m_dwIndex ++; iState = 10; } else { content << decoder.GetResult(); CFX_WideString dataStr = content.GetWideString(); if (!bCDATA && !m_bSaveSpaceChars) { dataStr.TrimRight((FX_LPCWSTR)L" \t\r\n"); } InsertContentSegment(bCDATA, dataStr, pElement); content.Clear(); decoder.Clear(); bCDATA = FALSE; iState = 0; m_dwIndex --; CXML_Element* pSubElement = ParseElement(pElement, TRUE); if (pSubElement == NULL) { break; } pSubElement->m_pParent = pElement; pElement->m_Children.Add((FX_LPVOID)CXML_Element::Element); pElement->m_Children.Add(pSubElement); SkipWhiteSpaces(); } break; case 2: if (ch == '[') { SkipLiterals(FX_BSTRC("]]>")); } else if (ch == '-') { m_dwIndex ++; SkipLiterals(FX_BSTRC("-->")); } else { SkipLiterals(FX_BSTRC(">")); } decoder.Clear(); SkipWhiteSpaces(); iState = 0; break; } if (iState == 10) { break; } } m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex; if (iState == 10 || m_dwIndex < m_dwBufferSize || IsEOF()) { break; } } while (ReadNextBlock()); content << decoder.GetResult(); CFX_WideString dataStr = content.GetWideString(); if (!m_bSaveSpaceChars) { dataStr.TrimRight((FX_LPCWSTR)L" \t\r\n"); } InsertContentSegment(bCDATA, dataStr, pElement); content.Clear(); decoder.Clear(); bCDATA = FALSE; return pElement; }
FX_DWORD CXML_Parser::GetCharRef() { m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex; if (IsEOF()) { return 0; } FX_BYTE ch; FX_INT32 iState = 0; CFX_ByteTextBuf buf; FX_DWORD code = 0; do { while (m_dwIndex < m_dwBufferSize) { ch = m_pBuffer[m_dwIndex]; switch (iState) { case 0: if (ch == '#') { m_dwIndex ++; iState = 2; break; } iState = 1; case 1: m_dwIndex ++; if (ch == ';') { CFX_ByteStringC ref = buf.GetByteString(); if (ref == FX_BSTRC("gt")) { code = '>'; } else if (ref == FX_BSTRC("lt")) { code = '<'; } else if (ref == FX_BSTRC("amp")) { code = '&'; } else if (ref == FX_BSTRC("apos")) { code = '\''; } else if (ref == FX_BSTRC("quot")) { code = '"'; } iState = 10; break; } buf.AppendByte(ch); break; case 2: if (ch == 'x') { m_dwIndex ++; iState = 4; break; } iState = 3; case 3: m_dwIndex ++; if (ch == ';') { iState = 10; break; } if (g_FXCRT_XML_IsDigital(ch)) { code = code * 10 + ch - '0'; } break; case 4: m_dwIndex ++; if (ch == ';') { iState = 10; break; } FX_BYTE nHex = g_FXCRT_XML_ByteTypes[ch] & FXCRTM_XML_CHARTYPE_HexChar; if (nHex) { if (nHex == FXCRTM_XML_CHARTYPE_HexDigital) { code = (code << 4) + ch - '0'; } else if (nHex == FXCRTM_XML_CHARTYPE_HexLowerLetter) { code = (code << 4) + ch - 87; } else { code = (code << 4) + ch - 55; } } break; } if (iState == 10) { break; } } m_nOffset = m_nBufferOffset + (FX_FILESIZE)m_dwIndex; if (iState == 10 || m_dwIndex < m_dwBufferSize || IsEOF()) { break; } } while (ReadNextBlock()); return code; }
void DataProvider::Resume() { if (!is_suspended_) return; is_suspended_ = false; ReadNextBlock(); }