size32_t CThorStreamDeserializerSource::readUtf8(ARowBuilder & target, size32_t offset, size32_t fixedSize, size32_t len)
{
//The len is the number of utf characters, size depends on which characters are included.
size32_t totalSize = 0;
loop
{
if (len == 0)
return totalSize;
size32_t available;
const byte * cur = doPeek(1, available);
size32_t copyLen;
for (copyLen = 0; copyLen < available;)
{
copyLen += readUtf8Size(cur+copyLen); // This function only accesses the first byte
if (--len == 0)
break;
}
byte * self = target.ensureCapacity(fixedSize + totalSize + copyLen, NULL);
doRead(copyLen, self+offset+totalSize);
totalSize += copyLen;
}
}
size32_t CThorContiguousRowBuffer::sizeUtf8(size32_t len)
{
if (len == 0)
return 0;
//The len is the number of utf characters, size depends on which characters are included.
size32_t nextOffset = readOffset;
while (len)
{
ensureAccessible(nextOffset+1);
for (;nextOffset < maxOffset;)
{
nextOffset += readUtf8Size(buffer+nextOffset); // This function only accesses the first byte
if (--len == 0)
break;
}
}
return nextOffset - readOffset;
}
void CThorStreamDeserializerSource::skipUtf8(size32_t len)
{
throwUnexpected();
loop
{
if (len == 0)
return;
size32_t available;
const byte * cur = doPeek(1, available);
size32_t copyLen;
for (copyLen = 0; copyLen < available;)
{
copyLen += readUtf8Size(cur+copyLen); // This function only accesses the first byte
len--;
}
in->skip(copyLen);
}
}