void
BinaryOutputStream::writeUInt64(uint64 u)
{
reserveBytes(8);
uint8 * convert = (uint8 *)&u;
if (m_swapBytes)
{
m_buffer[m_pos + 0] = convert[7];
m_buffer[m_pos + 1] = convert[6];
m_buffer[m_pos + 2] = convert[5];
m_buffer[m_pos + 3] = convert[4];
m_buffer[m_pos + 4] = convert[3];
m_buffer[m_pos + 5] = convert[2];
m_buffer[m_pos + 6] = convert[1];
m_buffer[m_pos + 7] = convert[0];
}
else
{
#ifdef DGP_ALLOW_UNALIGNED_WRITES
*(uint64 *)(m_buffer + m_pos) = u;
#else
m_buffer[m_pos + 0] = convert[0];
m_buffer[m_pos + 1] = convert[1];
m_buffer[m_pos + 2] = convert[2];
m_buffer[m_pos + 3] = convert[3];
m_buffer[m_pos + 4] = convert[4];
m_buffer[m_pos + 5] = convert[5];
m_buffer[m_pos + 6] = convert[6];
m_buffer[m_pos + 7] = convert[7];
#endif
}
m_pos += 8;
}
void
BinaryOutputStream::writeUInt32(uint32 u)
{
reserveBytes(4);
uint8 * convert = (uint8 *)&u;
debugAssertM(m_beginEndBits == 0, getNameStr() + ": Can't write non-bit data within beginBits/endBits block");
if (m_swapBytes)
{
m_buffer[m_pos + 0] = convert[3];
m_buffer[m_pos + 1] = convert[2];
m_buffer[m_pos + 2] = convert[1];
m_buffer[m_pos + 3] = convert[0];
}
else
{
#ifdef DGP_ALLOW_UNALIGNED_WRITES
*(uint32 *)(m_buffer + m_pos) = u;
#else
m_buffer[m_pos + 0] = convert[0];
m_buffer[m_pos + 1] = convert[1];
m_buffer[m_pos + 2] = convert[2];
m_buffer[m_pos + 3] = convert[3];
#endif
}
m_pos += 4;
}
std::size_t TupleOutputStream::startRows(int32_t partitionId)
{
writeInt(partitionId);
m_rowCount = 0;
m_rowCountPosition = reserveBytes(4);
m_totalBytesSerialized = 0;
return m_rowCountPosition;
}
void BinaryOutput::writeString(const char* s) {
// +1 is because strlen doesn't count the null
int len = strlen(s) + 1;
debugAssert(m_beginEndBits == 0);
reserveBytes(len);
System::memcpy(m_buffer + m_pos, s, len);
m_pos += len;
}
void BinaryOutput::writeString32(const char* s) {
// Write the NULL and count it
size_t len = strlen(s) + 1;
writeUInt32((uint32)len);
debugAssert(m_beginEndBits == 0);
reserveBytes(len);
System::memcpy(m_buffer + m_pos, s, len);
m_pos += len;
}
void
BinaryOutputStream::reserveBytesWhenOutOfMemory(size_t bytes)
{
if (m_path == "<memory>")
{
throw Error(getNameStr() + ": Out of memory while writing to memory (no RAM left)");
}
else if ((int64)bytes > m_bufferCapacity)
{
throw Error("Out of memory while writing to disk (could not create a large enough buffer)");
}
else
{
// Dump the contents to disk. In order to enable seeking backwards, we keep the last 10 MB in memory.
int64 bytesToWrite = m_bufferLen - 10 * 1024 * 1024;
if (bytesToWrite < m_bufferLen / 3)
{
// We're going to write less than 1/3 of the file; give up and just write the whole thing.
bytesToWrite = m_bufferLen;
}
debugAssertM(bytesToWrite > 0, getNameStr() + ": No bytes to write");
// Write to the file
char const * mode = (m_alreadyWritten > 0) ? "ab" : "wb";
FILE * file = fopen(m_path.c_str(), mode);
debugAssertM(file, getNameStr() + ": Could not open file for writing");
size_t count = fwrite(m_buffer, 1, (size_t)bytesToWrite, file);
debugAssertM((int64)count == bytesToWrite, getNameStr() + ": All bytes were not written");
(void)count; // avoid unused variable warning
fclose(file);
file = NULL;
// Record that we saved this data.
m_alreadyWritten += bytesToWrite;
m_bufferLen -= bytesToWrite;
m_pos -= bytesToWrite;
debugAssertM(m_bufferLen < m_bufferCapacity, getNameStr() + ": Buffer exceeds maximum size");
debugAssertM(m_bufferLen >= 0, getNameStr() + ": Buffer has negative size");
debugAssertM(m_pos >= 0, getNameStr() + ": Write position is negative");
debugAssertM(m_pos <= m_bufferLen, getNameStr() + ": Write position is beyond end of buffer");
// Shift the unwritten data back appropriately in the buffer.
std::memmove(m_buffer, m_buffer + bytesToWrite, (size_t)m_bufferLen);
// *Now* we allocate bytes (there should presumably be enough space in the buffer; if not, we'll come back through this code
// and dump the last 10MB to disk as well. Note that the bytes > maxBufferLen case above would already have triggered if
// this call couldn't succeed.
reserveBytes(bytes);
}
}
void BinaryOutput::reserveBytesWhenOutOfMemory(size_t bytes) {
if (m_filename == "<memory>") {
throw "Out of memory while writing to memory in BinaryOutput (no RAM left).";
} else if ((int)bytes > (int)m_maxBufferLen) {
throw "Out of memory while writing to disk in BinaryOutput (could not create a large enough buffer).";
} else {
// Dump the contents to disk. In order to enable seeking backwards,
// we keep the last 10 MB in memory.
size_t writeBytes = m_bufferLen - 10 * 1024 * 1024;
if (writeBytes < m_bufferLen / 3) {
// We're going to write less than 1/3 of the file;
// give up and just write the whole thing.
writeBytes = m_bufferLen;
}
debugAssert(writeBytes > 0);
//debugPrintf("Writing %d bytes to disk\n", writeBytes);
const char* mode = (m_alreadyWritten > 0) ? "ab" : "wb";
alwaysAssertM(m_filename != "<memory>", "Writing memory file");
FILE* file = FileSystem::fopen(m_filename.c_str(), mode);
debugAssert(file);
size_t count = fwrite(m_buffer, 1, writeBytes, file);
debugAssert(count == writeBytes); (void)count;
fclose(file);
file = NULL;
// Record that we saved this data.
m_alreadyWritten += writeBytes;
m_bufferLen -= writeBytes;
m_pos -= writeBytes;
debugAssert(m_bufferLen < m_maxBufferLen);
debugAssert(m_bufferLen >= 0);
debugAssert(m_pos >= 0);
debugAssert(m_pos <= (int64)m_bufferLen);
// Shift the unwritten data back appropriately in the buffer.
debugAssert(isValidHeapPointer(m_buffer));
System::memcpy(m_buffer, m_buffer + writeBytes, m_bufferLen);
debugAssert(isValidHeapPointer(m_buffer));
// *now* we allocate bytes (there should presumably be enough
// space in the buffer; if not, we'll come back through this
// code and dump the last 10MB to disk as well. Note that the
// bytes > maxBufferLen case above would already have triggered
// if this call couldn't succeed.
reserveBytes(bytes);
}
}
void BinaryOutput::writeStringEven(const char* s) {
// +1 is because strlen doesn't count the null
int len = strlen(s) + 1;
reserveBytes(len);
System::memcpy(m_buffer + m_pos, s, len);
m_pos += len;
// Pad with another NULL
if ((len % 2) == 1) {
writeUInt8(0);
}
}
void BinaryOutput::writeUInt16(uint16 u) {
reserveBytes(2);
uint8* convert = (uint8*)&u;
if (m_swapBytes) {
m_buffer[m_pos] = convert[1];
m_buffer[m_pos + 1] = convert[0];
} else {
*(uint16*)(m_buffer + m_pos) = u;
}
m_pos += 2;
}
void BinaryOutput::writeUInt32(uint32 u) {
reserveBytes(4);
uint8* convert = (uint8*)&u;
debugAssert(m_beginEndBits == 0);
if (m_swapBytes) {
m_buffer[m_pos] = convert[3];
m_buffer[m_pos + 1] = convert[2];
m_buffer[m_pos + 2] = convert[1];
m_buffer[m_pos + 3] = convert[0];
} else {
*(uint32*)(m_buffer + m_pos) = u;
}
m_pos += 4;
}
void BinaryOutput::writeUInt64(uint64 u) {
reserveBytes(8);
uint8* convert = (uint8*)&u;
if (m_swapBytes) {
m_buffer[m_pos] = convert[7];
m_buffer[m_pos + 1] = convert[6];
m_buffer[m_pos + 2] = convert[5];
m_buffer[m_pos + 3] = convert[4];
m_buffer[m_pos + 4] = convert[3];
m_buffer[m_pos + 5] = convert[2];
m_buffer[m_pos + 6] = convert[1];
m_buffer[m_pos + 7] = convert[0];
} else {
*(uint64*)(m_buffer + m_pos) = u;
}
m_pos += 8;
}
void
BinaryOutputStream::writeUInt16(uint16 u)
{
reserveBytes(2);
uint8 * convert = (uint8 *)&u;
if (m_swapBytes)
{
m_buffer[m_pos + 0] = convert[1];
m_buffer[m_pos + 1] = convert[0];
}
else
{
#ifdef DGP_ALLOW_UNALIGNED_WRITES
*(uint16 *)(m_buffer + m_pos) = u;
#else
m_buffer[m_pos + 0] = convert[0];
m_buffer[m_pos + 1] = convert[1];
#endif
}
m_pos += 2;
}
bool OctreePacketData::reserveBitMask() {
return reserveBytes(sizeof(unsigned char));
}
T* reserveType(ArenaLinear& arena){
return (T*)reserveBytes(arena, sizeof(T), alignof(T));
}
T* reserveTypeArray(ArenaLinear& arena, u32 length){
return (T*)reserveBytes(arena, sizeof(T) * length, alignof(T));
}
