QBStatus TapFile::_flush()
{
static const char *method = "TapFile::_flush()";
if (_dataOStream != NULL) {
//
// update the trailer
//
_trailer.set_data_block_count(_trailer.data_block_count() + 1);
_trailer.set_message_count(_trailer.message_count() + _index_entry.message_count());
_trailer.set_uncompressed_bytes(_trailer.uncompressed_bytes() +
_dataOStream->ByteCount());
if (_trailer.max_decomp_size() < _dataOStream->ByteCount())
_trailer.set_max_decomp_size(_dataOStream->ByteCount());
//
// close out the data block
//
// delete the data CodedOutputStream to flush to the Gzip stream
delete _dataOStream;
_dataOStream = NULL;
// close and delete the GzipOutputStream to flush to the underlying coded stream
_dataGzipOStream->Close();
delete _dataGzipOStream;
_dataGzipOStream = NULL;
// pad to the next sector boundary
_pad(_fileOStream, 0);
// update the size of this data block in the index
_index_entry.set_data_bytes(_fileOStream->ByteCount() - _index_entry.data_offset());
//
// now write out the index entry for this data block
//
_indexOStream->WriteVarint32(_index_entry.ByteSize());
_index_entry.SerializeWithCachedSizes(_indexOStream);
}
return(QB_SUCCESS);
}
Error PCKPacker::flush(bool p_verbose) {
if (!file) {
ERR_FAIL_COND_V(!file, ERR_INVALID_PARAMETER);
return ERR_INVALID_PARAMETER;
};
// write the index
file->store_32(files.size());
for (int i=0; i<files.size(); i++) {
file->store_pascal_string(files[i].path);
files[i].offset_offset = file->get_pos();
file->store_64(0); // offset
file->store_64(files[i].size); // size
// # empty md5
file->store_32(0);
file->store_32(0);
file->store_32(0);
file->store_32(0);
};
uint64_t ofs = file->get_pos();
ofs = _align(ofs, alignment);
_pad(file, ofs - file->get_pos());
const uint32_t buf_max = 65536;
uint8_t *buf = memnew_arr(uint8_t, buf_max);
int count = 0;
for (int i=0; i<files.size(); i++) {
FileAccess* src = FileAccess::open(files[i].src_path, FileAccess::READ);
uint64_t to_write = files[i].size;
while (to_write > 0) {
int read = src->get_buffer(buf, MIN(to_write, buf_max));
file->store_buffer(buf, read);
to_write -= read;
};
uint64_t pos = file->get_pos();
file->seek(files[i].offset_offset); // go back to store the file's offset
file->store_64(ofs);
file->seek(pos);
ofs = _align(ofs + files[i].size, alignment);
_pad(file, ofs - pos);
src->close();
memdelete(src);
count += 1;
if (p_verbose) {
if (count % 100 == 0) {
printf("%i/%i (%.2f)\r", count, files.size(), float(count) / files.size() * 100);
fflush(stdout);
};
};
};
if (p_verbose)
printf("\n");
file->close();
return OK;
};
QBStatus TapFile::close()
{
static const char *method = "TapFile::close()";
QBStatus status;
if (_fd == 0) return(QB_SUCCESS);
if (_mode != QB_READ) {
// flush out the last data block
_flush();
//
// close out and serialize the whole index
//
// yes making an extra copy of the entire index in a string is a bad idea
// but i had trouble getting to the actual buffer to copy
//
// this should probably actually go into a temporary File, and then get copied
// over with sendFile(). the data would usually all stay in the buffercache
//
// delete the coded output stream to flush its contents to the OstreamOutputStrean
delete _indexOStream;
_indexOStream = NULL;
// delete the OstreamOutputStream to flush its contents to the ostringstream
delete _indexOstreamOStream;
_indexOstreamOStream = NULL;
// extract the data
string s = _indexOstringstream->str();
// now delete the ostringstream to free memory
delete _indexOstringstream;
_indexOstringstream = NULL;
// output the wole index and pad to the next sector boundary
_trailer.set_index_offset(_fileOStream->ByteCount());
_raw(_fileOStream, s.c_str(), s.length());
_pad(_fileOStream, 0);
_trailer.set_index_bytes(_fileOStream->ByteCount() - _trailer.index_offset());
//
// write the trailer including signature at the end of the File
//
uint64_t trailerOffset = _fileOStream->ByteCount();
_oStream = new google::protobuf::io::CodedOutputStream(_fileOStream);
_oStream->WriteRaw("trai",4);
_oStream->WriteRaw("none",4);
_oStream->WriteVarint32(_trailer.ByteSize());
_trailer.SerializeWithCachedSizes(_oStream);
_pad(_oStream, 16); // pad to 16 bytes before the sector end
_oStream->WriteLittleEndian64(trailerOffset);
_oStream->WriteRaw("tapproto",8);
delete _oStream;
_oStream = NULL;
//
// final cleanup of the File
//
_fileOStream->Close();
delete _fileOStream;
_fileOStream = NULL;
}
else {
int retval = ::close(_fd);
if (retval < 0) {
qb_logerr(QB_FAILURE, __FILE__, __LINE__, method,
"Error from ::close File:%s, mode:%d, errno: %d",
_name.c_str(), _mode, errno);
return(QB_FAILURE);
}
}
_name = "";
_fd = 0;
_eof_flag = false;
return(QB_SUCCESS);
}
QBStatus TapFile::write(const QBKey &key, const QBEvent &event)
{
const char *method = "TapFile::write()";
QBStatus status;
string s;
if (!isOpen()) return(QB_STREAMNOTOPEN);
if (_mode == QB_READ) return(QB_WRONGACCESSMODE);
// setup header and trailer fields on the first record written
if (_first_write) {
// setup header
_header.set_message_name(event.GetTypeName());
_header.set_target_decomp_size(_target_block_size);
// write signature and header
_oStream = new google::protobuf::io::CodedOutputStream(_fileOStream);
_oStream->WriteRaw("tapproto",8);
_oStream->WriteRaw("head",4);
_oStream->WriteRaw("none",4);
_oStream->WriteVarint32(_header.ByteSize());
_header.SerializeWithCachedSizes(_oStream);
_pad(_oStream, 0);
delete _oStream;
_oStream = NULL;
// copy the header to the trailer and fill in what we know
_trailer.CopyFrom(_header);
_trailer.set_first_key(key.data().c_str(), key.size());
_trailer.set_message_count(0);
_trailer.set_data_block_count(0);
_trailer.set_uncompressed_bytes(0);
_trailer.set_max_decomp_size(0);
// add descriptors from imported .proto Files to the trailer
for (int i=0; i < event.GetDescriptor()->file()->dependency_count(); i++) {
const FileDescriptor *fd_p = event.GetDescriptor()->file()->dependency(i);
FileDescriptorProto fdproto;
fd_p->CopyTo(&fdproto);
fdproto.SerializeToString(&s);
_header.add_format_descriptor(s.data(), s.size());
}
// add the main File descriptor proto to the trailer
FileDescriptorProto fdproto;
event.GetDescriptor()->file()->CopyTo(&fdproto);
fdproto.SerializeToString(&s);
_header.add_format_descriptor(s.data(), s.size());
// create the ostringstream for the index, write the index block header
// this index can get really large, we'll want to automatically grow
// the data block size as the File gets larger to limit the growth of the index
_indexOstringstream = new ostringstream(ostringstream::binary|ostringstream::app|ostringstream::out);
_indexOstreamOStream = new google::protobuf::io::OstreamOutputStream(_indexOstringstream);
_indexOStream = new google::protobuf::io::CodedOutputStream(_indexOstreamOStream);
_indexOStream->WriteRaw("upix",4);
_indexOStream->WriteRaw("none",4);
// first-write preprocessing is done
_first_write = false;
}
// do we have a current open data block?
if (_dataOStream == NULL) {
// setup the index entry
_index_entry.Clear();
_index_entry.set_first_key(key.data().c_str(), key.size());
_index_entry.set_data_offset(_fileOStream->ByteCount());
_index_entry.set_message_count(0);
// write the data block header
_raw(_fileOStream, "data");
_raw(_fileOStream, "gzip");
// create the GzipOutputStream for the data block
google::protobuf::io::GzipOutputStream::Options options;
options.compression_level = 6;
_dataGzipOStream = new google::protobuf::io::GzipOutputStream(_fileOStream, options);
_dataOStream = new google::protobuf::io::CodedOutputStream(_dataGzipOStream);
}
// serialize the record
_dataOStream->WriteVarint32(key.size());
_dataOStream->WriteRaw(key.data().c_str(),key.size());
_dataOStream->WriteVarint32(event.ByteSize());
event.SerializeWithCachedSizes(_dataOStream);
_index_entry.set_message_count(_index_entry.message_count() + 1);
_trailer.set_last_key(key.data().c_str(), key.size());
// close out this data block if full
if (_dataOStream->ByteCount() >= _target_block_size) {
_flush();
}
return(QB_SUCCESS);
}
void ATIExecutableKernel::initializeSharedMemory()
{
report("Allocating shared memory");
typedef std::unordered_map<std::string, size_t> AllocationMap;
typedef std::unordered_set<std::string> StringSet;
typedef std::deque<ir::PTXOperand*> OperandVector;
typedef std::unordered_map<std::string,
ir::Module::GlobalMap::const_iterator> GlobalMap;
AllocationMap map;
GlobalMap sharedGlobals;
StringSet external;
OperandVector externalOperands;
unsigned int externalAlignment = 1;
size_t sharedSize = 0;
assert(module != 0);
// global shared variables
ir::Module::GlobalMap globals = module->globals();
ir::Module::GlobalMap::const_iterator global;
for (global = globals.begin() ; global != globals.end() ; global++)
{
ir::PTXStatement statement = global->second.statement;
if (statement.directive == ir::PTXStatement::Shared)
{
if (statement.attribute == ir::PTXStatement::Extern)
{
report("Found global external shared variable \""
<< statement.name << "\"");
assertM(external.count(statement.name) == 0,
"External global \"" << statement.name
<< "\" declared more than once.");
external.insert(statement.name);
externalAlignment = std::max(externalAlignment,
(unsigned int)statement.alignment);
externalAlignment = std::max(externalAlignment,
ir::PTXOperand::bytes(statement.type));
} else {
report("Found global shared variable \""
<< statement.name << "\"");
sharedGlobals.insert(
std::make_pair(statement.name, global));
}
}
}
// local shared variables
LocalMap::const_iterator local;
for (local = locals.begin() ; local != locals.end() ; local++)
{
if (local->second.space == ir::PTXInstruction::Shared)
{
if (local->second.attribute == ir::PTXStatement::Extern)
{
report("Found local external shared variable \""
<< local->second.name << "\"");
assertM(external.count(local->second.name) == 0,
"External local \"" << local->second.name
<< "\" declared more than once.");
external.insert(local->second.name);
externalAlignment = std::max(externalAlignment,
(unsigned int)local->second.alignment);
externalAlignment = std::max(externalAlignment,
ir::PTXOperand::bytes(local->second.type));
} else
{
report("Allocating local shared variable \""
<< local->second.name << "\" of size "
<< local->second.getSize());
_pad(sharedSize, local->second.alignment);
map.insert(std::make_pair(local->second.name, sharedSize));
sharedSize += local->second.getSize();
}
}
}
ir::ControlFlowGraph::iterator block;
for (block = cfg()->begin() ; block != cfg()->end() ; block++)
{
ir::ControlFlowGraph::InstructionList insts = block->instructions;
ir::ControlFlowGraph::InstructionList::iterator inst;
for (inst = insts.begin() ; inst != insts.end() ; inst++)
{
ir::PTXInstruction& ptx =
static_cast<ir::PTXInstruction&>(**inst);
if (ptx.opcode == ir::PTXInstruction::Mov ||
ptx.opcode == ir::PTXInstruction::Ld ||
ptx.opcode == ir::PTXInstruction::St)
{
ir::PTXOperand* operands[] = {&ptx.d, &ptx.a, &ptx.b,
&ptx.c};
for (unsigned int i = 0 ; i != 4 ; i++)
{
ir::PTXOperand* operand = operands[i];
if (operand->addressMode == ir::PTXOperand::Address)
{
StringSet::iterator si =
external.find(operand->identifier);
if (si != external.end())
{
report("For instruction \""
<< ptx.toString()
<< "\", mapping shared label \"" << *si
<< "\" to external shared memory.");
externalOperands.push_back(operand);
continue;
}
GlobalMap::iterator gi =
sharedGlobals.find(operand->identifier);
if (gi != sharedGlobals.end())
{
ir::Module::GlobalMap::const_iterator it =
gi->second;
sharedGlobals.erase(gi);
report("Allocating global shared variable \""
<< it->second.statement.name << "\"");
map.insert(std::make_pair(
it->second.statement.name,
sharedSize));
sharedSize += it->second.statement.bytes();
}
AllocationMap::iterator mapping =
map.find(operand->identifier);
if (mapping != map.end())
{
report("For instruction " << ptx.toString()
<< ", mapping shared label "
<< mapping->first << " to " <<
mapping->second);
operand->addressMode =
ir::PTXOperand::Immediate;
operand->imm_uint = mapping->second;
}
}
}
}
}
}
_pad(sharedSize, externalAlignment);
report("Mapping external shared variables.");
OperandVector::iterator operand;
for (operand = externalOperands.begin() ;
operand != externalOperands.end() ; operand++)
{
report("Mapping external shared label \""
<< (*operand)->identifier << "\" to " << sharedSize);
(*operand)->addressMode = ir::PTXOperand::Immediate;
(*operand)->imm_uint = sharedSize;
}
// allocate shared memory object
_sharedMemorySize = sharedSize;
report("Total shared memory size is " << _sharedMemorySize);
}
