void MultiChain::record() {
//h_config = H5Gopen(h_file,"MultiChain");
typedef Bead::RealType PosType;
typedef Bead::Buffer_t Buffer_t;
Buffer_t chain_buffer,bead_buffer;
(*(this->begin()))->registerData(bead_buffer);
chain_buffer.rewind();
copyToBuffer(chain_buffer);
HDFAttribIO<Buffer_t> mcout(chain_buffer);
mcout.overwrite(h_config,"state");
std::deque<Bead*>::iterator bead_it(this->begin());
std::deque<Bead*>::iterator bead_end(this->end());
//create the group and increment counter
char GrpName[128];
int ibead=0;
while(bead_it != bead_end) {
sprintf(GrpName,"bead%04d",ibead);
hid_t bead_id = H5Gopen(h_config,GrpName);
bead_buffer.rewind();
Bead& bead(**bead_it);
bead.copyToBuffer(bead_buffer);
HDFAttribIO<Buffer_t> bout(bead_buffer);
bout.overwrite(bead_id,"state");
H5Gclose(bead_id);
++bead_it;
++ibead;
}
}
void CellQueryResultsSerializer::write(std::ostream& out, const sserialize::CellQueryResult& cqr) const {
MyOstream fetchedIndices;
std::vector<uint32_t> partialIdxIds;
BinaryWriter bout(out);
uint32_t numFetchedIndices = 0;
sserialize::ItemIndex::Types cqrIdxType = cqr.defaultIndexType();
bout.putU32(cqr.cellCount());//CellCount
for(sserialize::CellQueryResult::const_iterator it(cqr.cbegin()), end(cqr.cend()); it != end; ++it) {
uint32_t rawDesc = it.rawDesc();
if (!(rawDesc & sserialize::CellQueryResult::const_iterator::RD_FULL_MATCH)) {
if (rawDesc & sserialize::CellQueryResult::const_iterator::RD_FETCHED) {
const sserialize::ItemIndex & idx = (*it);
assert(idx.type() == cqrIdxType);
IndexDB::writeIndexData(fetchedIndices, 0xFFFFFFFF, idx.data().asMemView());
++numFetchedIndices;
}
else {
partialIdxIds.push_back(it.idxId());
}
}
bout.putU32(rawDesc);//CellInfo
}
//write the ItemIndexSet header
IndexDB::writeHeader(out, cqrIdxType, sserialize::Static::ItemIndexStore::IndexCompressionType::IC_NONE, numFetchedIndices);
//and the index data
out.write(&(fetchedIndices.data[0]), fetchedIndices.data.size());
bout.putU32(partialIdxIds.size());//cellIdxCount
for(uint32_t x : partialIdxIds) {//cellIdxIds
bout.putU32(x);
}
}
/**
* - MultiChain
* -- Version
* -- NumberOfBeads
* -- state: number of beads, grotwh direction etc, see MultiChain::copyToBuffer
* -- bead0000
* -- state: everything is stored in a buffer using PooledData<t>
* R, Drift, Multiple Gradients,
* properties, weights etc, see Bead::copyToBuffer
*/
void MultiChain::open(const string& aroot) {
hid_t h_file=-1;
string h5file=aroot+".config.h5";
h_file = H5Fopen(h5file.c_str(),H5F_ACC_RDWR,H5P_DEFAULT);
h_config = H5Gcreate(h_file,"MultiChain",0);
int m_version=1;
HDFAttribIO<int> v(m_version);
v.write(h_config,"Version");
int nc=Beads.size();
HDFAttribIO<int> c(nc);
c.write(h_config,"NumberOfBeads");
//typedef Bead::PosType PosType;
typedef Bead::RealType PosType;
typedef Bead::Buffer_t Buffer_t;
Buffer_t chain_buffer,bead_buffer;
(*(this->begin()))->registerData(bead_buffer);
nc=bead_buffer.size();
HDFAttribIO<int> c2(nc);
c2.write(h_config,"BufferSize");
chain_buffer.rewind();
copyToBuffer(chain_buffer);
HDFAttribIO<Buffer_t> mcout(chain_buffer);
mcout.write(h_config,"state");
std::deque<Bead*>::iterator bead_it(this->begin());
std::deque<Bead*>::iterator bead_end(this->end());
//create the group and increment counter
char GrpName[128];
int ibead=0;
while(bead_it != bead_end) {
sprintf(GrpName,"bead%04d",ibead);
hid_t bead_id = H5Gcreate(h_config,GrpName,0);
bead_buffer.rewind();
Bead& bead(**bead_it);
bead.copyToBuffer(bead_buffer);
HDFAttribIO<Buffer_t> bout(bead_buffer);
bout.write(bead_id,"state");
H5Gclose(bead_id);
++bead_it;
++ibead;
}
if(h_file>-1) H5Fclose(h_file);
}
bool MultiChain::read(hid_t grp){
hid_t hgrp = H5Gopen(grp,"MultiChain");
int m_version=1;
HDFAttribIO<int> v(m_version);
v.read(hgrp,"Version");
int nc(0);
HDFAttribIO<int> c(nc);
c.read(hgrp,"NumberOfBeads");
if(nc != Beads.size()) {
WARNMSG("The number of chains is different. Previous = " << nc << " Current = " << Beads.size())
}
typedef Bead::RealType PosType;
typedef Bead::Buffer_t Buffer_t;
Buffer_t chain_buffer,bead_buffer;
(*(this->begin()))->registerData(bead_buffer);
HDFAttribIO<int> c2(nc);
c2.read(hgrp,"BufferSize");
if(nc != bead_buffer.size()) {
ERRORMSG("The buffer size is different. Ignore restart data")
H5Gclose(hgrp);
return false;
}
chain_buffer.rewind();
copyToBuffer(chain_buffer);
HDFAttribIO<Buffer_t> mcin(chain_buffer);
mcin.read(hgrp,"state");
chain_buffer.rewind();
copyFromBuffer(chain_buffer);
std::deque<Bead*>::iterator bead_it(this->begin());
std::deque<Bead*>::iterator bead_end(this->end());
//create the group and increment counter
char GrpName[128];
int ibead=0;
while(bead_it != bead_end) {
sprintf(GrpName,"bead%04d",ibead);
hid_t bead_id = H5Gopen(hgrp,GrpName);
Bead& bead(**bead_it);
HDFAttribIO<Buffer_t> bout(bead_buffer);
bout.read(bead_id,"state");
bead_buffer.rewind();
bead.copyFromBuffer(bead_buffer);
H5Gclose(bead_id);
++bead_it;
++ibead;
}
H5Gclose(hgrp);
return true;
}
void reader_writer::sync(){
AIO_PRE_CONDITION(valid());
if (!m_imp->dirty)
return;
bool zip64_enabled = false;
long_size_t cd_size = 0;
for (auto & i : m_imp->items){
file_header& h = i;
cd_size += K_fix_part_of_central_header
+ h.name.str().size()
+ h.comments.size();
auto need_zip64 = h.compressed_size >= uint32_t(-1)
|| h.uncompressed_size >= uint32_t(-1)
|| h.relative_offset_ >= uint32_t(-1);
zip64_enabled = zip64_enabled || need_zip64;
if (need_zip64)
cd_size += h.extra.size();
}
if (m_imp->items.size() >= uint16_t(-1)
|| cd_size >= uint32_t(-1)
|| m_imp->end_of_last >= uint32_t(-1))
zip64_enabled = true;
uint16_t num_entries = m_imp->items.size() >= uint16_t(-1) ? uint16_t(-1) : uint16_t(m_imp->items.size());
auto cd_and_end_size = cd_size + K_sizeof_cd_end;
if (zip64_enabled)
cd_and_end_size += K_sizeof_zip64_end_cd + K_sizeof_zip64_locator;
auto view = m_imp->archive.get<io::write_map>().view_wr(ext_heap::handle(m_imp->end_of_last, m_imp->end_of_last + cd_and_end_size));
auto address = view.get<io::write_view>().address();
io::fixed_buffer_io bout(address);
auto saver = io::exchange::as_sink(bout);
for (auto & i : m_imp->items){
file_header& h = i;
auto need_zip64 = h.compressed_size >= uint32_t(-1)
|| h.uncompressed_size >= uint32_t(-1)
|| h.relative_offset_ >= uint32_t(-1);
saver & K_sig_central_file_header
& h.creator_version
& h.reader_version
& h.flags
& h.method
& h.modified_time
& h.modified_date
& h.crc32;
if (need_zip64)
saver & uint32_t(-1) & uint32_t(-1);
else
saver & uint32_t(h.compressed_size) & uint32_t(h.uncompressed_size);
saver & uint16_t(h.name.str().size())
& uint16_t(need_zip64 ? h.extra.size() : 0)
& uint16_t(h.comments.size())
& uint16_t(0) //disk number start
& uint16_t(0) //internal file attributes
& h.external_attrs;
if (need_zip64)
saver & uint32_t(-1);
else
saver & uint32_t(h.relative_offset_);
bout.write(make_range(reinterpret_cast<const byte*>(h.name.str().begin())
, reinterpret_cast<const byte*>(h.name.str().end())));
if (need_zip64)
bout.write(to_range(h.extra));
bout.write(make_range(reinterpret_cast<const byte*>(h.comments.begin())
, reinterpret_cast<const byte*>(h.comments.end())));
}
if (zip64_enabled){
saver & K_sig_zip64_end_cd
& uint64_t(cd_size)
& uint16_t(20) //version made by
& uint16_t(20) //version needed to extract
& uint32_t(0) //number of this disk
& uint32_t(0) //number of the disk with the start of the central directory
& uint64_t(m_imp->items.size()) //total number of entries in the central directory on this disk
& uint64_t(m_imp->items.size()) //total number of entries in the central directory
& uint64_t(cd_size) //size of the central directory
& uint64_t(m_imp->end_of_last); //offset of start of central directory with respect to the starting disk number
saver & K_sig_zip64_end_of_cd_locator
& uint32_t(0) //number of the disk with the start of the zip64 end of central directory
& uint64_t(cd_size + m_imp->end_of_last) //relative offset of the zip64 end of central directory record
& uint32_t(1); //total number of disks
}
saver & K_sig_end_central_dir_signature
& uint16_t(0) //number of this disk
& uint16_t(0) //number of the disk with the start of the central directory
& num_entries //total number of entries in the central directory on this disk
& num_entries //total number of entries in the central directory
& (cd_size >= uint32_t(-1)? uint32_t(-1) : uint32_t(cd_size)) //size of the central directory
& (m_imp->end_of_last >= uint32_t(-1)? uint32_t(-1) : uint32_t(m_imp->end_of_last)) //offset of start of central directory with respect to the starting disk number
& uint16_t(0); //.ZIP file comment length
m_imp->archive.get<io::write_map>().sync();
m_imp->dirty = false;
}
const file_header* append_(IoType& ar, const file_header& h_, file_type type, zip_package_writer_imp * m_imp){
file_header h = h_;
AIO_PRE_CONDITION(m_imp);
AIO_PRE_CONDITION(h.method == cm_deflate || h.method == cm_store);
AIO_PRE_CONDITION(h.name.str().size() < uint16_t(-1) );
AIO_PRE_CONDITION(h.comments.size() < uint16_t(-1) );
if (m_imp->exist_(h.name)) return 0;
auto header_size = K_fix_part_of_local_header + h.name.str().size() + K_sizeof_zip64_extra_field;
auto dest_map = io::decorate<io::tail_archive
, io::tail_read_map_p
, io::tail_write_map_p
>(m_imp->archive, m_imp->end_of_last + header_size);
m_imp->dirty = true;
if (h.method == cm_deflate){
if (type == ft_raw){
deflate_writer d_writer(dest_map);
iref<io::writer> dest(d_writer);
io::copy_data(ar, dest.get<io::writer>());
d_writer.finish();
h.uncompressed_size = d_writer.uncompressed_size();
h.compressed_size = d_writer.size();
h.crc32 = d_writer.crc32();
}
else {// type == ft_defalted
io::copy_data(ar, dest_map);
}
}
else { // == cm_store
auto size = io::copy_data(ar, dest_map);
h.uncompressed_size = size;
h.compressed_size = size;
h.crc32 = crc32(dest_map);
}
setDateTime(h);
h.relative_offset_ = m_imp->end_of_last;
bool zip64_enabled = (h.uncompressed_size >= uint32_t(-1)
|| h.compressed_size >= uint32_t(-1)
|| h.relative_offset_ >= uint32_t(-1)
);
{ // fill extra data for zip64
io::buffer_out bout(h.extra);
auto saver = io::exchange::as_sink(bout);
save(saver, uint16_t(K_z64_extra_id)); //
save(saver, uint16_t(28)); //sizeof extra data
save(saver, uint64_t(h.uncompressed_size));
save(saver, uint64_t(h.compressed_size));
save(saver, uint64_t(h.relative_offset_));
save(saver, uint32_t(0)); //Disk Start Number
}
auto hview = m_imp->archive.get<io::write_map>().view_wr(ext_heap::handle(m_imp->end_of_last
, m_imp->end_of_last + header_size));
io::fixed_buffer_io bout(hview.get<io::write_view>().address());
auto saver = io::exchange::as_sink(bout);
saver & K_sig_local_eader;
saver & h.reader_version
& h.flags
& h.method
& h.modified_time
& h.modified_date
& h.crc32;
if (zip64_enabled)
saver & uint32_t(-1) & uint32_t(-1);
else
saver & uint32_t(h.compressed_size) & uint32_t(h.uncompressed_size);
saver & uint16_t(h.name.str().size()) & uint16_t(h.extra.size());
bout.write(make_range(reinterpret_cast<const byte*>(h.name.str().begin())
, reinterpret_cast<const byte*>(h.name.str().end())));
bout.write(to_range(h.extra));
h.local_header_size_ = header_size;
m_imp->items.push_back(h);
m_imp->end_of_last += header_size + h.compressed_size;
return &m_imp->items.back();
}
