static cnpy::NpArray load_the_npy_file(Handler<struct zip_file>& zipFile)
{
std::vector<size_t> shape;
size_t word_size;
bool fortran_order;
NpHeader header;
zip_int64_t nread = zip_fread(zipFile.handle(), &header, sizeof(NpHeader));
if(nread != sizeof(NpHeader))
throw std::runtime_error("Error reading npy header in npz file");
std::string dict;
dict.resize(header.dictSize, ' ');
nread = zip_fread(zipFile.handle(), &dict[0], header.dictSize);
if(nread != header.dictSize)
throw std::runtime_error("Error reading npy dict header in npz file");
char elType;
parseDictHeader(dict, word_size, shape, fortran_order, elType);
cnpy::NpArray arr(shape, word_size, descr2Type(elType, word_size), fortran_order);
nread = zip_fread(zipFile.handle(), arr.data(), arr.size());
if(nread != arr.size())
throw std::runtime_error("npy file read error: expected "+std::to_string(arr.size())+", read "+std::to_string(nread));
return arr;
}
static cnpy::NpArray load_the_npy_file(Handler<std::FILE>& npyFile)
{
std::vector<size_t> shape;
size_t word_size;
bool fortran_order;
NpHeader header;
zip_int64_t nread = std::fread(&header, sizeof(NpHeader), 1, npyFile.handle());
if(nread != 1)
throw std::runtime_error("Error reading npy header");
std::string dict;
dict.resize(header.dictSize, ' ');
nread = std::fread(&dict[0], sizeof(char), header.dictSize, npyFile.handle());
if(nread != header.dictSize)
throw std::runtime_error("Error reading npy dict header");
char elType;
parseDictHeader(dict, word_size, shape, fortran_order, elType);
cnpy::NpArray arr(shape, word_size, descr2Type(elType, word_size), fortran_order);
nread = std::fread(arr.data(), arr.elemSize(), arr.numElements(), npyFile.handle());
if(nread != arr.numElements())
throw std::runtime_error("npy file read error: expected "+std::to_string(arr.numElements())+", read "+std::to_string(nread));
return arr;
}
cnpy::NpArray cnpy::npz_load(const std::string& fname, const std::string& varname)
{
Handler<struct zip> zip = zip_open(fname.c_str(), ZIP_CHECKCONS, nullptr);
if(zip.handle()==nullptr)
throw std::runtime_error("Error opening npz file "+fname);
std::string key = varname + ".npy";
int nameLookup = zip_name_locate(zip.handle(), key.c_str(), 0);
if(nameLookup<0)
throw std::runtime_error("Variable name "+varname+" not found in "+fname);
Handler<struct zip_file> zipFile = zip_fopen_index(zip.handle(), nameLookup, 0);
NpArray array = load_the_npy_file(zipFile);
return array;
}
cnpy::NpArray cnpy::npy_load(const std::string& fname)
{
Handler<std::FILE> fp = std::fopen(fname.c_str(), "r");
if(fp.handle()==NULL)
throw std::runtime_error("Error opening npy file "+fname);
NpArray arr = load_the_npy_file(fp);
return arr;
}
void * doWork(void *vptr) {
Server* s;
s = (Server*) vptr;
while (1) {
int currClient = s->buffer.take();
Handler handler = Handler(currClient, &(s->messageList), s->debug, &(s->serverLock));
handler.handle();
close(currClient);
}
}
cnpy::NpArrayDict cnpy::npz_load(const std::string& fname)
{
Handler<struct zip> zip = zip_open(fname.c_str(), ZIP_CHECKCONS, nullptr);
if(zip.handle()==nullptr)
throw std::runtime_error("Error opening npz file "+fname);
NpArrayDict arrays;
zip_uint64_t numFiles = zip_get_num_entries(zip.handle(), 0);
for(zip_uint64_t fid=0; fid<numFiles; ++fid)
{
const char* arrName = zip_get_name(zip.handle(), fid, 0);
if(arrName==nullptr)
continue;
Handler<struct zip_file> zipFile = zip_fopen_index(zip.handle(), fid, 0);
std::string name = arrName;
name.erase(name.size()-4);
arrays.insert(NpArrayDictItem(name, load_the_npy_file(zipFile)));
}
return arrays;
}
void cnpy::npz_save_data(const std::string& zipname, const std::string& name,
const unsigned char* data, const cnpy::Type dtype,
const size_t elemSize, const std::vector<size_t>& shape,
const char mode)
{
//first, append a .npy to the fname
std::string fname(name);
fname += ".npy";
if(mode=='w' && std::ifstream(zipname).is_open())
{
// Remove the old file if present
if(std::remove(zipname.c_str())!=0)
throw std::runtime_error("Unable to overwrite "+zipname);
}
Handler<struct zip> zip = zip_open(zipname.c_str(), ZIP_CREATE, nullptr);
if(zip.handle()==nullptr)
throw std::runtime_error("Error opening npz file "+zipname);
// Remove the old array if present
int nameLookup = zip_name_locate(zip.handle(), fname.c_str(), 0);
if(nameLookup>=0 && zip_delete(zip.handle(), nameLookup)!=0)
throw std::runtime_error("Unable to overwrite "+name+" array");
std::vector<char> header = create_npy_header(dtype, elemSize, shape);
const int dataSize = std::accumulate(shape.cbegin(), shape.cend(), elemSize, std::multiplies<size_t>());
ZipSourceCallbackData cbData(header, data, dataSize);
Handler<struct zip_source> zipSource = zip_source_function(zip.handle(), zipSourceCallback, &cbData);
if(zipSource.handle()==nullptr)
throw std::runtime_error("Error creating "+name+" array");
zip_int64_t fid = zip_add(zip.handle(), fname.c_str(), zipSource.handle());
if(fid<0)
{
zip_source_free(zipSource.handle());
throw std::runtime_error("Error creating "+name+" array");
}
zip.close();
}
int Server::loop_once(){
const Fdevents::events_t *events;
events = fdes->wait(20);
if(events == NULL){
log_fatal("events.wait error: %s", strerror(errno));
return 0;
}
for(int i=0; i<(int)events->size(); i++){
const Fdevent *fde = events->at(i);
if(fde->data.ptr == serv_link){
this->accept_session();
}else if(fde->data.num == HANDLER_TYPE){
Handler *handler = (Handler *)fde->data.ptr;
while(Response *resp = handler->handle()){
Session *sess = this->get_session(resp->sess.id);
if(sess){
Link *link = sess->link;
link->send(resp->msg);
if(link && !link->output.empty()){
fdes->set(link->fd(), FDEVENT_OUT, DEFAULT_TYPE, sess);
}
}
delete resp;
}
}else{
Session *sess = (Session *)fde->data.ptr;
Link *link = sess->link;
if(fde->events & FDEVENT_IN){
if(this->read_session(sess) == -1){
continue;
}
}
if(fde->events & FDEVENT_OUT){
if(this->write_session(sess) == -1){
continue;
}
}
if(link && !link->output.empty()){
fdes->set(link->fd(), FDEVENT_OUT, DEFAULT_TYPE, sess);
}
}
}
return 0;
}
inline void readEnum(
std::streambuf & istrm, Handler & result,
const ReaderFunc & read, const UInt8 beginsig, const UInt8 endsig)
{
verifySignature(istrm, beginsig);
bool done = false;
while (!done)
{
try
{
result.handle(read(istrm));
}
catch (const BadSignatureException& e)
{
// read threw because we've read all the objects
verifySignature(istrm, endsig);
done = true;
}
}
}
