int r_io_zip_slurp_file(RIOZipFileObj *zfo) { int res = R_FALSE; struct zip_stat sb; struct zip_file *zFile = NULL; struct zip * zipArch ; if (!zfo) return res; zipArch = r_io_zip_open_archive ( zfo->archivename, zfo->flags, zfo->mode, zfo->rw); //eprintf("Slurping file"); if (zipArch && zfo && zfo->entry != -1) { zFile = zip_fopen_index (zipArch, zfo->entry, 0); if (!zfo->b) zfo->b = r_buf_new (); zip_stat_init (&sb); if (zFile && zfo->b && !zip_stat_index(zipArch, zfo->entry, 0, &sb) ) { ut8 *buf = malloc (sb.size); memset (buf, 0, sb.size); if (buf) { zip_fread (zFile, buf, sb.size); r_buf_set_bytes (zfo->b, buf, sb.size); res = zfo->opened = R_TRUE; free (buf); } } zip_fclose (zFile); } zip_close (zipArch); return res; }
/* The file can be a file in the archive or ::[num]. */ RIOZipFileObj* r_io_zip_alloc_zipfileobj(const char *archivename, const char *filename, ut32 perm, int mode, int rw) { RIOZipFileObj *zfo = NULL; ut64 i, num_entries; struct zip_stat sb; struct zip *zipArch = r_io_zip_open_archive (archivename, perm, mode, rw); if (!zipArch) { return NULL; } num_entries = zip_get_num_files (zipArch); for (i = 0; i < num_entries; i++) { zip_stat_init (&sb); zip_stat_index (zipArch, i, 0, &sb); if (sb.name != NULL) { if (strcmp (sb.name, filename) == 0) { zfo = r_io_zip_create_new_file ( archivename, filename, &sb, perm, mode, rw); r_io_zip_slurp_file (zfo); break; } } } if (!zfo) { zfo = r_io_zip_create_new_file (archivename, filename, NULL, perm, mode, rw); } zip_close (zipArch); return zfo; }
int ipsw_get_file_size(const char* ipsw, const char* infile, off_t* size) { ipsw_archive* archive = ipsw_open(ipsw); if (archive == NULL || archive->zip == NULL) { error("ERROR: Invalid archive\n"); return -1; } int zindex = zip_name_locate(archive->zip, infile, 0); if (zindex < 0) { error("ERROR: zip_name_locate: %s\n", infile); return -1; } struct zip_stat zstat; zip_stat_init(&zstat); if (zip_stat_index(archive->zip, zindex, 0, &zstat) != 0) { error("ERROR: zip_stat_index: %s\n", infile); return -1; } *size = zstat.size; ipsw_close(archive); return 0; }
RList * r_io_zip_get_files(char *archivename, ut32 flags, int mode, int rw) { ut64 num_entries = 0, i = 0; struct zip * zipArch = r_io_zip_open_archive(archivename, flags, mode, rw); struct zip_stat sb; RList *files = NULL; //eprintf("Slurping file"); if (zipArch) { files = r_list_new(); num_entries = zip_get_num_files(zipArch); for (i=0; i < num_entries; i++) { char *name = NULL; zip_stat_init(&sb ); zip_stat_index(zipArch, i, 0, &sb ); //eprintf("Comparing %s == %s = %d\n", sb.name, filename, strcmp(sb.name, filename)); name = strdup(sb.name); if (name) { r_list_append(files, name); } } } if (zipArch) zip_close(zipArch); return files; }
/************************************************* Function: readZipFileName Descroption: Input: 1.zip* z 2.filetype Output: Return: Other: *************************************************/ int readZipFileName(struct zip* z, char* filetype) { int i; struct zip_stat fstat; if(z != NULL) { zip_stat_init(&fstat); int c = zip_get_num_files(z); if(c > 0) { for (i = 0 ; i < c; i++) { const char* name = zip_get_name(z, i, 0); if(name != NULL) { zip_stat(z, name,0,&fstat); //LOGI("File %i:%s Size: %lld Size2: %lld\n", i,fstat.name,fstat.size ,fstat.comp_size); } } } } else { return -1; } return 0; }
static int r_io_zip_slurp_file(RIOZipFileObj *zfo) { struct zip_file *zFile = NULL; struct zip *zipArch; struct zip_stat sb; bool res = false; if (!zfo) { return res; } zipArch = r_io_zip_open_archive ( zfo->archivename, zfo->perm, zfo->mode, zfo->rw); if (zipArch && zfo && zfo->entry != -1) { zFile = zip_fopen_index (zipArch, zfo->entry, 0); if (!zfo->b) { zfo->b = r_buf_new (); } zip_stat_init (&sb); if (zFile && zfo->b && !zip_stat_index (zipArch, zfo->entry, 0, &sb)) { ut8 *buf = malloc (sb.size); memset (buf, 0, sb.size); if (buf) { zip_fread (zFile, buf, sb.size); r_buf_set_bytes (zfo->b, buf, sb.size); res = true; zfo->opened = true; free (buf); } } zip_fclose (zFile); } zip_close (zipArch); return res; }
ZIP_EXTERN int zip_source_stat(zip_source_t *src, zip_stat_t *st) { if (src->source_closed) { return -1; } if (st == NULL) { zip_error_set(&src->error, ZIP_ER_INVAL, 0); return -1; } zip_stat_init(st); if (ZIP_SOURCE_IS_LAYERED(src)) { if (zip_source_stat(src->src, st) < 0) { _zip_error_set_from_source(&src->error, src->src); return -1; } } if (_zip_source_call(src, st, sizeof(*st), ZIP_SOURCE_STAT) < 0) { return -1; } return 0; }
static VALUE zipruby_stat_alloc(VALUE klass) { struct zipruby_stat *p = ALLOC(struct zipruby_stat); p->sb = ALLOC(struct zip_stat); zip_stat_init(p->sb); return Data_Wrap_Struct(klass, 0, zipruby_stat_free, p); }
size_t ZIPProvider::ZIPHandle::fileSize(const FileName & file) { struct zip_stat sb; zip_stat_init(&sb); if (zip_stat(handle, file.getPath().c_str(), 0, &sb) == -1) { WARN(zip_strerror(handle)); return 0; } return static_cast<size_t>(sb.size); }
bool ZIPProvider::ZIPHandle::isFile(const FileName & file) { struct zip_stat sb; zip_stat_init(&sb); if (zip_stat(handle, file.getPath().c_str(), 0, &sb) == -1) { return false; } std::string entry(sb.name); return (entry.back() != '/' && sb.size != 0); }
void ZipByteReader::Register(size_t seqId, const std::string& path) { auto zipFile = m_zips.pop_or_create([this]() { return OpenZip(); }); zip_stat_t stat; zip_stat_init(&stat); int err = zip_stat(zipFile.get(), path.c_str(), 0, &stat); if (ZIP_ER_OK != err) RuntimeError("Failed to get file info of %s, zip library error: %s", path.c_str(), GetZipError(err).c_str()); m_seqIdToIndex[seqId] = std::make_pair(stat.index, stat.size); m_zips.push(std::move(zipFile)); }
static zip_int64_t zipSourceCallback(void* userdata, void* data, zip_uint64_t len, zip_source_cmd cmd) { ZipSourceCallbackData* cbData = reinterpret_cast<ZipSourceCallbackData*>(userdata); switch (cmd) { case ZIP_SOURCE_OPEN: cbData->offset = 0; cbData->headerIsDone = false; break; case ZIP_SOURCE_READ: { if(cbData->headerIsDone) { size_t remain = cbData->dataSize-cbData->offset; size_t toCopy = std::min((size_t)len, remain); if(toCopy>0) std::memcpy(data, &cbData->data[cbData->offset], toCopy); cbData->offset += toCopy; return toCopy; } else { size_t remain = cbData->header.size()-cbData->offset; size_t toCopy = std::min((size_t)len, remain); if(toCopy>0) std::memcpy(data, &cbData->header[cbData->offset], toCopy); cbData->offset += toCopy; if(cbData->offset==cbData->header.size()) { cbData->headerIsDone = true; cbData->offset = 0; } return toCopy; } } case ZIP_SOURCE_CLOSE: break; case ZIP_SOURCE_STAT: { struct zip_stat* zs = reinterpret_cast<struct zip_stat*>(data); zip_stat_init(zs); zs->encryption_method = ZIP_EM_NONE; zs->size = cbData->dataSize + cbData->header.size(); /* size of file (uncompressed) */ zs->mtime = std::time(nullptr); zs->comp_method = ZIP_CM_STORE; /* compression method used */ zs->valid = ZIP_STAT_SIZE | ZIP_STAT_MTIME | ZIP_STAT_COMP_METHOD | ZIP_STAT_ENCRYPTION_METHOD; return 0; } case ZIP_SOURCE_ERROR: break; default: break; } return 0; }
zip_t * _zip_open(zip_source_t *src, unsigned int flags, zip_error_t *error) { zip_t *za; zip_cdir_t *cdir; struct zip_stat st; zip_uint64_t len; zip_stat_init(&st); if (zip_source_stat(src, &st) < 0) { _zip_error_set_from_source(error, src); return NULL; } if ((st.valid & ZIP_STAT_SIZE) == 0) { zip_error_set(error, ZIP_ER_SEEK, EOPNOTSUPP); return NULL; } len = st.size; /* treat empty files as empty archives */ if (len == 0) { if ((za=_zip_allocate_new(src, flags, error)) == NULL) { zip_source_free(src); return NULL; } return za; } if ((za=_zip_allocate_new(src, flags, error)) == NULL) { return NULL; } if ((cdir = _zip_find_central_dir(za, len)) == NULL) { _zip_error_copy(error, &za->error); /* keep src so discard does not get rid of it */ zip_source_keep(src); zip_discard(za); return NULL; } za->entry = cdir->entry; za->nentry = cdir->nentry; za->nentry_alloc = cdir->nentry_alloc; za->comment_orig = cdir->comment; za->ch_flags = za->flags; free(cdir); return za; }
bool ZIPProvider::ZIPHandle::isDir(const FileName & directory) { if (directory.getDir().empty()) { // Always return true for the root archive directory. return true; } struct zip_stat sb; zip_stat_init(&sb); if (zip_stat(handle, directory.getDir().c_str(), 0, &sb) == -1) { return false; } std::string entry(sb.name); return (entry.back() == '/' && sb.size == 0); }
int ipsw_extract_to_memory(const char* ipsw, const char* infile, unsigned char** pbuffer, unsigned int* psize) { ipsw_archive* archive = ipsw_open(ipsw); if (archive == NULL || archive->zip == NULL) { error("ERROR: Invalid archive\n"); return -1; } int zindex = zip_name_locate(archive->zip, infile, 0); if (zindex < 0) { error("ERROR: zip_name_locate: %s\n", infile); return -1; } struct zip_stat zstat; zip_stat_init(&zstat); if (zip_stat_index(archive->zip, zindex, 0, &zstat) != 0) { error("ERROR: zip_stat_index: %s\n", infile); return -1; } struct zip_file* zfile = zip_fopen_index(archive->zip, zindex, 0); if (zfile == NULL) { error("ERROR: zip_fopen_index: %s\n", infile); return -1; } int size = zstat.size; unsigned char* buffer = (unsigned char*) malloc(size+1); if (buffer == NULL) { error("ERROR: Out of memory\n"); zip_fclose(zfile); return -1; } if (zip_fread(zfile, buffer, size) != size) { error("ERROR: zip_fread: %s\n", infile); zip_fclose(zfile); free(buffer); return -1; } buffer[size] = '\0'; zip_fclose(zfile); ipsw_close(archive); *pbuffer = buffer; *psize = size; return 0; }
/* * tests for file existence */ static exists_t _zip_file_exists(zip_source_t *src, zip_error_t *error) { struct zip_stat st; zip_stat_init(&st); if (zip_source_stat(src, &st) != 0) { zip_error_t *src_error = zip_source_error(src); if (zip_error_code_zip(src_error) == ZIP_ER_READ && zip_error_code_system(src_error) == ENOENT) { return EXISTS_NOT; } _zip_error_copy(error, src_error); return EXISTS_ERROR; } return (st.valid & ZIP_STAT_SIZE) && st.size == 0 ? EXISTS_EMPTY : EXISTS_NONEMPTY; }
static int zip_get_contents(struct zip *zf, const char *filename, int locate_flags, char **buffer, uint32_t *len) { struct zip_stat zs; struct zip_file *zfile; int zindex = zip_name_locate(zf, filename, locate_flags); *buffer = NULL; *len = 0; if (zindex < 0) { return -1; } zip_stat_init(&zs); if (zip_stat_index(zf, zindex, 0, &zs) != 0) { fprintf(stderr, "ERROR: zip_stat_index '%s' failed!\n", filename); return -2; } if (zs.size > 10485760) { fprintf(stderr, "ERROR: file '%s' is too large!\n", filename); return -3; } zfile = zip_fopen_index(zf, zindex, 0); if (!zfile) { fprintf(stderr, "ERROR: zip_fopen '%s' failed!\n", filename); return -4; } *buffer = malloc(zs.size); if (zs.size > LLONG_MAX || zip_fread(zfile, *buffer, zs.size) != (zip_int64_t)zs.size) { fprintf(stderr, "ERROR: zip_fread %" PRIu64 " bytes from '%s'\n", (uint64_t)zs.size, filename); free(*buffer); *buffer = NULL; zip_fclose(zfile); return -5; } *len = zs.size; zip_fclose(zfile); return 0; }
struct zip_source * _zip_source_file_or_p(struct zip *za, const char *fname, FILE *file, zip_uint64_t start, zip_int64_t len, int closep, const struct zip_stat *st) { struct read_file *f; struct zip_source *zs; if (file == NULL && fname == NULL) { _zip_error_set(&za->error, ZIP_ER_INVAL, 0); return NULL; } if ((f=(struct read_file *)malloc(sizeof(struct read_file))) == NULL) { _zip_error_set(&za->error, ZIP_ER_MEMORY, 0); return NULL; } f->fname = NULL; if (fname) { if ((f->fname=strdup(fname)) == NULL) { _zip_error_set(&za->error, ZIP_ER_MEMORY, 0); free(f); return NULL; } } f->f = file; f->off = start; f->len = (len ? len : -1); f->closep = f->fname ? 1 : closep; if (st) memcpy(&f->st, st, sizeof(f->st)); else zip_stat_init(&f->st); if ((zs=zip_source_function(za, read_file, f)) == NULL) { free(f); return NULL; } return zs; }
static int _zip_stat_win32(HANDLE h, zip_stat_t *st, _zip_source_win32_read_file_t *ctx) { FILETIME mtimeft; time_t mtime; LARGE_INTEGER size; int regularp; if (!GetFileTime(h, NULL, NULL, &mtimeft)) { zip_error_set(&ctx->error, ZIP_ER_READ, _zip_win32_error_to_errno(GetLastError())); return -1; } if (_zip_filetime_to_time_t(mtimeft, &mtime) < 0) { zip_error_set(&ctx->error, ZIP_ER_READ, ERANGE); return -1; } regularp = 0; if (GetFileType(h) == FILE_TYPE_DISK) { regularp = 1; } if (!GetFileSizeEx(h, &size)) { zip_error_set(&ctx->error, ZIP_ER_READ, _zip_win32_error_to_errno(GetLastError())); return -1; } zip_stat_init(st); st->mtime = mtime; st->valid |= ZIP_STAT_MTIME; if (ctx->end != 0) { st->size = ctx->end - ctx->start; st->valid |= ZIP_STAT_SIZE; } else if (regularp) { st->size = (zip_uint64_t)size.QuadPart; st->valid |= ZIP_STAT_SIZE; } return 0; }
void ZipByteReader::Register(const std::map<std::string, size_t>& sequences) { auto zipFile = m_zips.pop_or_create([this]() { return OpenZip(); }); zip_stat_t stat; zip_stat_init(&stat); size_t numberOfEntries = 0; size_t numEntries = zip_get_num_entries(zipFile.get(), 0); for (size_t i = 0; i < numEntries; ++i) { int err = zip_stat_index(zipFile.get(), i, 0, &stat); if (ZIP_ER_OK != err) RuntimeError("Failed to get file info for index %d, zip library error: %s", (int)i, GetZipError(err).c_str()); auto sequenceId = sequences.find(std::string(stat.name)); if (sequenceId == sequences.end()) { continue; } else { m_seqIdToIndex[sequenceId->second] = std::make_pair(stat.index, stat.size); numberOfEntries++; } } m_zips.push(std::move(zipFile)); if (numberOfEntries != sequences.size()) { // Not all sequences have been found. Let's print them out and throw. for (const auto& s : sequences) { auto index = m_seqIdToIndex.find(s.second); if (index == m_seqIdToIndex.end()) { fprintf(stderr, "Sequence %s is not found in container %s.\n", s.first.c_str(), m_zipPath.c_str()); } } RuntimeError("Cannot retrieve image data for some sequences. For more detail, please see the log file."); } }
/************************************************* Function: readDexFile Descroption: Input: 1.zip* z 2.filename 3.char* buf Output: Return: Other: *************************************************/ long readDexFile(struct zip* z, char* filename, unsigned char** buf) { int i; long ReadNum = 0; struct zip_stat fstat; if(z != NULL && NULL != buf) { zip_stat_init(&fstat); int c = zip_get_num_files(z); if(c > 0) { for (i = 0 ; i < c; i++) { const char* name = zip_get_name(z, i, 0); if(0 == strcmp(name,filename)) { zip_stat(z, name,0,&fstat); //LOGI("File %i:%s Size1: %lld Size2: %lld\n", i,fstat.name,fstat.size ,fstat.comp_size); struct zip_file* file = zip_fopen(z, filename, 0); if (file) { *buf =(unsigned char *)malloc(fstat.size+1); memset(*buf,0,(fstat.size+1)); ReadNum = zip_fread(file, *buf,fstat.size); zip_fclose(file); } break; } } } } else { return 0; } return ReadNum; }
SDL_RWops *SDL_RWFromZip(struct zip *z,const char *fname) { zip_file_t* zf; struct zip_stat st; Sint64 idx = zip_name_locate(z, fname, 0); if ( idx < 0){ SDL_SetError("Can't find file %s",fname); return NULL; } //printf("Found file %s with idx %ld\n",fname,idx); zf=zip_fopen_index(z,idx,ZIP_FL_UNCHANGED); if(zf == NULL ){ zip_error_t *error = zip_get_error(z); SDL_SetError("PCK_RWFromZip failed for idx=%ld:%s", idx,zip_error_strerror(error)); zip_error_fini(error); return NULL; } zip_stat_init(&st); if (zip_stat_index(z, idx, 0, &st) == 0) { } SDL_RWops *c = SDL_AllocRW(); if (c == NULL) return NULL; c->seek = sdl_zip_seekfunc; c->size = sdl_zip_sizefunc; c->read = sdl_zip_readfunc; c->write = sdl_zip_writefunc; c->close = sdl_zip_closefunc; c->type = SDL_RW_TAG_CURRENT; ZipInfo* zi = SDL_malloc(sizeof(ZipInfo)); zi->size = st.size; zi->zip = z; c->hidden.unknown.data1 = zf; c->hidden.unknown.data2 = zi; return c; }
RList * r_io_zip_get_files(char *archivename, ut32 perm, int mode, int rw) { struct zip *zipArch = r_io_zip_open_archive (archivename, perm, mode, rw); ut64 num_entries = 0, i = 0; RList *files = NULL; struct zip_stat sb; char *name; if (zipArch) { files = r_list_newf (free); if (!files) { zip_close (zipArch); return NULL; } num_entries = zip_get_num_files (zipArch); for (i = 0; i < num_entries; i++) { zip_stat_init (&sb); zip_stat_index (zipArch, i, 0, &sb); if ((name = strdup (sb.name))) { r_list_append (files, name); } } } zip_close (zipArch); return files; }
zip_source_t * _zip_source_file_or_p(const char *fname, FILE *file, zip_uint64_t start, zip_int64_t len, const zip_stat_t *st, zip_error_t *error) { struct read_file *ctx; zip_source_t *zs; if (file == NULL && fname == NULL) { zip_error_set(error, ZIP_ER_INVAL, 0); return NULL; } if ((ctx=(struct read_file *)malloc(sizeof(struct read_file))) == NULL) { zip_error_set(error, ZIP_ER_MEMORY, 0); return NULL; } ctx->fname = NULL; if (fname) { if ((ctx->fname=strdup(fname)) == NULL) { zip_error_set(error, ZIP_ER_MEMORY, 0); free(ctx); return NULL; } } ctx->f = file; ctx->start = start; ctx->end = (len < 0 ? 0 : start+(zip_uint64_t)len); if (st) { memcpy(&ctx->st, st, sizeof(ctx->st)); ctx->st.name = NULL; ctx->st.valid &= ~ZIP_STAT_NAME; } else { zip_stat_init(&ctx->st); } ctx->tmpname = NULL; ctx->fout = NULL; zip_error_init(&ctx->error); ctx->supports = ZIP_SOURCE_SUPPORTS_READABLE | zip_source_make_command_bitmap(ZIP_SOURCE_SUPPORTS, ZIP_SOURCE_TELL, -1); if (ctx->fname) { struct stat sb; if (stat(ctx->fname, &sb) < 0 || S_ISREG(sb.st_mode)) { ctx->supports = ZIP_SOURCE_SUPPORTS_WRITABLE; } } else if (fseeko(ctx->f, 0, SEEK_CUR) == 0) { ctx->supports = ZIP_SOURCE_SUPPORTS_SEEKABLE; } if ((zs=zip_source_function_create(read_file, ctx, error)) == NULL) { free(ctx->fname); free(ctx); return NULL; } return zs; }
AbstractFSProvider::status_t ZIPProvider::ZIPHandle::dir( const std::string & localDirectory, std::list<FileName> & result, const uint8_t flags) { int num = zip_get_num_files(handle); if (num == -1) { WARN(zip_strerror(handle)); return FAILURE; } struct zip_stat sb; zip_stat_init(&sb); // Iterate over indices. for (int i = 0; i < num; ++i) { if (zip_stat_index(handle, i, 0, &sb) == -1) { WARN(zip_strerror(handle)); return FAILURE; } FileName entryFileName(sb.name); // Determine if the entry is a file or a directory. if (entryFileName.getFile().empty()) { if(!(flags & FileUtils::DIR_DIRECTORIES)) { continue; } if (!(flags & FileUtils::DIR_RECURSIVE)) { std::string entryDirectory = entryFileName.getDir(); if(entryDirectory == localDirectory) { continue; } if(entryDirectory.back() == '/') { entryDirectory.resize(entryDirectory.size() - 1); } const auto slashPos = entryDirectory.find_last_of('/'); if(slashPos != std::string::npos) { entryDirectory = entryDirectory.substr(0, slashPos + 1); } else { entryDirectory.clear(); } // Compare the parent directory of the directory with the requested localDirectory. if(entryDirectory != localDirectory) { continue; } } } else { if(!(flags & FileUtils::DIR_FILES)) { continue; } // Compare the directory of the file with the requested localDirectory. if (!(flags & FileUtils::DIR_RECURSIVE) && entryFileName.getDir() != localDirectory) { continue; } } // Check for hidden files beginning with '.' (files only). if (entryFileName.getFile().front() == '.' && !(flags & FileUtils::DIR_HIDDEN_FILES)) { continue; } FileName f; f.setFSName(archiveRoot.getFSName()); f.setDir(archiveRoot.getDir() + entryFileName.getDir()); f.setFile(entryFileName.getFile()); result.push_back(f); } return OK; }
zip_source_t * _zip_source_win32_handle_or_name(const void *fname, HANDLE h, zip_uint64_t start, zip_int64_t len, int closep, const zip_stat_t *st, _zip_source_win32_file_ops_t *ops, zip_error_t *error) { _zip_source_win32_read_file_t *ctx; zip_source_t *zs; if (h == INVALID_HANDLE_VALUE && fname == NULL) { zip_error_set(error, ZIP_ER_INVAL, 0); return NULL; } if ((ctx = (_zip_source_win32_read_file_t *)malloc(sizeof(_zip_source_win32_read_file_t))) == NULL) { zip_error_set(error, ZIP_ER_MEMORY, 0); return NULL; } ctx->fname = NULL; if (fname) { if ((ctx->fname = ops->op_strdup(fname)) == NULL) { zip_error_set(error, ZIP_ER_MEMORY, 0); free(ctx); return NULL; } } ctx->ops = ops; ctx->h = h; ctx->start = start; ctx->end = (len < 0 ? 0 : start + (zip_uint64_t)len); ctx->closep = ctx->fname ? 1 : closep; if (st) { memcpy(&ctx->st, st, sizeof(ctx->st)); ctx->st.name = NULL; ctx->st.valid &= ~ZIP_STAT_NAME; } else { zip_stat_init(&ctx->st); } ctx->tmpname = NULL; ctx->hout = INVALID_HANDLE_VALUE; zip_error_init(&ctx->error); ctx->supports = ZIP_SOURCE_SUPPORTS_READABLE | zip_source_make_command_bitmap(ZIP_SOURCE_SUPPORTS, ZIP_SOURCE_TELL, -1); if (ctx->fname) { HANDLE th; th = ops->op_open(ctx); if (th == INVALID_HANDLE_VALUE || GetFileType(th) == FILE_TYPE_DISK) { ctx->supports = ZIP_SOURCE_SUPPORTS_WRITABLE; } if (th != INVALID_HANDLE_VALUE) { CloseHandle(th); } } else if (GetFileType(ctx->h) == FILE_TYPE_DISK) { ctx->supports = ZIP_SOURCE_SUPPORTS_SEEKABLE; } if ((zs = zip_source_function_create(_win32_read_file, ctx, error)) == NULL) { free(ctx->fname); free(ctx); return NULL; } return zs; }
static zip_int64_t read_data(void *state, void *data, zip_uint64_t len, zip_source_cmd_t cmd) { struct read_data *ctx = (struct read_data *)state; switch (cmd) { case ZIP_SOURCE_BEGIN_WRITE: if ((ctx->out = buffer_new_write(WRITE_FRAGMENT_SIZE)) == NULL) { zip_error_set(&ctx->error, ZIP_ER_MEMORY, 0); return -1; } return 0; case ZIP_SOURCE_CLOSE: return 0; case ZIP_SOURCE_COMMIT_WRITE: buffer_free(ctx->in); ctx->in = ctx->out; ctx->out = NULL; return 0; case ZIP_SOURCE_ERROR: return zip_error_to_data(&ctx->error, data, len); case ZIP_SOURCE_FREE: buffer_free(ctx->in); buffer_free(ctx->out); free(ctx); return 0; case ZIP_SOURCE_OPEN: ctx->in->offset = 0; return 0; case ZIP_SOURCE_READ: if (len > ZIP_INT64_MAX) { zip_error_set(&ctx->error, ZIP_ER_INVAL, 0); return -1; } return buffer_read(ctx->in, data, len); case ZIP_SOURCE_REMOVE: { buffer_t *empty = buffer_new_read(NULL, 0, 0); if (empty == 0) { zip_error_set(&ctx->error, ZIP_ER_MEMORY, 0); return -1; } buffer_free(ctx->in); ctx->in = empty; return 0; } case ZIP_SOURCE_ROLLBACK_WRITE: buffer_free(ctx->out); ctx->out = NULL; return 0; case ZIP_SOURCE_SEEK: return buffer_seek(ctx->in, data, len, &ctx->error); case ZIP_SOURCE_SEEK_WRITE: return buffer_seek(ctx->out, data, len, &ctx->error); case ZIP_SOURCE_STAT: { zip_stat_t *st; if (len < sizeof(*st)) { zip_error_set(&ctx->error, ZIP_ER_INVAL, 0); return -1; } st = (zip_stat_t *)data; zip_stat_init(st); st->mtime = ctx->mtime; st->size = ctx->in->size; st->comp_size = st->size; st->comp_method = ZIP_CM_STORE; st->encryption_method = ZIP_EM_NONE; st->valid = ZIP_STAT_MTIME|ZIP_STAT_SIZE|ZIP_STAT_COMP_SIZE|ZIP_STAT_COMP_METHOD|ZIP_STAT_ENCRYPTION_METHOD; return sizeof(*st); } case ZIP_SOURCE_SUPPORTS: return zip_source_make_command_bitmap(ZIP_SOURCE_OPEN, ZIP_SOURCE_READ, ZIP_SOURCE_CLOSE, ZIP_SOURCE_STAT, ZIP_SOURCE_ERROR, ZIP_SOURCE_FREE, ZIP_SOURCE_SEEK, ZIP_SOURCE_TELL, ZIP_SOURCE_BEGIN_WRITE, ZIP_SOURCE_COMMIT_WRITE, ZIP_SOURCE_REMOVE, ZIP_SOURCE_ROLLBACK_WRITE, ZIP_SOURCE_SEEK_WRITE, ZIP_SOURCE_TELL_WRITE, ZIP_SOURCE_WRITE, -1); case ZIP_SOURCE_TELL: if (ctx->in->offset > ZIP_INT64_MAX) { zip_error_set(&ctx->error, ZIP_ER_TELL, EOVERFLOW); return -1; } return (zip_int64_t)ctx->in->offset; case ZIP_SOURCE_TELL_WRITE: if (ctx->out->offset > ZIP_INT64_MAX) { zip_error_set(&ctx->error, ZIP_ER_TELL, EOVERFLOW); return -1; } return (zip_int64_t)ctx->out->offset; case ZIP_SOURCE_WRITE: if (len > ZIP_INT64_MAX) { zip_error_set(&ctx->error, ZIP_ER_INVAL, 0); return -1; } return buffer_write(ctx->out, data, len, &ctx->error); default: zip_error_set(&ctx->error, ZIP_ER_OPNOTSUPP, 0); return -1; } }
static zip_int64_t read_file(void *state, void *data, zip_uint64_t len, zip_source_cmd_t cmd) { struct read_file *ctx; char *buf; zip_uint64_t n; size_t i; ctx = (struct read_file *)state; buf = (char *)data; switch (cmd) { case ZIP_SOURCE_BEGIN_WRITE: if (ctx->fname == NULL) { zip_error_set(&ctx->error, ZIP_ER_OPNOTSUPP, 0); return -1; } return create_temp_output(ctx); case ZIP_SOURCE_COMMIT_WRITE: { mode_t mask; if (fclose(ctx->fout) < 0) { ctx->fout = NULL; zip_error_set(&ctx->error, ZIP_ER_WRITE, errno); } ctx->fout = NULL; if (rename(ctx->tmpname, ctx->fname) < 0) { zip_error_set(&ctx->error, ZIP_ER_RENAME, errno); return -1; } mask = umask(022); umask(mask); /* not much we can do if chmod fails except make the whole commit fail */ (void)chmod(ctx->fname, 0666&~mask); free(ctx->tmpname); ctx->tmpname = NULL; return 0; } case ZIP_SOURCE_CLOSE: if (ctx->fname) { fclose(ctx->f); ctx->f = NULL; } return 0; case ZIP_SOURCE_ERROR: return zip_error_to_data(&ctx->error, data, len); case ZIP_SOURCE_FREE: free(ctx->fname); free(ctx->tmpname); if (ctx->f) fclose(ctx->f); free(ctx); return 0; case ZIP_SOURCE_OPEN: if (ctx->fname) { if ((ctx->f=fopen(ctx->fname, "rb")) == NULL) { zip_error_set(&ctx->error, ZIP_ER_OPEN, errno); return -1; } } if (ctx->start > 0) { if (_zip_fseek_u(ctx->f, ctx->start, SEEK_SET, &ctx->error) < 0) { return -1; } } ctx->current = ctx->start; return 0; case ZIP_SOURCE_READ: if (ctx->end > 0) { n = ctx->end-ctx->current; if (n > len) { n = len; } } else { n = len; } if (n > SIZE_MAX) n = SIZE_MAX; if ((i=fread(buf, 1, (size_t)n, ctx->f)) == 0) { if (ferror(ctx->f)) { zip_error_set(&ctx->error, ZIP_ER_READ, errno); return -1; } } ctx->current += i; return (zip_int64_t)i; case ZIP_SOURCE_REMOVE: if (remove(ctx->fname) < 0) { zip_error_set(&ctx->error, ZIP_ER_REMOVE, errno); return -1; } return 0; case ZIP_SOURCE_ROLLBACK_WRITE: if (ctx->fout) { fclose(ctx->fout); ctx->fout = NULL; } (void)remove(ctx->tmpname); free(ctx->tmpname); ctx->tmpname = NULL; return 0; case ZIP_SOURCE_SEEK: { zip_int64_t new_current; int need_seek; zip_source_args_seek_t *args = ZIP_SOURCE_GET_ARGS(zip_source_args_seek_t, data, len, &ctx->error); if (args == NULL) return -1; need_seek = 1; switch (args->whence) { case SEEK_SET: new_current = args->offset; break; case SEEK_END: if (ctx->end == 0) { if (_zip_fseek(ctx->f, args->offset, SEEK_END, &ctx->error) < 0) { return -1; } if ((new_current = ftello(ctx->f)) < 0) { zip_error_set(&ctx->error, ZIP_ER_SEEK, errno); return -1; } need_seek = 0; } else { new_current = (zip_int64_t)ctx->end + args->offset; } break; case SEEK_CUR: new_current = (zip_int64_t)ctx->current + args->offset; break; default: zip_error_set(&ctx->error, ZIP_ER_INVAL, 0); return -1; } if (new_current < 0 || (zip_uint64_t)new_current < ctx->start || (ctx->end != 0 && (zip_uint64_t)new_current > ctx->end)) { zip_error_set(&ctx->error, ZIP_ER_INVAL, 0); return -1; } ctx->current = (zip_uint64_t)new_current; if (need_seek) { if (_zip_fseek_u(ctx->f, ctx->current, SEEK_SET, &ctx->error) < 0) { return -1; } } return 0; } case ZIP_SOURCE_SEEK_WRITE: { zip_source_args_seek_t *args; args = ZIP_SOURCE_GET_ARGS(zip_source_args_seek_t, data, len, &ctx->error); if (args == NULL) { return -1; } if (_zip_fseek(ctx->fout, args->offset, args->whence, &ctx->error) < 0) { return -1; } return 0; } case ZIP_SOURCE_STAT: { if (len < sizeof(ctx->st)) return -1; if (ctx->st.valid != 0) memcpy(data, &ctx->st, sizeof(ctx->st)); else { zip_stat_t *st; struct stat fst; int err; if (ctx->f) err = fstat(fileno(ctx->f), &fst); else err = stat(ctx->fname, &fst); if (err != 0) { zip_error_set(&ctx->error, ZIP_ER_READ, errno); return -1; } st = (zip_stat_t *)data; zip_stat_init(st); st->mtime = fst.st_mtime; st->valid |= ZIP_STAT_MTIME; if (ctx->end != 0) { st->size = ctx->end - ctx->start; st->valid |= ZIP_STAT_SIZE; } else if ((fst.st_mode&S_IFMT) == S_IFREG) { st->size = (zip_uint64_t)fst.st_size; st->valid |= ZIP_STAT_SIZE; } } return sizeof(ctx->st); } case ZIP_SOURCE_SUPPORTS: return ctx->supports; case ZIP_SOURCE_TELL: return (zip_int64_t)ctx->current; case ZIP_SOURCE_TELL_WRITE: { off_t ret = ftello(ctx->fout); if (ret < 0) { zip_error_set(&ctx->error, ZIP_ER_TELL, errno); return -1; } return ret; } case ZIP_SOURCE_WRITE: { size_t ret; clearerr(ctx->fout); ret = fwrite(data, 1, len, ctx->fout); if (ret != len || ferror(ctx->fout)) { zip_error_set(&ctx->error, ZIP_ER_WRITE, errno); return -1; } return (zip_int64_t)ret; } default: zip_error_set(&ctx->error, ZIP_ER_OPNOTSUPP, 0); return -1; } }
int main(int argc, char* argv[]) { // Single byte to be read from the data files char c; // Counters to keep track of cuts unsigned long waveformCtr = 0; unsigned long linearGateCtr = 0; unsigned long muonVetoCtr = 0; unsigned long overflowCtr = 0; unsigned long bgAnalysisCtr = 0; unsigned long sAnalysisCtr = 0; // Variables for the background output files int bg_counter = 0; int bg_file_number = 0; std::ofstream bg_out_file; // Variables for the signal output files int s_counter = 0; int s_file_number = 0; std::ofstream s_out_file; // Run Info output std::ofstream infoOut; // Saved waveforms std::ofstream waveformOut; // LabView headers int no_samples = 0; int no_channels = 0; // Timestamp of current trigger std::string timestamp; // Waveform buffers int csi [35000] = {} ; int csi_raw[35000] = {} ; int mv [35000] = {} ; // Buffer to store bit shifted samples int _tmpC = 0; // Medians of CsI and muon veto int med_csi = 0; int med_mv = 0; // Buffers used for median calculation as well as overflow detection unsigned int med_csi_sum = 0; unsigned int med_mv_sum = 0; unsigned int med_csi_arr [256] = {} ; unsigned int med_mv_arr [256] = {}; bool med_csi_found = false; bool med_mv_found = false; bool overflow = false; // Buffers for SPE integration int spe_charge_dist[300] = {}; int spe_integration_ctr = 0; int spe_integration_charge = 0; // Buffers for peaks in pretrace int bg_pe_pt[52] = {}; int s_pe_pt[52] = {}; // Buffer for baseline int baseline_hist[32] = {}; // Rising and falling threshold crossings used for linear gate detection int _previous_c = 0; int gate_down = 0; int gate_up = 0; // Waveform contains a gate or muon veto fired more than three times bool linear_gate = false; bool muon_veto_flag = false; // Photoelectron counter for all interesting regions // pt = pretrace, roi = region of interest, iw = integration window unsigned int s_pt_ct = 0; unsigned int bg_pt_ct = 0; unsigned int s_roi_ct = 0; unsigned int bg_roi_ct = 0; unsigned int s_iw_ct = 0; unsigned int bg_iw_ct = 0; unsigned int PE_max_PT = 10; // Buffers to store current peak width in CsI and muon veto waveform int above_pe_threshold = 0; int current_peak_width = 0; int current_pe_width = 0; int m_peak_width = 0; // Peak thresholds int peak_height_threshold = 3; int peak_width_threshold = 4; // Peak locations in CsI, Muon locations in muon veto std::vector<int> peaks; std::vector<int> peak_heights; int peak_height_dist[100] = {}; int peak_amplitude = 0; std::vector<int> pe_beginnings; std::vector<int> pe_endings; std::vector<int> muon_peaks; std::vector<int> muon_onset_arr; // Keep track of all peak/pe locations in the full minute int peak_distribution[350][350] = {}; int charge_distribution[350][350] = {}; for (int i1 = 0; i1 < 350; i1++) { for (int i2 = 0; i2 < 350; i2++) { peak_distribution[i1][i2] = 0; charge_distribution[i1][i2] = 0; } } // Get peak width distribution int peak_width_distribution[51] = {}; // Charge of PE in PT int current_spe_q = 0; // Integration window buffers int s_q_arr [1500] = {}; int bg_q_arr [1500] = {}; int running_charge = 0; // LogLikelihood prefactors & estimators double lnL_pf_real[1500] = {}; double lnL_pf_flat[1500] = {}; double lnL_real = 0; double lnL_flat = 0; // Big pulse detection bool bP_detected = false; int _t_bP_idx = 0; std::vector<int> bP_onset_arr; std::vector<int> bP_charge_arr; // Calculate prefactors for loglikelihood analysis // timing and fraction from NIMA paper double r = 0.41; double tf = 527.0; double ts = 5600.0; // total integration window is 3 us long -> tMax = 1500 double _tFast = 1.0 / ((1.0 + r) * tf * (1 - exp(-1500.0 / tf))); double _tSlow = r / ((1.0 + r) * ts * (1 - exp(-1500.0 / ts))); // Calculate prefactor for each time step for (int i = 0; i < 1500; i++) { lnL_pf_real[i] = log(_tFast * exp(-(double)i / tf) + _tSlow * exp(-(double)i / ts)); lnL_pf_flat[i] = log(1.0 / 1500.0); } // Threshold buffer and measured rise times double thresholds [3] = {}; double bg_rt [3] = {}; double s_rt [3] = {}; // Buffers used during window analysis int idx_0 = 0; int idx_w_offset = 0; int i_peak = 0; int i_pe = 0; int q_int = 0; double _t1 = 0.0; double _t2 = 0.0; std::string main_dir; int current_time; int single_time; std::string out_dir; std::string current_zip_file; std::string time_name_in_zip; // Save waveforms as ascii - Counter,cuts etc int save_wf_ctr = 0; int no_total_peaks[2] = { 100, 200 }; int minimum_no_peaks_iw = 6; int rt1090_bottom_left[2] = { 750, 1150 }; int rt050_bottom_left[2] = { 235, 345 }; int rt1090_upper_right[2] = { 1080, 1280 }; int rt050_upper_right[2] = { 520, 760 }; bool save_waveforms = false; bool passed_cuts_bg = false; bool passed_cuts_s = false; bool passed_cut = false; // Set main run directory, e.g. Run-15-10-02-27-32-23/151002 // Set current time to be analzyed as index of sorted number of total files in folder, e.g. 0-1439 for a full day // Set output directory, eg Output/ Run-15-10-02-27-32-23/151002 int data_set = 0; if (argc == 6) { data_set = atoi(argv[1]); main_dir = std::string(argv[2]); current_time = atoi(argv[3]); out_dir = std::string(argv[4]); single_time = atoi(argv[5]); } else { std::cout << "Arguments not matching! Aborting now!" << std::endl; return 1; } unsigned int BG_PT[2] = {}; unsigned int BG_ROI[2] = {}; unsigned int S_PT[2] = {}; unsigned int S_ROI[2] = {}; switch (data_set) { case 1: BG_PT[0] = 0; BG_PT[1] = 19975; BG_ROI[0] = 19975; BG_ROI[1] = 27475; S_PT[0] = 7500; S_PT[1] = 27475; S_ROI[0] = 27475; S_ROI[1] = 34975; PE_max_PT = 10; break; case 2: BG_PT[0] = 0; BG_PT[1] = 20000; BG_ROI[0] = 20000; BG_ROI[1] = 27400; S_PT[0] = 7400; S_PT[1] = 27400; S_ROI[0] = 27400; S_ROI[1] = 35000; PE_max_PT = 20; break; case 3: BG_PT[0] = 0; BG_PT[1] = 17500; BG_ROI[0] = 17500; BG_ROI[1] = 25000; S_PT[0] = 7500; S_PT[1] = 25000; S_ROI[0] = 25000; S_ROI[1] = 32500; PE_max_PT = 30; break; default: std::cout << "Arguments not matching! Aborting now!" << std::endl; return 1; } // Full analysis -> Converts $(Process) from condor submit to the current time file if (single_time == 0) { // Buffers used to process data files and to step through directories std::vector<std::string> time_files; DIR *dpdf; struct dirent *epdf; // Find all time files and sort them time_files.clear(); dpdf = opendir(main_dir.c_str()); if (dpdf != NULL) { std::string _tmp; while (epdf = readdir(dpdf)) { _tmp = epdf->d_name; if (_tmp != "." && _tmp != ".." && _tmp.substr(7) == "zip") { time_files.push_back(_tmp.substr(0, 6)); } } } // Sort time files by time in ascending order to convert current_time index to proper file std::sort(time_files.begin(), time_files.end(), timeSort); // Prepare paths to zipped and unzipped files current_zip_file = main_dir + "/" + time_files[current_time] + ".zip"; time_name_in_zip = time_files[current_time]; } // Single file analysis -> Directly convert input to current time file else { current_zip_file = main_dir + "/" + zeroPad(current_time,6) +".zip"; time_name_in_zip = zeroPad(current_time,6); } //Open the ZIP archive int err = 0; int zidx = 0; int fileSize = 0; zip *z = zip_open(current_zip_file.c_str(), 0, &err); //Search for the file of given name const char *name = time_name_in_zip.c_str(); struct zip_stat st; zip_stat_init(&st); zip_stat(z, name, 0, &st); //Alloc memory for its uncompressed contents char *contents = new char[st.size]; // Unzip the compressed file into memory zip_file *f = zip_fopen(z, time_name_in_zip.c_str(), 0); fileSize = st.size; zip_fread(f, contents, fileSize); zip_fclose(f); //And close the archive zip_close(z); // Create signal, background and info output files bg_out_file.open((out_dir + "/" + fileName(atoi(time_name_in_zip.c_str()), "B-")).c_str(), std::ofstream::out | std::ofstream::trunc); s_out_file.open((out_dir + "/" + fileName(atoi(time_name_in_zip.c_str()), "S-")).c_str(), std::ofstream::out | std::ofstream::trunc); infoOut.open((out_dir + "/" + fileName(atoi(time_name_in_zip.c_str()), "I-")).c_str(), std::ofstream::out | std::ofstream::trunc); if (save_waveforms) { waveformOut.open((out_dir + "/" + fileName(atoi(time_name_in_zip.c_str()), "W-")).c_str(), std::ofstream::out | std::ofstream::trunc); } // Begin data processing if file has been properly opened if(err == 0) { waveformCtr = 0; zidx = 0; // Begin reading byte-stream while (zidx < fileSize) { // Read LabView header and get the total number of samples written for each channel in the next chunk of data for (int i=0;i<4;i++) { c = contents[zidx++]; no_samples = no_samples << 8 | (unsigned char) c; } // Read LabView header and get the total number of channels written in next chunk of data (always 2 in our case) for (int i=0;i<4;i++) { c = contents[zidx++]; no_channels = no_channels << 8 | (unsigned char) c; } // Takes care of LabViews closing bit... if (no_samples > 350070) { break; } // ---------------------------------------------------------------- // Process XYZ consecutive waveforms without encountering another // LabView header inbetween // ---------------------------------------------------------------- for (int wf=0; wf < (int) (no_samples/35007); wf++) { // A new waveform begins waveformCtr += 1; // ------------------------------------------------------------- // Reset all major waveform specific variables // ------------------------------------------------------------- timestamp.clear(); overflow = false; linear_gate = false; muon_veto_flag = false; _previous_c = 128; gate_down = 0; gate_up = 0; memset(med_csi_arr,0,sizeof med_csi_arr); memset(med_mv_arr,0,sizeof med_mv_arr ); med_csi_found = false; med_mv_found = false; med_csi_sum = 0; med_mv_sum = 0; above_pe_threshold = 0; current_peak_width = 0; current_pe_width = 0; current_spe_q = 0; m_peak_width = 0; peaks.clear(); peak_heights.clear(); pe_beginnings.clear(); pe_endings.clear(); muon_peaks.clear(); spe_integration_ctr = 0; spe_integration_charge = 0; passed_cuts_s = false; passed_cuts_bg = false; passed_cut = false; peak_amplitude = 0; bP_detected = false; _t_bP_idx = 0; // ------------------------------------------------------------- // Read current timestamp // ------------------------------------------------------------- for(int i=0; i<7; i++) { c = contents[zidx++]; c = contents[zidx++]; timestamp += zeroPad((int) c, 2); } // --------------------------------------------------------------- // Read the full CsI and muon veto waveforms from the zip-stream // + Apply bit transformation // + Determine if a linear gate is present // --------------------------------------------------------------- for(int i=0; i<35000; i++) { //-------------- // CsI //-------------- c = contents[zidx++]; // bit transformation to get rid of empty bins _tmpC = (int) c - (int) floor(((double) c + 5.0)/11.0); if (i<20000){ med_csi_arr[_tmpC + 128] += 1; } csi[i] = _tmpC; csi_raw[i] = _tmpC; if (i == 0) { _previous_c = _tmpC; } // Gate check if (_tmpC <= 18 && _previous_c > 18) { gate_down++; } if (_previous_c <= 18 && _tmpC > 18) { gate_up++; } _previous_c = _tmpC; // Overflow check if (!overflow && (c >= 127 || c == -128)) { overflow = true; overflowCtr += 1; } //-------------- // Muon Veto //-------------- c = contents[zidx++]; _tmpC = (int) c + (int) ((signbit((int) c) ? -1 : 1 ) * floor((4.0 - abs((double) c))/11.0)); med_mv_arr[_tmpC + 128] += 1; mv[i] = _tmpC; } // --------------------------------------- // Calculate the median of both channels // --------------------------------------- for(int i=0; i<256; i++) { if (!med_csi_found) { med_csi_sum += med_csi_arr[i]; if (med_csi_sum >= 10000) { med_csi = i-128; med_csi_found=true; } } if (!med_mv_found) { med_mv_sum += med_mv_arr[i]; if (med_mv_sum >= 10000) { med_mv = i-128; med_mv_found=true; } } } // ---------------------------------------------- // Adjust linear gate counter if wf is gated // ---------------------------------------------- if (gate_down != gate_up || med_csi < 50) { linear_gate = true; linearGateCtr += 1; } // ----------------------------------------------- // Find peaks and photoelectrons in waveforms // ----------------------------------------------- current_peak_width = 0; peak_amplitude = 0; bP_detected = false; for (int i = 0; i < 35000; i++) { // ------------------------------------------- // Analyze CsI[Na] waveform // ------------------------------------------- csi[i] = med_csi - csi[i]; // Simple peak finder using threshold crossing with history if (csi[i] >= peak_height_threshold) { current_peak_width++; peak_amplitude = csi[i] > peak_amplitude ? csi[i] : peak_amplitude; } else { if (current_peak_width >= peak_width_threshold) { peaks.push_back(i - current_peak_width); pe_beginnings.push_back((i - current_peak_width - 2) >= 0 ? (i - current_peak_width - 2) : 0); pe_endings.push_back((i + 1) <= 34999 ? (i + 1) : 34999); peak_width_distribution[(current_peak_width < 50) ? current_peak_width : 50] += 1; peak_heights.push_back(peak_amplitude); // Check for large energy depositions if (!bP_detected && current_peak_width >= 35 && !linear_gate && !overflow) { bP_detected = true; bP_onset_arr.push_back(i - current_peak_width); } } current_peak_width = 0; peak_amplitude = 0; } // ------------------------------------------- // Analyze muon veto waveform // ------------------------------------------- mv[i] = med_mv - mv[i]; // Peak finder if (mv[i] >= 10) { m_peak_width++; } else { if (m_peak_width >= 3) { muon_peaks.push_back(i-m_peak_width); } m_peak_width = 0; } } // If there was a big pulse in the trace integrate it // But only if there was no linear gate or overflow if (bP_detected && !linear_gate && !overflow) { int _t_charge = 0; for (int i = bP_onset_arr.back(); i < (1500 + bP_onset_arr.back()); i++) { _t_charge += (csi[i] >= peak_height_threshold) ? csi[i] : 0; } bP_charge_arr.push_back(_t_charge); } // Raise muon veto flag if more than three muons have been found // If less than 3 have been found fill the vector with -1 for postprocessing int muons_found = muon_peaks.size(); if (muons_found > 0) { if (linear_gate) { muon_onset_arr.push_back(muon_peaks[0]); } else { for (std::vector<int>::size_type idx = 0; idx < muons_found; idx++) { muon_onset_arr.push_back(muon_peaks[idx]); } } } if (muons_found > 3) { muon_veto_flag = true; muonVetoCtr += 1; } else { muon_peaks.resize(3, -1); } // Add PE peaks to peak height distribution if (peak_heights.size() > 0 && !overflow && !linear_gate) { for (int idx = 0; idx < peak_heights.size(); idx++) { peak_height_dist[(peak_heights[idx] < 100) ? peak_heights[idx] : 99]++; } } // ======================================================================== // Check that there is at least one PE in the trace, there is no overflow, // no linear gate and no muon veto flag, otherwise continue to next // waveform without bothering to analyze this one // ======================================================================== bool analyze = false; if (data_set == 1 && !overflow && !linear_gate && !muon_veto_flag) { analyze = true; } if (data_set == 2 && !overflow && !linear_gate) { analyze = true; } if (data_set == 3 && !overflow && !linear_gate) { analyze = true; } if (analyze && pe_beginnings.size() > 0) { // ------------------------------------------------------------- // Integrate all SPE found in the PT and histogram their charge // ------------------------------------------------------------- for (std::vector<int>::size_type idx = 0; idx < pe_beginnings.size(); idx++) { if (pe_beginnings[idx] < BG_PT[1]) { current_spe_q = 0; for (int i = pe_beginnings[idx]; i <= pe_endings[idx]; i++) { current_spe_q += csi[i]; } if (current_spe_q >= -50 && current_spe_q < 250) { spe_charge_dist[(current_spe_q+50)] += 1; } } else { break; } } // ------------------------------------------------------------- // Determine number of PE in different regions // ------------------------------------------------------------- bg_pt_ct = 0; bg_roi_ct = 0; bg_iw_ct = 0; s_pt_ct = 0; s_roi_ct = 0; s_iw_ct = 0; if (peaks.size() > 0) { for (std::vector<int>::size_type idx = 0; idx < peaks.size(); idx++) { if (peaks[idx] >= BG_PT[0] && peaks[idx] < BG_PT[1]) { bg_pt_ct += 1; } if (peaks[idx] >= S_PT[0] && peaks[idx] < S_PT[1]) { s_pt_ct += 1; } if (peaks[idx] >= BG_ROI[0] && peaks[idx] < BG_ROI[1]) { bg_roi_ct += 1; } if (peaks[idx] >= S_ROI[0] && peaks[idx] < S_ROI[1]) { s_roi_ct += 1; } } // Distribution of charge (only add >= 3) if (true) { int sz = peaks.size() / 2; if (sz < 350) { int cpi = 0; for (int idx = 0; idx < 35000; idx++) { // Add sample if it is within one of the PE regions identified previously if (idx >= pe_beginnings[cpi] && idx <= pe_endings[cpi]) { charge_distribution[sz][idx / 100] += csi[idx]; if (idx >= pe_endings[cpi]) { if (++cpi >= pe_beginnings.size()){ break; } } } } } } } // Histogram baseline and peaks in pretrace if (!overflow && !linear_gate && !muon_veto_flag) { int _t_med_csi_bin = med_csi - 85; if (_t_med_csi_bin >= 0 && _t_med_csi_bin <=30) { baseline_hist[_t_med_csi_bin]++; } else { baseline_hist[31]++; } if (bg_pt_ct <= 50) { bg_pe_pt[bg_pt_ct]++; } else { bg_pe_pt[51]++; } if (s_pt_ct <= 50) { s_pe_pt[s_pt_ct]++; } else { s_pe_pt[51]++; } } // ------------------------------------------------------------- // Only analzye BG region if there are a maximum of PE_max_PT // & at least one PE in ROI // ------------------------------------------------------------- if (bg_pt_ct <= PE_max_PT && bg_roi_ct > 0) { bgAnalysisCtr += 1; // Reset window parameters idx_0 = 0; i_peak = 0; i_pe = 0; q_int = 0; lnL_real = 0.0; lnL_flat = 0.0; _t1 = 0.0; _t2 = 0.0; // ------------------------------------------------------------- // Determine onset of integration window by finding the first // PE in the region of interest // ------------------------------------------------------------- for (int i = 0; i < peaks.size(); i++) { if (peaks[i] >= BG_ROI[0]) { idx_0 = peaks[i]; i_peak = i; break; } } idx_0 -= 3; // ------------------------------------------------------------- // Only analyze if the full integration window is within the ROI // ------------------------------------------------------------- if (idx_0 < (BG_ROI[1] - 1500)) { // -------------------------------------------------------------- // Determine number of peaks in integration window (magic bullet) // -------------------------------------------------------------- for (int i = i_peak; i < peaks.size(); i++) { bg_iw_ct += (peaks[i] - idx_0 < 1500) ? 1 : 0; } // ------------------------------------------------------------- // Integrate over all PE found in the integration window // ------------------------------------------------------------- i_pe = i_peak; for (int i = 0; i < 1500; i++) { // Get proper 'real' index that includes the onset offset idx_w_offset = i + idx_0; // Add sample if it is within one of the PE regions identified previously as well as update loglikelihood estimators if (i_pe < pe_beginnings.size() && idx_w_offset >= pe_beginnings[i_pe] && idx_w_offset <= pe_endings[i_pe]) { q_int += csi[idx_w_offset]; lnL_real += lnL_pf_real[i] * csi[idx_w_offset]; lnL_flat += lnL_pf_flat[i] * csi[idx_w_offset]; if (idx_w_offset == pe_endings[i_pe]) { i_pe++; } } // Keep track of charge integration to determine rise times later bg_q_arr[i] = q_int; } // ------------------------------------------------------------- // Determine rise times // ------------------------------------------------------------- // Calculate charge thresholds thresholds[0] = 0.1*bg_q_arr[1499]; thresholds[1] = 0.5*bg_q_arr[1499]; thresholds[2] = 0.9*bg_q_arr[1499]; // Determine threshold crossing times for (int i = 0; i < 1499; i++) { _t1 = (double)bg_q_arr[i]; _t2 = (double)bg_q_arr[i + 1]; for (int j = 0; j < 3; j++) { if (_t1 < thresholds[j] && _t2 >= thresholds[j]){ bg_rt[j] = i + (thresholds[j] - _t1) / (_t2 - _t1); } } } // ------------------------------------------------------------- // If waveforms are being saved check whether cuts are passed // ------------------------------------------------------------- if (save_waveforms) { int rt1090 = (bg_rt[2] - bg_rt[0]); int rt050 = bg_rt[1]; bool bottom_left_cut = (rt1090 >= rt1090_bottom_left[0]) && (rt1090 <= rt1090_bottom_left[1]) && (rt050 >= rt050_bottom_left[0]) && (rt050 <= rt050_bottom_left[1]); bool upper_right_cut = (rt1090 >= rt1090_upper_right[0]) && (rt1090 <= rt1090_upper_right[1]) && (rt050 >= rt050_upper_right[0]) && (rt050 <= rt050_upper_right[1]); bool min_peak_cut = (bg_iw_ct >= minimum_no_peaks_iw); bool total_peak_cut = (peaks.size() >= no_total_peaks[0]) && (peaks.size() <= no_total_peaks[1]); passed_cuts_bg = (bottom_left_cut || upper_right_cut) && min_peak_cut && total_peak_cut; } // ------------------------------------------------------------- // Write analysis results to file // ------------------------------------------------------------- bg_out_file << timestamp << " " << med_csi << " " << med_mv << " "; bg_out_file << bg_pt_ct << " " << bg_roi_ct << " "; bg_out_file << bg_iw_ct << " " << idx_0 << " " << bg_q_arr[1499] << " " << lnL_real << " " << lnL_flat << " "; bg_out_file << bg_rt[0] << " " << bg_rt[1] << " " << bg_rt[2] << " "; bg_out_file << muon_peaks[0] << " " << muon_peaks[1] << " " << muon_peaks[2] << " " << peaks.size() << std::endl; // Keeps track of how many BG waveforms have actually been analyzed bg_counter++; } } // ------------------------------------------------------------- // Only analzye S region if there are a maximum of PE_max_PT // & at least one PE in ROI // ------------------------------------------------------------- if (s_pt_ct <= PE_max_PT && s_roi_ct > 0) { sAnalysisCtr += 1; // Reset window parameters idx_0 = 0; i_peak = 0; i_pe = 0; q_int = 0; lnL_real = 0.0; lnL_flat = 0.0; _t1 = 0.0; _t2 = 0.0; // ------------------------------------------------------------- // Determine onset of integration window by finding the first // PE in the region of interest // ------------------------------------------------------------- for (int i = 0; i < peaks.size(); i++) { if (peaks[i] >= S_ROI[0]) { idx_0 = peaks[i]; i_peak = i; break; } } idx_0 -= 3; // ------------------------------------------------------------- // Only analyze if the full integration window is within the ROI // ------------------------------------------------------------- if (idx_0 < (S_ROI[1] - 1500)) { // -------------------------------------------------------------- // Determine number of peaks in integration window (magic bullet) // -------------------------------------------------------------- for (int i = i_peak; i < peaks.size(); i++) { s_iw_ct += (peaks[i] - idx_0 < 1500) ? 1 : 0; } i_pe = i_peak; // ------------------------------------------------------------- // Integrate over all PE found in the integration window // ------------------------------------------------------------- for (int i = 0; i < 1500; i++) { // Get proper 'real' index that includes the onset offset idx_w_offset = i + idx_0; // Add sample if it is within one of the PE regions identified previously & update loglikelihood estimators if (i_pe < pe_beginnings.size() && idx_w_offset >= pe_beginnings[i_pe] && idx_w_offset <= pe_endings[i_pe]) { q_int += csi[idx_w_offset]; lnL_real += lnL_pf_real[i] * csi[idx_w_offset]; lnL_flat += lnL_pf_flat[i] * csi[idx_w_offset]; if (idx_w_offset == pe_endings[i_pe]) { i_pe++; } } // Keep track of charge integration to determine rise times later s_q_arr[i] = q_int; } // ------------------------------------------------------------- // Determine rise times // ------------------------------------------------------------- // Calculate charge thresholds thresholds[0] = 0.1*s_q_arr[1499]; thresholds[1] = 0.5*s_q_arr[1499]; thresholds[2] = 0.9*s_q_arr[1499]; // Determine threshold crossing times for (int i = 0; i < 1499; i++) { _t1 = (double)s_q_arr[i]; _t2 = (double)s_q_arr[i + 1]; for (int j = 0; j < 3; j++) { if (_t1 < thresholds[j] && _t2 >= thresholds[j]){ s_rt[j] = i + (thresholds[j] - _t1) / (_t2 - _t1); } } } // ------------------------------------------------------------- // If waveforms are being saved check whether cuts are passed // ------------------------------------------------------------- if (save_waveforms) { int rt1090 = (s_rt[2] - s_rt[0]); int rt050 = s_rt[1]; bool bottom_left_cut = (rt1090 >= rt1090_bottom_left[0]) && (rt1090 <= rt1090_bottom_left[1]) && (rt050 >= rt050_bottom_left[0]) && (rt050 <= rt050_bottom_left[1]); bool upper_right_cut = (rt1090 >= rt1090_upper_right[0]) && (rt1090 <= rt1090_upper_right[1]) && (rt050 >= rt050_upper_right[0]) && (rt050 <= rt050_upper_right[1]); bool min_peak_cut = (s_iw_ct >= minimum_no_peaks_iw); bool total_peak_cut = (peaks.size() >= no_total_peaks[0]) && (peaks.size() <= no_total_peaks[1]); passed_cuts_s = (bottom_left_cut || upper_right_cut) && min_peak_cut && total_peak_cut; } // ------------------------------------------------------------- // Write analysis results to file // ------------------------------------------------------------- s_out_file << timestamp << " " << med_csi << " " << med_mv << " "; s_out_file << s_pt_ct << " " << s_roi_ct << " "; s_out_file << s_iw_ct << " " << idx_0 << " " << s_q_arr[1499] << " " << lnL_real << " " << lnL_flat << " "; s_out_file << s_rt[0] << " " << s_rt[1] << " " << s_rt[2] << " "; s_out_file << muon_peaks[0] << " " << muon_peaks[1] << " " << muon_peaks[2] << " " << peaks.size() << std::endl; // Keeps track of how many BG waveforms have actually been analyzed s_counter++; } } } // ------------------------------------------------------------- // Save waveform if cuts have been passed for either ROI // ------------------------------------------------------------- if (save_waveforms && passed_cut) { for (int idx = 0; idx < 35000; idx++) { waveformOut << csi_raw[idx] << " "; } waveformOut << gate_up << " " << gate_down << " " << med_csi << " "; for (int idx = 0; idx < pe_beginnings.size(); idx++) { waveformOut << pe_beginnings[idx] << " " << pe_endings[idx] << " "; } waveformOut << std::endl; } } } } // Before exiting, make sure that both output files are properly closed to prevent data loss. if (bg_out_file.is_open()) { bg_out_file.close(); } if (s_out_file.is_open()) { s_out_file.close(); } // Write run info if (infoOut.is_open()) { infoOut << "Total number of Waveforms processed" << std::endl; infoOut << waveformCtr << std::endl; infoOut << "Number of triggers with linear gates in CsI channel" << std::endl; infoOut << linearGateCtr << std::endl; infoOut << "Number of triggers with overflows in either channel" << std::endl; infoOut << overflowCtr << std::endl; infoOut << "Number of triggers with more than 3 muons in muon veto channel" << std::endl; infoOut << muonVetoCtr << std::endl; infoOut << "Number of triggers that have actually been analyzed in the background window (less than x PE/peaks in PT + at least one PE/peak in ROI)" << std::endl; infoOut << bgAnalysisCtr << std::endl; infoOut << "Number of triggers that have actually been analyzed in the signal window (less than x PE/peaks in PT + at least one PE/peak in ROI)" << std::endl; infoOut << sAnalysisCtr << std::endl; infoOut << "Maximum number of PE/peaks in the pretrace" << std::endl; infoOut << PE_max_PT << std::endl; infoOut << "Background pretrace start" << std::endl; infoOut << BG_PT[0] << std::endl; infoOut << "Background pretrace stop" << std::endl; infoOut << BG_PT[1] << std::endl; infoOut << "Background ROI start" << std::endl; infoOut << BG_ROI[0] << std::endl; infoOut << "Background ROI stop" << std::endl; infoOut << BG_ROI[1] << std::endl; infoOut << "Signal pretrace start" << std::endl; infoOut << S_PT[0] << std::endl; infoOut << "Signal pretrace stop" << std::endl; infoOut << S_PT[1] << std::endl; infoOut << "Signal ROI start" << std::endl; infoOut << S_ROI[0] << std::endl; infoOut << "Signal ROI stop" << std::endl; infoOut << S_ROI[1] << std::endl; infoOut << "Unzipped file size" << std::endl; infoOut << fileSize << std::endl; infoOut << "SPE charge histogram" << std::endl; for (int idx = 0; idx < 300; idx++) { infoOut << spe_charge_dist[idx] << " "; } infoOut << std::endl; infoOut << "Background - Peaks in pretrace histogram" << std::endl; for (int idx = 0; idx < 52; idx++) { infoOut << bg_pe_pt[idx] << " "; } infoOut << std::endl; infoOut << "Signal - Peaks in pretrace histogram" << std::endl; for (int idx = 0; idx < 52; idx++) { infoOut << s_pe_pt[idx] << " "; } infoOut << std::endl; infoOut << "CsI baseline histogram" << std::endl; for (int idx = 0; idx < 32; idx++) { infoOut << baseline_hist[idx] << " "; } infoOut << std::endl; infoOut << "Peak width distribution" << std::endl; for (int idx = 0; idx < 51; idx++) { infoOut << peak_width_distribution[idx] << " "; } infoOut << std::endl; infoOut << "PE amplitudes" << std::endl; for (int idx = 0; idx < 100; idx++) { infoOut << peak_height_dist[idx] << " "; } infoOut << std::endl; infoOut << "Big pulse onsets" << std::endl; for (int idx = 0; idx < bP_onset_arr.size(); idx++) { infoOut << bP_onset_arr[idx] << " "; } infoOut << std::endl; infoOut << "Big pulse charges" << std::endl; for (int idx = 0; idx < bP_charge_arr.size(); idx++) { infoOut << bP_charge_arr[idx] << " "; } infoOut << std::endl; infoOut << "All muon onsets - If linear gate present only the first is recorded" << std::endl; for (int idx = 0; idx < muon_onset_arr.size(); idx++) { infoOut << muon_onset_arr[idx] << " "; } infoOut << std::endl; infoOut << "Charge distribution in full waveform" << std::endl; for (int idx_1 = 0; idx_1 < 350; idx_1++) { bool printLine = false; for (int idx_2 = 0; idx_2 < 350; idx_2++) { if (charge_distribution[idx_1][idx_2] > 0) { printLine = true; break; } } if (printLine) { infoOut << idx_1 << " "; for (int idx_2 = 0; idx_2 < 350; idx_2++) { if (charge_distribution[idx_1][idx_2] > 0) { infoOut << idx_2 << " " << charge_distribution[idx_1][idx_2] << " "; } } infoOut << std::endl; } } infoOut.close(); } return 0; }
BinaryInput::BinaryInput( const std::string& filename, G3DEndian fileEndian, bool compressed) : m_filename(filename), m_bitPos(0), m_bitString(0), m_beginEndBits(0), m_alreadyRead(0), m_length(0), m_bufferLength(0), m_buffer(NULL), m_pos(0), m_freeBuffer(true) { setEndian(fileEndian); // Update global file tracker _internal::currentFilesUsed.insert(m_filename); #if _HAVE_ZIP /* G3DFIX: Use ZIP-library only if defined */ std::string zipfile; if (FileSystem::inZipfile(m_filename, zipfile)) { // Load from zipfile // zipRead(filename, v, s); std::string internalFile = m_filename.substr(zipfile.length() + 1); struct zip* z = zip_open(zipfile.c_str(), ZIP_CHECKCONS, NULL); { struct zip_stat info; zip_stat_init( &info ); // TODO: Docs unclear if zip_stat_init is required. zip_stat(z, internalFile.c_str(), ZIP_FL_NOCASE, &info); m_bufferLength = m_length = info.size; // sets machines up to use MMX, if they want m_buffer = reinterpret_cast<uint8*>(System::alignedMalloc(m_length, 16)); struct zip_file* zf = zip_fopen( z, internalFile.c_str(), ZIP_FL_NOCASE ); { int64 test = zip_fread( zf, m_buffer, m_length ); debugAssertM(test == m_length, internalFile + " was corrupt because it unzipped to the wrong size."); (void)test; } zip_fclose( zf ); } zip_close( z ); if (compressed) { decompress(); } m_freeBuffer = true; return; } #endif // Figure out how big the file is and verify that it exists. m_length = FileSystem::size(m_filename); // Read the file into memory FILE* file = fopen(m_filename.c_str(), "rb"); if (! file || (m_length == -1)) { throw format("File not found: \"%s\"", m_filename.c_str()); return; } if (! compressed && (m_length > INITIAL_BUFFER_LENGTH)) { // Read only a subset of the file so we don't consume // all available memory. m_bufferLength = INITIAL_BUFFER_LENGTH; } else { // Either the length is fine or the file is compressed // and requires us to read the whole thing for zlib. m_bufferLength = m_length; } debugAssert(m_freeBuffer); m_buffer = (uint8*)System::alignedMalloc(m_bufferLength, 16); if (m_buffer == NULL) { if (compressed) { throw "Not enough memory to load compressed file. (1)"; } // Try to allocate a small array; not much memory is available. // Give up if we can't allocate even 1k. while ((m_buffer == NULL) && (m_bufferLength > 1024)) { m_bufferLength /= 2; m_buffer = (uint8*)System::alignedMalloc(m_bufferLength, 16); } } debugAssert(m_buffer); fread(m_buffer, m_bufferLength, sizeof(int8), file); fclose(file); file = NULL; if (compressed) { if (m_bufferLength != m_length) { throw "Not enough memory to load compressed file. (2)"; } decompress(); } }