void do_open_store(fs::path const& location) {
if (!fs::exists(location / "format")) {
BOOST_THROW_EXCEPTION(common_exception()
<< errinfo_rpc_code(::rpc_error::STORE_NOT_FOUND)
<< errinfo_message(str(boost::format("Store at %s does not exist")
% location)));
}
io::stream<io::file_source> store_format((location / "format").string());
int format;
store_format >> format;
store_format.close();
if (format != indexer::STORE_FORMAT) {
BOOST_THROW_EXCEPTION(common_exception()
<< errinfo_rpc_code(::rpc_error::INVALID_STORE)
<< errinfo_message(str(boost::format("Store at %s has invalid "
"format %d, expecting %d")
% location % format % indexer::STORE_FORMAT)));
}
this->index.reset(new indexer::index(location / "index"));
this->db.reset(new indexer::value_db("localhost", "index.postings"));
io::stream<io::file_source> store_info((location / "info").string());
this->format.ParseFromIstream(&store_info);
store_info.close();
this->store_root = location;
}
int extract_or_test_files() /* return PK-type error code */
{
uch *cd_inptr;
int cd_incnt, error, error_in_archive=PK_COOL;
int i, j, renamed, query, len, filnum=(-1), blknum=0;
int *fn_matched=NULL, *xn_matched=NULL;
ush members_remaining, num_skipped=0, num_bad_pwd=0;
long cd_bufstart, bufstart, inbuf_offset, request;
LONGINT old_extra_bytes=0L;
static min_info info[DIR_BLKSIZ];
/*---------------------------------------------------------------------------
The basic idea of this function is as follows. Since the central di-
rectory lies at the end of the zipfile and the member files lie at the
beginning or middle or wherever, it is not very desirable to simply
read a central directory entry, jump to the member and extract it, and
then jump back to the central directory. In the case of a large zipfile
this would lead to a whole lot of disk-grinding, especially if each mem-
ber file is small. Instead, we read from the central directory the per-
tinent information for a block of files, then go extract/test the whole
block. Thus this routine contains two small(er) loops within a very
large outer loop: the first of the small ones reads a block of files
from the central directory; the second extracts or tests each file; and
the outer one loops over blocks. There's some file-pointer positioning
stuff in between, but that's about it. Btw, it's because of this jump-
ing around that we can afford to be lenient if an error occurs in one of
the member files: we should still be able to go find the other members,
since we know the offset of each from the beginning of the zipfile.
---------------------------------------------------------------------------*/
pInfo = info;
members_remaining = ecrec.total_entries_central_dir;
#if (defined(CRYPT) || !defined(NO_ZIPINFO))
newzip = TRUE;
#endif
/* malloc space for CRC table and generate it */
if ((crc_32_tab = (ulg *)malloc(256*sizeof(ulg))) == (ulg *)NULL)
return PK_MEM2;
makecrc();
/* malloc space for check on unmatched filespecs (OK if one or both NULL) */
if (filespecs > 0 &&
(fn_matched=(int *)malloc(filespecs*sizeof(int))) != (int *)NULL)
for (i = 0; i < filespecs; ++i)
fn_matched[i] = FALSE;
if (xfilespecs > 0 &&
(xn_matched=(int *)malloc(xfilespecs*sizeof(int))) != (int *)NULL)
for (i = 0; i < xfilespecs; ++i)
xn_matched[i] = FALSE;
/*---------------------------------------------------------------------------
Begin main loop over blocks of member files. We know the entire central
directory is on this disk: we would not have any of this information un-
less the end-of-central-directory record was on this disk, and we would
not have gotten to this routine unless this is also the disk on which
the central directory starts. In practice, this had better be the ONLY
disk in the archive, but maybe someday we'll add multi-disk support.
---------------------------------------------------------------------------*/
while (members_remaining) {
j = 0;
/*
* Loop through files in central directory, storing offsets, file
* attributes, case-conversion and text-conversion flags until block
* size is reached.
*/
while (members_remaining && (j < DIR_BLKSIZ)) {
--members_remaining;
pInfo = &info[j];
if (readbuf(sig, 4) == 0) {
error_in_archive = PK_EOF;
members_remaining = 0; /* ...so no more left to do */
break;
}
if (strncmp(sig, central_hdr_sig, 4)) { /* just to make sure */
FPRINTF(stderr, LoadFarString(CentSigMsg), j); /* sig not found */
FPRINTF(stderr, LoadFarString(ReportMsg)); /* check binary transfers */
error_in_archive = PK_BADERR;
members_remaining = 0; /* ...so no more left to do */
break;
}
/* process_cdir_file_hdr() sets pInfo->hostnum, pInfo->lcflag */
if ((error = process_cdir_file_hdr()) != PK_COOL) {
error_in_archive = error; /* only PK_EOF defined */
members_remaining = 0; /* ...so no more left to do */
break;
}
if ((error = do_string(crec.filename_length,FILENAME)) != PK_COOL) {
if (error > error_in_archive)
error_in_archive = error;
if (error > PK_WARN) { /* fatal: no more left to do */
FPRINTF(stderr, LoadFarString(FilNamMsg), filename, "central");
members_remaining = 0;
break;
}
}
if ((error = do_string(crec.extra_field_length, EXTRA_FIELD)) != 0)
{
if (error > error_in_archive)
error_in_archive = error;
if (error > PK_WARN) { /* fatal */
FPRINTF(stderr, LoadFarString(ExtFieldMsg), filename, "central");
members_remaining = 0;
break;
}
}
if ((error = do_string(crec.file_comment_length,SKIP)) != PK_COOL) {
if (error > error_in_archive)
error_in_archive = error;
if (error > PK_WARN) { /* fatal */
FPRINTF(stderr, LoadFarString(BadFileCommLength),
filename);
members_remaining = 0;
break;
}
}
if (process_all_files) {
if (store_info())
++j; /* file is OK; info[] stored; continue with next */
else
++num_skipped;
} else {
int do_this_file = FALSE;
char **pfn = pfnames-1;
while (*++pfn)
if (match(filename, *pfn, C_flag)) {
do_this_file = TRUE; /* ^-- ignore case or not? */
if (fn_matched)
fn_matched[pfn-pfnames] = TRUE;
break; /* found match, so stop looping */
}
if (do_this_file) { /* check if this is an excluded file */
char **pxn = pxnames-1;
while (*++pxn)
if (match(filename, *pxn, C_flag)) {
do_this_file = FALSE; /* ^-- ignore case or not? */
if (xn_matched)
xn_matched[pxn-pxnames] = TRUE;
break;
}
}
if (do_this_file)
if (store_info())
++j; /* file is OK */
else
++num_skipped; /* unsupp. compression or encryption */
} /* end if (process_all_files) */
} /* end while-loop (adding files to current block) */
/* save position in central directory so can come back later */
cd_bufstart = cur_zipfile_bufstart;
cd_inptr = inptr;
cd_incnt = incnt;
/*-----------------------------------------------------------------------
Second loop: process files in current block, extracting or testing
each one.
-----------------------------------------------------------------------*/
for (i = 0; i < j; ++i) {
filnum = i + blknum*DIR_BLKSIZ;
pInfo = &info[i];
/* if the target position is not within the current input buffer
* (either haven't yet read far enough, or (maybe) skipping back-
* ward), skip to the target position and reset readbuf(). */
/* LSEEK(pInfo->offset): */
request = pInfo->offset + extra_bytes;
inbuf_offset = request % INBUFSIZ;
bufstart = request - inbuf_offset;
Trace((stderr, "\ndebug: request = %ld, inbuf_offset = %ld\n",
request, inbuf_offset));
Trace((stderr,
"debug: bufstart = %ld, cur_zipfile_bufstart = %ld\n",
bufstart, cur_zipfile_bufstart));
if (request < 0) {
FPRINTF(stderr, LoadFarStringSmall(SeekMsg), zipfn,
LoadFarString(ReportMsg));
error_in_archive = PK_ERR;
if (filnum == 0 && extra_bytes != 0L) {
FPRINTF(stderr, LoadFarString(AttemptRecompensate));
old_extra_bytes = extra_bytes;
extra_bytes = 0L;
request = pInfo->offset; /* could also check if this != 0 */
inbuf_offset = request % INBUFSIZ;
bufstart = request - inbuf_offset;
Trace((stderr, "debug: request = %ld, inbuf_offset = %ld\n",
request, inbuf_offset));
Trace((stderr,
"debug: bufstart = %ld, cur_zipfile_bufstart = %ld\n",
bufstart, cur_zipfile_bufstart));
} else {
error_in_archive = PK_BADERR;
continue; /* this one hosed; try next */
}
}
/* try again */
if (request < 0) {
Trace((stderr, "debug: recompensated request still < 0\n"));
FPRINTF(stderr, LoadFarStringSmall(SeekMsg), zipfn,
LoadFarString(ReportMsg));
error_in_archive = PK_BADERR;
continue;
} else if (bufstart != cur_zipfile_bufstart) {
Trace((stderr, "debug: bufstart != cur_zipfile_bufstart\n"));
cur_zipfile_bufstart = lseek(zipfd,(LONGINT)bufstart,SEEK_SET);
if ((incnt = read(zipfd,(char *)inbuf,INBUFSIZ)) <= 0) {
FPRINTF(stderr, LoadFarString(OffsetMsg), filnum, "lseek",
bufstart);
error_in_archive = PK_BADERR;
continue; /* can still do next file */
}
inptr = inbuf + (int)inbuf_offset;
incnt -= (int)inbuf_offset;
} else {
incnt += (inptr-inbuf) - (int)inbuf_offset;
inptr = inbuf + (int)inbuf_offset;
}
/* should be in proper position now, so check for sig */
if (readbuf(sig, 4) == 0) { /* bad offset */
FPRINTF(stderr, LoadFarString(OffsetMsg), filnum, "EOF",
request);
error_in_archive = PK_BADERR;
continue; /* but can still try next one */
}
if (strncmp(sig, local_hdr_sig, 4)) {
FPRINTF(stderr, LoadFarString(OffsetMsg), filnum,
LoadFarStringSmall(LocalHdrSig), request);
error_in_archive = PK_ERR;
if ((filnum == 0 && extra_bytes != 0L) ||
(extra_bytes == 0L && old_extra_bytes != 0L)) {
FPRINTF(stderr, LoadFarString(AttemptRecompensate));
if (extra_bytes) {
old_extra_bytes = extra_bytes;
extra_bytes = 0L;
} else
extra_bytes = old_extra_bytes; /* third attempt */
LSEEK(pInfo->offset)
if (readbuf(sig, 4) == 0) { /* bad offset */
FPRINTF(stderr, LoadFarString(OffsetMsg), filnum, "EOF",
request);
error_in_archive = PK_BADERR;
continue; /* but can still try next one */
}
if (strncmp(sig, local_hdr_sig, 4)) {
FPRINTF(stderr, LoadFarString(OffsetMsg), filnum,
LoadFarStringSmall(LocalHdrSig), request);
error_in_archive = PK_BADERR;
continue;
}
} else
continue; /* this one hosed; try next */
}
/*
* get_ap_raw_info - to collect the key info of ap for connecting
* @interface - wifi interface name, such as "ra0", "wlan0"
* @ap_cnt - total ap that were scanned
* NULL - just store the ap info
* @ap_list - the specific infomation of each ap
* NULL - just get the count of ap
* Returns: 0 if collect raw info succeed, -1 if fail
*/
int get_ap_raw_info(IN struct wifi_info **ap_list,
OUT int *ap_cnt, IN char *interface)
{
int cnt = 0;
int all_ap = 0;
int len = 0;
char *buf;
FILE *fp = NULL;
if (ap_list == NULL)
return -1;
fp = fopen("/dev/wifi/list_ap","r");
if(fp == NULL)
{
FPRINTF_CA(stderr, "cant find file /dev/wifi/list_ap\n");
return -1;
}
fseek(fp,0,SEEK_END);
len = ftell(fp);
if(len == 0)
{
FPRINTF_CA(stderr, "file len is zero \n");
fclose(fp);
return -1;
}
buf = malloc(len);
if(buf == NULL)
{
FPRINTF_CA(stderr, "malloc buff error \n");
fclose(fp);
return -1;
}
fseek(fp,0,SEEK_SET);
fread(buf,len,1,fp);
fclose(fp);
if (!strncmp(buf, "no such interface.", 18)) {
PRINTF_CA("%s", buf);
free(buf);
return -NOSUCHINF;
} else if (!strncmp(buf, "no ap found.", 12)) {
PRINTF_CA("%s", buf);
*ap_cnt = all_ap;
free(buf);
return -NOAPFOUND;
}
for (cnt = 0; cnt < strlen(buf); cnt++) {
if ('\n' == buf[cnt])
all_ap++;
}
if (ap_cnt != NULL) {
if (all_ap > *ap_cnt)
all_ap = *ap_cnt;
*ap_cnt = all_ap;
}
int count;
char **line_index;
char *space_index[MAX_PART + 1] = {NULL};
line_index = (char **)malloc(sizeof(char *) * (all_ap + 1));
extract_info(buf, '\n', line_index);
for (count = 0; count < all_ap; count++) {
extract_ap_info(line_index[count], ' ' , space_index);
store_info(&(*ap_list)[count], space_index);
}
free(line_index);
free(buf);
return 0;
}
tdb_error tdb_encode(tdb_cons *cons, const tdb_item *items)
{
char path[TDB_MAX_PATH_SIZE];
char grouped_path[TDB_MAX_PATH_SIZE];
char toc_path[TDB_MAX_PATH_SIZE];
char *root = cons->root;
char *read_buf = NULL;
struct field_stats *fstats = NULL;
uint64_t num_trails = 0;
uint64_t num_events = cons->events.next;
uint64_t num_fields = cons->num_ofields + 1;
uint64_t max_timestamp = 0;
uint64_t max_timedelta = 0;
uint64_t *field_cardinalities = NULL;
uint64_t i;
Pvoid_t unigram_freqs = NULL;
struct judy_128_map gram_freqs;
struct judy_128_map codemap;
Word_t tmp;
FILE *grouped_w = NULL;
FILE *grouped_r = NULL;
int fd, ret = 0;
TDB_TIMER_DEF
j128m_init(&gram_freqs);
j128m_init(&codemap);
if (!(field_cardinalities = calloc(cons->num_ofields, 8))){
ret = TDB_ERR_NOMEM;
goto done;
}
for (i = 0; i < cons->num_ofields; i++)
field_cardinalities[i] = jsm_num_keys(&cons->lexicons[i]);
/* 1. group events by trail, sort events of each trail by time,
and delta-encode timestamps */
TDB_TIMER_START
TDB_PATH(grouped_path, "%s/tmp.grouped.XXXXXX", root);
if ((fd = mkstemp(grouped_path)) == -1){
ret = TDB_ERR_IO_OPEN;
goto done;
}
if (!(grouped_w = fdopen(fd, "w"))){
ret = TDB_ERR_IO_OPEN;
goto done;
}
if (cons->events.data)
if ((ret = groupby_uuid(grouped_w,
(struct tdb_cons_event*)cons->events.data,
cons,
&num_trails,
&max_timestamp,
&max_timedelta)))
goto done;
/*
not the most clean separation of ownership here, but these objects
can be huge so keeping them around unecessarily is expensive
*/
free(cons->events.data);
cons->events.data = NULL;
j128m_free(&cons->trails);
TDB_CLOSE(grouped_w);
grouped_w = NULL;
TDB_OPEN(grouped_r, grouped_path, "r");
if (!(read_buf = malloc(READ_BUFFER_SIZE))){
ret = TDB_ERR_NOMEM;
goto done;
}
setvbuf(grouped_r, read_buf, _IOFBF, READ_BUFFER_SIZE);
TDB_TIMER_END("trail/groupby_uuid");
/* 2. store metatadata */
TDB_TIMER_START
TDB_PATH(path, "%s/info", root);
if ((ret = store_info(path,
num_trails,
num_events,
cons->min_timestamp,
max_timestamp,
max_timedelta)))
goto done;
TDB_TIMER_END("trail/info");
/* 3. collect value (unigram) freqs, including delta-encoded timestamps */
TDB_TIMER_START
unigram_freqs = collect_unigrams(grouped_r, num_events, items, num_fields);
if (num_events > 0 && !unigram_freqs){
ret = TDB_ERR_NOMEM;
goto done;
}
TDB_TIMER_END("trail/collect_unigrams");
/* 4. construct uni/bi-grams */
tdb_opt_value dont_build_bigrams;
tdb_cons_get_opt(cons, TDB_OPT_CONS_NO_BIGRAMS, &dont_build_bigrams);
TDB_TIMER_START
if ((ret = make_grams(grouped_r,
num_events,
items,
num_fields,
unigram_freqs,
&gram_freqs,
dont_build_bigrams.value)))
goto done;
TDB_TIMER_END("trail/gram_freqs");
/* 5. build a huffman codebook and stats struct for encoding grams */
TDB_TIMER_START
if ((ret = huff_create_codemap(&gram_freqs, &codemap)))
goto done;
if (!(fstats = huff_field_stats(field_cardinalities,
num_fields,
max_timedelta))){
ret = TDB_ERR_NOMEM;
goto done;
}
TDB_TIMER_END("trail/huff_create_codemap");
/* 6. encode and write trails to disk */
TDB_TIMER_START
TDB_PATH(path, "%s/trails.data", root);
TDB_PATH(toc_path, "%s/trails.toc", root);
if ((ret = encode_trails(items,
grouped_r,
num_events,
num_trails,
num_fields,
&codemap,
&gram_freqs,
fstats,
path,
toc_path)))
goto done;
TDB_TIMER_END("trail/encode_trails");
/* 7. write huffman codebook to disk */
TDB_TIMER_START
tdb_path(path, "%s/trails.codebook", root);
if ((ret = store_codebook(&codemap, path)))
goto done;
TDB_TIMER_END("trail/store_codebook");
done:
TDB_CLOSE_FINAL(grouped_w);
TDB_CLOSE_FINAL(grouped_r);
j128m_free(&gram_freqs);
j128m_free(&codemap);
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wsign-compare"
JLFA(tmp, unigram_freqs);
#pragma GCC diagnostic pop
unlink(grouped_path);
free(field_cardinalities);
free(read_buf);
free(fstats);
return ret;
out_of_memory:
return TDB_ERR_NOMEM;
}
