bool t_check_sort_()
{
fseek(key_f_, 0, SEEK_END);
const uint64_t KEY_FILE_SIZE = ftell(key_f_);
const uint32_t KEY_LOAD_TIMES = KEY_FILE_SIZE%buf_size_==0? KEY_FILE_SIZE/buf_size_: KEY_FILE_SIZE/buf_size_+1;
fseek(key_f_, 0, SEEK_SET);
for (uint32_t i=0; i<KEY_LOAD_TIMES; ++i)
{
uint32_t ss = KEY_FILE_SIZE - ftell(key_f_)>buf_size_? buf_size_: KEY_FILE_SIZE - ftell(key_f_);//loading size
uint32_t nn = ss/sizeof(PRE_KEY_STRUCT);
IASSERT(fread(buffer_, ss, 1, key_f_)==1);
for (uint32_t j=0; j<nn-1; ++j)
{
// if (((PRE_KEY_STRUCT*)buffer_)[j].PRE_KEY()+1!=((PRE_KEY_STRUCT*)buffer_)[j+1].PRE_KEY())
// {
// std::cout<<"+"<<((PRE_KEY_STRUCT*)buffer_)[j].PRE_KEY()<<"-"<<((PRE_KEY_STRUCT*)buffer_)[j+1].PRE_KEY();
// return false;
// }
if (((PRE_KEY_STRUCT*)buffer_)[j].compare(((PRE_KEY_STRUCT*)buffer_)[j+1], original_f_)>0)
{
std::cout<<((PRE_KEY_STRUCT*)buffer_)[j].PRE_KEY()<<"-"<<((PRE_KEY_STRUCT*)buffer_)[j+1].PRE_KEY()<<std::endl;
return false;
}
}
}
IASSERT((uint64_t)ftell(key_f_)==KEY_FILE_SIZE);
return true;
}
void output_(FILE* f)
{
IO_TYPE ioStream(f,"w");
uint64_t count = 0;
uint64_t nextStart = 0;
while (count< count_)
{
boost::mutex::scoped_lock lock(out_buf_mtx_);
while (out_buf_size_ == 0)
out_buf_con_.wait(lock);
IASSERT(fwrite(&out_buf_size_, sizeof(uint32_t), 1, f)==1);
IASSERT(fwrite(&out_buf_num_, sizeof(uint32_t), 1, f)==1);
//IASSERT(fwrite(&max_record_len_of_this_run_, sizeof(uint32_t), 1, f)==1); //TODO
uint64_t nextStartPos = ftell(f);
IASSERT(fwrite(&nextStart, sizeof(uint64_t), 1, f)==1);
//IASSERT(fwrite(out_buf_, out_buf_size_, 1, f)==1);
ioStream.write(out_buf_, out_buf_size_);
nextStart = ftell(f);
fseek(f, nextStartPos, SEEK_SET);
IASSERT(fwrite(&nextStart, sizeof(uint64_t), 1, f)==1);
fseek(f, nextStart, SEEK_SET);
IASSERT(t_check_sort_());
count += out_buf_num_;
out_buf_size_ = out_buf_num_ = 0;
out_buf_con_.notify_one();
}
std::cout<<"Outputting is over...\n";
}
size_t write_compressed(int f, T * tbuf, size_t nbytes) {
#ifndef GRAPHCHI_DISABLE_COMPRESSION
unsigned char * buf = (unsigned char*)tbuf;
int ret;
unsigned have;
z_stream strm;
int CHUNK = (int) std::max((size_t)4096 * 1024, nbytes);
unsigned char * out = (unsigned char *) malloc(CHUNK);
lseek(f, 0, SEEK_SET);
/* allocate deflate state */
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
ret = deflateInit(&strm, Z_BEST_SPEED);
if (ret != Z_OK)
assert(false);
/* compress until end of file */
strm.avail_in = (int) nbytes;
strm.next_in = buf;
int trerr = ftruncate(f, 0);
IASSERT (trerr == 0);
size_t totwritten = 0;
/* run deflate() on input until output buffer not full, finish
compression if all of source has been read in */
do {
strm.avail_out = CHUNK;
strm.next_out = out;
ret = deflate(&strm, Z_FINISH); /* no bad return value */
assert(ret != Z_STREAM_ERROR); /* state not clobbered */
have = CHUNK - strm.avail_out;
if (write(f, out, have) != have) {
(void)deflateEnd(&strm);
assert(false);
}
totwritten += have;
} while (strm.avail_out == 0);
IASSERT(strm.avail_in == 0); /* all input will be used */
IASSERT(ret == Z_STREAM_END); /* stream will be complete */
/* clean up and return */
(void)deflateEnd(&strm);
free(out);
return totwritten;
#else
writea(f, tbuf, nbytes);
return nbytes;
#endif
}
static void
ieee80211_remove_vap(struct ieee80211com *ic)
{
int s;
s = splnet();
SLIST_REMOVE(&ieee80211_list, ic, ieee80211com, ic_next);
IASSERT(ic->ic_vap < sizeof(ieee80211_vapmap)*NBBY,
("invalid vap id %d", ic->ic_vap));
IASSERT(isset(ieee80211_vapmap, ic->ic_vap),
("vap id %d not allocated", ic->ic_vap));
clrbit(ieee80211_vapmap, ic->ic_vap);
splx(s);
}
void checkarray_filesize(std::string fname, size_t nelements) {
// Check the vertex file is correct size
int f = open(fname.c_str(), O_RDWR | O_CREAT, S_IROTH | S_IWOTH | S_IWUSR | S_IRUSR);
if (f < 1) {
logstream(LOG_ERROR) << "Error initializing the data-file: " << fname << " error:" << strerror(errno) << std::endl; }
IASSERT(f>0);
int err = ftruncate(f, nelements * sizeof(T));
if (err != 0) {
logstream(LOG_ERROR) << "Error in adjusting file size: " << fname << " to size: " << nelements * sizeof(T)
<< " error:" << strerror(errno) << std::endl;
}
IASSERT(err == 0);
close(f);
}
/*
* Set the contents of the specified key.
*
* Locking must be handled by the caller using:
* ieee80211_key_update_begin(ic);
* ieee80211_key_update_end(ic);
*/
int
ieee80211_crypto_setkey(struct ieee80211com *ic, struct ieee80211_key *key,
const u_int8_t macaddr[IEEE80211_ADDR_LEN])
{
const struct ieee80211_cipher *cip = key->wk_cipher;
IASSERT(cip != NULL, ("No cipher!"));
IEEE80211_DPRINTF(ic, IEEE80211_MSG_CRYPTO,
"%s: %s keyix %u flags 0x%x mac %s rsc %ju tsc %ju len %u\n",
__func__, cip->ic_name, key->wk_keyix,
key->wk_flags, ether_sprintf(macaddr),
key->wk_keyrsc, key->wk_keytsc, key->wk_keylen);
/*
* Give cipher a chance to validate key contents.
* XXX should happen before modifying state.
*/
if (!cip->ic_setkey(key)) {
IEEE80211_DPRINTF(ic, IEEE80211_MSG_CRYPTO,
"%s: cipher %s rejected key index %u len %u flags 0x%x\n",
__func__, cip->ic_name, key->wk_keyix,
key->wk_keylen, key->wk_flags);
ic->ic_stats.is_crypto_setkey_cipher++;
return 0;
}
if (key->wk_keyix == IEEE80211_KEYIX_NONE) {
/* XXX nothing allocated, should not happen */
IEEE80211_DPRINTF(ic, IEEE80211_MSG_CRYPTO,
"%s: no key index; should not happen!\n", __func__);
ic->ic_stats.is_crypto_setkey_nokey++;
return 0;
}
return dev_key_set(ic, key, macaddr);
}
/*
* Remove the key (no locking, for internal use).
*/
static int
_ieee80211_crypto_delkey(struct ieee80211com *ic, struct ieee80211_key *key)
{
ieee80211_keyix keyix;
IASSERT(key->wk_cipher != NULL, ("No cipher!"));
IEEE80211_DPRINTF(ic, IEEE80211_MSG_CRYPTO,
"%s: %s keyix %u flags 0x%x rsc %ju tsc %ju len %u\n",
__func__, key->wk_cipher->ic_name,
key->wk_keyix, key->wk_flags,
key->wk_keyrsc, key->wk_keytsc, key->wk_keylen);
keyix = key->wk_keyix;
if (keyix != IEEE80211_KEYIX_NONE) {
/*
* Remove hardware entry.
*/
/* XXX key cache */
if (!dev_key_delete(ic, key)) {
IEEE80211_DPRINTF(ic, IEEE80211_MSG_CRYPTO,
"%s: driver did not delete key index %u\n",
__func__, keyix);
ic->ic_stats.is_crypto_delkey++;
/* XXX recovery? */
}
}
cipher_detach(key);
memset(key, 0, sizeof(*key));
ieee80211_crypto_resetkey(ic, key, IEEE80211_KEYIX_NONE);
return 1;
}
void run()
{
struct timeval tvafter, tvpre;
struct timezone tz;
new_buffer_();
gettimeofday (&tvpre , &tz);
FILE* f = fopen(filenm_.c_str(), "r");
IASSERT(f);
IASSERT(fread(&count_, sizeof(uint64_t), 1, f)==1);
boost::thread prefetch_thre(boost::bind(&self_t::prefetch_, this, f));
boost::thread sort_thre(boost::bind(&self_t::sort_, this));
FILE* out_f = fopen((filenm_+".out").c_str(), "w+");
IASSERT(out_f);
IASSERT(fwrite(&count_, sizeof(uint64_t), 1, out_f)==1);
boost::thread out_thre(boost::bind(&self_t::output_, this, out_f));
prefetch_thre.join();
sort_thre.join();
out_thre.join();
// fseek(f, 0, SEEK_END);
// fseek(out_f, 0, SEEK_END);
// IASSERT((uint64_t)ftell(f)+ run_num_*sizeof(uint32_t)*2== (uint64_t)ftell(out_f));
fseek(out_f, 0, SEEK_SET);
IASSERT(fwrite(&count_, sizeof(uint64_t), 1, out_f)==1);
fclose(f);
fclose(out_f);
if (boost::filesystem::exists(filenm_))
boost::filesystem::remove(filenm_);
if (boost::filesystem::exists(filenm_+".out"))
boost::filesystem::rename(filenm_+".out", filenm_);
gettimeofday (&tvafter , &tz);
std::cout<<"A run is over("<<count_
<<"): "<<((tvafter.tv_sec-tvpre.tv_sec)*1000+(tvafter.tv_usec-tvpre.tv_usec)/1000.)/60000<<" min\n";
}
/*
* Set the current phy mode and recalculate the active channel
* set based on the available channels for this mode. Also
* select a new default/current channel if the current one is
* inappropriate for this mode.
*/
int
ieee80211_setmode(struct ieee80211com *ic, enum ieee80211_phymode mode)
{
#define N(a) (sizeof(a) / sizeof(a[0]))
static const u_int chanflags[] = {
0, /* IEEE80211_MODE_AUTO */
IEEE80211_CHAN_A, /* IEEE80211_MODE_11A */
IEEE80211_CHAN_B, /* IEEE80211_MODE_11B */
IEEE80211_CHAN_PUREG, /* IEEE80211_MODE_11G */
IEEE80211_CHAN_FHSS, /* IEEE80211_MODE_FH */
IEEE80211_CHAN_T, /* IEEE80211_MODE_TURBO_A */
IEEE80211_CHAN_108G, /* IEEE80211_MODE_TURBO_G */
};
struct ieee80211_channel *c;
u_int modeflags;
int i;
/* validate new mode */
if ((ic->ic_modecaps & (1<<mode)) == 0) {
IEEE80211_DPRINTF(ic, IEEE80211_MSG_ANY,
"%s: mode %u not supported (caps 0x%x)\n",
__func__, mode, ic->ic_modecaps);
return EINVAL;
}
/*
* Verify at least one channel is present in the available
* channel list before committing to the new mode.
*/
IASSERT(mode < N(chanflags), ("Unexpected mode %u", mode));
modeflags = chanflags[mode];
for (i = 0; i <= IEEE80211_CHAN_MAX; i++) {
c = &ic->ic_channels[i];
if (c->ic_flags == 0)
continue;
if (mode == IEEE80211_MODE_AUTO) {
/* ignore turbo channels for autoselect */
if ((c->ic_flags & IEEE80211_CHAN_TURBO) == 0)
break;
} else {
if ((c->ic_flags & modeflags) == modeflags)
break;
}
}
if (i > IEEE80211_CHAN_MAX) {
IEEE80211_DPRINTF(ic, IEEE80211_MSG_ANY,
"%s: no channels found for mode %u\n", __func__, mode);
return EINVAL;
}
/*
* Calculate the active channel set.
*/
memset(ic->ic_chan_active, 0, sizeof(ic->ic_chan_active));
for (i = 0; i <= IEEE80211_CHAN_MAX; i++) {
c = &ic->ic_channels[i];
if (c->ic_flags == 0)
continue;
if (mode == IEEE80211_MODE_AUTO) {
/* take anything but pure turbo channels */
if ((c->ic_flags & IEEE80211_CHAN_TURBO) == 0)
setbit(ic->ic_chan_active, i);
} else {
if ((c->ic_flags & modeflags) == modeflags)
setbit(ic->ic_chan_active, i);
}
}
/*
* If no current/default channel is setup or the current
* channel is wrong for the mode then pick the first
* available channel from the active list. This is likely
* not the right one.
*/
if (ic->ic_ibss_chan == NULL ||
isclr(ic->ic_chan_active, ieee80211_chan2ieee(ic, ic->ic_ibss_chan))) {
for (i = 0; i <= IEEE80211_CHAN_MAX; i++)
if (isset(ic->ic_chan_active, i)) {
ic->ic_ibss_chan = &ic->ic_channels[i];
break;
}
IASSERT(ic->ic_ibss_chan != NULL &&
isset(ic->ic_chan_active,
ieee80211_chan2ieee(ic, ic->ic_ibss_chan)),
("Bad IBSS channel %u",
ieee80211_chan2ieee(ic, ic->ic_ibss_chan)));
}
/*
* If the desired channel is set but no longer valid then reset it.
*/
if (ic->ic_des_chan != IEEE80211_CHAN_ANYC &&
isclr(ic->ic_chan_active, ieee80211_chan2ieee(ic, ic->ic_des_chan)))
ic->ic_des_chan = IEEE80211_CHAN_ANYC;
/*
* Do mode-specific rate setup.
*/
if (mode == IEEE80211_MODE_11G) {
/*
* Use a mixed 11b/11g rate set.
*/
ieee80211_set11gbasicrates(&ic->ic_sup_rates[mode],
IEEE80211_MODE_11G);
} else if (mode == IEEE80211_MODE_11B) {
/*
* Force pure 11b rate set.
*/
ieee80211_set11gbasicrates(&ic->ic_sup_rates[mode],
IEEE80211_MODE_11B);
}
/*
* Setup an initial rate set according to the
* current/default channel selected above. This
* will be changed when scanning but must exist
* now so driver have a consistent state of ic_ibss_chan.
*/
if (ic->ic_bss) /* NB: can be called before lateattach */
ic->ic_bss->ni_rates = ic->ic_sup_rates[mode];
ic->ic_curmode = mode;
ieee80211_reset_erp(ic); /* reset ERP state */
ieee80211_wme_initparams(ic); /* reset WME stat */
return 0;
#undef N
}
void _readBytes(char* data, size_t len)
{
IASSERT(fread(data, len, 1, fd_) == 1);
}
void sort_()
{
uint64_t count = 0;
while (count< count_)
{
uint32_t pre_buf_size = 0;
uint32_t pre_buf_num = 0;
{
boost::mutex::scoped_lock lock(pre_buf_mtx_);
while (pre_buf_size_==0)
pre_buf_con_.wait(lock);
assert(pre_buf_size_ <= RUN_BUF_SIZE_);
memcpy(run_buf_, pre_buf_, pre_buf_size_);
pre_buf_size = pre_buf_size_;
pre_buf_num = pre_buf_num_;
count += pre_buf_num_;
pre_buf_num_ = pre_buf_size_ = 0;
pre_buf_con_.notify_one();
}
key_buf_ = (struct KEY_PTR*)realloc(key_buf_, pre_buf_num*sizeof(struct KEY_PTR));
uint32_t pos = 0;
for (uint32_t i = 0; i<pre_buf_num; ++i)
{
key_buf_[i] = KEY_PTR(pos);
assert(pos <= RUN_BUF_SIZE_);
pos += *(LEN_TYPE*)(run_buf_ + pos)+sizeof(LEN_TYPE);
IASSERT(pos<=pre_buf_size);
}
IASSERT(pos==pre_buf_size);
quick_sort_(0, pre_buf_num-1, pre_buf_num);
boost::mutex::scoped_lock lock(out_buf_mtx_);
while (out_buf_size_ != 0)
out_buf_con_.wait(lock);
out_buf_size_ = 0;
out_buf_num_ = 0;
LEN_TYPE max_len_of_this_run = (LEN_TYPE)0;
for (uint32_t i=0; i<pre_buf_num; ++i, ++out_buf_num_)
{
assert(key_buf_[i].pos <= RUN_BUF_SIZE_);
assert(out_buf_size_+key_buf_[i].LEN(run_buf_)+ sizeof(LEN_TYPE) <= RUN_BUF_SIZE_);
memcpy(out_buf_+out_buf_size_, run_buf_+ key_buf_[i].pos, key_buf_[i].LEN(run_buf_)+ sizeof(LEN_TYPE));
LEN_TYPE len = key_buf_[i].LEN(run_buf_) + sizeof(LEN_TYPE);
out_buf_size_ += len;
if(len > max_len_of_this_run) max_len_of_this_run = len;
}
max_record_len_of_this_run_ = max_len_of_this_run;
min_run_buff_size_for_merger_ += max_record_len_of_this_run_;
if(max_len_of_this_run > max_record_len_) max_record_len_ = (uint32_t)max_len_of_this_run;
IASSERT(out_buf_num_ == pre_buf_num);
IASSERT(out_buf_size_ == pre_buf_size);
out_buf_con_.notify_one();
}
std::cout<<"Sorting is over...\n";
}
void read_compressed(int f, T * tbuf, size_t nbytes) {
#ifndef GRAPHCHI_DISABLE_COMPRESSION
unsigned char * buf = (unsigned char*)tbuf;
int ret;
unsigned have;
z_stream strm;
int CHUNK = (int) std::max((size_t)4096 * 1024, nbytes);
size_t fsize = lseek(f, 0, SEEK_END);
unsigned char * in = (unsigned char *) malloc(fsize);
lseek(f, 0, SEEK_SET);
/* allocate inflate state */
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
strm.avail_in = 0;
strm.next_in = Z_NULL;
ret = inflateInit(&strm);
if (ret != Z_OK)
assert(false);
/* decompress until deflate stream ends or end of file */
do {
ssize_t a = 0;
do {
a = read(f, in + strm.avail_in, fsize - strm.avail_in); //fread(in, 1, CHUNK, source);
strm.avail_in += (int) a;
IASSERT(a != (ssize_t)(-1));
} while (a > 0);
if (strm.avail_in == 0)
break;
strm.next_in = in;
/* run inflate() on input until output buffer not full */
do {
strm.avail_out = CHUNK;
strm.next_out = buf;
ret = inflate(&strm, Z_NO_FLUSH);
IASSERT(ret != Z_STREAM_ERROR); /* state not clobbered */
switch (ret) {
case Z_NEED_DICT:
ret = Z_DATA_ERROR; /* and fall through */
case Z_DATA_ERROR:
case Z_MEM_ERROR:
assert(false);
}
have = CHUNK - strm.avail_out;
buf += have;
} while (strm.avail_out == 0);
/* done when inflate() says it's done */
} while (ret != Z_STREAM_END);
// std::cout << "Read: " << (buf - (unsigned char*)tbuf) << std::endl;
/* clean up and return */
(void)inflateEnd(&strm);
free(in);
#else
preada(f, tbuf, nbytes, 0);
#endif
}
/*
* Set the current phy mode and recalculate the active channel
* set based on the available channels for this mode. Also
* select a new default/current channel if the current one is
* inappropriate for this mode.
*/
int
ieee80211_setmode(struct ieee80211com *ic, enum ieee80211_phymode mode)
{
#define N(a) (sizeof(a) / sizeof(a[0]))
static const u_int chanflags[] = {
0, /* IEEE80211_MODE_AUTO */
IEEE80211_CHAN_A, /* IEEE80211_MODE_11A */
IEEE80211_CHAN_B, /* IEEE80211_MODE_11B */
IEEE80211_CHAN_PUREG, /* IEEE80211_MODE_11G */
IEEE80211_CHAN_FHSS, /* IEEE80211_MODE_FH */
IEEE80211_CHAN_T, /* IEEE80211_MODE_TURBO */
};
struct ieee80211_channel *c;
u_int modeflags;
int i;
/* validate new mode */
if ((ic->ic_modecaps & (1<<mode)) == 0) {
IEEE80211_DPRINTF(("%s: mode %u not supported (caps 0x%x)\n",
__func__, mode, ic->ic_modecaps));
return EINVAL;
}
/*
* Verify at least one channel is present in the available
* channel list before committing to the new mode.
*/
IASSERT(mode < N(chanflags), ("Unexpected mode %u\n", mode));
modeflags = chanflags[mode];
for (i = 0; i <= IEEE80211_CHAN_MAX; i++) {
c = &ic->ic_channels[i];
if (mode == IEEE80211_MODE_AUTO) {
/* ignore turbo channels for autoselect */
if ((c->ic_flags &~ IEEE80211_CHAN_TURBO) != 0)
break;
} else {
if ((c->ic_flags & modeflags) == modeflags)
break;
}
}
if (i > IEEE80211_CHAN_MAX) {
IEEE80211_DPRINTF(("%s: no channels found for mode %u\n",
__func__, mode));
return EINVAL;
}
/*
* Calculate the active channel set.
*/
memset(ic->ic_chan_active, 0, sizeof(ic->ic_chan_active));
for (i = 0; i <= IEEE80211_CHAN_MAX; i++) {
c = &ic->ic_channels[i];
if (mode == IEEE80211_MODE_AUTO) {
/* take anything but pure turbo channels */
if ((c->ic_flags &~ IEEE80211_CHAN_TURBO) != 0)
setbit(ic->ic_chan_active, i);
} else {
if ((c->ic_flags & modeflags) == modeflags)
setbit(ic->ic_chan_active, i);
}
}
/*
* If no current/default channel is setup or the current
* channel is wrong for the mode then pick the first
* available channel from the active list. This is likely
* not the right one.
*/
if (ic->ic_ibss_chan == NULL ||
isclr(ic->ic_chan_active, ieee80211_chan2ieee(ic, ic->ic_ibss_chan))) {
for (i = 0; i <= IEEE80211_CHAN_MAX; i++)
if (isset(ic->ic_chan_active, i)) {
ic->ic_ibss_chan = &ic->ic_channels[i];
break;
}
IASSERT(ic->ic_ibss_chan != NULL &&
isset(ic->ic_chan_active,
ieee80211_chan2ieee(ic, ic->ic_ibss_chan)),
("Bad IBSS channel %u\n",
ieee80211_chan2ieee(ic, ic->ic_ibss_chan)));
}
/*
* Set/reset state flags that influence beacon contents, etc.
*
* XXX what if we have stations already associated???
* XXX probably not right for autoselect?
*/
if (ic->ic_caps & IEEE80211_C_SHPREAMBLE)
ic->ic_flags |= IEEE80211_F_SHPREAMBLE;
if (mode == IEEE80211_MODE_11G) {
if (ic->ic_caps & IEEE80211_C_SHSLOT)
ic->ic_flags |= IEEE80211_F_SHSLOT;
} else
ic->ic_flags &= ~IEEE80211_F_SHSLOT;
ic->ic_curmode = mode;
return 0;
#undef N
}
