void CmdLine :: parse (int argc, char** argv, bool strict)
{
// prepare the parameters and call cmdline_s get_opt
Arglist arglist;
Optdict optdict;
// allocate storage for long options spec
unsigned longopts_no = 0;
KeysFormat::iterator kitr;
char longopts_spec_buf [MAX_LONGOPTS][MAX_LONGOPT_LEN];
char* longopts_spec [MAX_LONGOPTS];
for (kitr = keys_format_.begin (); kitr != keys_format_.end (); kitr ++)
longopts_no += (*kitr).longopts_.size ();
if (longopts_no + 1 >= MAX_LONGOPTS)
ers << "Too many command line options in command line format definition" << ThrowEx(InternalError);
// fill in options specs and make option->spec map
std::map <std::string, KeyFormat*> opt2spec;
std::string shortopts_spec (EMPTY_STR);
int lidx = 0;
for (kitr = keys_format_.begin (); kitr != keys_format_.end (); kitr ++)
{
// short
svec::iterator soi = (*kitr).shortopts_.begin ();
for (; soi != (*kitr).shortopts_.end (); soi ++)
{
const char* optstr = (*soi).c_str ();
shortopts_spec+= optstr;
if ((*kitr).has_arg_) shortopts_spec += ":";
std::string keystr ("-");
keystr.append (optstr);
opt2spec [keystr] = &(*kitr);
}
// long
svec::iterator loi = (*kitr).longopts_.begin ();
for (; loi != (*kitr).longopts_.end (); loi ++, lidx ++)
{
unsigned lolen = (*loi).length () + 1;
if ((*kitr).has_arg_) lolen ++;
if (lolen + 1 >= MAX_LONGOPT_LEN)
ers << "Command line option " << *loi << " is too long" << Throw;
// longopts_spec [lidx] = new char [lolen];
const char* optstr = (*loi).c_str ();
longopts_spec [lidx] = longopts_spec_buf [lidx];
strcpy (longopts_spec [lidx], optstr);
if ((*kitr).has_arg_) strcat (longopts_spec [lidx], "=");
std::string keystr ("--");
keystr.append (optstr);
opt2spec [keystr] = &(*kitr);
}
}
longopts_spec [lidx] = NULL;
// actually parse command line
get_opt (argc, argv, shortopts_spec.c_str (), arglist, optdict, longopts_spec);
// fill in the keys_
Optdict::iterator oi = optdict.begin ();
for (;oi != optdict.end (); oi ++)
{
const char* key = (*oi).first.c_str ();
// find what parameter this name corresponds to
std::map <std::string, KeyFormat*>::iterator k = opt2spec.find (key);
if (k != opt2spec.end ())
{
svec values = (*oi).second;
if ((*k).second->has_arg_)
{
if (values.size () == 0)
{
error_report_ += "\nParameter ";
error_report_ += key;
error_report_ += "(";
error_report_ += (*k).second->name_;
error_report_ += ") requires argument";
ok_ = false;
break;
}
const char* val = values [values.size () - 1].c_str ();
keys_ [(*k).second->name_] = val;
}
else
{
keys_ [(*k).second->name_] = EMPTY_STR;
}
// DEBUG
// printf ("CMDL PAR '%s' : '%s'", (*k).second->name_.c_str (), val);
}
}
// check weather all non-optional keys are found
if (ok_ && strict)
{
for (kitr = keys_format_.begin (); kitr != keys_format_.end (); kitr ++)
{
if (!kitr->optional_)
{
std::map<std::string, std::string>::iterator valitr = keys_.find (kitr->name_);
if (valitr == keys_.end ())
{
// add notes about missing ones to the error_report
error_report_ += "Required option missing from command line: ";
error_report_ += kitr->name_;
error_report_ += "\n";
ok_ = false;
}
}
}
}
// fill in the arguments_
if (ok_) std::copy (arglist.begin (), arglist.end (), std::back_inserter (arguments_));
// check if the number of arguments is valid
if (strict && ok_ && repeatable_pos_ == -1 && arglist.size () > args_format_.size ())
{
// add the note to error_report
error_report_ += "Too many command-line arguments\n";
ok_ = false;
}
unsigned min_arg_number = ((first_optional_pos_ != -1) ? first_optional_pos_ : 0)
+ args_format_.size () - ((last_optional_pos_ == -1) ? 0 : last_optional_pos_);
if (strict && ok_ && arguments_.size () < min_arg_number)
{
// add the note to error_report
error_report_ += "Too few command-line arguments. Please use -h for help.\n";
ok_ = false;
}
}
Config* PyGLPSParse2Config(PyObject *, PyObject *args, PyObject *kws)
{
PyObject *conf = NULL, *extra_defs = Py_None;
const char *path = NULL;
const char *pnames[] = {"config", "path", "extra", NULL};
if(!PyArg_ParseTupleAndKeywords(args, kws, "O|zO", (char**)pnames, &conf, &path, &extra_defs))
return NULL;
GLPSParser parser;
if(extra_defs==Py_None) {
// no-op
} else if(PyDict_Check(extra_defs)) {
PyObject *key, *value;
Py_ssize_t pos = 0;
while(PyDict_Next(extra_defs, &pos, &key, &value)) {
PyRef<> keyx(key, borrow());
PyCString keystr(keyx);
Config::value_t curval;
if(PyNumber_Check(value)) {
PyRef<> pyf(PyNumber_Float(value));
curval = PyFloat_AsDouble(pyf.py());
} else if(PyString_Check(value)) {
PyRef<> valuex(value, borrow());
PyCString valstr(valuex);
curval = valstr.c_str();
} else {
PyErr_SetString(PyExc_ValueError, "extra {} can contain only numbers or strings");
return NULL;
}
parser.setVar(keystr.c_str(), curval);
}
} else {
PyErr_SetString(PyExc_ValueError, "'extra' must be a dict");
return NULL;
}
PyGetBuf buf;
std::auto_ptr<Config> C;
PyRef<> listref;
if(PyObject_HasAttrString(conf, "read")) { // file-like
PyCString pyname;
if(!path && PyObject_HasAttrString(conf, "name")) {
path = pyname.c_str(pydirname(PyObject_GetAttrString(conf, "name")));
}
PyRef<> pybytes(PyObject_CallMethod(conf, "read", ""));
if(!buf.get(pybytes.py())) {
PyErr_SetString(PyExc_TypeError, "read() must return a buffer");
return NULL;
}
C.reset(parser.parse_byte((const char*)buf.data(), buf.size(), path));
} else if(buf.get(conf)) {
C.reset(parser.parse_byte((const char*)buf.data(), buf.size(), path));
#if PY_MAJOR_VERSION >= 3
} else if(PyUnicode_Check(conf)) { // py3 str (aka unicode) doesn't implement buffer iface
PyCString buf;
const char *cbuf = buf.c_str(conf);
C.reset(parser.parse_byte(cbuf, strlen(cbuf), path));
#endif
} else {
if(PyDict_Check(conf)) {
listref.reset(PyMapping_Items(conf));
conf = listref.py();
}
if(PyList_Check(conf)) {
C.reset(list2conf(conf));
} else {
throw std::invalid_argument("'config' must be dict, list of tuples, or byte buffer");
}
}
return C.release();
}
void hashTableGenerator(string ref_name) {
//Open and read the reference file
ifstream ref_file;
deque<int>::iterator iter;
int coor_counter = KEY_LENGTH*(-1);
int flag_ignore = 0;
int monitor_counter = 0; // For operation monitoring
int monitor_counter2 = 0; // For operation monitoring
ref_file.open(ref_name.c_str());
if (!ref_file.good()) {
cerr << "reference file cannot open." << endl;
exit(1);
}
//cout << "Hello0" << endl;
//Initialize hash_table.
// hash_table.resize(1 << (2 * KEY_LENGTH) );
// hash_table_counter = 0;
//cout << "Hello1" << endl;
//Read in the first key
char temp_key[KEY_LENGTH + 1];
char temp_char;
string keystr(KEY_LENGTH, 'A');
//cout << "keystr: " << keystr << endl;
//throw out the first several Ns
do {
ref_file >> temp_char;
coor_counter++;
//cout << "coor_counter: " << coor_counter << endl;
} while (temp_char == 'N');
//update the first non N character
temp_key[0] = temp_char;
//cout << "temp_char: " << temp_char << endl;
//Read the first 12 characters
for (int i = 1; i < KEY_LENGTH; i++) {
//cout << "i: " << i << endl;
ref_file >> temp_char;
coor_counter++;
//cout << "coor_counter: " << coor_counter << endl;
temp_key[i] = temp_char;
//Check if the file is too short
if (ref_file.eof()) {
cerr << "file too short";
exit(1);
}
//Check if next N come too close
if (temp_char == 'N') {
cerr << "Too short useful String A" << endl;
exit(1);
}
}
temp_key[KEY_LENGTH] = '\0';
keystr = temp_key;
//cout << "temp_key: " << temp_key << endl;
//cout << "keystr: " << keystr << endl;
//Read the ref file until the end of file.
do {
//Compute idx and hash_value
if (flag_ignore == 0) {
int hash_value = hashVal(keystr);
//cout << "hash_value: " << hash_value << endl;
//Add to hash_table
//cout << "coor_counter: " << coor_counter << endl;
hash_table[hash_value].push_front(coor_counter);
hash_table_counter++;
} else {
flag_ignore = 0;
for (int i = 0; i < KEY_LENGTH; i++ ) {
if (keystr[i] == 'N') {
flag_ignore = 1;
}
}
if (flag_ignore == 0) {
int hash_value = hashVal(keystr);
//hash_table[hash_value].push_front(coor_counter);
//hash_table_counter++;
}
}
/*
//Add to hash_table list
if (hash_table[idx][hash_value].size() == 0) {
//cout << "first saw" << endl;
hash_table[idx][hash_value].push_back(counter);
}
else {
iter = hash_table[idx][hash_value].begin();
while (iter != hash_table[idx][hash_value].end() && counter > *iter) {
//cout << "*iter: " << *iter << endl;
iter++;
}
hash_table[idx][hash_value].insert(iter, counter);
}
*/
//Read next character
ref_file >> temp_char;
//Increment counter
coor_counter++;
bool fresh_start = false; //indicater for a new start after large area of N
//If enters N area, throw them away
while (temp_char == 'N') {
if (ref_file.eof() ) {
ref_file.close();
return;
}
fresh_start = true;
ref_file >> temp_char;
coor_counter++;
//cout << "coor_counter: " << coor_counter << endl;
}
if (fresh_start) {
//cout << "In a fresh start!" << endl;
keystr[0] = temp_char;
for (int i = 1; i < KEY_LENGTH; i++) {
//cout << 'i' << i << endl;
ref_file >> temp_char;
coor_counter++;
//cout << "coor_counter: " << coor_counter << endl;
//Check if next N come too close
if (temp_char == 'N') {
cerr << "Too short useful String B" << endl;
flag_ignore = 1;
//exit(1);
}
keystr[i] = temp_char;
}
} else
keystr = keystr.substr(1, KEY_LENGTH - 1) + temp_char;
//cout << "keystr: " << keystr << endl;
// add process monitoring loop
monitor_counter = monitor_counter +1; // For operation monitoring
monitor_counter2 = monitor_counter2 +1;
if (monitor_counter >= 1000000) {
cout << "hash generator: " << monitor_counter2 << endl;
monitor_counter = 0;
}
} while (ref_file.good());
static void
print_keyrec(int number,struct keyrec *keyrec)
{
int i;
iobuf_writebyte(keyrec->uidbuf,0);
iobuf_flush_temp(keyrec->uidbuf);
es_printf ("(%d)\t%s ", number, iobuf_get_temp_buffer (keyrec->uidbuf));
if (keyrec->size>0)
es_printf ("%d bit ", keyrec->size);
if(keyrec->type)
{
const char *str = gcry_pk_algo_name (keyrec->type);
if(str)
es_printf ("%s ",str);
else
es_printf ("unknown ");
}
switch(keyrec->desc.mode)
{
/* If the keyserver helper gave us a short keyid, we have no
choice but to use it. Do check --keyid-format to add a 0x if
needed. */
case KEYDB_SEARCH_MODE_SHORT_KID:
es_printf ("key %s%08lX",
(opt.keyid_format==KF_0xSHORT
|| opt.keyid_format==KF_0xLONG)?"0x":"",
(ulong)keyrec->desc.u.kid[1]);
break;
/* However, if it gave us a long keyid, we can honor
--keyid-format */
case KEYDB_SEARCH_MODE_LONG_KID:
es_printf ("key %s",keystr(keyrec->desc.u.kid));
break;
case KEYDB_SEARCH_MODE_FPR16:
es_printf ("key ");
for(i=0;i<16;i++)
es_printf ("%02X",keyrec->desc.u.fpr[i]);
break;
case KEYDB_SEARCH_MODE_FPR20:
es_printf ("key ");
for(i=0;i<20;i++)
es_printf ("%02X", keyrec->desc.u.fpr[i]);
break;
default:
BUG();
break;
}
if(keyrec->createtime>0)
{
es_printf (", ");
es_printf (_("created: %s"), strtimestamp(keyrec->createtime));
}
if(keyrec->expiretime>0)
{
es_printf (", ");
es_printf (_("expires: %s"), strtimestamp(keyrec->expiretime));
}
if (keyrec->flags&1)
es_printf (" (%s)", _("revoked"));
if(keyrec->flags&2)
es_printf (" (%s)", _("disabled"));
if(keyrec->flags&4)
es_printf (" (%s)", _("expired"));
es_printf ("\n");
}
int
AsyncPutHandler::handle_exception(ACE_HANDLE fd)
{
S3Status st = status();
// Was this a successful completion?
if (st == S3StatusOK)
{
LOG(lgr, 6, (void *) this << ' '
<< keystr(m_keydata, m_keysize) << " SUCCESS");
// Update the blockstore accounting.
m_s3bs.update_put_stats(this);
// Call the completion handler.
m_cmpl.bp_complete(m_keydata, m_keysize, m_argp);
// IMPORTANT - We get destructed here; Don't touch *anything*
// after this!
//
m_s3bs.remove_handler(this);
}
// Retryable failure?
else if (S3_status_is_retryable(st))
{
if (--m_retries > 0)
{
LOG(lgr, 5, (void *) this << ' '
<< keystr(m_keydata, m_keysize) << ": " << st << ": RETRY");
rh_reset(); // Reset our state.
m_s3bs.initiate_put(this);
// This path doesn't destroy the handler ...
}
else
{
LOG(lgr, 2, (void *) this << ' '
<< keystr(m_keydata, m_keysize) << ": " << st
<< " TOO MANY RETRIES");
ostringstream errstrm;
errstrm << FILELINE << "too many S3 retries";
InternalError ex(errstrm.str().c_str());
m_cmpl.bp_error(m_keydata, m_keysize, m_argp, ex);
// IMPORTANT - We get destructed here; Don't touch *anything*
// after this!
//
m_s3bs.remove_handler(this);
}
}
// Permanent failure ... bitter ...
else
{
// Call the error handler.
ostringstream errstrm;
LOG(lgr, 2, (void *) this << ' '
<< keystr(m_keydata, m_keysize) << " UNEXPECTED: " << st);
errstrm << FILELINE << "unexpected S3 error: " << st;
InternalError ex(errstrm.str().c_str());
m_cmpl.bp_error(m_keydata, m_keysize, m_argp, ex);
// IMPORTANT - We get destructed here; Don't touch *anything*
// after this!
//
m_s3bs.remove_handler(this);
}
return 0;
}
AsyncPutHandler::~AsyncPutHandler()
{
LOG(lgr, 6, (void *) this << ' '
<< keystr(m_keydata, m_keysize) << " DTOR");
}
int Ardb::GeoSearchByOptions(Context& ctx, ValueObject& meta, GeoSearchOptions& options)
{
uint64 start_time = get_current_epoch_micros();
int ret = 0;
double x = options.x, y = options.y;
if (options.by_member)
{
Data element, score;
element.SetString(options.member, true);
ret = ZSetScore(ctx, meta, element, score);
if (0 != ret || score.IsNil())
{
return -1;
}
GeoHashHelper::GetMercatorXYByHash(score.value.iv, x, y);
}
else
{
if (options.coord_type != GEO_MERCATOR_TYPE)
{
x = GeoHashHelper::GetMercatorX(options.x);
y = GeoHashHelper::GetMercatorY(options.y);
}
}
DEBUG_LOG("####Step1: Cost %lluus", get_current_epoch_micros() - start_time);
GeoPointArray points;
ZSetRangeByScoreOptions fetch_options;
fetch_options.withscores = false;
fetch_options.op = OP_GET;
fetch_options.fill_reply = false;
fetch_options.fetch_geo_location = true;
if (options.in_members)
{
StringSet::iterator it = options.submembers.begin();
while (it != options.submembers.end())
{
//GeoPoint point;
Data element, score;
element.SetString(*it, true);
Location loc;
ret = ZSetScore(ctx, meta, element, score, &loc);
if (0 == ret)
{
fetch_options.results.push_back(element);
//fetch_options.results.push_back(score);
fetch_options.locs.push_back(loc);
}
it++;
}
}
else
{
GeoHashBitsSet ress;
GeoHashHelper::GetAreasByRadiusV2(GEO_MERCATOR_TYPE, y, x, options.radius, ress);
/*
* Merge areas if possible to avoid disk search
*/
std::vector<ZRangeSpec> range_array;
GeoHashBitsSet::iterator rit = ress.begin();
typedef TreeMap<uint64, uint64>::Type HashRangeMap;
HashRangeMap tmp;
while (rit != ress.end())
{
GeoHashBits& hash = *rit;
GeoHashBits next = hash;
next.bits++;
tmp[GeoHashHelper::Allign60Bits(hash)] = GeoHashHelper::Allign60Bits(next);
rit++;
}
HashRangeMap::iterator tit = tmp.begin();
HashRangeMap::iterator nit = tmp.begin();
nit++;
while (tit != tmp.end())
{
ZRangeSpec range;
range.contain_min = true;
range.contain_max = true;
range.min.SetInt64(tit->first);
range.max.SetInt64(tit->second);
while (nit != tmp.end() && nit->first == range.max.value.iv)
{
range.max.SetInt64(nit->second);
nit++;
tit++;
}
range_array.push_back(range);
nit++;
tit++;
}
DEBUG_LOG("After areas merging, reduce searching area size from %u to %u", ress.size(), range_array.size());
std::vector<ZRangeSpec>::iterator hit = range_array.begin();
ZSetIterator* iter = NULL;
while (hit != range_array.end())
{
ZRangeSpec& range = *hit;
uint64 t1 = get_current_epoch_millis();
ZSetRangeByScore(ctx, meta, range, fetch_options, iter);
uint64 t2 = get_current_epoch_millis();
DEBUG_LOG("####Cost %llums to range fetch", t2 - t1);
hit++;
}
DELETE(iter);
}
DEBUG_LOG("####Step2: Cost %lluus", get_current_epoch_micros() - start_time);
uint32 outrange = 0;
LocationDeque::iterator lit = fetch_options.locs.begin();
DataArray::iterator vit = fetch_options.results.begin();
while (vit != fetch_options.results.end())
{
Location& loc = *lit;
GeoPoint point;
point.x = loc.x;
point.y = loc.y;
/*
* distance accuracy is 0.2m
*/
if (GeoHashHelper::GetDistanceSquareIfInRadius(GEO_MERCATOR_TYPE, x, y, point.x, point.y, options.radius,
point.distance, 0.2))
{
vit->GetDecodeString(point.value);
/*
* filter by exclude/include
*/
if (!options.includes.empty() || !options.excludes.empty())
{
Data subst;
subst.SetString(point.value, false);
bool matched = options.includes.empty() ? true : false;
if (!options.includes.empty())
{
StringStringMap::const_iterator sit = options.includes.begin();
while (sit != options.includes.end())
{
Data mv;
if (0 != MatchValueByPattern(ctx, sit->first, sit->second, subst, mv))
{
matched = false;
break;
}
else
{
matched = true;
}
sit++;
}
}
if (matched && !options.excludes.empty())
{
StringStringMap::const_iterator sit = options.excludes.begin();
while (sit != options.excludes.end())
{
Data mv;
if (0 == MatchValueByPattern(ctx, sit->first, sit->second, subst, mv))
{
matched = false;
break;
}
else
{
matched = true;
}
sit++;
}
}
if (matched)
{
points.push_back(point);
}
}
else
{
points.push_back(point);
}
}
else
{
outrange++;
}
vit++;
lit++;
}
DEBUG_LOG("###Result size:%d, outrange:%d", points.size(), outrange);
DEBUG_LOG("####Step3: Cost %lluus", get_current_epoch_micros() - start_time);
if (!options.nosort)
{
std::sort(points.begin(), points.end(), options.asc ? less_by_distance : great_by_distance);
}
DEBUG_LOG("####Step3.5: Cost %lluus", get_current_epoch_micros() - start_time);
if (options.offset > 0)
{
if ((uint32) options.offset > points.size())
{
points.clear();
}
else
{
GeoPointArray::iterator start = points.begin() + options.offset;
points.erase(points.begin(), start);
}
}
if (options.limit > 0)
{
if ((uint32) options.limit < points.size())
{
GeoPointArray::iterator end = points.begin() + options.limit;
points.erase(end, points.end());
}
}
DEBUG_LOG("####Step4: Cost %lluus", get_current_epoch_micros() - start_time);
ValueObjectMap meta_cache;
GeoPointArray::iterator pit = points.begin();
while (pit != points.end())
{
RedisReply& r = ctx.reply.AddMember();
fill_str_reply(r, pit->value);
GeoGetOptionArray::const_iterator ait = options.get_patterns.begin();
while (ait != options.get_patterns.end())
{
if (ait->get_distances)
{
RedisReply& rr = ctx.reply.AddMember();
rr.type = REDIS_REPLY_STRING;
char dbuf[128];
int dlen = snprintf(dbuf, sizeof(dbuf), "%.2f", sqrt(pit->distance));
rr.str.assign(dbuf, dlen);
}
else if (ait->get_coodinates)
{
if (options.coord_type == GEO_WGS84_TYPE)
{
pit->x = GeoHashHelper::GetWGS84X(pit->x);
pit->y = GeoHashHelper::GetWGS84Y(pit->y);
}
RedisReply& rr1 = ctx.reply.AddMember();
RedisReply& rr2 = ctx.reply.AddMember();
if (options.coord_type == GEO_WGS84_TYPE)
{
fill_double_reply(rr1, pit->x);
fill_double_reply(rr2, pit->y);
}
else
{
char dbuf[128];
int dlen = snprintf(dbuf, sizeof(dbuf), "%.2f", pit->x);
rr1.type = REDIS_REPLY_STRING;
rr1.str.assign(dbuf, dlen);
dlen = snprintf(dbuf, sizeof(dbuf), "%.2f", pit->y);
rr2.type = REDIS_REPLY_STRING;
rr2.str.assign(dbuf, dlen);
}
}
else if (ait->hgetall)
{
std::string keystr(ait->get_pattern.data(), ait->get_pattern.size());
string_replace(keystr, "*", pit->value);
RedisReply& rr = ctx.reply.AddMember();
rr.type = REDIS_REPLY_ARRAY;
HashGetAll(ctx, keystr, rr);
}
else
{
Data v, attr;
v.SetString(pit->value, false);
GetValueByPattern(ctx, ait->get_pattern, v, attr, &meta_cache);
RedisReply& rr = ctx.reply.AddMember();
fill_value_reply(rr, attr);
}
ait++;
}
pit++;
}
DEBUG_LOG("####Step5: Cost %lluus", get_current_epoch_micros() - start_time);
uint64 end_time = get_current_epoch_micros();
DEBUG_LOG("Cost %llu microseconds to search.", end_time - start_time);
return points.size();
}
