void CmdLine :: parse (int argc, char** argv, bool strict) { // prepare the parameters and call cmdline_s get_opt Arglist arglist; Optdict optdict; // allocate storage for long options spec unsigned longopts_no = 0; KeysFormat::iterator kitr; char longopts_spec_buf [MAX_LONGOPTS][MAX_LONGOPT_LEN]; char* longopts_spec [MAX_LONGOPTS]; for (kitr = keys_format_.begin (); kitr != keys_format_.end (); kitr ++) longopts_no += (*kitr).longopts_.size (); if (longopts_no + 1 >= MAX_LONGOPTS) ers << "Too many command line options in command line format definition" << ThrowEx(InternalError); // fill in options specs and make option->spec map std::map <std::string, KeyFormat*> opt2spec; std::string shortopts_spec (EMPTY_STR); int lidx = 0; for (kitr = keys_format_.begin (); kitr != keys_format_.end (); kitr ++) { // short svec::iterator soi = (*kitr).shortopts_.begin (); for (; soi != (*kitr).shortopts_.end (); soi ++) { const char* optstr = (*soi).c_str (); shortopts_spec+= optstr; if ((*kitr).has_arg_) shortopts_spec += ":"; std::string keystr ("-"); keystr.append (optstr); opt2spec [keystr] = &(*kitr); } // long svec::iterator loi = (*kitr).longopts_.begin (); for (; loi != (*kitr).longopts_.end (); loi ++, lidx ++) { unsigned lolen = (*loi).length () + 1; if ((*kitr).has_arg_) lolen ++; if (lolen + 1 >= MAX_LONGOPT_LEN) ers << "Command line option " << *loi << " is too long" << Throw; // longopts_spec [lidx] = new char [lolen]; const char* optstr = (*loi).c_str (); longopts_spec [lidx] = longopts_spec_buf [lidx]; strcpy (longopts_spec [lidx], optstr); if ((*kitr).has_arg_) strcat (longopts_spec [lidx], "="); std::string keystr ("--"); keystr.append (optstr); opt2spec [keystr] = &(*kitr); } } longopts_spec [lidx] = NULL; // actually parse command line get_opt (argc, argv, shortopts_spec.c_str (), arglist, optdict, longopts_spec); // fill in the keys_ Optdict::iterator oi = optdict.begin (); for (;oi != optdict.end (); oi ++) { const char* key = (*oi).first.c_str (); // find what parameter this name corresponds to std::map <std::string, KeyFormat*>::iterator k = opt2spec.find (key); if (k != opt2spec.end ()) { svec values = (*oi).second; if ((*k).second->has_arg_) { if (values.size () == 0) { error_report_ += "\nParameter "; error_report_ += key; error_report_ += "("; error_report_ += (*k).second->name_; error_report_ += ") requires argument"; ok_ = false; break; } const char* val = values [values.size () - 1].c_str (); keys_ [(*k).second->name_] = val; } else { keys_ [(*k).second->name_] = EMPTY_STR; } // DEBUG // printf ("CMDL PAR '%s' : '%s'", (*k).second->name_.c_str (), val); } } // check weather all non-optional keys are found if (ok_ && strict) { for (kitr = keys_format_.begin (); kitr != keys_format_.end (); kitr ++) { if (!kitr->optional_) { std::map<std::string, std::string>::iterator valitr = keys_.find (kitr->name_); if (valitr == keys_.end ()) { // add notes about missing ones to the error_report error_report_ += "Required option missing from command line: "; error_report_ += kitr->name_; error_report_ += "\n"; ok_ = false; } } } } // fill in the arguments_ if (ok_) std::copy (arglist.begin (), arglist.end (), std::back_inserter (arguments_)); // check if the number of arguments is valid if (strict && ok_ && repeatable_pos_ == -1 && arglist.size () > args_format_.size ()) { // add the note to error_report error_report_ += "Too many command-line arguments\n"; ok_ = false; } unsigned min_arg_number = ((first_optional_pos_ != -1) ? first_optional_pos_ : 0) + args_format_.size () - ((last_optional_pos_ == -1) ? 0 : last_optional_pos_); if (strict && ok_ && arguments_.size () < min_arg_number) { // add the note to error_report error_report_ += "Too few command-line arguments. Please use -h for help.\n"; ok_ = false; } }
Config* PyGLPSParse2Config(PyObject *, PyObject *args, PyObject *kws) { PyObject *conf = NULL, *extra_defs = Py_None; const char *path = NULL; const char *pnames[] = {"config", "path", "extra", NULL}; if(!PyArg_ParseTupleAndKeywords(args, kws, "O|zO", (char**)pnames, &conf, &path, &extra_defs)) return NULL; GLPSParser parser; if(extra_defs==Py_None) { // no-op } else if(PyDict_Check(extra_defs)) { PyObject *key, *value; Py_ssize_t pos = 0; while(PyDict_Next(extra_defs, &pos, &key, &value)) { PyRef<> keyx(key, borrow()); PyCString keystr(keyx); Config::value_t curval; if(PyNumber_Check(value)) { PyRef<> pyf(PyNumber_Float(value)); curval = PyFloat_AsDouble(pyf.py()); } else if(PyString_Check(value)) { PyRef<> valuex(value, borrow()); PyCString valstr(valuex); curval = valstr.c_str(); } else { PyErr_SetString(PyExc_ValueError, "extra {} can contain only numbers or strings"); return NULL; } parser.setVar(keystr.c_str(), curval); } } else { PyErr_SetString(PyExc_ValueError, "'extra' must be a dict"); return NULL; } PyGetBuf buf; std::auto_ptr<Config> C; PyRef<> listref; if(PyObject_HasAttrString(conf, "read")) { // file-like PyCString pyname; if(!path && PyObject_HasAttrString(conf, "name")) { path = pyname.c_str(pydirname(PyObject_GetAttrString(conf, "name"))); } PyRef<> pybytes(PyObject_CallMethod(conf, "read", "")); if(!buf.get(pybytes.py())) { PyErr_SetString(PyExc_TypeError, "read() must return a buffer"); return NULL; } C.reset(parser.parse_byte((const char*)buf.data(), buf.size(), path)); } else if(buf.get(conf)) { C.reset(parser.parse_byte((const char*)buf.data(), buf.size(), path)); #if PY_MAJOR_VERSION >= 3 } else if(PyUnicode_Check(conf)) { // py3 str (aka unicode) doesn't implement buffer iface PyCString buf; const char *cbuf = buf.c_str(conf); C.reset(parser.parse_byte(cbuf, strlen(cbuf), path)); #endif } else { if(PyDict_Check(conf)) { listref.reset(PyMapping_Items(conf)); conf = listref.py(); } if(PyList_Check(conf)) { C.reset(list2conf(conf)); } else { throw std::invalid_argument("'config' must be dict, list of tuples, or byte buffer"); } } return C.release(); }
void hashTableGenerator(string ref_name) { //Open and read the reference file ifstream ref_file; deque<int>::iterator iter; int coor_counter = KEY_LENGTH*(-1); int flag_ignore = 0; int monitor_counter = 0; // For operation monitoring int monitor_counter2 = 0; // For operation monitoring ref_file.open(ref_name.c_str()); if (!ref_file.good()) { cerr << "reference file cannot open." << endl; exit(1); } //cout << "Hello0" << endl; //Initialize hash_table. // hash_table.resize(1 << (2 * KEY_LENGTH) ); // hash_table_counter = 0; //cout << "Hello1" << endl; //Read in the first key char temp_key[KEY_LENGTH + 1]; char temp_char; string keystr(KEY_LENGTH, 'A'); //cout << "keystr: " << keystr << endl; //throw out the first several Ns do { ref_file >> temp_char; coor_counter++; //cout << "coor_counter: " << coor_counter << endl; } while (temp_char == 'N'); //update the first non N character temp_key[0] = temp_char; //cout << "temp_char: " << temp_char << endl; //Read the first 12 characters for (int i = 1; i < KEY_LENGTH; i++) { //cout << "i: " << i << endl; ref_file >> temp_char; coor_counter++; //cout << "coor_counter: " << coor_counter << endl; temp_key[i] = temp_char; //Check if the file is too short if (ref_file.eof()) { cerr << "file too short"; exit(1); } //Check if next N come too close if (temp_char == 'N') { cerr << "Too short useful String A" << endl; exit(1); } } temp_key[KEY_LENGTH] = '\0'; keystr = temp_key; //cout << "temp_key: " << temp_key << endl; //cout << "keystr: " << keystr << endl; //Read the ref file until the end of file. do { //Compute idx and hash_value if (flag_ignore == 0) { int hash_value = hashVal(keystr); //cout << "hash_value: " << hash_value << endl; //Add to hash_table //cout << "coor_counter: " << coor_counter << endl; hash_table[hash_value].push_front(coor_counter); hash_table_counter++; } else { flag_ignore = 0; for (int i = 0; i < KEY_LENGTH; i++ ) { if (keystr[i] == 'N') { flag_ignore = 1; } } if (flag_ignore == 0) { int hash_value = hashVal(keystr); //hash_table[hash_value].push_front(coor_counter); //hash_table_counter++; } } /* //Add to hash_table list if (hash_table[idx][hash_value].size() == 0) { //cout << "first saw" << endl; hash_table[idx][hash_value].push_back(counter); } else { iter = hash_table[idx][hash_value].begin(); while (iter != hash_table[idx][hash_value].end() && counter > *iter) { //cout << "*iter: " << *iter << endl; iter++; } hash_table[idx][hash_value].insert(iter, counter); } */ //Read next character ref_file >> temp_char; //Increment counter coor_counter++; bool fresh_start = false; //indicater for a new start after large area of N //If enters N area, throw them away while (temp_char == 'N') { if (ref_file.eof() ) { ref_file.close(); return; } fresh_start = true; ref_file >> temp_char; coor_counter++; //cout << "coor_counter: " << coor_counter << endl; } if (fresh_start) { //cout << "In a fresh start!" << endl; keystr[0] = temp_char; for (int i = 1; i < KEY_LENGTH; i++) { //cout << 'i' << i << endl; ref_file >> temp_char; coor_counter++; //cout << "coor_counter: " << coor_counter << endl; //Check if next N come too close if (temp_char == 'N') { cerr << "Too short useful String B" << endl; flag_ignore = 1; //exit(1); } keystr[i] = temp_char; } } else keystr = keystr.substr(1, KEY_LENGTH - 1) + temp_char; //cout << "keystr: " << keystr << endl; // add process monitoring loop monitor_counter = monitor_counter +1; // For operation monitoring monitor_counter2 = monitor_counter2 +1; if (monitor_counter >= 1000000) { cout << "hash generator: " << monitor_counter2 << endl; monitor_counter = 0; } } while (ref_file.good());
static void print_keyrec(int number,struct keyrec *keyrec) { int i; iobuf_writebyte(keyrec->uidbuf,0); iobuf_flush_temp(keyrec->uidbuf); es_printf ("(%d)\t%s ", number, iobuf_get_temp_buffer (keyrec->uidbuf)); if (keyrec->size>0) es_printf ("%d bit ", keyrec->size); if(keyrec->type) { const char *str = gcry_pk_algo_name (keyrec->type); if(str) es_printf ("%s ",str); else es_printf ("unknown "); } switch(keyrec->desc.mode) { /* If the keyserver helper gave us a short keyid, we have no choice but to use it. Do check --keyid-format to add a 0x if needed. */ case KEYDB_SEARCH_MODE_SHORT_KID: es_printf ("key %s%08lX", (opt.keyid_format==KF_0xSHORT || opt.keyid_format==KF_0xLONG)?"0x":"", (ulong)keyrec->desc.u.kid[1]); break; /* However, if it gave us a long keyid, we can honor --keyid-format */ case KEYDB_SEARCH_MODE_LONG_KID: es_printf ("key %s",keystr(keyrec->desc.u.kid)); break; case KEYDB_SEARCH_MODE_FPR16: es_printf ("key "); for(i=0;i<16;i++) es_printf ("%02X",keyrec->desc.u.fpr[i]); break; case KEYDB_SEARCH_MODE_FPR20: es_printf ("key "); for(i=0;i<20;i++) es_printf ("%02X", keyrec->desc.u.fpr[i]); break; default: BUG(); break; } if(keyrec->createtime>0) { es_printf (", "); es_printf (_("created: %s"), strtimestamp(keyrec->createtime)); } if(keyrec->expiretime>0) { es_printf (", "); es_printf (_("expires: %s"), strtimestamp(keyrec->expiretime)); } if (keyrec->flags&1) es_printf (" (%s)", _("revoked")); if(keyrec->flags&2) es_printf (" (%s)", _("disabled")); if(keyrec->flags&4) es_printf (" (%s)", _("expired")); es_printf ("\n"); }
int AsyncPutHandler::handle_exception(ACE_HANDLE fd) { S3Status st = status(); // Was this a successful completion? if (st == S3StatusOK) { LOG(lgr, 6, (void *) this << ' ' << keystr(m_keydata, m_keysize) << " SUCCESS"); // Update the blockstore accounting. m_s3bs.update_put_stats(this); // Call the completion handler. m_cmpl.bp_complete(m_keydata, m_keysize, m_argp); // IMPORTANT - We get destructed here; Don't touch *anything* // after this! // m_s3bs.remove_handler(this); } // Retryable failure? else if (S3_status_is_retryable(st)) { if (--m_retries > 0) { LOG(lgr, 5, (void *) this << ' ' << keystr(m_keydata, m_keysize) << ": " << st << ": RETRY"); rh_reset(); // Reset our state. m_s3bs.initiate_put(this); // This path doesn't destroy the handler ... } else { LOG(lgr, 2, (void *) this << ' ' << keystr(m_keydata, m_keysize) << ": " << st << " TOO MANY RETRIES"); ostringstream errstrm; errstrm << FILELINE << "too many S3 retries"; InternalError ex(errstrm.str().c_str()); m_cmpl.bp_error(m_keydata, m_keysize, m_argp, ex); // IMPORTANT - We get destructed here; Don't touch *anything* // after this! // m_s3bs.remove_handler(this); } } // Permanent failure ... bitter ... else { // Call the error handler. ostringstream errstrm; LOG(lgr, 2, (void *) this << ' ' << keystr(m_keydata, m_keysize) << " UNEXPECTED: " << st); errstrm << FILELINE << "unexpected S3 error: " << st; InternalError ex(errstrm.str().c_str()); m_cmpl.bp_error(m_keydata, m_keysize, m_argp, ex); // IMPORTANT - We get destructed here; Don't touch *anything* // after this! // m_s3bs.remove_handler(this); } return 0; }
AsyncPutHandler::~AsyncPutHandler() { LOG(lgr, 6, (void *) this << ' ' << keystr(m_keydata, m_keysize) << " DTOR"); }
int Ardb::GeoSearchByOptions(Context& ctx, ValueObject& meta, GeoSearchOptions& options) { uint64 start_time = get_current_epoch_micros(); int ret = 0; double x = options.x, y = options.y; if (options.by_member) { Data element, score; element.SetString(options.member, true); ret = ZSetScore(ctx, meta, element, score); if (0 != ret || score.IsNil()) { return -1; } GeoHashHelper::GetMercatorXYByHash(score.value.iv, x, y); } else { if (options.coord_type != GEO_MERCATOR_TYPE) { x = GeoHashHelper::GetMercatorX(options.x); y = GeoHashHelper::GetMercatorY(options.y); } } DEBUG_LOG("####Step1: Cost %lluus", get_current_epoch_micros() - start_time); GeoPointArray points; ZSetRangeByScoreOptions fetch_options; fetch_options.withscores = false; fetch_options.op = OP_GET; fetch_options.fill_reply = false; fetch_options.fetch_geo_location = true; if (options.in_members) { StringSet::iterator it = options.submembers.begin(); while (it != options.submembers.end()) { //GeoPoint point; Data element, score; element.SetString(*it, true); Location loc; ret = ZSetScore(ctx, meta, element, score, &loc); if (0 == ret) { fetch_options.results.push_back(element); //fetch_options.results.push_back(score); fetch_options.locs.push_back(loc); } it++; } } else { GeoHashBitsSet ress; GeoHashHelper::GetAreasByRadiusV2(GEO_MERCATOR_TYPE, y, x, options.radius, ress); /* * Merge areas if possible to avoid disk search */ std::vector<ZRangeSpec> range_array; GeoHashBitsSet::iterator rit = ress.begin(); typedef TreeMap<uint64, uint64>::Type HashRangeMap; HashRangeMap tmp; while (rit != ress.end()) { GeoHashBits& hash = *rit; GeoHashBits next = hash; next.bits++; tmp[GeoHashHelper::Allign60Bits(hash)] = GeoHashHelper::Allign60Bits(next); rit++; } HashRangeMap::iterator tit = tmp.begin(); HashRangeMap::iterator nit = tmp.begin(); nit++; while (tit != tmp.end()) { ZRangeSpec range; range.contain_min = true; range.contain_max = true; range.min.SetInt64(tit->first); range.max.SetInt64(tit->second); while (nit != tmp.end() && nit->first == range.max.value.iv) { range.max.SetInt64(nit->second); nit++; tit++; } range_array.push_back(range); nit++; tit++; } DEBUG_LOG("After areas merging, reduce searching area size from %u to %u", ress.size(), range_array.size()); std::vector<ZRangeSpec>::iterator hit = range_array.begin(); ZSetIterator* iter = NULL; while (hit != range_array.end()) { ZRangeSpec& range = *hit; uint64 t1 = get_current_epoch_millis(); ZSetRangeByScore(ctx, meta, range, fetch_options, iter); uint64 t2 = get_current_epoch_millis(); DEBUG_LOG("####Cost %llums to range fetch", t2 - t1); hit++; } DELETE(iter); } DEBUG_LOG("####Step2: Cost %lluus", get_current_epoch_micros() - start_time); uint32 outrange = 0; LocationDeque::iterator lit = fetch_options.locs.begin(); DataArray::iterator vit = fetch_options.results.begin(); while (vit != fetch_options.results.end()) { Location& loc = *lit; GeoPoint point; point.x = loc.x; point.y = loc.y; /* * distance accuracy is 0.2m */ if (GeoHashHelper::GetDistanceSquareIfInRadius(GEO_MERCATOR_TYPE, x, y, point.x, point.y, options.radius, point.distance, 0.2)) { vit->GetDecodeString(point.value); /* * filter by exclude/include */ if (!options.includes.empty() || !options.excludes.empty()) { Data subst; subst.SetString(point.value, false); bool matched = options.includes.empty() ? true : false; if (!options.includes.empty()) { StringStringMap::const_iterator sit = options.includes.begin(); while (sit != options.includes.end()) { Data mv; if (0 != MatchValueByPattern(ctx, sit->first, sit->second, subst, mv)) { matched = false; break; } else { matched = true; } sit++; } } if (matched && !options.excludes.empty()) { StringStringMap::const_iterator sit = options.excludes.begin(); while (sit != options.excludes.end()) { Data mv; if (0 == MatchValueByPattern(ctx, sit->first, sit->second, subst, mv)) { matched = false; break; } else { matched = true; } sit++; } } if (matched) { points.push_back(point); } } else { points.push_back(point); } } else { outrange++; } vit++; lit++; } DEBUG_LOG("###Result size:%d, outrange:%d", points.size(), outrange); DEBUG_LOG("####Step3: Cost %lluus", get_current_epoch_micros() - start_time); if (!options.nosort) { std::sort(points.begin(), points.end(), options.asc ? less_by_distance : great_by_distance); } DEBUG_LOG("####Step3.5: Cost %lluus", get_current_epoch_micros() - start_time); if (options.offset > 0) { if ((uint32) options.offset > points.size()) { points.clear(); } else { GeoPointArray::iterator start = points.begin() + options.offset; points.erase(points.begin(), start); } } if (options.limit > 0) { if ((uint32) options.limit < points.size()) { GeoPointArray::iterator end = points.begin() + options.limit; points.erase(end, points.end()); } } DEBUG_LOG("####Step4: Cost %lluus", get_current_epoch_micros() - start_time); ValueObjectMap meta_cache; GeoPointArray::iterator pit = points.begin(); while (pit != points.end()) { RedisReply& r = ctx.reply.AddMember(); fill_str_reply(r, pit->value); GeoGetOptionArray::const_iterator ait = options.get_patterns.begin(); while (ait != options.get_patterns.end()) { if (ait->get_distances) { RedisReply& rr = ctx.reply.AddMember(); rr.type = REDIS_REPLY_STRING; char dbuf[128]; int dlen = snprintf(dbuf, sizeof(dbuf), "%.2f", sqrt(pit->distance)); rr.str.assign(dbuf, dlen); } else if (ait->get_coodinates) { if (options.coord_type == GEO_WGS84_TYPE) { pit->x = GeoHashHelper::GetWGS84X(pit->x); pit->y = GeoHashHelper::GetWGS84Y(pit->y); } RedisReply& rr1 = ctx.reply.AddMember(); RedisReply& rr2 = ctx.reply.AddMember(); if (options.coord_type == GEO_WGS84_TYPE) { fill_double_reply(rr1, pit->x); fill_double_reply(rr2, pit->y); } else { char dbuf[128]; int dlen = snprintf(dbuf, sizeof(dbuf), "%.2f", pit->x); rr1.type = REDIS_REPLY_STRING; rr1.str.assign(dbuf, dlen); dlen = snprintf(dbuf, sizeof(dbuf), "%.2f", pit->y); rr2.type = REDIS_REPLY_STRING; rr2.str.assign(dbuf, dlen); } } else if (ait->hgetall) { std::string keystr(ait->get_pattern.data(), ait->get_pattern.size()); string_replace(keystr, "*", pit->value); RedisReply& rr = ctx.reply.AddMember(); rr.type = REDIS_REPLY_ARRAY; HashGetAll(ctx, keystr, rr); } else { Data v, attr; v.SetString(pit->value, false); GetValueByPattern(ctx, ait->get_pattern, v, attr, &meta_cache); RedisReply& rr = ctx.reply.AddMember(); fill_value_reply(rr, attr); } ait++; } pit++; } DEBUG_LOG("####Step5: Cost %lluus", get_current_epoch_micros() - start_time); uint64 end_time = get_current_epoch_micros(); DEBUG_LOG("Cost %llu microseconds to search.", end_time - start_time); return points.size(); }