//
// Main Routine
//
int main(void) {
#ifdef DEBUG
CSerial ser; // declare a Serial object
ser.enable();
CDebug dbg(ser); // Debug stream use the UART object
dbg.start();
#endif
//
// SoftDevice
//
bleDevice ble;
ble.enable(); // enable BLE SoftDevice stack
// GAP
ble.m_gap.settings(DEVICE_NAME, 10, 50); // set Device Name on GAP, conn. interval min=10ms, max=50ms
ble.m_gap.tx_power(BLE_TX_0dBm); // set Output power
//
// Add BLE UART Service
//
bleServiceUART nus(ble); // declare a BLE "Nordic UART Service" (NUS) object
//
// Add "connection parameters update" negotiation. (optional)
//
bleConnParams conn(ble);
//
// BLE Advertising
//
ble.m_advertising.commpany_identifier(APP_COMPANY_IDENTIFIER);// add company identifier
ble.m_advertising.update(); // update advertising data
// Start advertising
ble.m_advertising.interval(APP_ADV_INTERVAL); // set advertising interval
ble.m_advertising.start();
//
// my command parse class
//
cmdParse cmd;
//
// LED output enable
//
ledLeft.output(); // set ledLeft as an output pin
ledRight.output(); // set ledRight as an output pin
uint8_t ch= 0;
//
// Enable Tickless Technology
//
#ifndef DEBUG
CPowerSave::tickless(true);
#endif
//
// Enter main loop.
//
while (1) {
//
// Uart Service
//
if ( ble.isConnected() ) {
// block in the read() to wait a char.
// Also, block task will save the power when tickless enabled.
while ( nus.readable() ) {
ch = nus.read();
if ( ch ) {
cmd.input(ch);
}
}
} else {
//
// alternate led when disconnected (idle)
//
ch = (ch ? 0 : 1);
ledRight = (ch ? LED_ON : LED_OFF);
ledLeft = (ch ? LED_OFF : LED_ON);
sleep(10); // blink a short time (10ms)
ledRight = LED_OFF;
ledLeft = LED_OFF;
sleep(990); // save power with a long time sleep (990ms)
}
// Negotiate the "Connect Parameters Update"
conn.negotiate();
}
}
int conn(void)
{
int syserr = -1;
int sock = -1;
sock = socket(AF_INET, SOCK_STREAM, 0);
if (sock == -1)
{
perror("socket");
goto EXIT_LABEL;
}
//u_short port = (time(NULL) % 2) + PORTNO;
u_short port = PORTNO;
struct sockaddr_in addr =
{
.sin_family=AF_INET,
.sin_port=htons(port),
.sin_addr.s_addr=inet_addr("10.96.157.84"),
};
//fprintf(stderr, "connect to port=%u\n", port);
syserr = connect(sock, &addr, sizeof addr);
if (syserr == -1)
{
perror("connect");
goto EXIT_LABEL;
}
syserr = 0;
EXIT_LABEL:
if (syserr)
{
close(sock);
sock = -1;
}
return sock;
}
int send_data(int loop, int start, const char* wstr)
{
int syserr = -1;
int sock = -1;
char* rstr = NULL;
const size_t wstr_len = strlen(wstr);
//fprintf(stderr, "wstr=[%s]\n", wstr);
sock = conn();
if (sock == -1)
{
goto EXIT_LABEL;
}
struct AppHeader_st header = { .type=1, .version=95, .payload_len=wstr_len, };
//fprintf(stderr, "sizeof(header) = %zu\n", sizeof(header));
const size_t hdrlen = sizeof(header);
ssize_t nw = write(sock, &header, hdrlen);
if (nw != (ssize_t)hdrlen)
{
perror("write header");
goto EXIT_LABEL;
}
//fprintf(stderr, "write header success\n");
nw = write(sock, wstr, wstr_len);
if (nw != (ssize_t)wstr_len)
{
perror("write payload");
goto EXIT_LABEL;
}
//fprintf(stderr, "write payload success\n");
ssize_t nr = read(sock, &header, sizeof(header));
//fprintf(stderr, "nr=%zu\n", nr);
if (nr != (ssize_t)hdrlen)
{
perror("read header");
goto EXIT_LABEL;
}
//fprintf(stderr, "read header success\n");
//fprintf(stderr, "payload len=%zu\n", header.payload_len);
if (header.payload_len != wstr_len)
{
perror("read payload-len");
goto EXIT_LABEL;
}
rstr = malloc(wstr_len + 1);
assert(rstr);
char* pos = rstr;
size_t rest = wstr_len;
do
{
nr = read(sock, pos, rest);
if (nr == -1)
{
perror("read payload");
goto EXIT_LABEL;
}
//fprintf(stderr, "read payload %zd:%zu\n", nr, wstr_len);
rest -= nr;
pos += nr;
if (! rest)
{
break;
}
}
while (nr);
rstr[wstr_len] = '\0';
//fprintf(stderr, "payload(%zu) = [%s]\n", strlen(rstr), rstr);
ljbx_strrev(rstr);
//fprintf(stderr, "rev payload=[%s]\n", rstr);
//fprintf(stderr, "loop=%d start=%d) ", loop, start);
if (strcmp(rstr, wstr) == 0)
{
if (CHILDS_NUM >= 10 && SENDLOOP_NUM >= 100)
;
else
fprintf(stderr, "* match\n");
}
else
{
fprintf(stderr, "!! unmatch\n");
}
syserr = 0;
EXIT_LABEL:
free(rstr);
rstr = NULL;
if (sock != -1)
{
close(sock);
sock = -1;
}
return syserr;
}
void child_proc(void* data, int start)
{
int suc = 0;
int err = 0;
char* buf = malloc(BUFSTRLEN + 1);
assert(buf);
for (int i=0; i<SENDLOOP_NUM; i++)
{
memset(buf, 0, BUFSTRLEN + 1);
//buf[BUFSTRLEN - 1] = '\n';
char* wpos = buf;
char* wend = &buf[BUFSTRLEN - 1];
for (char* rpos=&((char*)data)[start]; wpos != wend; rpos++)
{
*wpos++ = ((*rpos > 0) ? 'A' : 'a') + (abs(*rpos) % 26);
}
//fprintf(stderr, "buf=[%s]\n", buf);
int syserr = send_data(i, start, buf);
if (syserr == 0)
{
suc++;
}
else
{
err++;
}
}
free(buf);
_exit(suc);
}
void radiosity_lightmap::generate(world_lightmap_access& data,
const chunk_coordinates& pos,
const surface& s, lightmap_hr &lightchunk,
unsigned int phase) const
{
std::array<float, 6> irr_sun, irr_ambient, irr_artificial;
vector half {0.5f, 0.5f, 0.5f};
auto lmi = std::begin(lightchunk);
for (const faces& f : s)
{
fill(irr_sun, 0.f);
fill(irr_ambient, 0.f);
fill(irr_artificial, 0.f);
auto surr (surroundings(f.pos, phase + 1));
for (auto sp : surr)
{
if (sp == f.pos)
continue;
auto found (find(s, sp));
if (found == s.end())
continue;
auto& other (*found);
for (int i = 0; i < 6; ++i)
{
if (!f[i])
continue;
vector this_face (half + f.pos + vector(dir_vector[i]) * 0.52f);
for (int j = 0; j < 6; ++j)
{
if (i == j || !other[j])
continue;
vector that_face (half + other.pos + vector(dir_vector[j])* 0.52f);
vector conn (that_face - this_face);
vector norm_conn (normalize(conn));
float dp1 (dot_prod<vector>(dir_vector[j], -norm_conn));
if (dp1 <= 0)
continue;
float dp2 = dot_prod<vector>(dir_vector[i], norm_conn);
if (dp2 <= 0)
continue;
float intensity = dp1 * dp2 / squared_length(conn);
auto dist = std::distance(std::begin(s), found);
auto& olm = lightchunk.data[dist];
irr_sun[i] += olm.sunlight * intensity;
irr_ambient[i] += olm.ambient * intensity;
irr_artificial[i] += olm.artificial * intensity;
}
}
}
for (int i (0); i < 6; ++i)
{
if (!f[i])
continue;
lmi->r_sunlight = irr_sun[i] * 255.f + 0.49f;
lmi->r_ambient = irr_ambient[i] * 255.f + 0.49f;
lmi->r_artificial = irr_artificial[i] * 255.f + 0.49f;
++lmi;
}
}
}
Status ClusterAggregate::runAggregate(OperationContext* txn,
const Namespaces& namespaces,
BSONObj cmdObj,
int options,
BSONObjBuilder* result) {
auto scopedShardDbStatus = ScopedShardDatabase::getExisting(txn, namespaces.executionNss.db());
if (!scopedShardDbStatus.isOK()) {
appendEmptyResultSet(
*result, scopedShardDbStatus.getStatus(), namespaces.requestedNss.ns());
return Status::OK();
}
auto request = AggregationRequest::parseFromBSON(namespaces.executionNss, cmdObj);
if (!request.isOK()) {
return request.getStatus();
}
const auto conf = scopedShardDbStatus.getValue().db();
// Determine the appropriate collation and 'resolve' involved namespaces to make the
// ExpressionContext.
// We won't try to execute anything on a mongos, but we still have to populate this map so that
// any $lookups, etc. will be able to have a resolved view definition. It's okay that this is
// incorrect, we will repopulate the real resolved namespace map on the mongod. Note that we
// need to check if any involved collections are sharded before forwarding an aggregation
// command on an unsharded collection.
StringMap<ExpressionContext::ResolvedNamespace> resolvedNamespaces;
LiteParsedPipeline liteParsedPipeline(request.getValue());
for (auto&& ns : liteParsedPipeline.getInvolvedNamespaces()) {
uassert(28769, str::stream() << ns.ns() << " cannot be sharded", !conf->isSharded(ns.ns()));
resolvedNamespaces[ns.coll()] = {ns, std::vector<BSONObj>{}};
}
if (!conf->isSharded(namespaces.executionNss.ns())) {
return aggPassthrough(txn, namespaces, conf, cmdObj, result, options);
}
auto chunkMgr = conf->getChunkManager(txn, namespaces.executionNss.ns());
std::unique_ptr<CollatorInterface> collation;
if (!request.getValue().getCollation().isEmpty()) {
collation = uassertStatusOK(CollatorFactoryInterface::get(txn->getServiceContext())
->makeFromBSON(request.getValue().getCollation()));
} else if (chunkMgr->getDefaultCollator()) {
collation = chunkMgr->getDefaultCollator()->clone();
}
boost::intrusive_ptr<ExpressionContext> mergeCtx = new ExpressionContext(
txn, request.getValue(), std::move(collation), std::move(resolvedNamespaces));
mergeCtx->inRouter = true;
// explicitly *not* setting mergeCtx->tempDir
// Parse and optimize the pipeline specification.
auto pipeline = Pipeline::parse(request.getValue().getPipeline(), mergeCtx);
if (!pipeline.isOK()) {
return pipeline.getStatus();
}
pipeline.getValue()->optimizePipeline();
// If the first $match stage is an exact match on the shard key (with a simple collation or
// no string matching), we only have to send it to one shard, so send the command to that
// shard.
BSONObj firstMatchQuery = pipeline.getValue()->getInitialQuery();
BSONObj shardKeyMatches;
shardKeyMatches = uassertStatusOK(
chunkMgr->getShardKeyPattern().extractShardKeyFromQuery(txn, firstMatchQuery));
bool singleShard = false;
if (!shardKeyMatches.isEmpty()) {
auto chunk = chunkMgr->findIntersectingChunk(
txn, shardKeyMatches, request.getValue().getCollation());
if (chunk.isOK()) {
singleShard = true;
}
}
// Don't need to split pipeline if the first $match is an exact match on shard key, unless
// there is a stage that needs to be run on the primary shard.
const bool needPrimaryShardMerger = pipeline.getValue()->needsPrimaryShardMerger();
const bool needSplit = !singleShard || needPrimaryShardMerger;
// Split the pipeline into pieces for mongod(s) and this mongos. If needSplit is true,
// 'pipeline' will become the merger side.
boost::intrusive_ptr<Pipeline> shardPipeline(needSplit ? pipeline.getValue()->splitForSharded()
: pipeline.getValue());
// Create the command for the shards. The 'fromRouter' field means produce output to be
// merged.
MutableDocument commandBuilder(request.getValue().serializeToCommandObj());
commandBuilder[AggregationRequest::kPipelineName] = Value(shardPipeline->serialize());
if (needSplit) {
commandBuilder[AggregationRequest::kFromRouterName] = Value(true);
commandBuilder[AggregationRequest::kCursorName] =
Value(DOC(AggregationRequest::kBatchSizeName << 0));
}
// These fields are not part of the AggregationRequest since they are not handled by the
// aggregation subsystem, so we serialize them separately.
const std::initializer_list<StringData> fieldsToPropagateToShards = {
"$queryOptions", "readConcern", QueryRequest::cmdOptionMaxTimeMS,
};
for (auto&& field : fieldsToPropagateToShards) {
commandBuilder[field] = Value(cmdObj[field]);
}
BSONObj shardedCommand = commandBuilder.freeze().toBson();
BSONObj shardQuery = shardPipeline->getInitialQuery();
// Run the command on the shards
// TODO need to make sure cursors are killed if a retry is needed
std::vector<Strategy::CommandResult> shardResults;
Strategy::commandOp(txn,
namespaces.executionNss.db().toString(),
shardedCommand,
options,
namespaces.executionNss.ns(),
shardQuery,
request.getValue().getCollation(),
&shardResults);
if (mergeCtx->isExplain) {
// This must be checked before we start modifying result.
uassertAllShardsSupportExplain(shardResults);
if (needSplit) {
*result << "needsPrimaryShardMerger" << needPrimaryShardMerger << "splitPipeline"
<< DOC("shardsPart" << shardPipeline->writeExplainOps() << "mergerPart"
<< pipeline.getValue()->writeExplainOps());
} else {
*result << "splitPipeline" << BSONNULL;
}
BSONObjBuilder shardExplains(result->subobjStart("shards"));
for (size_t i = 0; i < shardResults.size(); i++) {
shardExplains.append(shardResults[i].shardTargetId,
BSON("host" << shardResults[i].target.toString() << "stages"
<< shardResults[i].result["stages"]));
}
return Status::OK();
}
if (!needSplit) {
invariant(shardResults.size() == 1);
invariant(shardResults[0].target.getServers().size() == 1);
auto executorPool = Grid::get(txn)->getExecutorPool();
const BSONObj reply =
uassertStatusOK(storePossibleCursor(shardResults[0].target.getServers()[0],
shardResults[0].result,
namespaces.requestedNss,
executorPool->getArbitraryExecutor(),
Grid::get(txn)->getCursorManager()));
result->appendElements(reply);
return getStatusFromCommandResult(reply);
}
pipeline.getValue()->addInitialSource(
DocumentSourceMergeCursors::create(parseCursors(shardResults), mergeCtx));
MutableDocument mergeCmd(request.getValue().serializeToCommandObj());
mergeCmd["pipeline"] = Value(pipeline.getValue()->serialize());
mergeCmd["cursor"] = Value(cmdObj["cursor"]);
if (cmdObj.hasField("$queryOptions")) {
mergeCmd["$queryOptions"] = Value(cmdObj["$queryOptions"]);
}
if (cmdObj.hasField(QueryRequest::cmdOptionMaxTimeMS)) {
mergeCmd[QueryRequest::cmdOptionMaxTimeMS] =
Value(cmdObj[QueryRequest::cmdOptionMaxTimeMS]);
}
mergeCmd.setField("writeConcern", Value(cmdObj["writeConcern"]));
mergeCmd.setField("readConcern", Value(cmdObj["readConcern"]));
// If the user didn't specify a collation already, make sure there's a collation attached to
// the merge command, since the merging shard may not have the collection metadata.
if (mergeCmd.peek()["collation"].missing()) {
mergeCmd.setField("collation",
mergeCtx->getCollator()
? Value(mergeCtx->getCollator()->getSpec().toBSON())
: Value(Document{CollationSpec::kSimpleSpec}));
}
std::string outputNsOrEmpty;
if (DocumentSourceOut* out =
dynamic_cast<DocumentSourceOut*>(pipeline.getValue()->getSources().back().get())) {
outputNsOrEmpty = out->getOutputNs().ns();
}
// Run merging command on random shard, unless a stage needs the primary shard. Need to use
// ShardConnection so that the merging mongod is sent the config servers on connection init.
auto& prng = txn->getClient()->getPrng();
const auto& mergingShardId =
(needPrimaryShardMerger || internalQueryAlwaysMergeOnPrimaryShard.load())
? conf->getPrimaryId()
: shardResults[prng.nextInt32(shardResults.size())].shardTargetId;
const auto mergingShard =
uassertStatusOK(Grid::get(txn)->shardRegistry()->getShard(txn, mergingShardId));
ShardConnection conn(mergingShard->getConnString(), outputNsOrEmpty);
BSONObj mergedResults =
aggRunCommand(txn, conn.get(), namespaces, mergeCmd.freeze().toBson(), options);
conn.done();
if (auto wcErrorElem = mergedResults["writeConcernError"]) {
appendWriteConcernErrorToCmdResponse(mergingShardId, wcErrorElem, *result);
}
// Copy output from merging (primary) shard to the output object from our command.
// Also, propagates errmsg and code if ok == false.
result->appendElementsUnique(mergedResults);
return getStatusFromCommandResult(result->asTempObj());
}
void appendReplicationInfo(BSONObjBuilder& result, int level) {
if ( replSet ) {
if( theReplSet == 0 || theReplSet->state().shunned() ) {
result.append("ismaster", false);
result.append("secondary", false);
result.append("info", ReplSet::startupStatusMsg.get());
result.append( "isreplicaset" , true );
}
else {
theReplSet->fillIsMaster(result);
}
return;
}
if ( replAllDead ) {
result.append("ismaster", 0);
string s = string("dead: ") + replAllDead;
result.append("info", s);
}
else {
result.appendBool("ismaster", _isMaster() );
}
if ( level && replSet ) {
result.append( "info" , "is replica set" );
}
else if ( level ) {
BSONObjBuilder sources( result.subarrayStart( "sources" ) );
int n = 0;
list<BSONObj> src;
{
Client::ReadContext ctx("local.sources", storageGlobalParams.dbpath);
auto_ptr<Runner> runner(InternalPlanner::collectionScan("local.sources"));
BSONObj obj;
Runner::RunnerState state;
while (Runner::RUNNER_ADVANCED == (state = runner->getNext(&obj, NULL))) {
src.push_back(obj);
}
}
for( list<BSONObj>::const_iterator i = src.begin(); i != src.end(); i++ ) {
BSONObj s = *i;
BSONObjBuilder bb;
bb.append( s["host"] );
string sourcename = s["source"].valuestr();
if ( sourcename != "main" )
bb.append( s["source"] );
{
BSONElement e = s["syncedTo"];
BSONObjBuilder t( bb.subobjStart( "syncedTo" ) );
t.appendDate( "time" , e.timestampTime() );
t.append( "inc" , e.timestampInc() );
t.done();
}
if ( level > 1 ) {
wassert( !Lock::isLocked() );
// note: there is no so-style timeout on this connection; perhaps we should have one.
ScopedDbConnection conn(s["host"].valuestr());
DBClientConnection *cliConn = dynamic_cast< DBClientConnection* >( &conn.conn() );
if ( cliConn && replAuthenticate(cliConn) ) {
BSONObj first = conn->findOne( (string)"local.oplog.$" + sourcename,
Query().sort( BSON( "$natural" << 1 ) ) );
BSONObj last = conn->findOne( (string)"local.oplog.$" + sourcename,
Query().sort( BSON( "$natural" << -1 ) ) );
bb.appendDate( "masterFirst" , first["ts"].timestampTime() );
bb.appendDate( "masterLast" , last["ts"].timestampTime() );
double lag = (double) (last["ts"].timestampTime() - s["syncedTo"].timestampTime());
bb.append( "lagSeconds" , lag / 1000 );
}
conn.done();
}
sources.append( BSONObjBuilder::numStr( n++ ) , bb.obj() );
}
sources.done();
}
}
int ConfigServer::dbConfigVersion() {
ScopedDbConnection conn( _primary );
int version = dbConfigVersion( conn.conn() );
conn.done();
return version;
}
bool run(const string& , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool) {
string target = cmdObj.firstElement().valuestrsafe();
Shard s = Shard::make( target );
if ( ! grid.knowAboutShard( s.getConnString() ) ) {
errmsg = "unknown shard";
return false;
}
ScopedDbConnection conn( configServer.getPrimary() );
if (conn->count("config.shards", BSON("_id" << NE << s.getName() << ShardFields::draining(true)))){
conn.done();
errmsg = "Can't have more than one draining shard at a time";
return false;
}
if (conn->count("config.shards", BSON("_id" << NE << s.getName())) == 0){
conn.done();
errmsg = "Can't remove last shard";
return false;
}
// If the server is not yet draining chunks, put it in draining mode.
BSONObj searchDoc = BSON( "_id" << s.getName() );
BSONObj drainingDoc = BSON( "_id" << s.getName() << ShardFields::draining(true) );
BSONObj shardDoc = conn->findOne( "config.shards", drainingDoc );
if ( shardDoc.isEmpty() ) {
// TODO prevent move chunks to this shard.
log() << "going to start draining shard: " << s.getName() << endl;
BSONObj newStatus = BSON( "$set" << BSON( ShardFields::draining(true) ) );
conn->update( "config.shards" , searchDoc , newStatus, false /* do no upsert */);
errmsg = conn->getLastError();
if ( errmsg.size() ) {
log() << "error starting remove shard: " << s.getName() << " err: " << errmsg << endl;
return false;
}
Shard::reloadShardInfo();
result.append( "msg" , "draining started successfully" );
result.append( "state" , "started" );
result.append( "shard" , s.getName() );
conn.done();
return true;
}
// If the server has been completely drained, remove it from the ConfigDB.
// Check not only for chunks but also databases.
BSONObj shardIDDoc = BSON( "shard" << shardDoc[ "_id" ].str() );
long long chunkCount = conn->count( "config.chunks" , shardIDDoc );
BSONObj primaryDoc = BSON( "primary" << shardDoc[ "_id" ].str() );
long long dbCount = conn->count( "config.databases" , primaryDoc );
if ( ( chunkCount == 0 ) && ( dbCount == 0 ) ) {
log() << "going to remove shard: " << s.getName() << endl;
conn->remove( "config.shards" , searchDoc );
errmsg = conn->getLastError();
if ( errmsg.size() ) {
log() << "error concluding remove shard: " << s.getName() << " err: " << errmsg << endl;
return false;
}
Shard::removeShard( shardDoc[ "_id" ].str() );
Shard::reloadShardInfo();
result.append( "msg" , "removeshard completed successfully" );
result.append( "state" , "completed" );
result.append( "shard" , s.getName() );
conn.done();
return true;
}
// If the server is already in draining mode, just report on its progress.
// Report on databases (not just chunks) that are left too.
result.append( "msg" , "draining ongoing" );
result.append( "state" , "ongoing" );
BSONObjBuilder inner;
inner.append( "chunks" , chunkCount );
inner.append( "dbs" , dbCount );
result.append( "remaining" , inner.obj() );
conn.done();
return true;
}
void PDOResource::persistentSave() {
String serialized = HHVM_FN(serialize)(def_stmt_ctor_args);
conn()->serialized_def_stmt_ctor_args = serialized.toCppString();
def_stmt_ctor_args.releaseForSweep(); // we're called from requestShutdown
}
void PDOResource::persistentRestore() {
auto const serialized = conn()->serialized_def_stmt_ctor_args;
if (!serialized.empty()) {
def_stmt_ctor_args = unserialize_from_string(serialized);
}
}
int main()
{
int content_length = -1;
char method[1024];
char query_string[1024];
char post_data[4096];
memset(method, '\0', sizeof(method));
memset(query_string, '\0', sizeof(query_string));
memset(post_data, '\0', sizeof(post_data));
std::cout<<"<html>"<<std::endl;
std::cout<<"<head>Register The Car's Information Result</head><br/>"<<std::endl;
std::cout<<"<body>"<<std::endl;
strcpy(method, getenv("REQUEST_METHOD"));
if( strcasecmp("GET", method) == 0 )
{
strcpy(query_string, getenv("QUERY_STRING"));
regis_ter(query_string);
}
else if( strcasecmp("POST", method) == 0 )
{
content_length = atoi(getenv("CONTENT_LENGTH"));
int i = 0;
for(; i < content_length; i++ )
{
read(0, &post_data[i], 1);
}
post_data[i] = '\0';
regis_ter(post_data);
}
std::string _sql_data[1024][5];
std::string header[5];
int curr_row = -1;
sql_connecter conn(_remote_ip,_remote_user,_remote_passwd,_remote_db);
conn.begin_connect();
conn.select_sql(header,_sql_data,curr_row);
std::cout<<"<table border=\"1\">"<<std::endl;
std::cout<<"<tr>"<<std::endl;
for(int i = 0;i < 5;i++)
{
std::cout<<"<th>"<<header[i]<<"</th>"<<std::endl;
}
std::cout<<"</tr>"<<std::endl;
for(int i = 0;i < curr_row; i++)
{
std::cout<<"<tr>"<<std::endl;
for(int j =0;j < 5;j++)
{
std::cout<<"<td>"<<_sql_data[i][j]<<"</td>"<<std::endl;
}
std::cout<<"</tr>"<<std::endl;
}
std::cout<<"</table>"<<std::endl;
std::cout<<"</body>"<<std::endl;
std::cout<<"</html>"<<std::endl;
return 0;
}
void PDOResource::sweep() {
assert(!conn()->is_persistent);
def_stmt_ctor_args.releaseForSweep();
this->~PDOResource();
}
bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool) {
string dbname = cc().database()->name; // this has to come before dbtemprelease
dbtemprelease temprelease; // we don't touch the db directly
string shardedOutputCollection = cmdObj["shardedOutputCollection"].valuestrsafe();
MRSetup mr( dbname , cmdObj.firstElement().embeddedObjectUserCheck() , false );
set<ServerAndQuery> servers;
BSONObjBuilder shardCounts;
map<string,long long> counts;
BSONObj shards = cmdObj["shards"].embeddedObjectUserCheck();
vector< auto_ptr<DBClientCursor> > shardCursors;
BSONObjIterator i( shards );
while ( i.more() ) {
BSONElement e = i.next();
string shard = e.fieldName();
BSONObj res = e.embeddedObjectUserCheck();
uassert( 10078 , "something bad happened" , shardedOutputCollection == res["result"].valuestrsafe() );
servers.insert( shard );
shardCounts.appendAs( res["counts"] , shard.c_str() );
BSONObjIterator j( res["counts"].embeddedObjectUserCheck() );
while ( j.more() ) {
BSONElement temp = j.next();
counts[temp.fieldName()] += temp.numberLong();
}
}
BSONObj sortKey = BSON( "_id" << 1 );
ParallelSortClusteredCursor cursor( servers , dbname + "." + shardedOutputCollection ,
Query().sort( sortKey ) );
auto_ptr<Scope> s = globalScriptEngine->getPooledScope( ns );
ScriptingFunction reduceFunction = s->createFunction( mr.reduceCode.c_str() );
ScriptingFunction finalizeFunction = 0;
if ( mr.finalizeCode.size() )
finalizeFunction = s->createFunction( mr.finalizeCode.c_str() );
BSONList values;
result.append( "result" , mr.finalShort );
DBDirectClient db;
while ( cursor.more() ) {
BSONObj t = cursor.next().getOwned();
if ( values.size() == 0 ) {
values.push_back( t );
continue;
}
if ( t.woSortOrder( *(values.begin()) , sortKey ) == 0 ) {
values.push_back( t );
continue;
}
db.insert( mr.tempLong , reduceValues( values , s.get() , reduceFunction , 1 , finalizeFunction ) );
values.clear();
values.push_back( t );
}
if ( values.size() )
db.insert( mr.tempLong , reduceValues( values , s.get() , reduceFunction , 1 , finalizeFunction ) );
long long finalCount = mr.renameIfNeeded( db );
log(0) << " mapreducefinishcommand " << mr.finalLong << " " << finalCount << endl;
for ( set<ServerAndQuery>::iterator i=servers.begin(); i!=servers.end(); i++ ) {
ScopedDbConnection conn( i->_server );
conn->dropCollection( dbname + "." + shardedOutputCollection );
}
result.append( "shardCounts" , shardCounts.obj() );
{
BSONObjBuilder c;
for ( map<string,long long>::iterator i=counts.begin(); i!=counts.end(); i++ ) {
c.append( i->first , i->second );
}
result.append( "counts" , c.obj() );
}
return 1;
}
bool ClientInfo::getLastError( const string& dbName,
const BSONObj& options,
BSONObjBuilder& result,
string& errmsg,
bool fromWriteBackListener)
{
scoped_ptr<TimerHolder> gleTimerHolder;
if ( ! fromWriteBackListener ) {
bool doTiming = false;
const BSONElement& e = options["w"];
if ( e.isNumber() ) {
doTiming = e.numberInt() > 1;
}
else if ( e.type() == String ) {
doTiming = true;
}
if ( doTiming ) {
gleTimerHolder.reset( new TimerHolder( &gleWtimeStats ) );
}
}
set<string> * shards = getPrev();
if ( shards->size() == 0 ) {
result.appendNull( "err" );
return true;
}
vector<WBInfo> writebacks;
//
// TODO: These branches should be collapsed into a single codepath
//
// handle single server
if ( shards->size() == 1 ) {
string theShard = *(shards->begin() );
BSONObj res;
bool ok = false;
{
LOG(5) << "gathering response for gle from: " << theShard << endl;
ShardConnection conn( theShard , "" );
try {
ok = conn->runCommand( dbName , options , res );
}
catch( std::exception &e ) {
string message =
str::stream() << "could not get last error from shard " << theShard
<< causedBy( e );
warning() << message << endl;
errmsg = message;
// Catch everything that happens here, since we need to ensure we return our connection when we're
// finished.
conn.done();
return false;
}
res = res.getOwned();
conn.done();
}
_addWriteBack( writebacks, res, true );
LOG(4) << "gathering writebacks from " << sinceLastGetError().size() << " hosts for"
<< " gle (" << theShard << ")" << endl;
// hit other machines just to block
for ( set<string>::const_iterator i=sinceLastGetError().begin(); i!=sinceLastGetError().end(); ++i ) {
string temp = *i;
if ( temp == theShard )
continue;
LOG(5) << "gathering writebacks for single-shard gle from: " << temp << endl;
try {
ShardConnection conn( temp , "" );
ON_BLOCK_EXIT_OBJ( conn, &ShardConnection::done );
_addWriteBack( writebacks, conn->getLastErrorDetailed(), false );
}
catch( std::exception &e ){
warning() << "could not clear last error from shard " << temp << causedBy( e ) << endl;
}
}
clearSinceLastGetError();
// We never need to handle writebacks if we're coming from the wbl itself
if ( writebacks.size() && !fromWriteBackListener ){
LOG(4) << "checking " << writebacks.size() << " writebacks for"
<< " gle (" << theShard << ")" << endl;
vector<BSONObj> v = _handleWriteBacks( writebacks , false );
// this will usually be 1
// it can be greater than 1 if a write to a different shard
// than the last write op had a writeback
// all we're going to report is the first
// since that's the current write
// but we block for all
verify( v.size() >= 1 );
if ( res["writebackSince"].numberInt() > 0 ) {
// got writeback from older op
// ignore the result from it, just needed to wait
result.appendElements( res );
}
else if ( writebacks[0].fromLastOperation ) {
result.appendElements( v[0] );
result.appendElementsUnique( res );
result.append( "writebackGLE" , v[0] );
result.append( "initialGLEHost" , theShard );
result.append( "initialGLE", res );
}
else {
// there was a writeback
// but its from an old operations
// so all that's important is that we block, not that we return stats
result.appendElements( res );
}
}
else {
result.append( "singleShard" , theShard );
result.appendElements( res );
}
return ok;
}
BSONArrayBuilder bbb( result.subarrayStart( "shards" ) );
BSONObjBuilder shardRawGLE;
long long n = 0;
int updatedExistingStat = 0; // 0 is none, -1 has but false, 1 has true
// hit each shard
vector<string> errors;
vector<BSONObj> errorObjects;
for ( set<string>::iterator i = shards->begin(); i != shards->end(); i++ ) {
string theShard = *i;
bbb.append( theShard );
LOG(5) << "gathering a response for gle from: " << theShard << endl;
boost::scoped_ptr<ShardConnection> conn;
BSONObj res;
bool ok = false;
try {
conn.reset( new ShardConnection( theShard , "" ) ); // constructor can throw if shard is down
ok = (*conn)->runCommand( dbName , options , res );
shardRawGLE.append( theShard , res );
}
catch( std::exception &e ){
// Safe to return here, since we haven't started any extra processing yet, just collecting
// responses.
string message =
str::stream() << "could not get last error from a shard " << theShard
<< causedBy( e );
warning() << message << endl;
errmsg = message;
if (conn)
conn->done();
return false;
}
_addWriteBack( writebacks, res, true );
string temp = DBClientWithCommands::getLastErrorString( res );
if ( (*conn)->type() != ConnectionString::SYNC && ( ok == false || temp.size() ) ) {
errors.push_back( temp );
errorObjects.push_back( res );
}
n += res["n"].numberLong();
if ( res["updatedExisting"].type() ) {
if ( res["updatedExisting"].trueValue() )
updatedExistingStat = 1;
else if ( updatedExistingStat == 0 )
updatedExistingStat = -1;
}
conn->done();
}
bbb.done();
result.append( "shardRawGLE" , shardRawGLE.obj() );
result.appendNumber( "n" , n );
if ( updatedExistingStat )
result.appendBool( "updatedExisting" , updatedExistingStat > 0 );
LOG(4) << "gathering writebacks from " << sinceLastGetError().size() << " hosts for"
<< " gle (" << shards->size() << " shards)" << endl;
// hit other machines just to block
for ( set<string>::const_iterator i=sinceLastGetError().begin(); i!=sinceLastGetError().end(); ++i ) {
string temp = *i;
if ( shards->count( temp ) )
continue;
LOG(5) << "gathering writebacks for multi-shard gle from: " << temp << endl;
ShardConnection conn( temp , "" );
try {
_addWriteBack( writebacks, conn->getLastErrorDetailed(), false );
}
catch( std::exception &e ){
warning() << "could not clear last error from a shard " << temp << causedBy( e ) << endl;
}
conn.done();
}
clearSinceLastGetError();
LOG(4) << "checking " << writebacks.size() << " writebacks for"
<< " gle (" << shards->size() << " shards)" << endl;
// Multi-shard results from the writeback listener implicitly means that:
// A) no versioning was used (multi-update/delete)
// B) internal GLE was used (bulk insert)
if ( errors.size() == 0 ) {
result.appendNull( "err" );
_handleWriteBacks( writebacks , fromWriteBackListener );
return true;
}
result.append( "err" , errors[0].c_str() );
{
// errs
BSONArrayBuilder all( result.subarrayStart( "errs" ) );
for ( unsigned i=0; i<errors.size(); i++ ) {
all.append( errors[i].c_str() );
}
all.done();
}
{
// errObjects
BSONArrayBuilder all( result.subarrayStart( "errObjects" ) );
for ( unsigned i=0; i<errorObjects.size(); i++ ) {
all.append( errorObjects[i] );
}
all.done();
}
_handleWriteBacks( writebacks , fromWriteBackListener );
return true;
}
void reload() {
list<BSONObj> all;
{
scoped_ptr<ScopedDbConnection> conn(
ScopedDbConnection::getInternalScopedDbConnection(
configServer.getPrimary().getConnString(), 30));
auto_ptr<DBClientCursor> c = conn->get()->query(ShardType::ConfigNS , Query());
massert( 13632 , "couldn't get updated shard list from config server" , c.get() );
while ( c->more() ) {
all.push_back( c->next().getOwned() );
}
conn->done();
}
scoped_lock lk( _mutex );
// We use the _lookup table for all shards and for the primary config DB. The config DB info,
// however, does not come from the ShardNS::shard. So when cleaning the _lookup table we leave
// the config state intact. The rationale is that this way we could drop shards that
// were removed without reinitializing the config DB information.
ShardMap::iterator i = _lookup.find( "config" );
if ( i != _lookup.end() ) {
ShardPtr config = i->second;
_lookup.clear();
_lookup[ "config" ] = config;
}
else {
_lookup.clear();
}
_rsLookup.clear();
for ( list<BSONObj>::iterator i=all.begin(); i!=all.end(); ++i ) {
BSONObj o = *i;
string name = o[ ShardType::name() ].String();
string host = o[ ShardType::host() ].String();
long long maxSize = 0;
BSONElement maxSizeElem = o[ ShardType::maxSize.name() ];
if ( ! maxSizeElem.eoo() ) {
maxSize = maxSizeElem.numberLong();
}
bool isDraining = false;
BSONElement isDrainingElem = o[ ShardType::draining.name() ];
if ( ! isDrainingElem.eoo() ) {
isDraining = isDrainingElem.Bool();
}
ShardPtr s( new Shard( name , host , maxSize , isDraining ) );
if ( o[ ShardType::tags() ].type() == Array ) {
vector<BSONElement> v = o[ ShardType::tags() ].Array();
for ( unsigned j=0; j<v.size(); j++ ) {
s->addTag( v[j].String() );
}
}
_lookup[name] = s;
_installHost( host , s );
}
}
bool DBConfig::dropDatabase( string& errmsg ) {
/**
* 1) make sure everything is up
* 2) update config server
* 3) drop and reset sharded collections
* 4) drop and reset primary
* 5) drop everywhere to clean up loose ends
*/
log() << "DBConfig::dropDatabase: " << _name << endl;
configServer.logChange( "dropDatabase.start" , _name , BSONObj() );
// 1
if ( ! configServer.allUp( errmsg ) ) {
log(1) << "\t DBConfig::dropDatabase not all up" << endl;
return 0;
}
// 2
grid.removeDB( _name );
{
ScopedDbConnection conn( configServer.modelServer() );
conn->remove( ShardNS::database , BSON( "_id" << _name ) );
errmsg = conn->getLastError();
if ( ! errmsg.empty() ) {
log() << "could not drop '" << _name << "': " << errmsg << endl;
conn.done();
return false;
}
conn.done();
}
if ( ! configServer.allUp( errmsg ) ) {
log() << "error removing from config server even after checking!" << endl;
return 0;
}
log(1) << "\t removed entry from config server for: " << _name << endl;
set<Shard> allServers;
// 3
while ( true ) {
int num = 0;
if ( ! _dropShardedCollections( num , allServers , errmsg ) )
return 0;
log() << " DBConfig::dropDatabase: " << _name << " dropped sharded collections: " << num << endl;
if ( num == 0 )
break;
}
// 4
{
ScopedDbConnection conn( _primary );
BSONObj res;
if ( ! conn->dropDatabase( _name , &res ) ) {
errmsg = res.toString();
return 0;
}
conn.done();
}
// 5
for ( set<Shard>::iterator i=allServers.begin(); i!=allServers.end(); i++ ) {
ScopedDbConnection conn( *i );
BSONObj res;
if ( ! conn->dropDatabase( _name , &res ) ) {
errmsg = res.toString();
return 0;
}
conn.done();
}
log(1) << "\t dropped primary db for: " << _name << endl;
configServer.logChange( "dropDatabase" , _name , BSONObj() );
return true;
}
int main(int argc, char **argv)
{
toConfigurationNew::setQSettingsEnv();
/*! \warning: Keep the code before QApplication init as small
as possible. There could be serious display issues when
you construct some Qt classes before QApplication.
It's the same for global static stuff - some instances can
break it (e.g. qscintilla lexers etc.).
*/
QApplication app(argc, argv);
QString style(toConfigurationNewSingle::Instance().option(ToConfiguration::Global::Style).toString());
if (!style.isEmpty())
QApplication::setStyle(QStyleFactory::create(style));
try
{
toQValue::setNumberFormat(
toConfigurationNewSingle::Instance().option(ToConfiguration::Database::NumberFormatInt).toInt(),
toConfigurationNewSingle::Instance().option(ToConfiguration::Database::NumberDecimalsInt).toInt()
);
qRegisterMetaType<toQColumnDescriptionList>("toQColumnDescriptionList&");
qRegisterMetaType<ValuesList>("ValuesList&");
qRegisterMetaType<toConnection::exception>("toConnection::exception");
if (argc == 1)
usage();
QString connect = QString::fromLatin1(argv[1]);
QString user, password, database, schema, table;
QStringList slashList, atList = connect.split("@", QString::SkipEmptyParts);
if ( atList.size() == 1)
database = QString::fromLatin1(qgetenv("ORACLE_SID"));
if ( atList.size() > 2)
usage();
if ( atList.at(0).contains("/"))
{
slashList = atList.at(0).split("/", QString::SkipEmptyParts);
user = slashList.at(0);
password = slashList.at(1);
database = atList.at(1);
}
else
{
slashList = atList.at(1).split("/", QString::SkipEmptyParts);
user = atList.at(0);
database = slashList.at(0);
password = slashList.at(1);
}
if ( argc == 2)
{
schema = user;
}
else
{
schema = QString::fromLatin1(argv[2]);
}
if ( argc == 4)
{
table = QString::fromLatin1(argv[3]) ;
}
else
{
table = QString::fromLatin1("T_CHILD6");
}
// List of all connection provider finders
std::vector<std::string> finders = ConnectionProviderFinderFactory::Instance().keys();
// Resulting list of all the providers found
QList<toConnectionProviderFinder::ConnectionProvirerParams> allProviders;
// Loop over all finders and all ther find method each of them can return more locations
for (std::vector<std::string>::const_iterator i = finders.begin(); i != finders.end(); ++i)
{
TLOG(5, toDecorator, __HERE__) << "Looking for client: " << *i << std::endl;
std::unique_ptr<toConnectionProviderFinder> finder = ConnectionProviderFinderFactory::Instance().create(*i, 0);
QList<toConnectionProviderFinder::ConnectionProvirerParams> l = finder->find();
allProviders.append(l);
}
QDir oHome = toConfigurationNewSingle::Instance().option(ToConfiguration::Global::OracleHomeDirectory).toString();
foreach(toConnectionProviderFinder::ConnectionProvirerParams const& params, allProviders)
{
if ( params.value("PROVIDER").toString() != ORACLE_PROVIDER)
continue;
QDir pHome(params.value("ORACLE_HOME").toString());
if (oHome != pHome)
continue;
QString providerName = params.value("PROVIDER").toString();
if (providerName == "Oracle")
{
toConnectionProviderRegistrySing::Instance().load(params);
break;
}
}
foreach(toConnectionProviderFinder::ConnectionProvirerParams const& params, allProviders)
{
QString providerName = params.value("PROVIDER").toString();
if (params.value("PROVIDER").toString() != ORACLE_PROVIDER)
continue;
if (params.value("IS INSTANT").toBool() == true)
continue;
if (toConnectionProviderRegistrySing::Instance().providers().contains(providerName))
continue;
if (providerName == "Oracle")
{
toConnectionProviderRegistrySing::Instance().load(params);
break;
}
}
QSet<QString> options;
QPointer<toConnection> oraCon = new toConnection(
QString("Oracle"),
user,
password,
"",
database,
"",
"",
options);
TLOG(0,toDecorator,__HERE__) << "Version: " << oraCon->version() << std::endl;
// uncomment this if you want to run cache refresh in synchronous mode
// i.e. run it in the current main thread
// toCache::cacheObjects co(*oraCon);
// co.run();
// uncomment this if you want to run cache refresh in async mode
// i.e. run it the background thread
//oraCon->getCache().readCache(); // rereadCache()
//oraCon->getCache().wait4BGThread(); // wait for bg thread to finish.
QString ConstrainsSQL = QString::fromLatin1(
" SELECT \n"
" c.constraint_name \n" // c1
" -- max(a.constraint_name) as constraint_name \n"
" -- , c.constraint_name \n"
" , max(r.constraint_name) as r_constraint_name \n" // c2
" , max(c.owner) as owner \n" // c3
" , max(c.table_name) as table_name \n" // c4
" , c.column_name as column_name \n" // c5
" , max(r.owner) as r_owner \n" // c6
" , max(r.table_name) as r_table_name \n" // c7
" , max(r.column_name) as r_column_name \n" // c8
" , max(a.constraint_type) \n" // c9
" FROM sys.all_constraints a \n"
" JOIN sys.all_cons_columns c ON (c.constraint_name = a.constraint_name AND c.owner = a.owner) \n"
" JOIN sys.all_cons_columns r ON (r.constraint_name = a.r_constraint_name AND r.owner = a.r_owner AND r.position = c.position) \n"
" WHERE \n"
" a.owner = :f1<char[101]> \n"
" AND a.table_name = :f2<char[101]> \n"
" AND a.constraint_type = 'R' \n"
" GROUP BY ROLLUP (c.constraint_name, c.column_name) \n"
);
QString ReferencesSQL = QString::fromLatin1(
" SELECT \n"
" c.constraint_name \n" // c1
" -- max(a.constraint_name) as constraint_name \n"
" -- , c.constraint_name \n"
" , max(r.constraint_name) as r_constraint_name \n" // c2
" , max(c.owner) as owner \n" // c3
" , max(c.table_name) as table_name \n" // c4
" , c.column_name as column_name \n" // c5
" , max(r.owner) as r_owner \n" // c6
" , max(r.table_name) as r_table_name \n" // c7
" , max(r.column_name) as r_column_name \n" // c8
" , max(a.constraint_type) \n" // c9
" FROM sys.all_constraints a \n"
" JOIN sys.all_cons_columns c ON (c.constraint_name = a.constraint_name AND c.owner = a.owner) \n"
" JOIN sys.all_cons_columns r ON (r.constraint_name = a.r_constraint_name AND r.owner = a.r_owner AND r.position = c.position) \n"
" WHERE \n"
" a.r_owner = :f1<char[101]> \n"
" AND r.table_name = :f2<char[101]> \n"
" AND a.constraint_type = 'R' \n"
" GROUP BY ROLLUP (c.constraint_name, c.column_name) \n"
);
//QList<toCache::CacheEntry const*> tables = oraCon->getCache().getEntriesInSchema(schema, toCache::TABLE);
//Q_FOREACH(toCache::CacheEntry const*e, tables)
{
toConnectionSubLoan conn(*oraCon);
toQValue c1, c2, c3, c4, c5, c6, c7, c8, c9;
//toCacheEntryTable const* f = static_cast<toCacheEntryTable const*>(e);
//oraCon->getCache().describeEntry(f);
toQuery QueryC(conn, ConstrainsSQL, toQueryParams() << schema.toUpper() << table.toUpper());
while (!QueryC.eof())
{
c1 = QueryC.readValue();
c2 = QueryC.readValue();
c3 = QueryC.readValue();
c4 = QueryC.readValue();
c5 = QueryC.readValue();
c6 = QueryC.readValue();
c7 = QueryC.readValue();
c8 = QueryC.readValue();
c9 = QueryC.readValue();
if ( c5.isNull() && c1.isNull())
{
// Here collect FK?details (column list for compound keys)
}
if ( c5.isNull() && !c1.isNull())
{
TLOG(0, toNoDecorator, __HERE__)
<< c3.displayData() << '.' << c4.displayData()
<< " => "
<< c6.displayData() << '.' << c7.displayData()
<< std::endl;
}
}
toQuery QueryR(conn, ReferencesSQL, toQueryParams() << schema.toUpper() << table.toUpper());
while (!QueryR.eof())
{
c1 = QueryR.readValue();
c2 = QueryR.readValue();
c3 = QueryR.readValue();
c4 = QueryR.readValue();
c5 = QueryR.readValue();
c6 = QueryR.readValue();
c7 = QueryR.readValue();
c8 = QueryR.readValue();
c9 = QueryR.readValue();
if ( c5.isNull() && !c1.isNull())
{
TLOG(0, toNoDecorator, __HERE__)
<< c3.displayData() << '.' << c4.displayData()
<< " => "
<< c6.displayData() << '.' << c7.displayData()
<< std::endl;
}
}
}
delete oraCon;
}
bool ConfigServer::checkConfigServersConsistent( string& errmsg , int tries ) const {
if ( _config.size() == 1 )
return true;
if ( tries <= 0 )
return false;
unsigned firstGood = 0;
int up = 0;
vector<BSONObj> res;
for ( unsigned i=0; i<_config.size(); i++ ) {
BSONObj x;
try {
ScopedDbConnection conn( _config[i] );
if ( ! conn->simpleCommand( "config" , &x , "dbhash" ) )
x = BSONObj();
else {
x = x.getOwned();
if ( up == 0 )
firstGood = i;
up++;
}
conn.done();
}
catch ( std::exception& ) {
log(LL_WARNING) << " couldn't check on config server:" << _config[i] << " ok for now" << endl;
}
res.push_back(x);
}
if ( up == 0 ) {
errmsg = "no config servers reachable";
return false;
}
if ( up == 1 ) {
log( LL_WARNING ) << "only 1 config server reachable, continuing" << endl;
return true;
}
BSONObj base = res[firstGood];
for ( unsigned i=firstGood+1; i<res.size(); i++ ) {
if ( res[i].isEmpty() )
continue;
string c1 = base.getFieldDotted( "collections.chunks" );
string c2 = res[i].getFieldDotted( "collections.chunks" );
string d1 = base.getFieldDotted( "collections.databases" );
string d2 = res[i].getFieldDotted( "collections.databases" );
if ( c1 == c2 && d1 == d2 )
continue;
stringstream ss;
ss << "config servers " << _config[firstGood] << " and " << _config[i] << " differ";
log( LL_WARNING ) << ss.str();
if ( tries <= 1 ) {
ss << "\n" << c1 << "\t" << c2 << "\n" << d1 << "\t" << d2;
errmsg = ss.str();
return false;
}
return checkConfigServersConsistent( errmsg , tries - 1 );
}
return true;
}
void appendReplicationInfo(OperationContext* txn, BSONObjBuilder& result, int level) {
ReplicationCoordinator* replCoord = getGlobalReplicationCoordinator();
if (replCoord->getSettings().usingReplSets()) {
IsMasterResponse isMasterResponse;
replCoord->fillIsMasterForReplSet(&isMasterResponse);
result.appendElements(isMasterResponse.toBSON());
return;
}
// TODO(dannenberg) replAllDead is bad and should be removed when master slave is removed
if (replAllDead) {
result.append("ismaster", 0);
string s = string("dead: ") + replAllDead;
result.append("info", s);
}
else {
result.appendBool("ismaster",
getGlobalReplicationCoordinator()->isMasterForReportingPurposes());
}
if (level) {
BSONObjBuilder sources( result.subarrayStart( "sources" ) );
int n = 0;
list<BSONObj> src;
{
const char* localSources = "local.sources";
Client::ReadContext ctx(txn, localSources);
auto_ptr<PlanExecutor> exec(
InternalPlanner::collectionScan(txn,
localSources,
ctx.ctx().db()->getCollection(txn,
localSources)));
BSONObj obj;
PlanExecutor::ExecState state;
while (PlanExecutor::ADVANCED == (state = exec->getNext(&obj, NULL))) {
src.push_back(obj);
}
}
for( list<BSONObj>::const_iterator i = src.begin(); i != src.end(); i++ ) {
BSONObj s = *i;
BSONObjBuilder bb;
bb.append( s["host"] );
string sourcename = s["source"].valuestr();
if ( sourcename != "main" )
bb.append( s["source"] );
{
BSONElement e = s["syncedTo"];
BSONObjBuilder t( bb.subobjStart( "syncedTo" ) );
t.appendDate( "time" , e.timestampTime() );
t.append( "inc" , e.timestampInc() );
t.done();
}
if ( level > 1 ) {
wassert(!txn->lockState()->isLocked());
// note: there is no so-style timeout on this connection; perhaps we should have one.
ScopedDbConnection conn(s["host"].valuestr());
DBClientConnection *cliConn = dynamic_cast< DBClientConnection* >( &conn.conn() );
if ( cliConn && replAuthenticate(cliConn) ) {
BSONObj first = conn->findOne( (string)"local.oplog.$" + sourcename,
Query().sort( BSON( "$natural" << 1 ) ) );
BSONObj last = conn->findOne( (string)"local.oplog.$" + sourcename,
Query().sort( BSON( "$natural" << -1 ) ) );
bb.appendDate( "masterFirst" , first["ts"].timestampTime() );
bb.appendDate( "masterLast" , last["ts"].timestampTime() );
double lag = (double) (last["ts"].timestampTime() - s["syncedTo"].timestampTime());
bb.append( "lagSeconds" , lag / 1000 );
}
conn.done();
}
sources.append( BSONObjBuilder::numStr( n++ ) , bb.obj() );
}
sources.done();
}
}
void CursorCache::gotKillCursors(Message& m ) {
DbMessage dbmessage(m);
int n = dbmessage.pullInt();
if ( n > 2000 ) {
( n < 30000 ? warning() : error() ) << "receivedKillCursors, n=" << n << endl;
}
uassert( 13286 , "sent 0 cursors to kill" , n >= 1 );
uassert( 13287 , "too many cursors to kill" , n < 30000 );
massert( 18632 , str::stream() << "bad kill cursors size: " << m.dataSize(),
m.dataSize() == 8 + ( 8 * n ) );
const long long* cursors = dbmessage.getArray(n);
ClientBasic* client = ClientBasic::getCurrent();
AuthorizationSession* authSession = client->getAuthorizationSession();
for ( int i=0; i<n; i++ ) {
long long id = cursors[i];
LOG(_myLogLevel) << "CursorCache::gotKillCursors id: " << id << endl;
if ( ! id ) {
warning() << " got cursor id of 0 to kill" << endl;
continue;
}
string server;
{
scoped_lock lk( _mutex );
MapSharded::iterator i = _cursors.find( id );
if ( i != _cursors.end() ) {
const bool isAuthorized = authSession->isAuthorizedForActionsOnNamespace(
NamespaceString(i->second->getNS()), ActionType::killCursors);
audit::logKillCursorsAuthzCheck(
client,
NamespaceString(i->second->getNS()),
id,
isAuthorized ? ErrorCodes::OK : ErrorCodes::Unauthorized);
if (isAuthorized) {
_cursorsMaxTimeMS.erase( i->second->getId() );
_cursors.erase( i );
}
continue;
}
MapNormal::iterator refsIt = _refs.find(id);
MapNormal::iterator refsNSIt = _refsNS.find(id);
if (refsIt == _refs.end()) {
warning() << "can't find cursor: " << id << endl;
continue;
}
verify(refsNSIt != _refsNS.end());
const bool isAuthorized = authSession->isAuthorizedForActionsOnNamespace(
NamespaceString(refsNSIt->second), ActionType::killCursors);
audit::logKillCursorsAuthzCheck(
client,
NamespaceString(refsNSIt->second),
id,
isAuthorized ? ErrorCodes::OK : ErrorCodes::Unauthorized);
if (!isAuthorized) {
continue;
}
server = refsIt->second;
_refs.erase(refsIt);
_refsNS.erase(refsNSIt);
}
LOG(_myLogLevel) << "CursorCache::found gotKillCursors id: " << id << " server: " << server << endl;
verify( server.size() );
ScopedDbConnection conn(server);
conn->killCursor( id );
conn.done();
}
}
void Balancer::run() {
// this is the body of a BackgroundJob so if we throw here we're basically ending the balancer thread prematurely
while ( ! inShutdown() ) {
if ( ! _init() ) {
log() << "will retry to initialize balancer in one minute" << endl;
sleepsecs( 60 );
continue;
}
break;
}
// getConnectioString and dist lock constructor does not throw, which is what we expect on while
// on the balancer thread
ConnectionString config = configServer.getConnectionString();
DistributedLock balanceLock( config , "balancer" );
while ( ! inShutdown() ) {
try {
ScopedDbConnection conn( config );
// ping has to be first so we keep things in the config server in sync
_ping( conn.conn() );
// now make sure we should even be running
if ( ! grid.shouldBalance() ) {
LOG(1) << "skipping balancing round because balancing is disabled" << endl;
conn.done();
sleepsecs( 30 );
continue;
}
uassert( 13258 , "oids broken after resetting!" , _checkOIDs() );
// use fresh shard state
Shard::reloadShardInfo();
{
dist_lock_try lk( &balanceLock , "doing balance round" );
if ( ! lk.got() ) {
LOG(1) << "skipping balancing round because another balancer is active" << endl;
conn.done();
sleepsecs( 30 ); // no need to wake up soon
continue;
}
LOG(1) << "*** start balancing round" << endl;
vector<CandidateChunkPtr> candidateChunks;
_doBalanceRound( conn.conn() , &candidateChunks );
if ( candidateChunks.size() == 0 ) {
LOG(1) << "no need to move any chunk" << endl;
}
else {
_balancedLastTime = _moveChunks( &candidateChunks );
}
LOG(1) << "*** end of balancing round" << endl;
}
conn.done();
sleepsecs( _balancedLastTime ? 5 : 10 );
}
catch ( std::exception& e ) {
log() << "caught exception while doing balance: " << e.what() << endl;
// Just to match the opening statement if in log level 1
LOG(1) << "*** End of balancing round" << endl;
sleepsecs( 30 ); // sleep a fair amount b/c of error
continue;
}
}
}
bool run( const string& dbname,
BSONObj& cmdObj,
int,
string& errmsg,
BSONObjBuilder& result,
bool ) {
string ns = parseNs(dbname, cmdObj);
if ( ns.size() == 0 ) {
errmsg = "no namespace specified";
return false;
}
vector<BSONObj> bounds;
if ( !FieldParser::extract( cmdObj, boundsField, &bounds, &errmsg ) ) {
return false;
}
if ( bounds.size() == 0 ) {
errmsg = "no bounds were specified";
return false;
}
if ( bounds.size() != 2 ) {
errmsg = "only a min and max bound may be specified";
return false;
}
BSONObj minKey = bounds[0];
BSONObj maxKey = bounds[1];
if ( minKey.isEmpty() ) {
errmsg = "no min key specified";
return false;
}
if ( maxKey.isEmpty() ) {
errmsg = "no max key specified";
return false;
}
// This refreshes the chunk metadata if stale.
refreshChunkCache( NamespaceString( ns ) );
ShardPtr mergeShard = guessMergeShard( NamespaceString( ns ), minKey );
if ( !mergeShard ) {
errmsg = (string)"could not find shard for merge range starting at "
+ minKey.toString();
return false;
}
BSONObjBuilder remoteCmdObjB;
remoteCmdObjB.append( cmdObj[ ClusterMergeChunksCommand::nsField() ] );
remoteCmdObjB.append( cmdObj[ ClusterMergeChunksCommand::boundsField() ] );
remoteCmdObjB.append( ClusterMergeChunksCommand::configField(),
configServer.getPrimary().getAddress().toString() );
remoteCmdObjB.append( ClusterMergeChunksCommand::shardNameField(),
mergeShard->getName() );
BSONObj remoteResult;
// Throws, but handled at level above. Don't want to rewrap to preserve exception
// formatting.
ScopedDbConnection conn( mergeShard->getAddress() );
bool ok = conn->runCommand( "admin", remoteCmdObjB.obj(), remoteResult );
conn.done();
result.appendElements( remoteResult );
return ok;
}
void WriteBackListener::run() {
int secsToSleep = 0;
scoped_ptr<ChunkVersion> lastNeededVersion;
int lastNeededCount = 0;
bool needsToReloadShardInfo = false;
while ( ! inShutdown() ) {
if ( ! Shard::isAShardNode( _addr ) ) {
LOG(1) << _addr << " is not a shard node" << endl;
sleepsecs( 60 );
continue;
}
try {
if (needsToReloadShardInfo) {
// It's possible this shard was removed
Shard::reloadShardInfo();
needsToReloadShardInfo = false;
}
ScopedDbConnection conn(_addr);
BSONObj result;
{
BSONObjBuilder cmd;
cmd.appendOID( "writebacklisten" , &serverID ); // Command will block for data
if ( ! conn->runCommand( "admin" , cmd.obj() , result ) ) {
result = result.getOwned();
log() << "writebacklisten command failed! " << result << endl;
conn.done();
continue;
}
}
conn.done();
LOG(1) << "writebacklisten result: " << result << endl;
BSONObj data = result.getObjectField( "data" );
if ( data.getBoolField( "writeBack" ) ) {
string ns = data["ns"].valuestrsafe();
ConnectionIdent cid( "" , 0 );
OID wid;
if ( data["connectionId"].isNumber() && data["id"].type() == jstOID ) {
string s = "";
if ( data["instanceIdent"].type() == String )
s = data["instanceIdent"].String();
cid = ConnectionIdent( s , data["connectionId"].numberLong() );
wid = data["id"].OID();
}
else {
warning() << "mongos/mongod version mismatch (1.7.5 is the split)" << endl;
}
int len; // not used, but needed for next call
Message msg( (void*)data["msg"].binData( len ) , false );
massert( 10427 , "invalid writeback message" , msg.header()->valid() );
DBConfigPtr db = grid.getDBConfig( ns );
ChunkVersion needVersion = ChunkVersion::fromBSON( data, "version" );
//
// TODO: Refactor the sharded strategy to correctly handle all sharding state changes itself,
// we can't rely on WBL to do this for us b/c anything could reset our state in-between.
// We should always reload here for efficiency when possible, but staleness is also caught in the
// loop below.
//
ChunkManagerPtr manager;
ShardPtr primary;
db->getChunkManagerOrPrimary( ns, manager, primary );
ChunkVersion currVersion;
if( manager ) currVersion = manager->getVersion();
LOG(1) << "connectionId: " << cid << " writebackId: " << wid << " needVersion : " << needVersion.toString()
<< " mine : " << currVersion.toString() << endl;
LOG(1) << msg.toString() << endl;
//
// We should reload only if we need to update our version to be compatible *and* we
// haven't already done so. This avoids lots of reloading when we remove/add a sharded collection
//
bool alreadyReloaded = lastNeededVersion &&
lastNeededVersion->isEquivalentTo( needVersion );
if( alreadyReloaded ){
LOG(1) << "wbl already reloaded config information for version "
<< needVersion << ", at version " << currVersion << endl;
}
else if( lastNeededVersion ) {
log() << "new version change detected to " << needVersion.toString()
<< ", " << lastNeededCount << " writebacks processed at "
<< lastNeededVersion->toString() << endl;
lastNeededCount = 0;
}
//
// Set our lastNeededVersion for next time
//
lastNeededVersion.reset( new ChunkVersion( needVersion ) );
lastNeededCount++;
//
// Determine if we should reload, if so, reload
//
bool shouldReload = ! needVersion.isWriteCompatibleWith( currVersion ) &&
! alreadyReloaded;
if( shouldReload && currVersion.isSet()
&& needVersion.isSet()
&& currVersion.hasCompatibleEpoch( needVersion ) )
{
//
// If we disagree about versions only, reload the chunk manager
//
db->getChunkManagerIfExists( ns, true );
}
else if( shouldReload ){
//
// If we disagree about anything else, reload the full db
//
warning() << "reloading config data for " << db->getName() << ", "
<< "wanted version " << needVersion.toString()
<< " but currently have version " << currVersion.toString() << endl;
db->reload();
}
// do request and then call getLastError
// we have to call getLastError so we can return the right fields to the user if they decide to call getLastError
BSONObj gle;
int attempts = 0;
while ( true ) {
attempts++;
try {
Request r( msg , 0 );
r.init();
r.d().reservedField() |= Reserved_FromWriteback;
ClientInfo * ci = r.getClientInfo();
if (AuthorizationManager::isAuthEnabled()) {
ci->getAuthorizationSession()->grantInternalAuthorization(
UserName("_writebackListener", "local"));
}
ci->noAutoSplit();
r.process( attempts );
ci->newRequest(); // this so we flip prev and cur shards
BSONObjBuilder b;
string errmsg;
if ( ! ci->getLastError( "admin",
BSON( "getLastError" << 1 ),
b,
errmsg,
true ) )
{
b.appendBool( "commandFailed" , true );
if( ! b.hasField( "errmsg" ) ){
b.append( "errmsg", errmsg );
gle = b.obj();
}
else if( errmsg.size() > 0 ){
// Rebuild GLE object with errmsg
// TODO: Make this less clumsy by improving GLE interface
gle = b.obj();
if( gle["errmsg"].type() == String ){
BSONObj gleNoErrmsg =
gle.filterFieldsUndotted( BSON( "errmsg" << 1 ),
false );
BSONObjBuilder bb;
bb.appendElements( gleNoErrmsg );
bb.append( "errmsg", gle["errmsg"].String() +
" ::and:: " +
errmsg );
gle = bb.obj().getOwned();
}
}
}
else{
gle = b.obj();
}
if ( gle["code"].numberInt() == 9517 ) {
log() << "new version change detected, "
<< lastNeededCount << " writebacks processed previously" << endl;
lastNeededVersion.reset();
lastNeededCount = 1;
log() << "writeback failed because of stale config, retrying attempts: " << attempts << endl;
LOG(1) << "writeback error : " << gle << endl;
//
// Bringing this in line with the similar retry logic elsewhere
//
// TODO: Reloading the chunk manager may not help if we dropped a
// collection, but we don't actually have that info in the writeback
// error
//
if( attempts <= 2 ){
db->getChunkManagerIfExists( ns, true );
}
else{
versionManager.forceRemoteCheckShardVersionCB( ns );
sleepsecs( attempts - 1 );
}
uassert( 15884, str::stream()
<< "Could not reload chunk manager after "
<< attempts << " attempts.", attempts <= 4 );
continue;
}
ci->clearSinceLastGetError();
}
catch ( DBException& e ) {
error() << "error processing writeback: " << e << endl;
BSONObjBuilder b;
e.getInfo().append( b, "err", "code" );
gle = b.obj();
}
break;
}
{
scoped_lock lk( _seenWritebacksLock );
WBStatus& s = _seenWritebacks[cid];
s.id = wid;
s.gle = gle;
}
}
else if ( result["noop"].trueValue() ) {
// no-op
}
else {
log() << "unknown writeBack result: " << result << endl;
}
secsToSleep = 0;
continue;
}
catch ( std::exception& e ) {
// Attention! Do not call any method that would throw an exception
// (or assert) in this block.
if ( inShutdown() ) {
// we're shutting down, so just clean up
return;
}
log() << "WriteBackListener exception : " << e.what() << endl;
needsToReloadShardInfo = true;
}
catch ( ... ) {
log() << "WriteBackListener uncaught exception!" << endl;
}
secsToSleep++;
sleepsecs(secsToSleep);
if ( secsToSleep > 10 )
secsToSleep = 0;
}
log() << "WriteBackListener exiting : address no longer in cluster " << _addr;
}
void OplogReader::connectToSyncSource(OperationContext* txn,
OpTime lastOpTimeFetched,
ReplicationCoordinator* replCoord) {
const OpTime sentinel(Milliseconds(curTimeMillis64()).total_seconds(), 0);
OpTime oldestOpTimeSeen = sentinel;
invariant(conn() == NULL);
while (true) {
HostAndPort candidate = replCoord->chooseNewSyncSource();
if (candidate.empty()) {
if (oldestOpTimeSeen == sentinel) {
// If, in this invocation of connectToSyncSource(), we did not successfully
// connect to any node ahead of us,
// we apparently have no sync sources to connect to.
// This situation is common; e.g. if there are no writes to the primary at
// the moment.
return;
}
// Connected to at least one member, but in all cases we were too stale to use them
// as a sync source.
error() << "RS102 too stale to catch up";
log() << "our last optime : " << lastOpTimeFetched.toStringLong();
log() << "oldest available is " << oldestOpTimeSeen.toStringLong();
log() << "See http://dochub.mongodb.org/core/resyncingaverystalereplicasetmember";
setMinValid(txn, oldestOpTimeSeen);
bool worked = replCoord->setFollowerMode(MemberState::RS_RECOVERING);
if (!worked) {
warning() << "Failed to transition into "
<< MemberState(MemberState::RS_RECOVERING)
<< ". Current state: " << replCoord->getMemberState();
}
return;
}
if (!connect(candidate)) {
LOG(2) << "can't connect to " << candidate.toString() <<
" to read operations";
resetConnection();
replCoord->blacklistSyncSource(candidate, Date_t(curTimeMillis64() + 10*1000));
continue;
}
// Read the first (oldest) op and confirm that it's not newer than our last
// fetched op. Otherwise, we have fallen off the back of that source's oplog.
BSONObj remoteOldestOp(findOne(rsoplog, Query()));
BSONElement tsElem(remoteOldestOp["ts"]);
if (tsElem.type() != Timestamp) {
// This member's got a bad op in its oplog.
warning() << "oplog invalid format on node " << candidate.toString();
resetConnection();
replCoord->blacklistSyncSource(candidate,
Date_t(curTimeMillis64() + 600*1000));
continue;
}
OpTime remoteOldOpTime = tsElem._opTime();
if (lastOpTimeFetched < remoteOldOpTime) {
// We're too stale to use this sync source.
resetConnection();
replCoord->blacklistSyncSource(candidate,
Date_t(curTimeMillis64() + 600*1000));
if (oldestOpTimeSeen > remoteOldOpTime) {
warning() << "we are too stale to use " << candidate.toString() <<
" as a sync source";
oldestOpTimeSeen = remoteOldOpTime;
}
continue;
}
// Got a valid sync source.
return;
} // while (true)
}
bool handleSpecialNamespaces( Request& r , QueryMessage& q ) {
const char * ns = r.getns();
ns = strstr( r.getns() , ".$cmd.sys." );
if ( ! ns )
return false;
ns += 10;
BSONObjBuilder b;
vector<Shard> shards;
if ( strcmp( ns , "inprog" ) == 0 ) {
Shard::getAllShards( shards );
BSONArrayBuilder arr( b.subarrayStart( "inprog" ) );
for ( unsigned i=0; i<shards.size(); i++ ) {
Shard shard = shards[i];
ScopedDbConnection conn( shard );
BSONObj temp = conn->findOne( r.getns() , BSONObj() );
if ( temp["inprog"].isABSONObj() ) {
BSONObjIterator i( temp["inprog"].Obj() );
while ( i.more() ) {
BSONObjBuilder x;
BSONObjIterator j( i.next().Obj() );
while( j.more() ) {
BSONElement e = j.next();
if ( str::equals( e.fieldName() , "opid" ) ) {
stringstream ss;
ss << shard.getName() << ':' << e.numberInt();
x.append( "opid" , ss.str() );
}
else if ( str::equals( e.fieldName() , "client" ) ) {
x.appendAs( e , "client_s" );
}
else {
x.append( e );
}
}
arr.append( x.obj() );
}
}
conn.done();
}
arr.done();
}
else if ( strcmp( ns , "killop" ) == 0 ) {
BSONElement e = q.query["op"];
if ( strstr( r.getns() , "admin." ) != 0 ) {
b.append( "err" , "unauthorized" );
}
else if ( e.type() != String ) {
b.append( "err" , "bad op" );
b.append( e );
}
else {
b.append( e );
string s = e.String();
string::size_type i = s.find( ':' );
if ( i == string::npos ) {
b.append( "err" , "bad opid" );
}
else {
string shard = s.substr( 0 , i );
int opid = atoi( s.substr( i + 1 ).c_str() );
b.append( "shard" , shard );
b.append( "shardid" , opid );
log() << "want to kill op: " << e << endl;
Shard s(shard);
ScopedDbConnection conn( s );
conn->findOne( r.getns() , BSON( "op" << opid ) );
conn.done();
}
}
}
else if ( strcmp( ns , "unlock" ) == 0 ) {
b.append( "err" , "can't do unlock through mongos" );
}
else {
log( LL_WARNING ) << "unknown sys command [" << ns << "]" << endl;
return false;
}
BSONObj x = b.done();
replyToQuery(0, r.p(), r.m(), x);
return true;
}
bool Tool::isMongos() {
// TODO: when mongos supports QueryOption_Exaust add a version check (SERVER-2628)
BSONObj isdbgrid;
conn("true").simpleCommand("admin", &isdbgrid, "isdbgrid");
return isdbgrid["isdbgrid"].trueValue();
}
Status MetadataLoader::initChunks( const string& ns,
const string& shard,
const CollectionMetadata* oldMetadata,
CollectionMetadata* metadata ) const
{
map<string, ChunkVersion> versionMap;
// Preserve the epoch
versionMap[shard] = metadata->_shardVersion;
OID epoch = metadata->getCollVersion().epoch();
bool fullReload = true;
// Check to see if we should use the old version or not.
if ( oldMetadata ) {
// If our epochs are compatible, it's useful to use the old metadata for diffs
if ( oldMetadata->getCollVersion().hasCompatibleEpoch( epoch ) ) {
fullReload = false;
dassert( oldMetadata->isValid() );
versionMap[shard] = oldMetadata->_shardVersion;
metadata->_collVersion = oldMetadata->_collVersion;
// TODO: This could be made more efficient if copying not required, but
// not as frequently reloaded as in mongos.
metadata->_chunksMap = oldMetadata->_chunksMap;
LOG( 2 ) << "loading new chunks for collection " << ns
<< " using old metadata w/ version " << oldMetadata->getShardVersion()
<< " and " << metadata->_chunksMap.size() << " chunks" << endl;
}
else {
warning() << "reloading collection metadata for " << ns << " with new epoch "
<< epoch.toString() << ", the current epoch is "
<< oldMetadata->getCollVersion().epoch().toString() << endl;
}
}
// Exposes the new metadata's range map and version to the "differ," who
// would ultimately be responsible of filling them up.
SCMConfigDiffTracker differ( shard );
differ.attach( ns, metadata->_chunksMap, metadata->_collVersion, versionMap );
try {
ScopedDbConnection conn( _configLoc.toString(), 30 );
auto_ptr<DBClientCursor> cursor = conn->query( ChunkType::ConfigNS,
differ.configDiffQuery() );
if ( !cursor.get() ) {
// Make our metadata invalid
metadata->_collVersion = ChunkVersion( 0, 0, OID() );
metadata->_chunksMap.clear();
conn.done();
return Status( ErrorCodes::HostUnreachable,
"problem opening chunk metadata cursor" );
}
//
// The diff tracker should always find at least one chunk (the highest chunk we saw
// last time). If not, something has changed on the config server (potentially between
// when we read the collection data and when we read the chunks data).
//
int diffsApplied = differ.calculateConfigDiff( *cursor );
if ( diffsApplied > 0 ) {
// Chunks found, return ok
LOG(2) << "loaded " << diffsApplied << " chunks into new metadata for " << ns
<< " with version " << metadata->_collVersion << endl;
metadata->_shardVersion = versionMap[shard];
metadata->fillRanges();
conn.done();
dassert( metadata->isValid() );
return Status::OK();
}
else if ( diffsApplied == 0 ) {
// No chunks found, the collection is dropping or we're confused
// If this is a full reload, assume it is a drop for backwards compatibility
// TODO: drop the config.collections entry *before* the chunks and eliminate this
// ambiguity
string errMsg =
str::stream() << "no chunks found when reloading " << ns
<< ", previous version was "
<< metadata->_collVersion.toString()
<< ( fullReload ? ", this is a drop" : "" );
warning() << errMsg << endl;
metadata->_collVersion = ChunkVersion( 0, 0, OID() );
metadata->_chunksMap.clear();
conn.done();
return fullReload ? Status( ErrorCodes::NamespaceNotFound, errMsg ) :
Status( ErrorCodes::RemoteChangeDetected, errMsg );
}
else {
// Invalid chunks found, our epoch may have changed because we dropped/recreated
// the collection.
string errMsg = // br
str::stream() << "invalid chunks found when reloading " << ns
<< ", previous version was "
<< metadata->_collVersion.toString()
<< ", this should be rare";
warning() << errMsg << endl;
metadata->_collVersion = ChunkVersion( 0, 0, OID() );
metadata->_chunksMap.clear();
conn.done();
return Status( ErrorCodes::RemoteChangeDetected, errMsg );
}
}
catch ( const DBException& e ) {
string errMsg = str::stream() << "problem querying chunks metadata" << causedBy( e );
// We deliberately do not return connPtr to the pool, since it was involved
// with the error here.
return Status( ErrorCodes::HostUnreachable, errMsg );
}
}
void do_step(void)
{
int syserr = -1;
int sock = -1;
my_pause("do connect");
sock = conn();
if (sock == -1)
{
goto EXIT_LABEL;
}
my_pause("do write");
ssize_t nw = write_bigdata(sock);
if (nw == -1)
{
perror("write");
goto EXIT_LABEL;
}
my_pause("do shutdown");
shutdown(sock, SHUT_WR);
my_pause("do read");
ssize_t nr = 0;
do
{
char buf[10] = { 0 };
nr = read(sock, buf, sizeof(buf) - 1);
if (nr)
{
buf[nr] = '\0';
printf("[%s]\n", buf);
}
}
while (nr);
my_pause("do close");
close(sock);
sock = -1;
syserr = 0;
EXIT_LABEL:
if (sock != -1)
{
close(sock);
sock = -1;
}
printf("syserr = %d\n", syserr);
}
Status MetadataLoader::initCollection( const string& ns,
const string& shard,
CollectionMetadata* metadata ) const
{
//
// Bring collection entry from the config server.
//
BSONObj collDoc;
{
try {
ScopedDbConnection conn( _configLoc.toString(), 30 );
collDoc = conn->findOne( CollectionType::ConfigNS, QUERY(CollectionType::ns()<<ns));
conn.done();
}
catch ( const DBException& e ) {
string errMsg = str::stream() << "could not query collection metadata"
<< causedBy( e );
// We deliberately do not return conn to the pool, since it was involved
// with the error here.
return Status( ErrorCodes::HostUnreachable, errMsg );
}
}
string errMsg;
if ( collDoc.isEmpty() ) {
errMsg = str::stream() << "could not load metadata, collection " << ns << " not found";
warning() << errMsg << endl;
return Status( ErrorCodes::NamespaceNotFound, errMsg );
}
CollectionType collInfo;
if ( !collInfo.parseBSON( collDoc, &errMsg ) || !collInfo.isValid( &errMsg ) ) {
errMsg = str::stream() << "could not parse metadata for collection " << ns
<< causedBy( errMsg );
warning() << errMsg << endl;
return Status( ErrorCodes::FailedToParse, errMsg );
}
if ( collInfo.isDroppedSet() && collInfo.getDropped() ) {
errMsg = str::stream() << "could not load metadata, collection " << ns
<< " was dropped";
warning() << errMsg << endl;
return Status( ErrorCodes::NamespaceNotFound, errMsg );
}
if ( collInfo.isKeyPatternSet() && !collInfo.getKeyPattern().isEmpty() ) {
// Sharded collection, need to load chunks
metadata->_keyPattern = collInfo.getKeyPattern();
metadata->_shardVersion = ChunkVersion( 0, 0, collInfo.getEpoch() );
metadata->_collVersion = ChunkVersion( 0, 0, collInfo.getEpoch() );
return Status::OK();
}
else if ( collInfo.isPrimarySet() && collInfo.getPrimary() == shard ) {
// A collection with a non-default primary
// Empty primary field not allowed if set
dassert( collInfo.getPrimary() != "" );
metadata->_keyPattern = BSONObj();
metadata->_shardVersion = ChunkVersion( 1, 0, collInfo.getEpoch() );
metadata->_collVersion = metadata->_shardVersion;
return Status::OK();
}
else {
// A collection with a primary that doesn't match this shard or is empty, the primary
// may have changed before we loaded.
errMsg = // br
str::stream() << "collection " << ns << " does not have a shard key "
<< "and primary "
<< ( collInfo.isPrimarySet() ? collInfo.getPrimary() : "" )
<< " does not match this shard " << shard;
warning() << errMsg << endl;
metadata->_collVersion = ChunkVersion( 0, 0, OID() );
return Status( ErrorCodes::RemoteChangeDetected, errMsg );
}
}
void Balancer::run() {
// this is the body of a BackgroundJob so if we throw here we're basically ending the balancer thread prematurely
while ( ! inShutdown() ) {
if ( ! _init() ) {
log() << "will retry to initialize balancer in one minute" << endl;
sleepsecs( 60 );
continue;
}
break;
}
int sleepTime = 10;
// getConnectioString and dist lock constructor does not throw, which is what we expect on while
// on the balancer thread
ConnectionString config = configServer.getConnectionString();
DistributedLock balanceLock( config , "balancer" );
while ( ! inShutdown() ) {
try {
ScopedDbConnection conn(config.toString(), 30);
// ping has to be first so we keep things in the config server in sync
_ping();
// use fresh shard state
Shard::reloadShardInfo();
// refresh chunk size (even though another balancer might be active)
Chunk::refreshChunkSize();
SettingsType balancerConfig;
string errMsg;
if (!grid.getBalancerSettings(&balancerConfig, &errMsg)) {
warning() << errMsg;
return ;
}
// now make sure we should even be running
if (balancerConfig.isKeySet() && // balancer config doc exists
!grid.shouldBalance(balancerConfig)) {
LOG(1) << "skipping balancing round because balancing is disabled" << endl;
// Ping again so scripts can determine if we're active without waiting
_ping( true );
conn.done();
sleepsecs( sleepTime );
continue;
}
uassert( 13258 , "oids broken after resetting!" , _checkOIDs() );
{
dist_lock_try lk( &balanceLock , "doing balance round" );
if ( ! lk.got() ) {
LOG(1) << "skipping balancing round because another balancer is active" << endl;
// Ping again so scripts can determine if we're active without waiting
_ping( true );
conn.done();
sleepsecs( sleepTime ); // no need to wake up soon
continue;
}
if ( !isConfigServerConsistent() ) {
conn.done();
warning() << "Skipping balancing round because data inconsistency"
<< " was detected amongst the config servers." << endl;
sleepsecs( sleepTime );
continue;
}
const bool waitForDelete = (balancerConfig.isWaitForDeleteSet() ?
balancerConfig.getWaitForDelete() : false);
scoped_ptr<WriteConcernOptions> writeConcern;
if (balancerConfig.isKeySet()) { // if balancer doc exists.
StatusWith<WriteConcernOptions*> extractStatus =
balancerConfig.extractWriteConcern();
if (extractStatus.isOK()) {
writeConcern.reset(extractStatus.getValue());
}
else {
warning() << extractStatus.toString();
}
}
LOG(1) << "*** start balancing round. "
<< "waitForDelete: " << waitForDelete
<< ", secondaryThrottle: "
<< (writeConcern.get() ? writeConcern->toBSON().toString() : "default")
<< endl;
vector<CandidateChunkPtr> candidateChunks;
_doBalanceRound( conn.conn() , &candidateChunks );
if ( candidateChunks.size() == 0 ) {
LOG(1) << "no need to move any chunk" << endl;
_balancedLastTime = 0;
}
else {
_balancedLastTime = _moveChunks(&candidateChunks,
writeConcern.get(),
waitForDelete );
}
LOG(1) << "*** end of balancing round" << endl;
}
// Ping again so scripts can determine if we're active without waiting
_ping( true );
conn.done();
sleepsecs( _balancedLastTime ? sleepTime / 10 : sleepTime );
}
catch ( std::exception& e ) {
log() << "caught exception while doing balance: " << e.what() << endl;
// Just to match the opening statement if in log level 1
LOG(1) << "*** End of balancing round" << endl;
sleepsecs( sleepTime ); // sleep a fair amount b/c of error
continue;
}
}
}
virtual bool run(const string& dbName, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool) {
LastError *le = lastError.disableForCommand();
{
assert( le );
if ( le->msg.size() && le->nPrev == 1 ){
le->appendSelf( result );
return true;
}
}
ClientInfo * client = ClientInfo::get();
set<string> * shards = client->getPrev();
if ( shards->size() == 0 ){
result.appendNull( "err" );
return true;
}
//log() << "getlasterror enter: " << shards->size() << endl;
vector<OID> writebacks;
// handle single server
if ( shards->size() == 1 ){
string theShard = *(shards->begin() );
result.append( "theshard" , theShard.c_str() );
ShardConnection conn( theShard , "" );
BSONObj res;
bool ok = conn->runCommand( dbName , cmdObj , res );
//log() << "\t" << res << endl;
result.appendElements( res );
conn.done();
result.append( "singleShard" , theShard );
addWriteBack( writebacks , res );
// hit other machines just to block
for ( set<string>::const_iterator i=client->sinceLastGetError().begin(); i!=client->sinceLastGetError().end(); ++i ){
string temp = *i;
if ( temp == theShard )
continue;
ShardConnection conn( temp , "" );
addWriteBack( writebacks , conn->getLastErrorDetailed() );
conn.done();
}
client->clearSinceLastGetError();
handleWriteBacks( writebacks );
return ok;
}
BSONArrayBuilder bbb( result.subarrayStart( "shards" ) );
long long n = 0;
// hit each shard
vector<string> errors;
vector<BSONObj> errorObjects;
for ( set<string>::iterator i = shards->begin(); i != shards->end(); i++ ){
string theShard = *i;
bbb.append( theShard );
ShardConnection conn( theShard , "" );
BSONObj res;
bool ok = conn->runCommand( dbName , cmdObj , res );
addWriteBack( writebacks, res );
string temp = DBClientWithCommands::getLastErrorString( res );
if ( ok == false || temp.size() ){
errors.push_back( temp );
errorObjects.push_back( res );
}
n += res["n"].numberLong();
conn.done();
}
bbb.done();
result.appendNumber( "n" , n );
// hit other machines just to block
for ( set<string>::const_iterator i=client->sinceLastGetError().begin(); i!=client->sinceLastGetError().end(); ++i ){
string temp = *i;
if ( shards->count( temp ) )
continue;
ShardConnection conn( temp , "" );
addWriteBack( writebacks, conn->getLastErrorDetailed() );
conn.done();
}
client->clearSinceLastGetError();
if ( errors.size() == 0 ){
result.appendNull( "err" );
handleWriteBacks( writebacks );
return true;
}
result.append( "err" , errors[0].c_str() );
{ // errs
BSONArrayBuilder all( result.subarrayStart( "errs" ) );
for ( unsigned i=0; i<errors.size(); i++ ){
all.append( errors[i].c_str() );
}
all.done();
}
{ // errObjects
BSONArrayBuilder all( result.subarrayStart( "errObjects" ) );
for ( unsigned i=0; i<errorObjects.size(); i++ ){
all.append( errorObjects[i] );
}
all.done();
}
handleWriteBacks( writebacks );
return true;
}
