bool
do_command(atrt_config& config){
#ifdef _WIN32
return true;
#endif
MYSQL* mysql= find_atrtdb_client(config);
if (!mysql)
return true;
AtrtClient atrtdb(mysql);
SqlResultSet command;
if (!atrtdb.doQuery("SELECT * FROM command " \
"WHERE state = 'new' ORDER BY id LIMIT 1", command)){
g_logger.critical("query failed");
return false;
}
if (command.numRows() == 0)
return true;
uint id= command.columnAsInt("id");
uint cmd= command.columnAsInt("cmd");
g_logger.info("Got command, id: %d, cmd: %d", id, cmd);
// command.print();
// Set state of command to running
if (!ack_command(atrtdb, id, "running"))
return false;
switch (cmd){
case AtrtClient::ATCT_CHANGE_VERSION:
if (!do_change_version(config, command, atrtdb))
return false;
break;
case AtrtClient::ATCT_RESET_PROC:
if (!do_reset_proc(config, command, atrtdb))
return false;
break;
default:
command.print();
g_logger.error("got unknown command: %d", cmd);
return false;
}
// Set state of command to done
if (!ack_command(atrtdb, id, "done"))
return false;
g_logger.info("done!");
return true;
}
int
verifySlave(BaseString& sqlStm, BaseString& db, BaseString& column)
{
SqlResultSet result;
float masterSum;
float slaveSum;
//Create SQL Objects
DbUtil master(db.c_str());
DbUtil slave(db.c_str(), ".1.slave");
if(!syncSlaveWithMaster())
{
g_err << "Verify Slave -> Syncing with slave failed"
<< endl;
return NDBT_FAILED;
}
//Login to Master
if (!master.connect())
{
g_err << "Verify Slave -> connect master failed"
<< endl;
return NDBT_FAILED;
}
if(!master.doQuery(sqlStm.c_str(),result))
{
return NDBT_FAILED;
}
masterSum = result.columnAsInt(column.c_str());
//Login to slave
if (!slave.connect())
{
return NDBT_FAILED;
}
if(!slave.doQuery(sqlStm.c_str(),result))
{
return NDBT_FAILED;
}
slaveSum = result.columnAsInt(column.c_str());
if(masterSum != slaveSum)
{
g_err << "VerifySlave -> masterSum != slaveSum..." << endl;
return NDBT_FAILED;
}
return NDBT_OK;
}
int runCheckStarted(NDBT_Context* ctx, NDBT_Step* step) {
// Check cluster is started
NdbRestarter restarter;
if(restarter.waitClusterStarted() != 0) {
g_err << "All nodes was not started " << endl;
return NDBT_FAILED;
}
// Check atrtclient is started
AtrtClient atrt;
if(!atrt.waitConnected()) {
g_err << "atrt server was not started " << endl;
return NDBT_FAILED;
}
// Make sure atrt assigns nodeid != -1
SqlResultSet procs;
if (!atrt.doQuery("SELECT * FROM process where type <> \'mysql\'", procs))
return NDBT_FAILED;
while (procs.next())
{
if (procs.columnAsInt("node_id") == (unsigned)-1) {
ndbout << "Found one process with node_id -1, "
<< "use --fix-nodeid=1 to atrt to fix this" << endl;
return NDBT_FAILED;
}
}
return NDBT_OK;
}
int
rollingRestart(NDBT_Context* ctx, NDBT_Step* step)
{
// Assuming 2 replicas
AtrtClient atrt;
SqlResultSet clusters;
if (!atrt.getClusters(clusters))
return NDBT_FAILED;
while (clusters.next())
{
uint clusterId= clusters.columnAsInt("id");
SqlResultSet tmp_result;
if (!atrt.getConnectString(clusterId, tmp_result))
return NDBT_FAILED;
NdbRestarter restarter(tmp_result.column("connectstring"));
if (restarter.rollingRestart())
return NDBT_FAILED;
}
return NDBT_OK;
}
unsigned long
DbUtil::selectCountTable(const char * table)
{
BaseString query;
SqlResultSet result;
query.assfmt("select count(*) as count from %s", table);
if (!doQuery(query, result)) {
printError("select count(*) failed");
return -1;
}
return result.columnAsInt("count");
}
static
bool
do_reset_proc(atrt_config& config, SqlResultSet& command,
AtrtClient& atrtdb){
uint process_id= command.columnAsInt("process_id");
g_logger.info("Reset process: %d", process_id);
// Get the process
if (process_id > config.m_processes.size()){
g_logger.critical("Invalid process id %d", process_id);
return false;
}
atrt_process& proc= *config.m_processes[process_id];
g_logger.info("stopping process...");
if (!stop_process(proc))
return false;
if (proc.m_save.m_saved)
{
ndbout << "before: " << proc << endl;
proc.m_proc= proc.m_save.m_proc;
proc.m_save.m_saved= false;
proc.m_proc.m_id= -1;
ndbout << "after: " << proc << endl;
}
else
{
ndbout << "process has not changed" << endl;
}
g_logger.info("starting process...");
if (!start_process(proc))
return false;
return true;
}
int runTestDbUtil(NDBT_Context* ctx, NDBT_Step* step){
DbUtil sql;
{
// Select all rows from mysql.user
SqlResultSet result;
if (!sql.doQuery("SELECT * FROM mysql.user", result))
return NDBT_FAILED;
// result.print();
while(result.next())
{
ndbout << result.column("host") << ", "
<< result.column("uSer") << ", "
<< result.columnAsInt("max_updates") << ", "
<< endl;
}
result.reset();
while(result.next())
{
ndbout << result.column("host") << endl;
}
}
{
// No column name, query should fail
Properties args;
SqlResultSet result;
if (sql.doQuery("SELECT * FROM mysql.user WHERE name=?", args, result))
return NDBT_FAILED;
result.print();
}
{
// Select nonexisiting rows from mysql.user
Properties args;
SqlResultSet result;
args.put("0", "no_such_host");
if (!sql.doQuery("SELECT * FROM mysql.user WHERE host=?", args, result))
return NDBT_FAILED;
ndbout << "no rows" << endl;
result.print();
// Change args to an find one row
args.clear();
args.put("0", "localhost");
if (!sql.doQuery("SELECT host, user FROM mysql.user WHERE host=?",
args, result))
return NDBT_FAILED;
result.print();
}
{
if (!sql.doQuery("CREATE TABLE sql_client_test (a int, b varchar(255))"))
return NDBT_FAILED;
if (!sql.doQuery("INSERT INTO sql_client_test VALUES(1, 'hello'), (2, 'bye')"))
return NDBT_FAILED;
// Select all rows from sql_client_test
SqlResultSet result;
if (!sql.doQuery("SELECT * FROM sql_client_test", result))
return NDBT_FAILED;
// result.print();
while(result.next())
{
}
// Select second row from sql_client_test
Properties args;
args.put("0", 2);
if (!sql.doQuery("SELECT * FROM sql_client_test WHERE a=?", args,result))
return NDBT_FAILED;
result.print();
result.reset();
while(result.next())
{
ndbout << "a: " << result.columnAsInt("a") << endl;
ndbout << "b: " << result.column("b") << endl;
if (result.columnAsInt("a") != 2){
ndbout << "hepp1" << endl;
return NDBT_FAILED;
}
if (strcmp(result.column("b"), "bye")){
ndbout << "hepp2" << endl;
return NDBT_FAILED;
}
}
if (sql.selectCountTable("sql_client_test") != 2)
{
ndbout << "Got wrong count" << endl;
return NDBT_FAILED;
}
if (!sql.doQuery("DROP TABLE sql_client_test"))
return NDBT_FAILED;
}
return NDBT_OK;
}
static
bool
do_change_version(atrt_config& config, SqlResultSet& command,
AtrtClient& atrtdb){
/**
* TODO make option to restart "not" initial
*/
uint process_id= command.columnAsInt("process_id");
const char* process_args= command.column("process_args");
g_logger.info("Change version for process: %d, args: %s",
process_id, process_args);
// Get the process
if (process_id > config.m_processes.size()){
g_logger.critical("Invalid process id %d", process_id);
return false;
}
atrt_process& proc= *config.m_processes[process_id];
const char* new_prefix= g_prefix1 ? g_prefix1 : g_prefix;
const char* old_prefix= g_prefix;
const char *start= strstr(proc.m_proc.m_path.c_str(), old_prefix);
if (!start){
/* Process path does not contain old prefix.
* Perhaps it contains the new prefix - e.g. is already
* upgraded?
*/
if (strstr(proc.m_proc.m_path.c_str(), new_prefix))
{
/* Process is already upgraded, *assume* that this
* is ok
* Alternatives could be - error, or downgrade.
*/
g_logger.info("Process already upgraded");
return true;
}
g_logger.critical("Could not find '%s' in '%s'",
old_prefix, proc.m_proc.m_path.c_str());
return false;
}
// Save current proc state
if (proc.m_save.m_saved == false)
{
proc.m_save.m_proc= proc.m_proc;
proc.m_save.m_saved= true;
}
g_logger.info("stopping process...");
if (!stop_process(proc))
return false;
BaseString newEnv = set_env_var(proc.m_proc.m_env,
BaseString("MYSQL_BASE_DIR"),
BaseString(new_prefix));
proc.m_proc.m_env.assign(newEnv);
BaseString suffix(proc.m_proc.m_path.substr(strlen(old_prefix)));
proc.m_proc.m_path.assign(new_prefix).append(suffix);
if (process_args && strlen(process_args))
{
/* Beware too long args */
proc.m_proc.m_args.append(" ");
proc.m_proc.m_args.append(process_args);
}
ndbout << proc << endl;
g_logger.info("starting process...");
if (!start_process(proc))
return false;
return true;
}
static
bool
do_change_version(atrt_config& config, SqlResultSet& command,
AtrtClient& atrtdb){
/**
* TODO make option to restart "not" initial
*/
uint process_id= command.columnAsInt("process_id");
const char* process_args= command.column("process_args");
g_logger.info("Change version for process: %d, args: %s",
process_id, process_args);
// Get the process
if (process_id > config.m_processes.size()){
g_logger.critical("Invalid process id %d", process_id);
return false;
}
atrt_process& proc= *config.m_processes[process_id];
const char* new_prefix= g_prefix1 ? g_prefix1 : g_prefix;
const char* old_prefix= g_prefix;
const char *start= strstr(proc.m_proc.m_path.c_str(), old_prefix);
if (!start){
/* Process path does not contain old prefix.
* Perhaps it contains the new prefix - e.g. is already
* upgraded?
*/
if (strstr(proc.m_proc.m_path.c_str(), new_prefix))
{
/* Process is already upgraded, *assume* that this
* is ok
* Alternatives could be - error, or downgrade.
*/
g_logger.info("Process already upgraded");
return true;
}
g_logger.critical("Could not find '%s' in '%s'",
old_prefix, proc.m_proc.m_path.c_str());
return false;
}
// Save current proc state
if (proc.m_save.m_saved == false)
{
proc.m_save.m_proc= proc.m_proc;
proc.m_save.m_saved= true;
}
g_logger.info("stopping process...");
if (!stop_process(proc))
return false;
BaseString newEnv = set_env_var(proc.m_proc.m_env,
BaseString("MYSQL_BASE_DIR"),
BaseString(new_prefix));
proc.m_proc.m_env.assign(newEnv);
ssize_t pos = proc.m_proc.m_path.lastIndexOf('/') + 1;
BaseString exename(proc.m_proc.m_path.substr(pos));
char * exe = find_bin_path(new_prefix, exename.c_str());
proc.m_proc.m_path = exe;
if (exe)
{
free(exe);
}
if (process_args && strlen(process_args))
{
/* Beware too long args */
proc.m_proc.m_args.append(" ");
proc.m_proc.m_args.append(process_args);
}
{
/**
* In 5.5...binaries aren't compiled with rpath
* So we need an explicit LD_LIBRARY_PATH
* So when upgrading..we need to change LD_LIBRARY_PATH
* So I hate 5.5...
*/
#if defined(__MACH__)
ssize_t p0 = proc.m_proc.m_env.indexOf(" DYLD_LIBRARY_PATH=");
#else
ssize_t p0 = proc.m_proc.m_env.indexOf(" LD_LIBRARY_PATH=");
#endif
ssize_t p1 = proc.m_proc.m_env.indexOf(' ', p0 + 1);
BaseString part0 = proc.m_proc.m_env.substr(0, p0);
BaseString part1 = proc.m_proc.m_env.substr(p1);
proc.m_proc.m_env.assfmt("%s%s",
part0.c_str(),
part1.c_str());
BaseString lib(g_libmysqlclient_so_path);
ssize_t pos = lib.lastIndexOf('/') + 1;
BaseString libname(lib.substr(pos));
char * exe = find_bin_path(new_prefix, libname.c_str());
char * dir = dirname(exe);
#if defined(__MACH__)
proc.m_proc.m_env.appfmt(" DYLD_LIBRARY_PATH=%s", dir);
#else
proc.m_proc.m_env.appfmt(" LD_LIBRARY_PATH=%s", dir);
#endif
free(exe);
free(dir);
}
ndbout << proc << endl;
g_logger.info("starting process...");
if (!start_process(proc))
return false;
return true;
}
static
int
runUpgrade_Half(NDBT_Context* ctx, NDBT_Step* step)
{
// Assuming 2 replicas
AtrtClient atrt;
const bool waitNode = ctx->getProperty("WaitNode", Uint32(0)) != 0;
const bool event = ctx->getProperty("CreateDropEvent", Uint32(0)) != 0;
const char * args = "";
if (ctx->getProperty("KeepFS", Uint32(0)) != 0)
{
args = "--initial=0";
}
NodeSet mgmdNodeSet = (NodeSet) ctx->getProperty("MgmdNodeSet", Uint32(0));
NodeSet ndbdNodeSet = (NodeSet) ctx->getProperty("NdbdNodeSet", Uint32(0));
SqlResultSet clusters;
if (!atrt.getClusters(clusters))
return NDBT_FAILED;
while (clusters.next())
{
uint clusterId= clusters.columnAsInt("id");
SqlResultSet tmp_result;
if (!atrt.getConnectString(clusterId, tmp_result))
return NDBT_FAILED;
NdbRestarter restarter(tmp_result.column("connectstring"));
restarter.setReconnect(true); // Restarting mgmd
g_err << "Cluster '" << clusters.column("name")
<< "@" << tmp_result.column("connectstring") << "'" << endl;
if(restarter.waitClusterStarted())
return NDBT_FAILED;
// Restart ndb_mgmd(s)
SqlResultSet mgmds;
if (!atrt.getMgmds(clusterId, mgmds))
return NDBT_FAILED;
uint mgmdCount = mgmds.numRows();
uint restartCount = getNodeCount(mgmdNodeSet, mgmdCount);
ndbout << "Restarting "
<< restartCount << " of " << mgmdCount
<< " mgmds" << endl;
while (mgmds.next() && restartCount --)
{
ndbout << "Restart mgmd" << mgmds.columnAsInt("node_id") << endl;
if (!atrt.changeVersion(mgmds.columnAsInt("id"), ""))
return NDBT_FAILED;
if(restarter.waitConnected())
return NDBT_FAILED;
}
NdbSleep_SecSleep(5); // TODO, handle arbitration
// Restart one ndbd in each node group
SqlResultSet ndbds;
if (!atrt.getNdbds(clusterId, ndbds))
return NDBT_FAILED;
Vector<NodeInfo> nodes;
while (ndbds.next())
{
struct NodeInfo n;
n.nodeId = ndbds.columnAsInt("node_id");
n.processId = ndbds.columnAsInt("id");
n.nodeGroup = restarter.getNodeGroup(n.nodeId);
nodes.push_back(n);
}
uint ndbdCount = ndbds.numRows();
restartCount = getNodeCount(ndbdNodeSet, ndbdCount);
ndbout << "Restarting "
<< restartCount << " of " << ndbdCount
<< " ndbds" << endl;
int nodesarray[256];
int cnt= 0;
Bitmask<4> seen_groups;
Bitmask<4> restarted_nodes;
for (Uint32 i = 0; (i<nodes.size() && restartCount); i++)
{
int nodeId = nodes[i].nodeId;
int processId = nodes[i].processId;
int nodeGroup= nodes[i].nodeGroup;
if (seen_groups.get(nodeGroup))
{
// One node in this node group already down
continue;
}
seen_groups.set(nodeGroup);
restarted_nodes.set(nodeId);
ndbout << "Restart node " << nodeId << endl;
if (!atrt.changeVersion(processId, args))
return NDBT_FAILED;
if (waitNode)
{
restarter.waitNodesNoStart(&nodeId, 1);
}
nodesarray[cnt++]= nodeId;
restartCount--;
}
if (!waitNode)
{
if (restarter.waitNodesNoStart(nodesarray, cnt))
return NDBT_FAILED;
}
ndbout << "Starting and wait for started..." << endl;
if (restarter.startAll())
return NDBT_FAILED;
if (restarter.waitClusterStarted())
return NDBT_FAILED;
if (event && createDropEvent(ctx, step))
{
return NDBT_FAILED;
}
ndbout << "Half started" << endl;
if (ctx->getProperty("HalfStartedHold", (Uint32)0) != 0)
{
while (ctx->getProperty("HalfStartedHold", (Uint32)0) != 0)
{
ndbout << "Half started holding..." << endl;
ctx->setProperty("HalfStartedDone", (Uint32)1);
NdbSleep_SecSleep(30);
}
ndbout << "Got half started continue..." << endl;
}
// Restart the remaining nodes
cnt= 0;
for (Uint32 i = 0; (i<nodes.size() && restartCount); i++)
{
int nodeId = nodes[i].nodeId;
int processId = nodes[i].processId;
if (restarted_nodes.get(nodeId))
continue;
ndbout << "Restart node " << nodeId << endl;
if (!atrt.changeVersion(processId, args))
return NDBT_FAILED;
if (waitNode)
{
restarter.waitNodesNoStart(&nodeId, 1);
}
nodesarray[cnt++]= nodeId;
restartCount --;
}
if (!waitNode)
{
if (restarter.waitNodesNoStart(nodesarray, cnt))
return NDBT_FAILED;
}
ndbout << "Starting and wait for started..." << endl;
if (restarter.startAll())
return NDBT_FAILED;
if (restarter.waitClusterStarted())
return NDBT_FAILED;
if (event && createDropEvent(ctx, step))
{
return NDBT_FAILED;
}
}
return NDBT_OK;
}
int runUpgrade_NR1(NDBT_Context* ctx, NDBT_Step* step) {
AtrtClient atrt;
NodeSet mgmdNodeSet = (NodeSet) ctx->getProperty("MgmdNodeSet", Uint32(0));
NodeSet ndbdNodeSet = (NodeSet) ctx->getProperty("NdbdNodeSet", Uint32(0));
SqlResultSet clusters;
if (!atrt.getClusters(clusters))
return NDBT_FAILED;
while (clusters.next())
{
uint clusterId= clusters.columnAsInt("id");
SqlResultSet tmp_result;
if (!atrt.getConnectString(clusterId, tmp_result))
return NDBT_FAILED;
NdbRestarter restarter(tmp_result.column("connectstring"));
restarter.setReconnect(true); // Restarting mgmd
g_err << "Cluster '" << clusters.column("name")
<< "@" << tmp_result.column("connectstring") << "'" << endl;
if (restarter.waitClusterStarted())
return NDBT_FAILED;
// Restart ndb_mgmd(s)
SqlResultSet mgmds;
if (!atrt.getMgmds(clusterId, mgmds))
return NDBT_FAILED;
uint mgmdCount = mgmds.numRows();
uint restartCount = getNodeCount(mgmdNodeSet, mgmdCount);
ndbout << "Restarting "
<< restartCount << " of " << mgmdCount
<< " mgmds" << endl;
while (mgmds.next() && restartCount --)
{
ndbout << "Restart mgmd " << mgmds.columnAsInt("node_id") << endl;
if (!atrt.changeVersion(mgmds.columnAsInt("id"), ""))
return NDBT_FAILED;
if (restarter.waitConnected())
return NDBT_FAILED;
ndbout << "Connected to mgmd"<< endl;
}
ndbout << "Waiting for started"<< endl;
if (restarter.waitClusterStarted())
return NDBT_FAILED;
ndbout << "Started"<< endl;
// Restart ndbd(s)
SqlResultSet ndbds;
if (!atrt.getNdbds(clusterId, ndbds))
return NDBT_FAILED;
uint ndbdCount = ndbds.numRows();
restartCount = getNodeCount(ndbdNodeSet, ndbdCount);
ndbout << "Restarting "
<< restartCount << " of " << ndbdCount
<< " ndbds" << endl;
while(ndbds.next() && restartCount --)
{
int nodeId = ndbds.columnAsInt("node_id");
int processId = ndbds.columnAsInt("id");
ndbout << "Restart node " << nodeId << endl;
if (!atrt.changeVersion(processId, ""))
return NDBT_FAILED;
if (restarter.waitNodesNoStart(&nodeId, 1))
return NDBT_FAILED;
if (restarter.startNodes(&nodeId, 1))
return NDBT_FAILED;
if (restarter.waitNodesStarted(&nodeId, 1))
return NDBT_FAILED;
if (createDropEvent(ctx, step))
return NDBT_FAILED;
}
}
ctx->stopTest();
return NDBT_OK;
}
static
int
runUpgrade_SR(NDBT_Context* ctx, NDBT_Step* step)
{
/* System restart upgrade.
* Stop all data nodes
* Change versions
* Restart em together.
*/
AtrtClient atrt;
NodeSet mgmdNodeSet = All;
const char * args = "";
bool skipMgmds = (ctx->getProperty("SkipMgmds", Uint32(0)) != 0);
SqlResultSet clusters;
if (!atrt.getClusters(clusters))
return NDBT_FAILED;
while (clusters.next())
{
uint clusterId= clusters.columnAsInt("id");
SqlResultSet tmp_result;
if (!atrt.getConnectString(clusterId, tmp_result))
return NDBT_FAILED;
NdbRestarter restarter(tmp_result.column("connectstring"));
restarter.setReconnect(true); // Restarting mgmd
g_err << "Cluster '" << clusters.column("name")
<< "@" << tmp_result.column("connectstring") << "'" << endl;
if(restarter.waitClusterStarted())
return NDBT_FAILED;
/* Now restart to nostart state, prior to SR */
g_err << "Restarting all data nodes-nostart" << endl;
if (restarter.restartAll2(NdbRestarter::NRRF_NOSTART) != 0)
{
g_err << "Failed to restart all" << endl;
return NDBT_FAILED;
}
ndbout << "Waiting for no-start state" << endl;
if (restarter.waitClusterNoStart() != 0)
{
g_err << "Failed waiting for NoStart state" << endl;
return NDBT_FAILED;
}
// Restart ndb_mgmd(s)
SqlResultSet mgmds;
if (!atrt.getMgmds(clusterId, mgmds))
return NDBT_FAILED;
uint mgmdCount = mgmds.numRows();
uint restartCount = getNodeCount(mgmdNodeSet, mgmdCount);
if (!skipMgmds)
{
ndbout << "Restarting "
<< restartCount << " of " << mgmdCount
<< " mgmds" << endl;
while (mgmds.next() && restartCount --)
{
ndbout << "Restart mgmd" << mgmds.columnAsInt("node_id") << endl;
if (!atrt.changeVersion(mgmds.columnAsInt("id"), ""))
return NDBT_FAILED;
if(restarter.waitConnected())
return NDBT_FAILED;
}
NdbSleep_SecSleep(5); // TODO, handle arbitration
}
else
{
ndbout << "Skipping MGMD upgrade" << endl;
}
// Restart all ndbds
SqlResultSet ndbds;
if (!atrt.getNdbds(clusterId, ndbds))
return NDBT_FAILED;
uint ndbdCount = ndbds.numRows();
restartCount = ndbdCount;
ndbout << "Upgrading "
<< restartCount << " of " << ndbdCount
<< " ndbds" << endl;
while (ndbds.next())
{
uint nodeId = ndbds.columnAsInt("node_id");
uint processId = ndbds.columnAsInt("id");
ndbout << "Upgrading node " << nodeId << endl;
if (!atrt.changeVersion(processId, args))
return NDBT_FAILED;
}
ndbout << "Waiting for no-start state" << endl;
if (restarter.waitClusterNoStart() != 0)
{
g_err << "Failed waiting for NoStart state" << endl;
return NDBT_FAILED;
}
ndbout << "Starting cluster (SR)" << endl;
if (restarter.restartAll2(0) != 0)
{
g_err << "Error restarting all nodes" << endl;
return NDBT_FAILED;
}
ndbout << "Waiting for cluster to start" << endl;
if (restarter.waitClusterStarted() != 0)
{
g_err << "Failed waiting for Cluster start" << endl;
return NDBT_FAILED;
}
ndbout << "Cluster started." << endl;
}
return NDBT_OK;
}
