static void nemaweaver_s_sdata (int ignore ATTRIBUTE_UNUSED)
{
#ifdef OBJ_ELF
obj_elf_change_section (".sdata", SHT_PROGBITS, SHF_ALLOC+SHF_WRITE, 0, 0, 0, 0);
#else
s_data (ignore);
#endif
}
/*
* auth_sock_callback - called when an event occurs on the socket
*/
static void auth_sock_callback(struct Event* ev)
{
struct AuthRequest* auth;
assert(0 != ev_socket(ev));
assert(0 != s_data(ev_socket(ev)));
auth = s_data(ev_socket(ev));
switch (ev_type(ev)) {
case ET_DESTROY: /* being destroyed */
auth->flags &= ~AM_SOCKET;
if (!(auth->flags & AM_FREE_MASK)) {
Debug((DEBUG_LIST, "Freeing auth from sock callback; %p [%p]", auth,
ev_socket(ev)));
MyFree(auth); /* done with it finally */
}
break;
case ET_CONNECT: /* socket connection completed */
Debug((DEBUG_LIST, "Connection completed for auth %p [%p]; sending query",
auth, ev_socket(ev)));
socket_state(&auth->socket, SS_CONNECTED);
send_auth_query(auth);
break;
case ET_READ: /* socket is readable */
case ET_EOF: /* end of file on socket */
case ET_ERROR: /* error on socket */
Debug((DEBUG_LIST, "Auth socket %p [%p] readable", auth, ev_socket(ev)));
read_auth_reply(auth);
break;
default:
#ifndef NDEBUG
abort(); /* unrecognized event */
#endif
break;
}
}
/** Callback for socket activity on an outbound uping socket.
* @param[in] ev I/O event for socket.
*/
static void uping_read_callback(struct Event* ev)
{
struct UPing *pptr;
assert(0 != ev_socket(ev));
assert(0 != s_data(ev_socket(ev)));
pptr = (struct UPing*) s_data(ev_socket(ev));
Debug((DEBUG_SEND, "uping_read_callback called, %p (%d)", pptr,
ev_type(ev)));
if (ev_type(ev) == ET_DESTROY) { /* being destroyed */
pptr->freeable &= ~UPING_PENDING_SOCKET;
if (!pptr->freeable)
MyFree(pptr); /* done with it, finally */
} else {
assert(ev_type(ev) == ET_READ || ev_type(ev) == ET_ERROR);
uping_read(pptr); /* read uping response */
}
}
nsresult nsNativeAudioStream::Write(const AudioDataValue* aBuf, uint32_t aFrames)
{
NS_ASSERTION(!mPaused, "Don't write audio when paused, you'll block");
if (mInError)
return NS_ERROR_FAILURE;
uint32_t samples = aFrames * mChannels;
nsAutoArrayPtr<short> s_data(new short[samples]);
float scaled_volume = float(GetVolumeScale() * mVolume);
ConvertAudioSamplesWithScale(aBuf, s_data.get(), samples, scaled_volume);
if (sa_stream_write(static_cast<sa_stream_t*>(mAudioHandle),
s_data.get(),
samples * sizeof(short)) != SA_SUCCESS)
{
PR_LOG(gAudioStreamLog, PR_LOG_ERROR, ("nsNativeAudioStream: sa_stream_write error"));
mInError = true;
return NS_ERROR_FAILURE;
}
return NS_OK;
}
static void
s_seg()
{
if (strncmp(input_line_pointer, "\"text\"", 6) == 0) {
input_line_pointer += 6;
s_text();
return;
}
if (strncmp(input_line_pointer, "\"data\"", 6) == 0) {
input_line_pointer += 6;
s_data();
return;
}
if (strncmp(input_line_pointer, "\"data1\"", 7) == 0) {
input_line_pointer += 7;
s_data1();
return;
}
as_warn("Unknown segment type");
demand_empty_rest_of_line();
return;
}
void bidiag_gkl_restart(
int locked, int l, int n,
CAX && Ax, CATX && Atx, CD && D, CE && E, CRho && rho, CP && P, CQ && Q, int s_indx, int t_s_indx) {
// enhancements version from SLEPc
const double eta = 1.e-10;
double t_start = 0.0, t_end = 0.0;
double local_start = 0.0, local_end = 0.0;
double t_total3 = 0.0, t_total4 = 0.0, t_total5 = 0.0, t_total6 = 0.0, t_total7 = 0.0;
int rank, nprocs;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
// Step 1
int recv_len = (int)P.dim0() * nprocs;
vec_container<double> tmp(Ax.dim0());
vec_container<double> recv_tmp(recv_len);
auto m_Ax = make_gemv_ax(&Ax);
auto m_Atx = make_gemv_ax(&Atx);
m_Ax(Q.col(l), tmp, P.dim0() > 1000);
vec_container<double> send_data(P.dim0(),0);
for(size_t i = s_indx; i < s_indx + Ax.dim0(); ++i)
send_data[i] = tmp.get(i-s_indx);
MPI_Gather(&send_data[0], P.dim0(), MPI_DOUBLE, &recv_tmp[0], P.dim0(), MPI_DOUBLE, 0, MPI_COMM_WORLD);
P.col(l) = 0;
// Generate truly P.col(l)
if(rank == 0) {
local_union(P, recv_tmp, l, nprocs);
// Step 2 & also in rank 0
for (int j = locked; j < l; ++j) {
P.col(l) += -rho(j) * P.col(j);
}
}
MPI_Bcast(&(P.col(0)[0]), P.size(), MPI_DOUBLE, 0, MPI_COMM_WORLD);
//MPI_Bcast(&(P.col(l)[0]), P.size(), MPI_DOUBLE, 0, MPI_COMM_WORLD);
// Main loop
vec_container<double> T(n);
int recv_l = Q.dim0() * nprocs;
vec_container<double> recv_t(recv_l);
for (int j = l; j < n; ++j) {
// Step 3
vec_container<double> tmp2(Atx.dim0());
/* for print */
if(rank == 0)
t_start = currenttime();
local_start = currenttime();
m_Atx(P.col(j), tmp2, Q.dim0() > 1000);
local_end = currenttime();
std::cout << "parallel mv time cost is " << (local_end - local_start) / 1.0e6 << std::endl;
vec_container<double> s_data(Q.dim0(), 0);
for(size_t i = t_s_indx; i < t_s_indx + Atx.dim0(); ++i)
s_data[i] = tmp2[i-t_s_indx];
MPI_Gather(&s_data[0], Q.dim0(), MPI_DOUBLE, &recv_t[0], Q.dim0(), MPI_DOUBLE, 0, MPI_COMM_WORLD);
local_start = currenttime();
std::cout << "parallel mv time cost2 is " << (local_start - local_end) / 1.0e6 << std::endl;
//Q.col(j+1) = 0;
if(rank == 0) {
// Generate truly Q.col(j+1)
local_union(Q, recv_t, j + 1, nprocs);
local_end = currenttime();
t_end = currenttime();
std::cout << "parallel mv time cost3 is " << (local_end - local_start) / 1.0e6 << std::endl;
std::cout << "time of step 3 is : " << (t_end - t_start) / 1.0e6 << std::endl;
t_total3 += (t_end - t_start) / 1.0e6;
}
// Step 4
for(size_t aa = 0; aa < Q.dim0(); ++aa) // row
MPI_Bcast(&(Q.row(aa)[0]), j + 2, MPI_DOUBLE, 0, MPI_COMM_WORLD);
// MPI_Bcast(&(Q.col(0)[0]), Q.size(), MPI_DOUBLE, 0, MPI_COMM_WORLD);
if(rank == 0)
t_end = currenttime();
auto Qj = mat_cols(Q, 0, j + 1);
auto Tj = make_vec(&T, j + 1);
//Tj.assign(gemv(Qj.trans(), Q.col(j + 1)), j >= 3);
parallel_gemv_task(Qj.trans(), Q.col(j+1), Tj);
if(rank == 0) {
t_start = currenttime();
t_total4 += (t_start - t_end) / 1.0e6;
std::cout << "time of step 4 is : " << (t_start - t_end) / 1.0e6 << std::endl;
}
// Step 5
if(rank == 0) {
double r = Q.col(j + 1).norm2();
D[j] = vec_unit(P.col(j));
Q.col(j + 1).scale(1. / D[j]);
Tj = Tj / D[j];
r /= D[j];
Q.col(j + 1).plus_assign(- gemv(Qj, Tj), Q.dim0() > 1000);
t_end = currenttime();
t_total5 += (t_end - t_start) / 1.0e6;
std::cout << "time of step 5 is : " << (t_end - t_start) / 1.0e6 << std::endl;
// Step 6
double beta = r * r - Tj.square_sum();
if (beta < eta * r * r) {
Tj.assign(gemv(Qj.trans(), Q.col(j + 1)), Q.dim0() > 1000);
r = Q.col(j + 1).square_sum();
Q.col(j + 1).plus_assign(-gemv(Qj, Tj), Q.dim0() > 1000);
beta = r * r - Tj.square_sum();
}
beta = std::sqrt(beta);
E[j] = beta;
Q.col(j + 1).scale(1. / E[j]);
t_start = currenttime();
t_total6 += (t_start - t_end) / 1.0e6;
std::cout << "time of step 6 is : " << (t_start - t_end) / 1.0e6 << std::endl;
}
// Step 7
// MPI_Bcast(&(Q.col(j+1)[0]), Q.size(), MPI_DOUBLE, 0, MPI_COMM_WORLD);
// MPI_Bcast(&(Q.col(0)[0]), Q.size(), MPI_DOUBLE, 0, MPI_COMM_WORLD);
for(size_t aa = 0; aa < Q.dim0(); ++aa)
MPI_Bcast(&(Q.col(j+1)[aa]), 1, MPI_DOUBLE, 0, MPI_COMM_WORLD);
if (j + 1 < n) {
if(rank == 0)
t_start = currenttime();
vec_container<double> tmp3(Ax.dim0());
vec_container<double> se_data(P.dim0(), 0);
m_Ax(Q.col(j + 1), tmp3, P.dim0() > 1000);
for(size_t k1 = s_indx; k1 < s_indx + Ax.dim0(); ++k1)
se_data[k1] = tmp3[k1-s_indx];
MPI_Gather(&se_data[0], P.dim0(), MPI_DOUBLE, &recv_tmp[0], P.dim0(), MPI_DOUBLE, 0, MPI_COMM_WORLD);
// P.col(j+1) = 0;
if(rank == 0) {
local_union(P, recv_tmp, j + 1, nprocs);
P.col(j + 1).plus_assign(- E[j] * P.col(j), P.dim0() > 1000);
}
/* for print */
if(rank == 0) {
t_end = currenttime();
t_total7 += (t_end - t_start) / 1.0e6;
std::cout << "time of step 7 is : " << (t_end - t_start) / 1.0e6 << std::endl;
}
// MPI_Bcast(&(P.col(l)[0]), P.size(), MPI_DOUBLE, 0, MPI_COMM_WORLD);
// MPI_Bcast(&(P.col(0)[0]), P.size(), MPI_DOUBLE, 0, MPI_COMM_WORLD);
for(size_t aa = 0; aa < P.dim0(); ++aa)
MPI_Bcast(&(P.col(j+1)[aa]), 1, MPI_DOUBLE, 0, MPI_COMM_WORLD);
} // end if
} // end while
/* for print time of each step. */
if(rank == 0) {
std::cout << "total step 3 time is : " << t_total3 << std::endl;
std::cout << "total step 4 time is : " << t_total4 << std::endl;
std::cout << "total step 5 time is : " << t_total5 << std::endl;
std::cout << "total step 6 time is : " << t_total6 << std::endl;
std::cout << "total step 7 time is : " << t_total7 << std::endl;
}
return ;
}
void LBFGSSolver::solve(const Function& function,
SolverResults* results) const
{
double global_start_time = wall_time();
// Dimension of problem.
size_t n = function.get_number_of_scalars();
if (n == 0) {
results->exit_condition = SolverResults::FUNCTION_TOLERANCE;
return;
}
// Current point, gradient and Hessian.
double fval = std::numeric_limits<double>::quiet_NaN();
double fprev = std::numeric_limits<double>::quiet_NaN();
double normg0 = std::numeric_limits<double>::quiet_NaN();
double normg = std::numeric_limits<double>::quiet_NaN();
double normdx = std::numeric_limits<double>::quiet_NaN();
Eigen::VectorXd x, g;
// Copy the user state to the current point.
function.copy_user_to_global(&x);
Eigen::VectorXd x2(n);
// L-BFGS history.
std::vector<Eigen::VectorXd> s_data(this->lbfgs_history_size),
y_data(this->lbfgs_history_size);
std::vector<Eigen::VectorXd*> s(this->lbfgs_history_size),
y(this->lbfgs_history_size);
for (int h = 0; h < this->lbfgs_history_size; ++h) {
s_data[h].resize(function.get_number_of_scalars());
s_data[h].setZero();
y_data[h].resize(function.get_number_of_scalars());
y_data[h].setZero();
s[h] = &s_data[h];
y[h] = &y_data[h];
}
Eigen::VectorXd rho(this->lbfgs_history_size);
rho.setZero();
Eigen::VectorXd alpha(this->lbfgs_history_size);
alpha.setZero();
Eigen::VectorXd q(n);
Eigen::VectorXd r(n);
// Needed from the previous iteration.
Eigen::VectorXd x_prev(n), s_tmp(n), y_tmp(n);
CheckExitConditionsCache exit_condition_cache;
//
// START MAIN ITERATION
//
results->startup_time += wall_time() - global_start_time;
results->exit_condition = SolverResults::INTERNAL_ERROR;
int iter = 0;
bool last_iteration_successful = true;
int number_of_line_search_failures = 0;
int number_of_restarts = 0;
while (true) {
//
// Evaluate function and derivatives.
//
double start_time = wall_time();
// y[0] should contain the difference between the gradient
// in this iteration and the gradient from the previous.
// Therefore, update y before and after evaluating the
// function.
if (iter > 0) {
y_tmp = -g;
}
fval = function.evaluate(x, &g);
normg = std::max(g.maxCoeff(), -g.minCoeff());
if (iter == 0) {
normg0 = normg;
}
results->function_evaluation_time += wall_time() - start_time;
//
// Update history
//
start_time = wall_time();
if (iter > 0 && last_iteration_successful) {
s_tmp = x - x_prev;
y_tmp += g;
double sTy = s_tmp.dot(y_tmp);
if (sTy > 1e-16) {
// Shift all pointers one step back, discarding the oldest one.
Eigen::VectorXd* sh = s[this->lbfgs_history_size - 1];
Eigen::VectorXd* yh = y[this->lbfgs_history_size - 1];
for (int h = this->lbfgs_history_size - 1; h >= 1; --h) {
s[h] = s[h - 1];
y[h] = y[h - 1];
rho[h] = rho[h - 1];
}
// Reuse the storage of the discarded data for the new data.
s[0] = sh;
y[0] = yh;
*y[0] = y_tmp;
*s[0] = s_tmp;
rho[0] = 1.0 / sTy;
}
}
results->lbfgs_update_time += wall_time() - start_time;
//
// Test stopping criteriea
//
start_time = wall_time();
if (iter > 1 && this->check_exit_conditions(fval, fprev, normg,
normg0, x.norm(), normdx,
last_iteration_successful,
&exit_condition_cache, results)) {
break;
}
if (iter >= this->maximum_iterations) {
results->exit_condition = SolverResults::NO_CONVERGENCE;
break;
}
if (this->callback_function) {
CallbackInformation information;
information.objective_value = fval;
information.x = &x;
information.g = &g;
if (!callback_function(information)) {
results->exit_condition = SolverResults::USER_ABORT;
break;
}
}
results->stopping_criteria_time += wall_time() - start_time;
//
// Compute search direction via L-BGFS two-loop recursion.
//
start_time = wall_time();
bool should_restart = false;
double H0 = 1.0;
if (iter > 0) {
// If the gradient is identical two iterations in a row,
// y will be the zero vector and H0 will be NaN. In this
// case the line search will fail and L-BFGS will be restarted
// with a steepest descent step.
H0 = s[0]->dot(*y[0]) / y[0]->dot(*y[0]);
// If isinf(H0) || isnan(H0)
if (H0 == std::numeric_limits<double>::infinity() ||
H0 == -std::numeric_limits<double>::infinity() ||
H0 != H0) {
should_restart = true;
}
}
q = -g;
for (int h = 0; h < this->lbfgs_history_size; ++h) {
alpha[h] = rho[h] * s[h]->dot(q);
q = q - alpha[h] * (*y[h]);
}
r = H0 * q;
for (int h = this->lbfgs_history_size - 1; h >= 0; --h) {
double beta = rho[h] * y[h]->dot(r);
r = r + (*s[h]) * (alpha[h] - beta);
}
// If the function improves very little, the approximated Hessian
// might be very bad. If this is the case, it is better to discard
// the history once in a while. This allows the solver to correctly
// solve some badly scaled problems.
double restart_test = std::fabs(fval - fprev) /
(std::fabs(fval) + std::fabs(fprev));
if (iter > 0 && iter % 100 == 0 && restart_test
< this->lbfgs_restart_tolerance) {
should_restart = true;
}
if (! last_iteration_successful) {
should_restart = true;
}
if (should_restart) {
if (this->log_function) {
char str[1024];
if (number_of_restarts <= 10) {
std::sprintf(str, "Restarting: fval = %.3e, deltaf = %.3e, max|g_i| = %.3e, test = %.3e",
fval, std::fabs(fval - fprev), normg, restart_test);
this->log_function(str);
}
if (number_of_restarts == 10) {
this->log_function("NOTE: No more restarts will be reported.");
}
number_of_restarts++;
}
r = -g;
for (int h = 0; h < this->lbfgs_history_size; ++h) {
(*s[h]).setZero();
(*y[h]).setZero();
}
rho.setZero();
alpha.setZero();
// H0 is not used, but its value will be printed.
H0 = std::numeric_limits<double>::quiet_NaN();
}
results->lbfgs_update_time += wall_time() - start_time;
//
// Perform line search.
//
start_time = wall_time();
double start_alpha = 1.0;
// In the first iteration, start with a much smaller step
// length. (heuristic used by e.g. minFunc)
if (iter == 0) {
double sumabsg = 0.0;
for (size_t i = 0; i < n; ++i) {
sumabsg += std::fabs(g[i]);
}
start_alpha = std::min(1.0, 1.0 / sumabsg);
}
double alpha_step = this->perform_linesearch(function, x, fval, g,
r, &x2, start_alpha);
if (alpha_step <= 0) {
if (this->log_function) {
this->log_function("Line search failed.");
char str[1024];
std::sprintf(str, "%4d %+.3e %9.3e %.3e %.3e %.3e %.3e",
iter, fval, std::fabs(fval - fprev), normg, alpha_step, H0, rho[0]);
this->log_function(str);
}
if (! last_iteration_successful || number_of_line_search_failures++ > 10) {
// This happens quite seldom. Every time it has happened, the function
// was actually converged to a solution.
results->exit_condition = SolverResults::GRADIENT_TOLERANCE;
break;
}
last_iteration_successful = false;
}
else {
// Record length of this step.
normdx = alpha_step * r.norm();
// Compute new point.
x_prev = x;
x = x + alpha_step * r;
last_iteration_successful = true;
}
results->backtracking_time += wall_time() - start_time;
//
// Log the results of this iteration.
//
start_time = wall_time();
int log_interval = 1;
if (iter > 30) {
log_interval = 10;
}
if (iter > 200) {
log_interval = 100;
}
if (iter > 2000) {
log_interval = 1000;
}
if (this->log_function && iter % log_interval == 0) {
if (iter == 0) {
this->log_function("Itr f deltaf max|g_i| alpha H0 rho");
}
this->log_function(
to_string(
std::setw(4), iter, " ",
std::setw(10), std::setprecision(3), std::scientific, std::showpos, fval, std::noshowpos, " ",
std::setw(9), std::setprecision(3), std::scientific, std::fabs(fval - fprev), " ",
std::setw(9), std::setprecision(3), std::setprecision(3), std::scientific, normg, " ",
std::setw(9), std::setprecision(3), std::scientific, alpha_step, " ",
std::setw(9), std::setprecision(3), std::scientific, H0, " ",
std::setw(9), std::setprecision(3), std::scientific, rho[0]
)
);
}
results->log_time += wall_time() - start_time;
fprev = fval;
iter++;
}
function.copy_global_to_user(x);
results->total_time += wall_time() - global_start_time;
if (this->log_function) {
char str[1024];
std::sprintf(str, " end %+.3e %.3e", fval, normg);
this->log_function(str);
}
}
/*-------------------------------------------------------------------------*/
void AzPrepText2::gen_regions_parsup(int argc, const char *argv[]) const
{
const char *eyec = "AzPrepText2::gen_regions_parsup";
AzPrepText2_gen_regions_parsup_Param p(argc, argv, out);
check_batch_id(p.s_batch_id);
AzMats_file<AzSmat> mfile;
int feat_data_num = mfile.reset_for_read(p.s_feat_fn.c_str());
AzStrPool sp_typ(10,10); sp_typ.put(kw_bow, kw_seq);
AzXi::check_input(p.s_xtyp.c_str(), &sp_typ, eyec, kw_xtyp);
bool do_xseq = p.s_xtyp.equals(kw_seq);
bool do_skip_stopunk = (do_xseq) ? false : true;
AzDic dic(p.s_xdic_fn.c_str());
AzX::throw_if((dic.size() <= 0), AzInputError, eyec, "No vocabulary");
/*--- scan files to determine buffer size and #data ---*/
AzOut noout;
AzStrPool sp_list;
AzIntArr ia_data_num;
int buff_size = AzTools_text::scan_files_in_list(p.s_inp_fn.c_str(), p.s_txt_ext.c_str(),
noout, &sp_list, &ia_data_num);
int data_num = ia_data_num.sum();
AzX::throw_if ((data_num != feat_data_num), eyec, "#data mismatch");
/*--- read data and generate features ---*/
AzDataArr<AzSmat> am_x(data_num), am_y(data_num);
buff_size += 256;
AzBytArr s_buff;
AzByte *buff = s_buff.reset(buff_size, 0);
int no_data = 0, data_no = 0, cnum = 0, cnum_before_reduce = 0;
feat_info fi[2];
for (int fx = 0; fx < sp_list.size(); ++fx) { /* for each file */
AzBytArr s_fn(sp_list.c_str(fx), p.s_txt_ext.c_str());
const char *fn = s_fn.c_str();
AzTimeLog::print(fn, log_out);
AzFile file(fn);
file.open("rb");
int num_in_file = ia_data_num.get(fx);
int inc = num_in_file / 50, milestone = inc;
int dx = 0;
for ( ; ; ++dx) { /* for each doc */
AzTools::check_milestone(milestone, dx, inc);
int len = file.gets(buff, buff_size);
if (len <= 0) break;
/*--- X ---*/
AzBytArr s_data(buff, len);
int my_len = s_data.length();
AzIntArr ia_tokno;
int nn = 1;
AzTools_text::tokenize(s_data.point_u(), my_len, &dic, nn, p.do_lower, p.do_utf8dashes, &ia_tokno);
AzIntArr ia_pos;
bool do_allow_zero = false;
if (do_xseq) gen_X_seq(ia_tokno, dic.size(), p.pch_sz, p.pch_step, p.padding,
do_allow_zero, do_skip_stopunk, am_x.point_u(data_no), &ia_pos);
else gen_X_bow(ia_tokno, dic.size(), p.pch_sz, p.pch_step, p.padding,
do_skip_stopunk, am_x.point_u(data_no), &ia_pos);
AzSmat m_feat;
mfile.read(&m_feat);
if (am_x.point(data_no)->colNum() <= 0) {
++no_data;
continue;
}
if (p.top_num_each > 0 || p.top_num_total > 0 || p.scale_y > 0) {
double min_ifeat = m_feat.min();
AzX::no_support((min_ifeat < 0), eyec, "Negative values for internal-feature components.");
}
/*--- Y (ifeat: internal features generated by a supervised model) ---*/
gen_Y_ifeat(p.top_num_each, p.top_num_total, &m_feat, &ia_tokno, &ia_pos,
p.pch_sz, -p.dist, p.dist, p.do_nolr,
p.f_pch_sz, p.f_pch_step, p.f_padding,
am_y.point_u(data_no), fi);
if (p.min_yval > 0) {
am_y.point_u(data_no)->cut(p.min_yval);
}
cnum_before_reduce += am_x.point(data_no)->colNum();
reduce_xy(p.min_x, p.min_y, am_x.point_u(data_no), am_y.point_u(data_no));
if (am_x.point(data_no)->colNum() <= 0) {
++no_data;
continue;
}
cnum += am_x.point(data_no)->colNum();
++data_no;
} /* for each doc */
AzTools::finish_milestone(milestone);
AzBytArr s(" #data="); s << data_no << " no_data=" << no_data << " #col=" << cnum;
AzPrint::writeln(out, s);
} /* for each file */
mfile.done();
AzBytArr s("#data="); s << data_no << " no_data=" << no_data << " #col=" << cnum << " #col_all=" << cnum_before_reduce;
AzPrint::writeln(out, s);
s.reset("all:"); fi[0].show(s); AzPrint::writeln(out, s);
s.reset("top:"); fi[1].show(s); AzPrint::writeln(out, s);
if (p.do_binarize) {
AzTimeLog::print("Binarizing Y ... ", log_out);
for (int dx = 0; dx < data_no; ++dx) am_y(dx)->binarize(); /* (x>0) ? 1 : (x<0) ? -1 : 0 */
}
else if (p.scale_y > 0) {
double max_top = fi[1].max_val;
double scale = 1;
if (max_top < p.scale_y) for ( ; ; scale *= 2) if (max_top*scale >= p.scale_y) break;
if (max_top > p.scale_y*2) for ( ; ; scale /= 2) if (max_top*scale <= p.scale_y*2) break;
s.reset("Multiplying Y with "); s << scale; AzPrint::writeln(out, s);
for (int dx = 0; dx < data_no; ++dx) am_y(dx)->multiply(scale);
}
const char *outnm = p.s_rnm.c_str();
AzTimeLog::print("Generating X ... ", out);
write_XY(am_x, data_no, p.s_batch_id, outnm, p.s_x_ext.c_str(), &dic, xtext_ext);
AzTimeLog::print("Generating Y ... ", out);
write_XY(am_y, data_no, p.s_batch_id, outnm, p.s_y_ext.c_str());
}
/* Note: X and Y use different dictionaries */
void AzPrepText2::gen_regions_unsup(int argc, const char *argv[]) const
{
const char *eyec = "AzPrepText2::gen_regions_unsup";
AzPrepText2_gen_regions_unsup_Param p(argc, argv, out);
check_batch_id(p.s_batch_id);
bool do_xseq = p.s_xtyp.equals(kw_seq);
bool do_skip_stopunk = (do_xseq) ? false : true;
AzDic ydic(p.s_ydic_fn.c_str());
int ydic_nn = ydic.get_max_n();
AzPrint::writeln(out, "y dic n=", ydic_nn);
AzX::throw_if((ydic.size() <= 0), AzInputError, eyec, "No Y (target) vocabulary.");
AzDic xdic(p.s_xdic_fn.c_str());
int xdic_nn = xdic.get_max_n();
AzPrint::writeln(out, "x dic n=", xdic_nn);
AzX::throw_if((xdic.size() <= 0), AzInputError, eyec, "No vocabulary.");
AzX::no_support((xdic_nn > 1 && do_xseq), eyec, "X with multi-word vocabulary and Seq option");
/*--- scan files to determine buffer size and #data ---*/
AzOut noout;
AzStrPool sp_list;
AzIntArr ia_data_num;
int buff_size = AzTools_text::scan_files_in_list(p.s_inp_fn.c_str(), p.s_txt_ext.c_str(),
noout, &sp_list, &ia_data_num);
int data_num = ia_data_num.sum();
/*--- read data and generate features ---*/
AzDataArr<AzSmat> am_x(data_num), am_y(data_num);
buff_size += 256;
AzBytArr s_buff;
AzByte *buff = s_buff.reset(buff_size, 0);
int no_data = 0, data_no = 0, cnum = 0, cnum_before_reduce = 0;
int l_dist = -p.dist, r_dist = p.dist;
AzIntArr ia_xnn; for (int ix = 1; ix <= xdic_nn; ++ix) ia_xnn.put(ix);
AzIntArr ia_ynn; for (int ix = 1; ix <= ydic_nn; ++ix) ia_ynn.put(ix);
for (int fx = 0; fx < sp_list.size(); ++fx) { /* for each file */
AzBytArr s_fn(sp_list.c_str(fx), p.s_txt_ext.c_str());
const char *fn = s_fn.c_str();
AzTimeLog::print(fn, out);
AzFile file(fn);
file.open("rb");
int num_in_file = ia_data_num.get(fx);
int inc = num_in_file / 50, milestone = inc;
int dx = 0;
for ( ; ; ++dx) { /* for each doc */
AzTools::check_milestone(milestone, dx, inc);
int len = file.gets(buff, buff_size);
if (len <= 0) break;
/*--- X ---*/
AzIntArr ia_pos;
AzBytArr s_data(buff, len);
int my_len = s_data.length();
bool do_allow_zero = false;
if (p.do_no_skip) {
do_allow_zero = true;
do_skip_stopunk = false;
}
int xtok_num = 0;
if (xdic_nn > 1) { /* n-grams */
AzDataArr<AzIntArr> aia_xtokno;
AzTools_text::tokenize(s_data.point_u(), my_len, &xdic, ia_xnn, p.do_lower, p.do_utf8dashes, aia_xtokno);
if (aia_xtokno.size() > 0) xtok_num = aia_xtokno[0]->size();
gen_X_ngram_bow(ia_xnn, aia_xtokno, xdic.size(), p.pch_sz, p.pch_step, p.padding, do_allow_zero,
am_x.point_u(data_no), &ia_pos);
}
else { /* words */
AzIntArr ia_xtokno;
int nn = 1;
AzTools_text::tokenize(s_data.point_u(), my_len, &xdic, nn, p.do_lower, p.do_utf8dashes, &ia_xtokno);
xtok_num = ia_xtokno.size();
if (do_xseq) gen_X_seq(ia_xtokno, xdic.size(), p.pch_sz, p.pch_step, p.padding,
do_allow_zero, do_skip_stopunk, am_x.point_u(data_no), &ia_pos);
else gen_X_bow(ia_xtokno, xdic.size(), p.pch_sz, p.pch_step, p.padding,
do_skip_stopunk, am_x.point_u(data_no), &ia_pos);
}
if (am_x.point(data_no)->colNum() <= 0) {
++no_data;
continue;
}
/*--- Y ---*/
s_data.reset(buff, len);
my_len = s_data.length();
if (ydic_nn > 1) { /* n-grams */
AzDataArr<AzIntArr> aia_ytokno;
AzTools_text::tokenize(s_data.point_u(), my_len, &ydic, ia_ynn, p.do_lower, p.do_utf8dashes, aia_ytokno);
int ytok_num = (aia_ytokno.size() > 0) ? aia_ytokno[0]->size() : 0;
AzX::throw_if((xtok_num != ytok_num), eyec, "conflict in the numbers of X tokens and Y tokens");
gen_Y_ngram_bow(ia_ynn, aia_ytokno, ydic.size(), ia_pos,
p.pch_sz, l_dist, r_dist, p.do_nolr, am_y.point_u(data_no));
}
else { /* words */
int nn = 1;
AzIntArr ia_ytokno;
AzTools_text::tokenize(s_data.point_u(), my_len, &ydic, nn, p.do_lower, p.do_utf8dashes, &ia_ytokno);
AzX::throw_if((xtok_num != ia_ytokno.size()), eyec, "conflict in the numbers of X tokens and Y tokens");
if (p.do_nolr) gen_Y_nolr(ia_ytokno, ydic.size(), ia_pos,
p.pch_sz, l_dist, r_dist, am_y.point_u(data_no));
else gen_Y(ia_ytokno, ydic.size(), ia_pos,
p.pch_sz, l_dist, r_dist, am_y.point_u(data_no));
}
cnum_before_reduce += am_x.point(data_no)->colNum();
reduce_xy(p.min_x, p.min_y, am_x.point_u(data_no), am_y.point_u(data_no));
if (am_x.point(data_no)->colNum() <= 0) {
++no_data;
continue;
}
cnum += am_x.point(data_no)->colNum();
++data_no;
} /* for each doc */
AzTools::finish_milestone(milestone);
AzBytArr s(" #data="); s << data_no << " no_data=" << no_data << " #col=" << cnum; AzPrint::writeln(out, s);
} /* for each file */
AzBytArr s("#data="); s << data_no << " no_data=" << no_data << " #col=" << cnum << " #col_all=" << cnum_before_reduce;
AzPrint::writeln(out, s);
const char *outnm = p.s_rnm.c_str();
AzTimeLog::print("Generating X ... ", out);
write_XY(am_x, data_no, p.s_batch_id, outnm, p.s_x_ext.c_str(), &xdic, xtext_ext);
AzTimeLog::print("Generating Y ... ", out);
write_XY(am_y, data_no, p.s_batch_id, outnm, p.s_y_ext.c_str(), &ydic, ytext_ext);
AzTimeLog::print("Done ... ", out);
}
int main (int argc, char ** argv) {
if (argc < 5) {
cerr << "Usage: " << endl;
cerr << "\tHDP_MEDOIDS [dataFile] [nRuns] [lambda_global] [lambda_local]" << endl;
exit(-1);
}
// PARSE arguments
char* dataFile = argv[1];
int nRuns = atoi(argv[2]);
vector<double> LAMBDAs (2, 0.0);
LAMBDAs[0] = atof(argv[3]); // lambda_global
LAMBDAs[1] = atof(argv[4]); // lambda_local
objmin_trace << "time objective" << endl;
// read in data
int FIX_DIM;
Parser parser;
vector<Instance*>* pdata;
vector<Instance*> data;
pdata = parser.parseSVM(dataFile, FIX_DIM);
data = *pdata;
// init lookup_tables
vector< pair<int,int> > doc_lookup;
get_doc_lookup (data, doc_lookup);
Lookups lookup_tables;
lookup_tables.doc_lookup = &doc_lookup;
lookup_tables.nWords = data.size();
lookup_tables.nDocs = lookup_tables.doc_lookup->size();
int seed = time(NULL);
srand (seed);
cerr << "###########################################" << endl;
cerr << "nDocs = " << lookup_tables.nDocs << endl; // # documents
cerr << "nWords = " << lookup_tables.nWords << endl; // # words
cerr << "lambda_global = " << LAMBDAs[0] << endl;
cerr << "lambda_local = " << LAMBDAs[1] << endl;
cerr << "TRIM_THRESHOLD = " << TRIM_THRESHOLD << endl;
cerr << "seed = " << seed << endl;
cerr << "###########################################" << endl;
// Run sparse convex clustering
int N = lookup_tables.nWords;
int D = lookup_tables.nDocs;
double** W = mat_init (N, N);
// dist_mat computation and output
dist_func df = L2norm;
double** dist_mat = mat_init (N, N);
compute_dist_mat (data, dist_mat, N, FIX_DIM, df, true);
start_time = omp_get_wtime();
double min_obj = INF;
vector< vector<double> > min_means;
for (int j = 0; j < nRuns; j ++) {
vector< vector<double> > means;
// inner-doc shuffle
for (int d = 0; d < D; d++) {
int begin_i = doc_lookup[d].first;
int end_i = doc_lookup[d].second;
random_shuffle(data.begin()+begin_i, data.begin()+end_i);
}
// between-doc shuffle
vector<pair<int,int> > s_doc_lookup (doc_lookup);
random_shuffle(s_doc_lookup.begin(), s_doc_lookup.end());
vector<Instance*> s_data (N, NULL);
int p = 0;
for (int d = 0; d < D; d ++) {
for (int i = s_doc_lookup[d].first; i < s_doc_lookup[d].second; i ++) {
s_data[p] = data[i];
p ++;
}
}
lookup_tables.doc_lookup = &s_doc_lookup;
double obj = HDP_MEDOIDS (s_data, means, &lookup_tables, LAMBDAs, df, FIX_DIM);
lookup_tables.doc_lookup = &doc_lookup;
cerr << "###################################################" << endl;
if (obj < min_obj) {
min_obj = obj;
min_means = means;
}
}
/* Output objective */
output_objective (min_obj);
ofstream model_out ("opt_model");
for (int i = 0; i < min_means.size(); i ++) {
model_out << "mean[" << i << "] ";
for (int j = 0; j < min_means[i].size(); j ++) {
model_out << min_means[i][j] << " " ;
}
model_out << endl;
}
model_out.close();
/* Output cluster centroids */
// output_model (W, &lookup_tables);
/* Output assignment */
// output_assignment (W, &lookup_tables);
/* reallocation */
mat_free (W, N, N);
mat_free (dist_mat, N, N);
objmin_trace.close();
}
/** Read a 'packet' of data from a connection and process it. Read in
* 8k chunks to give a better performance rating (for server
* connections). Do some tricky stuff for client connections to make
* sure they don't do any flooding >:-) -avalon
* @param cptr Client from which to read data.
* @param socket_ready If non-zero, more data can be read from the client's socket.
* @return Positive number on success, zero on connection-fatal failure, negative
* if user is killed.
*/
static int read_packet(struct Client *cptr, int socket_ready)
{
unsigned int dolen = 0;
unsigned int length = 0;
if (socket_ready &&
!(IsUser(cptr) &&
DBufLength(&(cli_recvQ(cptr))) > feature_uint(FEAT_CLIENT_FLOOD))) {
#if defined(USE_SSL)
switch (client_recv(cptr, readbuf, sizeof(readbuf), &length)) {
#else
switch (os_recv_nonb(cli_fd(cptr), readbuf, sizeof(readbuf), &length)) {
#endif
case IO_SUCCESS:
if (length)
{
cli_lasttime(cptr) = CurrentTime;
ClearPingSent(cptr);
ClrFlag(cptr, FLAG_NONL);
if (cli_lasttime(cptr) > cli_since(cptr))
cli_since(cptr) = cli_lasttime(cptr);
}
break;
case IO_BLOCKED:
break;
case IO_FAILURE:
cli_error(cptr) = errno;
/* SetFlag(cptr, FLAG_DEADSOCKET); */
return 0;
}
}
/*
* For server connections, we process as many as we can without
* worrying about the time of day or anything :)
*/
if (length > 0 && IsServer(cptr))
return server_dopacket(cptr, readbuf, length);
else if (length > 0 && (IsHandshake(cptr) || IsConnecting(cptr)))
return connect_dopacket(cptr, readbuf, length);
else
{
/*
* Before we even think of parsing what we just read, stick
* it on the end of the receive queue and do it when its
* turn comes around.
*/
if (length > 0 && dbuf_put(cptr, &(cli_recvQ(cptr)), readbuf, length) == 0)
return exit_client(cptr, cptr, &me, "dbuf_put fail");
if ((DBufLength(&(cli_recvQ(cptr))) > feature_uint(FEAT_CLIENT_FLOOD))
&& !IsChannelService(cptr))
return exit_client(cptr, cptr, &me, "Excess Flood");
while (DBufLength(&(cli_recvQ(cptr))) && !NoNewLine(cptr) &&
(IsTrusted(cptr) || IsChannelService(cptr) || cli_since(cptr) - CurrentTime < 10))
{
dolen = dbuf_getmsg(&(cli_recvQ(cptr)), cli_buffer(cptr), BUFSIZE);
/*
* Devious looking...whats it do ? well..if a client
* sends a *long* message without any CR or LF, then
* dbuf_getmsg fails and we pull it out using this
* loop which just gets the next 512 bytes and then
* deletes the rest of the buffer contents.
* -avalon
*/
if (dolen == 0)
{
if (DBufLength(&(cli_recvQ(cptr))) < 510)
SetFlag(cptr, FLAG_NONL);
else
{
/* More than 512 bytes in the line - drop the input and yell
* at the client.
*/
DBufClear(&(cli_recvQ(cptr)));
send_reply(cptr, ERR_INPUTTOOLONG);
}
}
else if (client_dopacket(cptr, dolen) == CPTR_KILLED)
return CPTR_KILLED;
/*
* If it has become registered as a Server
* then skip the per-message parsing below.
*/
if (IsHandshake(cptr) || IsServer(cptr))
{
while (-1)
{
dolen = dbuf_get(&(cli_recvQ(cptr)), readbuf, sizeof(readbuf));
if (dolen <= 0)
return 1;
else if (dolen == 0)
{
if (DBufLength(&(cli_recvQ(cptr))) < 510)
SetFlag(cptr, FLAG_NONL);
else {
DBufClear(&(cli_recvQ(cptr)));
/* send_reply(cptr, ERR_INPUTTOOLONG); */
}
}
else if ((IsServer(cptr) &&
server_dopacket(cptr, readbuf, dolen) == CPTR_KILLED) ||
(!IsServer(cptr) &&
connect_dopacket(cptr, readbuf, dolen) == CPTR_KILLED))
return CPTR_KILLED;
}
}
}
/* If there's still data to process, wait 2 seconds first */
if (DBufLength(&(cli_recvQ(cptr))) && !NoNewLine(cptr) &&
!t_onqueue(&(cli_proc(cptr))))
{
Debug((DEBUG_LIST, "Adding client process timer for %C", cptr));
cli_freeflag(cptr) |= FREEFLAG_TIMER;
timer_add(&(cli_proc(cptr)), client_timer_callback, cli_connect(cptr),
TT_RELATIVE, 2);
}
}
return 1;
}
/** Start a connection to another server.
* @param aconf Connect block data for target server.
* @param by Client who requested the connection (if any).
* @return Non-zero on success; zero on failure.
*/
int connect_server(struct ConfItem* aconf, struct Client* by)
{
struct Client* cptr = 0;
assert(0 != aconf);
if (aconf->dns_pending) {
sendto_opmask(0, SNO_OLDSNO, "Server %s connect DNS pending",
aconf->name);
return 0;
}
Debug((DEBUG_NOTICE, "Connect to %s[@%s]", aconf->name,
ircd_ntoa(&aconf->address.addr)));
if ((cptr = FindClient(aconf->name))) {
if (IsServer(cptr) || IsMe(cptr)) {
sendto_opmask(0, SNO_OLDSNO, "Server %s already present from %s",
aconf->name, cli_name(cli_from(cptr)));
if (by && IsUser(by) && !MyUser(by)) {
sendcmdto_one(&me, CMD_NOTICE, by, "%C :Server %s already present "
"from %s", by, aconf->name, cli_name(cli_from(cptr)));
}
return 0;
}
else if (IsHandshake(cptr) || IsConnecting(cptr)) {
if (by && IsUser(by)) {
sendcmdto_one(&me, CMD_NOTICE, by, "%C :Connection to %s already in "
"progress", by, cli_name(cptr));
}
return 0;
}
}
/*
* If we don't know the IP# for this host and it is a hostname and
* not a ip# string, then try and find the appropriate host record.
*/
if (!irc_in_addr_valid(&aconf->address.addr)
&& !ircd_aton(&aconf->address.addr, aconf->host)) {
char buf[HOSTLEN + 1];
host_from_uh(buf, aconf->host, HOSTLEN);
gethost_byname(buf, connect_dns_callback, aconf);
aconf->dns_pending = 1;
return 0;
}
cptr = make_client(NULL, STAT_UNKNOWN_SERVER);
/*
* Copy these in so we have something for error detection.
*/
ircd_strncpy(cli_name(cptr), aconf->name, HOSTLEN);
ircd_strncpy(cli_sockhost(cptr), aconf->host, HOSTLEN);
/*
* Attach config entries to client here rather than in
* completed_connection. This to avoid null pointer references
*/
attach_confs_byhost(cptr, aconf->host, CONF_SERVER);
if (!find_conf_byhost(cli_confs(cptr), aconf->host, CONF_SERVER)) {
sendto_opmask(0, SNO_OLDSNO, "Host %s is not enabled for "
"connecting: no Connect block", aconf->name);
if (by && IsUser(by) && !MyUser(by)) {
sendcmdto_one(&me, CMD_NOTICE, by, "%C :Connect to host %s failed: no "
"Connect block", by, aconf->name);
}
det_confs_butmask(cptr, 0);
free_client(cptr);
return 0;
}
/*
* attempt to connect to the server in the conf line
*/
if (!connect_inet(aconf, cptr)) {
if (by && IsUser(by) && !MyUser(by)) {
sendcmdto_one(&me, CMD_NOTICE, by, "%C :Couldn't connect to %s", by,
cli_name(cptr));
}
det_confs_butmask(cptr, 0);
free_client(cptr);
return 0;
}
/*
* NOTE: if we're here we have a valid C:Line and the client should
* have started the connection and stored the remote address/port and
* ip address name in itself
*
* The socket has been connected or connect is in progress.
*/
make_server(cptr);
if (by && IsUser(by)) {
ircd_snprintf(0, cli_serv(cptr)->by, sizeof(cli_serv(cptr)->by), "%s%s",
NumNick(by));
assert(0 == cli_serv(cptr)->user);
cli_serv(cptr)->user = cli_user(by);
cli_user(by)->refcnt++;
}
else {
*(cli_serv(cptr))->by = '\0';
/* strcpy(cptr->serv->by, "Auto"); */
}
cli_serv(cptr)->up = &me;
SetConnecting(cptr);
if (cli_fd(cptr) > HighestFd)
HighestFd = cli_fd(cptr);
LocalClientArray[cli_fd(cptr)] = cptr;
Count_newunknown(UserStats);
/* Actually we lie, the connect hasn't succeeded yet, but we have a valid
* cptr, so we register it now.
* Maybe these two calls should be merged.
*/
add_client_to_list(cptr);
hAddClient(cptr);
/* nextping = CurrentTime; */
return (s_state(&cli_socket(cptr)) == SS_CONNECTED) ?
completed_connection(cptr) : 1;
}
/** Find the real hostname for the host running the server (or one which
* matches the server's name) and its primary IP#. Hostname is stored
* in the client structure passed as a pointer.
*/
void init_server_identity(void)
{
const struct LocalConf* conf = conf_get_local();
assert(0 != conf);
ircd_strncpy(cli_name(&me), conf->name, HOSTLEN);
SetYXXServerName(&me, conf->numeric);
}
/** Process events on a client socket.
* @param ev Socket event structure that has a struct Connection as
* its associated data.
*/
static void client_sock_callback(struct Event* ev)
{
struct Client* cptr;
struct Connection* con;
char *fmt = "%s";
char *fallback = 0;
assert(0 != ev_socket(ev));
assert(0 != s_data(ev_socket(ev)));
con = (struct Connection*) s_data(ev_socket(ev));
assert(0 != con_client(con) || ev_type(ev) == ET_DESTROY);
cptr = con_client(con);
assert(0 == cptr || con == cli_connect(cptr));
switch (ev_type(ev)) {
case ET_DESTROY:
con_freeflag(con) &= ~FREEFLAG_SOCKET;
if (!con_freeflag(con) && !cptr)
free_connection(con);
#if defined(USE_SSL)
ssl_free(ev_socket(ev));
#endif
break;
case ET_CONNECT: /* socket connection completed */
if (!completed_connection(cptr) || IsDead(cptr))
fallback = cli_info(cptr);
break;
case ET_ERROR: /* an error occurred */
fallback = cli_info(cptr);
cli_error(cptr) = ev_data(ev);
/* If the OS told us we have a bad file descriptor, we should
* record that for future reference.
*/
if (cli_error(cptr) == EBADF)
cli_fd(cptr) = -1;
if (s_state(&(con_socket(con))) == SS_CONNECTING) {
completed_connection(cptr);
/* for some reason, the os_get_sockerr() in completed_connection()
* can return 0 even when ev_data(ev) indicates a real error, so
* re-assign the client error here.
*/
cli_error(cptr) = ev_data(ev);
break;
}
/*FALLTHROUGH*/
case ET_EOF: /* end of file on socket */
Debug((DEBUG_ERROR, "READ ERROR: fd = %d %d", cli_fd(cptr),
cli_error(cptr)));
SetFlag(cptr, FLAG_DEADSOCKET);
if ((IsServer(cptr) || IsHandshake(cptr)) && cli_error(cptr) == 0) {
exit_client_msg(cptr, cptr, &me, "Server %s closed the connection (%s)",
cli_name(cptr), cli_serv(cptr)->last_error_msg);
return;
} else {
fmt = "Read error: %s";
fallback = "EOF from client";
}
break;
case ET_WRITE: /* socket is writable */
ClrFlag(cptr, FLAG_BLOCKED);
if (cli_listing(cptr) && MsgQLength(&(cli_sendQ(cptr))) < 2048)
list_next_channels(cptr);
Debug((DEBUG_SEND, "Sending queued data to %C", cptr));
send_queued(cptr);
break;
case ET_READ: /* socket is readable */
if (!IsDead(cptr)) {
Debug((DEBUG_DEBUG, "Reading data from %C", cptr));
if (read_packet(cptr, 1) == 0) /* error while reading packet */
fallback = "EOF from client";
}
break;
default:
assert(0 && "Unrecognized socket event in client_sock_callback()");
break;
}
assert(0 == cptr || 0 == cli_connect(cptr) || con == cli_connect(cptr));
if (fallback) {
const char* msg = (cli_error(cptr)) ? strerror(cli_error(cptr)) : fallback;
if (!msg)
msg = "Unknown error";
exit_client_msg(cptr, cptr, &me, fmt, msg);
}
}
/** Process a timer on client socket.
* @param ev Timer event that has a struct Connection as its
* associated data.
*/
static void client_timer_callback(struct Event* ev)
{
struct Client* cptr;
struct Connection* con;
assert(0 != ev_timer(ev));
assert(0 != t_data(ev_timer(ev)));
assert(ET_DESTROY == ev_type(ev) || ET_EXPIRE == ev_type(ev));
con = (struct Connection*) t_data(ev_timer(ev));
assert(0 != con_client(con) || ev_type(ev) == ET_DESTROY);
cptr = con_client(con);
assert(0 == cptr || con == cli_connect(cptr));
if (ev_type(ev)== ET_DESTROY) {
con_freeflag(con) &= ~FREEFLAG_TIMER; /* timer has expired... */
if (!con_freeflag(con) && !cptr)
free_connection(con); /* client is being destroyed */
} else {
Debug((DEBUG_LIST, "Client process timer for %C expired; processing",
cptr));
read_packet(cptr, 0); /* read_packet will re-add timer if needed */
}
assert(0 == cptr || 0 == cli_connect(cptr) || con == cli_connect(cptr));
}
/** Process events on a client socket.
* @param ev Socket event structure that has a struct Connection as
* its associated data.
*/
static void client_sock_callback(struct Event* ev)
{
struct Client* cptr;
struct Connection* con;
char *fmt = "%s";
char *fallback = 0;
assert(0 != ev_socket(ev));
assert(0 != s_data(ev_socket(ev)));
con = (struct Connection*) s_data(ev_socket(ev));
assert(0 != con_client(con) || ev_type(ev) == ET_DESTROY);
cptr = con_client(con);
assert(0 == cptr || con == cli_connect(cptr));
switch (ev_type(ev)) {
case ET_DESTROY:
con_freeflag(con) &= ~FREEFLAG_SOCKET;
if (!con_freeflag(con) && !cptr)
free_connection(con);
break;
case ET_CONNECT: /* socket connection completed */
if (!completed_connection(cptr) || IsDead(cptr))
fallback = cli_info(cptr);
break;
case ET_ERROR: /* an error occurred */
fallback = cli_info(cptr);
cli_error(cptr) = ev_data(ev);
if (s_state(&(con_socket(con))) == SS_CONNECTING) {
completed_connection(cptr);
/* for some reason, the os_get_sockerr() in completed_connect()
* can return 0 even when ev_data(ev) indicates a real error, so
* re-assign the client error here.
*/
cli_error(cptr) = ev_data(ev);
break;
}
/*FALLTHROUGH*/
case ET_EOF: /* end of file on socket */
Debug((DEBUG_ERROR, "READ ERROR: fd = %d %d", cli_fd(cptr),
cli_error(cptr)));
SetFlag(cptr, FLAG_DEADSOCKET);
if ((IsServer(cptr) || IsHandshake(cptr)) && cli_error(cptr) == 0) {
exit_client_msg(cptr, cptr, &me, "Server %s closed the connection (%s)",
cli_name(cptr), cli_serv(cptr)->last_error_msg);
return;
} else {
fmt = "Read error: %s";
fallback = "EOF from client";
}
break;
case ET_WRITE: /* socket is writable */
ClrFlag(cptr, FLAG_BLOCKED);
if (cli_listing(cptr) && MsgQLength(&(cli_sendQ(cptr))) < 2048)
list_next_channels(cptr);
Debug((DEBUG_SEND, "Sending queued data to %C", cptr));
send_queued(cptr);
break;
case ET_READ: /* socket is readable */
if (!IsDead(cptr)) {
Debug((DEBUG_DEBUG, "Reading data from %C", cptr));
if (read_packet(cptr, 1) == 0) /* error while reading packet */
fallback = "EOF from client";
}
break;
default:
assert(0 && "Unrecognized socket event in client_sock_callback()");
break;
}
assert(0 == cptr || 0 == cli_connect(cptr) || con == cli_connect(cptr));
if (fallback) {
const char* msg = (cli_error(cptr)) ? strerror(cli_error(cptr)) : fallback;
if (!msg)
msg = "Unknown error";
exit_client_msg(cptr, cptr, &me, fmt, msg);
}
}
