void print_array(int myid)
{
int i, j, k;
double **buf;
int ii, jj;
int edge;
int ibs, jbs, skip;
buf = (double **)ARMCI_Malloc_local(nblocks*nblocks*sizeof(double *));
for(i=0; i<nblocks; i++)
for(j=0; j<nblocks; j++)
if(block_owner(i, j) == myid)
buf[i+j*nblocks] = a[i+j*nblocks];
else {
buf[i+j*nblocks] = (double *)ARMCI_Malloc_local(block_size*block_size*
sizeof(double));
get_remote(buf[i+j*nblocks], i, j);
}
/* copied from lu.C */
edge = n%block_size;
for (i=0; i<n; i++) {
for (j=0; j<n; j++) {
if ((n - i) <= edge) {
ibs = edge;
ibs = n-edge;
skip = edge;
} else {
ibs = block_size;
skip = block_size;
}
if ((n - j) <= edge) {
jbs = edge;
jbs = n-edge;
} else {
jbs = block_size;
}
ii = (i/block_size) + (j/block_size)*nblocks;
jj = (i%ibs)+(j%jbs)*skip;
printf("%8.1f ", buf[ii][jj]);
}
printf("\n");
}
fflush(stdout);
for(i=0; i<nblocks; i++)
for(j=0; j<nblocks; j++)
if(block_owner(i, j) != myid) ARMCI_Free_local(buf[i+j*nblocks]);
ARMCI_Free_local(buf);
}
void lu(int n, int bs, int me)
{
int i, il, j, jl, k, kl;
int I, J, K;
double *A, *B, *C, *D;
int dimI, dimJ, dimK;
int strI, strJ, strK;
unsigned int t1, t2, t3, t4, t11, t22;
int diagowner;
double *buf1, *buf2;
/* temporary memories */
buf1 = (double *)malloc(block_size*block_size*sizeof(double));
buf2 = (double *)malloc(block_size*block_size*sizeof(double));
for (k=0, K=0; k<n; k+=bs, K++) {
kl = k + bs;
if (kl > n) {
kl = n;
strK = kl - k;
} else {
strK = bs;
}
/* factor diagonal block */
diagowner = block_owner(K, K);
if (diagowner == me) {
A = a[K+K*nblocks];
lu0(A, strK, strK);
}
MP_BARRIER();
/* divide column k by diagonal block */
if(block_owner(K, K) == me)
D = a[K+K*nblocks];
else {
D = buf1;
get_remote(D, K, K);
}
for (i=kl, I=K+1; i<n; i+=bs, I++) {
if (block_owner(I, K) == me) { /* parcel out blocks */
il = i + bs;
if (il > n) {
il = n;
strI = il - i;
} else {
strI = bs;
}
A = a[I+K*nblocks];
bdiv(A, D, strI, strK, strI, strK);
}
}
/* modify row k by diagonal block */
for (j=kl, J=K+1; j<n; j+=bs, J++) {
if (block_owner(K, J) == me) { /* parcel out blocks */
jl = j+bs;
if (jl > n) {
jl = n;
strJ = jl - j;
} else {
strJ = bs;
}
A = a[K+J*nblocks];
bmodd(D, A, strK, strJ, strK, strK);
}
}
MP_BARRIER();
/* modify subsequent block columns */
for (i=kl, I=K+1; i<n; i+=bs, I++) {
il = i+bs;
if (il > n) {
il = n;
strI = il - i;
} else {
strI = bs;
}
if(block_owner(I,K) == me)
A = a[I+K*nblocks];
else {
A = buf1;
get_remote(A, I, K);
}
for (j=kl, J=K+1; j<n; j+=bs, J++) {
jl = j + bs;
if (jl > n) {
jl = n;
strJ= jl - j;
} else {
strJ = bs;
}
if (block_owner(I, J) == me) { /* parcel out blocks */
if(block_owner(K,J) == me)
B = a[K+J*nblocks];
else {
B = buf2;
get_remote(B, K, J);
}
C = a[I+J*nblocks];
bmod(A, B, C, strI, strJ, strK, strI, strK, strI);
}
}
}
}
free(buf1);
free(buf2);
}
/*
* Handles an incoming RPC connection on a socket. This method will fork(2)
* a copy of this program, if appropriate, for handling incoming RPC messages.
*
* sock The socket with the incoming RPC connection.
*/
static void handle_connection(
int sock)
{
struct sockaddr_in raddr;
socklen_t len;
int xp_sock;
pid_t pid;
SVCXPRT *xprt;
int status = 1; /* EXIT_FAILURE assumed unless one_svc_run() success */
peer_info* remote = get_remote();
again: len = sizeof(raddr);
(void) memset(&raddr, 0, len);
xp_sock = accept(sock, (struct sockaddr *) &raddr, &len);
(void) exitIfDone(0);
if (xp_sock < 0) {
if (errno == EINTR) {
errno = 0;
goto again;
}
/* else */
serror("accept");
return;
}
/*
* Don't bother continuing if no more clients are allowed.
*/
if (cps_count() >= maxClients) {
setremote(&raddr, xp_sock);
unotice("Denying connection from [%s] because too many clients",
remote->astr);
(void) close(xp_sock);
return;
}
pid = ldmfork();
if (pid == -1) {
log_add("Couldn't fork process to handle incoming connection");
log_log(LOG_ERR);
/* TODO: try again?*/
(void) close(xp_sock);
return;
}
if (pid > 0) {
/* parent */
/* unotice("child %d", pid); */
(void) close(xp_sock);
if (cps_add(pid))
serror("Couldn't add child PID to set");
return;
}
/* else child */
setremote(&raddr, xp_sock);
/* Access control */
if (!lcf_isHostOk(remote)) {
ensureRemoteName(&raddr);
if (!lcf_isHostOk(remote)) {
if (remote->printname == remote->astr) {
unotice("Denying connection from [%s] because not "
"allowed", remote->astr);
}
else {
unotice("Denying connection from \"%s\" because not "
"allowed", remote_name());
}
/*
* Try to tell the other guy.
* TODO: Why doesn't this work?
*/
xprt = svcfd_create(xp_sock, remote->sendsz, remote->recvsz);
if (xprt != NULL ) {
xprt->xp_raddr = raddr;
xprt->xp_addrlen = (int) len;
svcerr_weakauth(xprt);
svc_destroy(xprt);
}
goto unwind_sock;
}
}
/* else */
endpriv();
portIsMapped = 0; /* don't call pmap_unset() from child */
(void) close(sock);
/* Set the ulog identifier, optional. */
set_abbr_ident(remote_name(), NULL );
uinfo("Connection from %s", remote_name());
xprt = svcfd_create(xp_sock, remote->sendsz, remote->recvsz);
if (xprt == NULL ) {
uerror("Can't create fd service.");
goto unwind_sock;
}
/* hook up the remote address to the xprt. */
/* xprt->xp_raddr = raddr; */
xprt->xp_raddr = raddr;
xprt->xp_addrlen = (int) len;
if (!svc_register(xprt, LDMPROG, 4, ldmprog_4, 0)) {
uerror("unable to register LDM-4 service.");
svc_destroy(xprt);
goto unwind_sock;
}
if (!svc_register(xprt, LDMPROG, FIVE, ldmprog_5, 0)) {
uerror("unable to register LDM-5 service.");
svc_destroy(xprt);
goto unwind_sock;
}
if (!svc_register(xprt, LDMPROG, SIX, ldmprog_6, 0)) {
uerror("unable to register LDM-6 service.");
svc_destroy(xprt);
goto unwind_sock;
}
#if WANT_MULTICAST
if (!svc_register(xprt, LDMPROG, SEVEN, ldmprog_7, 0)) {
uerror("unable to register LDM-7 service.");
svc_destroy(xprt);
goto unwind_sock;
}
#endif
/*
* handle rpc requests
*/
{
const unsigned TIMEOUT = 2*interval;
status = one_svc_run(xp_sock, TIMEOUT);
(void) exitIfDone(0);
if (status == 0) {
log_add("Done");
log_log(LOG_INFO);
}
else if (status == ETIMEDOUT) {
log_add("Connection from client LDM silent for %u seconds",
TIMEOUT);
log_log(LOG_NOTICE);
}
else { /* connection to client lost */
log_add("Connection with client LDM closed");
log_log(LOG_INFO);
status = 0; /* EXIT_SUCCESS */
}
}
/* svc_destroy(xprt); done by svc_getreqset() */
unwind_sock: (void) close(xp_sock);
exit(status);
}
static int handle_key (int sock, struct allnet_header * hp,
char * data, int dsize, char * contact,
char * secret1, char * secret2, int max_hops)
{
#ifdef DEBUG_PRINT
printf ("in handle_key (%s, %s, %s)\n", contact, secret1, secret2);
#endif /* DEBUG_PRINT */
if ((contact == NULL) || (secret1 == NULL))
return 0;
if (hp->hops > max_hops)
return 0;
keyset * keys;
int nkeys = all_keys (contact, &keys);
if (nkeys < 1) {
printf ("error '%s'/%d: create own key before calling handle_key\n",
contact, nkeys);
return 0;
}
#ifdef DEBUG_PRINT
printf ("handle_key received key\n");
#endif /* DEBUG_PRINT */
unsigned char peer_addr [ADDRESS_SIZE];
int peer_bits;
int key_index = -1;
int i;
for (i = 0; i < nkeys; i++) {
allnet_rsa_pubkey key;
peer_bits = get_remote (keys [i], peer_addr);
if ((get_contact_pubkey (keys [i], &key) <= 0) &&
((peer_bits <= 0) ||
(matches (peer_addr, peer_bits, hp->source, hp->src_nbits) > 0))) {
#ifdef DEBUG_PRINT
printf ("handle_key matches at index %d/%d\n", i, nkeys);
#endif /* DEBUG_PRINT */
key_index = i;
break;
}
}
if (key_index < 0) {
if (nkeys <= 0)
printf ("handle_key error: contact '%s' has %d keys\n", contact, nkeys);
/* it is fairly normal to get multiple copies of the key. Ignore.
else
printf ("handle_key: contact '%s' has %d keys, no unmatched key found\n",
contact, nkeys);
*/
return 0;
}
char * received_key = data;
int ksize = dsize - SHA512_SIZE;
/* check to see if it is my own key */
for (i = 0; i < nkeys; i++) {
allnet_rsa_pubkey k;
get_my_pubkey (keys [i], &k);
char test_key [ALLNET_MTU];
int pub_ksize = allnet_pubkey_to_raw (k, test_key, sizeof (test_key));
if ((pub_ksize == ksize) && (memcmp (received_key, test_key, ksize) == 0)) {
/* it is fairly normal to get my own key back. Ignore.
*/
printf ("handle_key: got my own key\n");
return 0;
}
}
char * received_hmac = data + ksize;
char hmac [SHA512_SIZE];
sha512hmac (received_key, ksize, secret1, strlen (secret1), hmac);
int found1 = (memcmp (hmac, received_hmac, SHA512_SIZE) == 0);
int found2 = 0;
if ((! found1) && (secret2 != NULL)) {
sha512hmac (received_key, ksize, secret2, strlen (secret2), hmac);
found2 = (memcmp (hmac, received_hmac, SHA512_SIZE) == 0);
}
#ifdef DEBUG_PRINT
printf ("hmac gives %d/%d\n", found1, found2);
#endif /* DEBUG_PRINT */
if ((found1) || (found2)) {
#ifdef DEBUG_PRINT
printf ("received valid public key %p/%d for '%s'/%d\n", received_key,
ksize, contact, key_index);
print_buffer (received_key, ksize, "key", 10, 1);
#endif /* DEBUG_PRINT */
if (set_contact_pubkey (keys [key_index], received_key, ksize)) {
if (hp->src_nbits > 0)
set_contact_remote_addr (keys [key_index], hp->src_nbits, hp->source);
/* send the key to the peer -- may be redundant, but may be useful */
char * secret = secret1;
if (found2) /* use peer's secret */
secret = secret2;
/* else peer sent us a valid key, must know our secret1 */
printf ("sending back key with secret %s\n", secret);
if (! send_key (sock, contact, keys [key_index], secret,
hp->source, hp->src_nbits, max_hops))
printf ("send_key failed for key index %d/%d\n", key_index, nkeys);
return -1; /* successful key exchange */
}
printf ("handle_key error: set_contact_pubkey returned 0\n");
return 0;
}
#ifdef DEBUG_PRINT
printf ("public key does not check with secrets %s %s\n", secret1, secret2);
#endif /* DEBUG_PRINT */
return 0;
}
void lu(int n, int bs, int me)
{
int i, il, j, jl, k, kl;
int I, J, K;
double *A, *B, *C, *D;
int dimI, dimJ, dimK;
int strI, strJ, strK;
unsigned int t1, t2, t3, t4, t11, t22;
int diagowner, destp, hc, m;
double *dbuf;
armci_hdl_t handle[2*MAXPROC];
int saved[MAXPROC];
dbuf = (double *)ARMCI_Malloc_local((armci_size_t) block_size*block_size*sizeof(double));
for (k=0, K=0; k<n; k+=bs, K++) {
kl = k + bs;
if (kl > n) {
kl = n;
strK = kl - k;
} else {
strK = bs;
}
/* factor diagonal block */
diagowner = block_owner(K, K);
if (diagowner == me) {
A = a[K+K*nblocks];
lu0(A, strK, strK); /* impl algo on this diag block */
}
MP_BARRIER();
/* divide column k by diagonal block */
if(block_owner(K, K) == me)
D = a[K+K*nblocks];
else {
D = dbuf;
get_remote(D, K, K);
}
for (i=kl, I=K+1; i<n; i+=bs, I++) {
if (block_owner(I, K) == me) { /* parcel out blocks */
il = i + bs;
if (il > n) {
il = n;
strI = il - i;
} else {
strI = bs;
}
A = a[I+K*nblocks];
bdiv(A, D, strI, strK, strI, strK);
/* Pre-put this block to the block-owners of all blocks on the I-th row with a non-blocking put*/
memset (saved, 0, sizeof(saved));
for (m = K+1; m < nblocks; m++) {
destp = block_owner (I, m);
if (destp != me && !saved[destp]) {
ARMCI_NbPut(A, bufc[destp*nblocks + I], strI*strK*sizeof(double), destp, NULL);
saved[destp] = 1;
}
}
}
} /* end of for (i=k1, I=K+1...) */
/* modify row k by diagonal block */
for (j=kl, J=K+1; j<n; j+=bs, J++) {
if (block_owner(K, J) == me) { /* parcel out blocks */
jl = j+bs;
if (jl > n) {
jl = n;
strJ = jl - j;
} else {
strJ = bs;
}
A = a[K+J*nblocks];
bmodd(D, A, strK, strJ, strK, strK);
/* Pre-put this block to the block-owners of all blocks on the J-th column with a non-blocking put*/
memset (saved, 0, sizeof(saved));
for (m = K+1; m < nblocks; m++) {
destp = block_owner (m, J);
if (destp != me && !saved[destp]) {
ARMCI_NbPut(A, bufr[destp*nblocks + J], strK*strJ*sizeof(double), destp, NULL);
saved[destp] = 1;
}
}
}
}
ARMCI_WaitAll();
ARMCI_AllFence();
MP_BARRIER();
/* modify subsequent block columns */
for (i=kl, I=K+1; i<n; i+=bs, I++) {
il = i+bs;
if (il > n) {
il = n;
strI = il - i;
} else {
strI = bs;
}
for (j=kl, J=K+1; j<n; j+=bs, J++) {
jl = j + bs;
if (jl > n) {
jl = n;
strJ= jl - j;
} else {
strJ = bs;
}
if (block_owner(I, J) == me) { /* parcel out blocks */
if(block_owner(I,K) == me)
A = a[I+K*nblocks];
else {
A = bufc[me*nblocks+I];
}
if(block_owner(K,J) == me)
B = a[K+J*nblocks];
else
B = bufr[me*nblocks + J];
C = a[I+J*nblocks];
bmod(A, B, C, strI, strJ, strK, strI, strK, strI);
}
}
}
}
ARMCI_Free_local(dbuf);
}
void *lu(void *lu_arg)
{
int n, bs, th_idx;
int i, il, j, jl, k, kl;
int I, J, K;
double *A, *B, *C, *D;
int dimI, dimJ, dimK;
int strI, strJ, strK;
unsigned int t1, t2, t3, t4, t11, t22;
int diagowner;
double *buf1, *buf2;
n = ((int *)lu_arg)[0];
bs = ((int *)lu_arg)[1];
th_idx = ((int *)lu_arg)[2];
#ifdef DEBUG
printf("DBG: starting thread %d(idx=%d) on node %d\n", me_th[th_idx], th_idx, me); fflush(stdout);
#endif
/* temporary memories */
buf1 = (double *)malloc(block_size*block_size*sizeof(double));
buf2 = (double *)malloc(block_size*block_size*sizeof(double));
for (k=0, K=0; k<n; k+=bs, K++) {
kl = k + bs;
if (kl > n) {
kl = n;
strK = kl - k;
} else {
strK = bs;
}
/* factor diagonal block */
diagowner = block_owner(K, K);
if (diagowner == me_th[th_idx]) {
A = a[K+K*nblocks];
print_block_dbg(A, "th=%d, idx=%d: before lu0 a[%d]:\n", me_th[th_idx], th_idx, K+K*nblocks);
lu0(A, strK, strK);
}
MT_BARRIER();
/* divide column k by diagonal block */
if(block_owner(K, K) == me_th[th_idx])
D = a[K+K*nblocks];
else {
D = buf1;
get_remote(D, K, K);
}
for (i=kl, I=K+1; i<n; i+=bs, I++) {
if (block_owner(I, K) == me_th[th_idx]) { /* parcel out blocks */
il = i + bs;
if (il > n) {
il = n;
strI = il - i;
} else {
strI = bs;
}
A = a[I+K*nblocks];
bdiv(A, D, strI, strK, strI, strK);
}
}
/* modify row k by diagonal block */
for (j=kl, J=K+1; j<n; j+=bs, J++) {
if (block_owner(K, J) == me_th[th_idx]) { /* parcel out blocks */
jl = j+bs;
if (jl > n) {
jl = n;
strJ = jl - j;
} else {
strJ = bs;
}
A = a[K+J*nblocks];
bmodd(D, A, strK, strJ, strK, strK);
}
}
MT_BARRIER();
/* modify subsequent block columns */
for (i=kl, I=K+1; i<n; i+=bs, I++) {
il = i+bs;
if (il > n) {
il = n;
strI = il - i;
} else {
strI = bs;
}
if(block_owner(I,K) == me_th[th_idx])
A = a[I+K*nblocks];
else {
A = buf1;
get_remote(A, I, K);
}
for (j=kl, J=K+1; j<n; j+=bs, J++) {
jl = j + bs;
if (jl > n) {
jl = n;
strJ= jl - j;
} else {
strJ = bs;
}
if (block_owner(I, J) == me_th[th_idx]) { /* parcel out blocks */
if(block_owner(K,J) == me_th[th_idx])
B = a[K+J*nblocks];
else {
B = buf2;
get_remote(B, K, J);
}
C = a[I+J*nblocks];
bmod(A, B, C, strI, strJ, strK, strI, strK, strI);
}
}
}
}
free(buf1);
free(buf2);
return lu_arg;
}
