//connect to scheduler(host)
static void add_host(gpointer data){
int ihost=GPOINTER_TO_INT(data);
int todo=0;
LOCK(mhost);
if(hsock[ihost]==-1){
hsock[ihost]--;//make it -2 so no concurrent access.
todo=1;
}
UNLOCK(mhost);
if(todo){
int sock=connect_port(hosts[ihost], PORT, 0, 0);
if(sock>-1){
int cmd[2];
cmd[0]=CMD_MONITOR;
cmd[1]=scheduler_version;
if(stwriteintarr(sock, cmd, 2)){//write failed.
warning("Rare event: Failed to write to scheduler at %s\n", hosts[ihost]);
close(sock);
LOCK(mhost);
hsock[ihost]=-1;
UNLOCK(mhost);
}else{
host_added(ihost, sock);
}
}else{
LOCK(mhost);
hsock[ihost]=-1;
UNLOCK(mhost);
}
}
}
/**
called by monitor to talk to scheduler.
*/
int scheduler_cmd(int host,int pid, int command){
if(command==CMD_DISPLAY){
return scheduler_display(host, pid);
}else{
int sock=hsock[host];
if(sock==-1){
add_host_wrap(host);
sleep(1);
}
if(sock==-1) return 1;
int cmd[2];
cmd[0]=command;
cmd[1]=pid;/*pid */
int ans=stwriteintarr(sock,cmd,2);
if(ans){/*communicated failed.*/
close(sock);
}
return ans;
}
}
/**
called by monitor to let a MAOS job remotely draw on this computer
*/
int scheduler_display(int ihost, int pid){
/*connect to scheduler with a new port. The schedule pass the other
end of the port to drawdaemon so we can communicate with it.*/
int sock=connect_port(hosts[ihost], PORT, 0, 0);
int ans=1;
int cmd[2]={CMD_DISPLAY, pid};
if(stwriteintarr(sock, cmd, 2)){
warning("Failed to communicate to scheduler\n");
close(sock);
}
if(streadintarr(sock, cmd, 1)){
warning("Read response failed\n");
}else if(cmd[0]){
warning("The scheduler failed to talk to maos\n");
}else{
if(spawn_drawdaemon(sock)){
warning("spwn drawdaemon failed\n");
}else{
ans=0;
}
close(sock);
}
return ans;
}
int main(int argc, char *argv[]){
enum{
P_EXE,
P_FRAC,
P_NSTEP,
P_TOT,
};
if(argc!=P_TOT){
info2("Usage: \n\tenv MVM_CLIENT=hostname MVM_PORT=port MVM_SASTEP=sastep ./mvm_cpu fraction nstep\n");
_Exit(0);
}
int fraction=strtol(argv[P_FRAC], NULL, 10);
int nstep=strtol(argv[P_NSTEP], NULL, 10);
int nstep0=nstep>1?20:0;//warm up
dmat *d_saind=dread("NFIRAOS_saind");
const int nsa=(d_saind->nx-1)/fraction;
int *saind=mymalloc((1+nsa),int);
for(int i=0; i<nsa+1; i++){
saind[i]=(int)d_saind->p[i];
}
dfree(d_saind);
const int totpix=saind[nsa];
const int nact=6981;//active subapertures.
int ng=nsa*2;
float FSMdelta=-0.2;
smat *dm=snew(nact,1);
smat *mvm=snew(nact, ng);
smat *mtch=snew(totpix*2,1);
smat *grad=snew(ng,1);
smat *im0=snew(totpix,3);
short *pix=mymalloc(totpix,short);
short *pixbias=mymalloc(totpix,short);
{
rand_t rseed;
seed_rand(&rseed, 1);
srandu(mvm, 1e-7, &rseed);
srandu(mtch, 1, &rseed);
for(int i=0; i<totpix; i++){
pix[i]=(short)(randu(&rseed)*25565);
pixbias[i]=(short)(randu(&rseed)*1000);
}
}
smat *mvmt=strans(mvm);
int sastep=200;//how many subapertures each time
int nrep=1;
if(getenv("MVM_NREP")){
nrep=strtol(getenv("MVM_NREP"), NULL, 10);
}
if(getenv("MVM_SECT")){
sastep=nsa/strtol(getenv("MVM_SECT"), NULL, 10);
}
if(getenv("MVM_TRANS")){
use_trans=strtol(getenv("MVM_TRANS"), NULL, 10);
}
if(getenv("MVM_SASTEP")){
sastep=strtol(getenv("MVM_SASTEP"), NULL, 10);
}
info2("use_trans=%d, nrep=%d, sastep=%d\n", use_trans, nrep, sastep);
int sock=-1;
char* MVM_CLIENT=getenv("MVM_CLIENT");
if(MVM_CLIENT){
short port=(short)strtol(getenv("MVM_PORT"), NULL, 10);
sock=connect_port(MVM_CLIENT, port, 0 ,1);
if(sock!=-1) {
info2("Connected\n");
int cmd[7];
cmd[0]=nact;
cmd[1]=nsa;
cmd[2]=sastep;
cmd[3]=totpix;
cmd[4]=nstep;
cmd[5]=nstep0;
cmd[6]=2;
if(stwriteintarr(sock, cmd, 7)
|| stwriteintarr(sock, saind, nsa+1)
|| stwrite(sock, pix, sizeof(short)*totpix)){
close(sock); sock=-1;
warning("Failed: %s\n", strerror(errno));
}
}
}
int ready=0;
if(sock!=-1 && stwriteint(sock, ready)){
warning("error send ready signal: %s\n", strerror(errno));
close(sock); sock=-1;
}
smat *timing=snew(nstep, 1);
TIC;
float timtot=0, timmax=0, timmin=INFINITY;
set_realtime(-1, -20);
for(int jstep=-nstep0; jstep<nstep; jstep++){
int istep=jstep<0?0:jstep;
tic;
double theta=M_PI*0.5*istep+FSMdelta;
float cd=cos(theta);
float sd=cos(theta);
szero(dm);
for(int isa=0; isa<nsa; isa+=sastep){
int npixleft;
int nsaleft;
if(nsa<isa+sastep){//terminate
npixleft=totpix-saind[isa];
nsaleft=nsa-isa;
}else{
npixleft=saind[isa+sastep]-saind[isa];
nsaleft=sastep;
}
short *pcur=pix+saind[isa];
if(sock!=-1){
if(stread(sock, pcur, sizeof(short)*npixleft)){
warning("failed: %s\n", strerror(errno));
close(sock); sock=-1;
_Exit(1);
}
if(isa==0) tic;
}
//Matched filter
mtch_do(mtch->p, pix, pixbias,
grad->p+isa*2, im0->p, im0->p+totpix, im0->p+totpix*2,
saind+isa, nsaleft, cd, sd);
//MVM
for(int irep=0; irep<nrep; irep++){
if(use_trans){
mvmt_do(mvmt->p+isa*2, grad->p+isa*2,dm->p, nact, nsaleft*2, ng);
}else{
mvm_do(mvm->p+isa*2*nact, grad->p+isa*2, dm->p, nact, nsaleft*2);
}
}
}//for isa
if(sock!=-1){
if(stwrite(sock, dm->p, sizeof(float)*nact)){
warning("error write dmres: %s\n", strerror(errno));
close(sock); sock=-1;
_Exit(1);
}
if(streadint(sock, &ready)){//acknowledgement.
warning("error read ack failed: %s\n", strerror(errno));
close(sock), sock=-1;
_Exit(1);
}
timing->p[istep]=ready*1.e-6;
}else{
timing->p[istep]=toc3;//do not tic.
}
if(jstep==istep){
timtot+=timing->p[istep];
if(timmax<timing->p[istep]){
timmax=timing->p[istep];
}
if(timmin>timing->p[istep]){
timmin=timing->p[istep];
}
}
}//for istep
float timmean=timtot/nstep;
info2("Timing is mean %.3f, max %.3f min %.3f. BW is %.1f of 51.2GB/s\n",
timmean*1e3, timmax*1e3, timmin*1e3, nrep*(nact*ng+nact+ng)*sizeof(float)/timmean/(1024*1024*1024));
writebin(timing, "cpu_timing_%s", HOST);
if(nstep==1){
writearr("cpu_pix", 1, sizeof(short), M_INT16, NULL, pix, totpix, 1);
writearr("cpu_pixbias", 1, sizeof(short), M_INT16, NULL, pixbias, totpix, 1);
writebin(dm, "cpu_dm");
writebin(grad, "cpu_grad");
writebin(mvm, "cpu_mvm");
writebin(mtch, "cpu_mtch");
}
}
void add_host_wrap(int ihost){
int cmd[3]={MON_ADDHOST, ihost, 0};
stwriteintarr(sock_main[1], cmd, 3);
}
