Ejemplo n.º 1
0
static int gasnetc_init(int *argc, char ***argv) {
    int ret;
    int rank, size;
    
    /*  check system sanity */
    gasnetc_check_config();

    if (gasneti_init_done) 
        GASNETI_RETURN_ERRR(NOT_INIT, "GASNet already initialized");

    gasneti_freezeForDebugger();

#if GASNET_DEBUG_VERBOSE
    /* note - can't call trace macros during gasnet_init because trace system not yet initialized */
    fprintf(stderr,"gasnetc_init(): about to spawn...\n"); fflush(stderr);
#endif

#if GASNET_DEBUG_VERBOSE
    fprintf(stderr,"gasnetc_init(): spawn successful - node %i/%i starting...\n", 
            gasneti_mynode, gasneti_nodes); fflush(stderr);
#endif

    /* gasnetc_p4_init has barrier semantics such that
       gasnetc_bootstrapBarrier and gasnetc_bootstrapExchange will
       work upon return of init. */
    ret = gasnetc_p4_init(&rank, &size);
    if (GASNET_OK != ret) gasneti_fatalerror("gasnetc_p4_init failed: %d\n", ret);

    gasneti_mynode = rank;
    gasneti_nodes = size;

    /* node map set in gasnetc_p4_init */

#if GASNET_PSHM
    gasneti_pshm_init(&gasnetc_bootstrapExchange, 0);
#endif

#if GASNET_SEGMENT_FAST || GASNET_SEGMENT_LARGE
  { uintptr_t limit;
    limit = gasneti_mmapLimit((uintptr_t)-1, (uint64_t)-1,
                              &gasnetc_bootstrapExchange,
                              &gasnetc_bootstrapBarrier);
    gasneti_segmentInit(limit, &gasnetc_bootstrapExchange);
  }
#elif GASNET_SEGMENT_EVERYTHING
    /* segment is everything - nothing to do */
#else
#error Bad segment config
#endif

    gasneti_init_done = 1;  

    gasneti_auxseg_init(); /* adjust max seg values based on auxseg */

    return GASNET_OK;
}
Ejemplo n.º 2
0
static int gasnetc_init(int *argc, char ***argv) 
{
  int ret = GASNET_OK;
  /*  check system sanity */
  gasnetc_check_config();

  if (gasneti_init_done) 
    GASNETI_RETURN_ERRR(NOT_INIT, "GASNet already initialized");

  gasneti_freezeForDebugger();

  #if GASNET_DEBUG_VERBOSE
    /* note - can't call trace macros during gasnet_init because trace system not yet initialized */
    fprintf(stderr,"gasnetc_init(): about to spawn...\n"); fflush(stderr);
  #endif

  /* bootstrap the nodes for ofi conduit */
  ret = gasnetc_ofi_init(argc, argv, &gasneti_nodes, &gasneti_mynode);
  if (GASNET_OK != ret)
	 return ret;

  #if GASNET_DEBUG_VERBOSE
    fprintf(stderr,"gasnetc_init(): spawn successful - node %i/%i starting...\n", 
      gasneti_mynode, gasneti_nodes); fflush(stderr);
  #endif

  #if GASNET_PSHM
  gasneti_pshm_init(gasneti_bootstrapSNodeBroadcast, 0);
  #endif

  #if GASNET_SEGMENT_FAST || GASNET_SEGMENT_LARGE
  { uintptr_t limit;
    limit = gasneti_mmapLimit((uintptr_t)-1, (uint64_t)-1,
                              gasneti_bootstrapExchange,
                              gasneti_bootstrapBarrier);
    gasneti_segmentInit(limit, gasneti_bootstrapExchange);
  }
  #elif GASNET_SEGMENT_EVERYTHING
    /* segment is everything - nothing to do */
  #else
    #error Bad segment config
  #endif

  gasneti_init_done = 1;  

  gasneti_auxseg_init(); /* adjust max seg values based on auxseg */

  return GASNET_OK;
}
Ejemplo n.º 3
0
static int gasnetc_init(int *argc, char ***argv) {
  int retval = GASNET_OK;

  /*  check system sanity */
  gasnetc_check_config();

  /* --------- begin Master code ------------ */
  if (!AMUDP_SPMDIsWorker(argv?*argv:NULL)) {
    /* assume we're an implicit master 
       (we don't currently support explicit workers spawned 
        without using the AMUDP SPMD API)   
     */
    int num_nodes;
    int i;
    char spawnfn;
    amudp_spawnfn_t fp = (amudp_spawnfn_t)NULL;

    if (!argv) {
      gasneti_fatalerror("implicit-master without argv not supported - use amudprun");
    }

    /* pretend we're node 0, for purposes of verbose env reporting */
    gasneti_init_done = 1;
    gasneti_mynode = 0;

    #if defined(GASNET_CSPAWN_CMD)
    { /* set configure default cspawn cmd */
      const char *cmd = gasneti_getenv_withdefault("GASNET_CSPAWN_CMD",GASNET_CSPAWN_CMD);
      gasneti_setenv("GASNET_CSPAWN_CMD",cmd);
    }
    #endif

    /* parse node count from command line */
    if (*argc < 2) {
      fprintf(stderr, "GASNet: Missing parallel node count\n");
      fprintf(stderr, "GASNet: Specify node count as first argument, or use upcrun/tcrun spawner script to start job\n");
      fprintf(stderr, "GASNet: Usage '%s <num_nodes> {program arguments}'\n", (*argv)[0]);
      exit(-1);
    }
    /*
     * argv[1] is number of nodes; argv[0] is program name; argv is
     * list of arguments including program name and number of nodes.
     * We need to remove argv[1] when the argument array is passed
     * to the tic_main().
     */
    num_nodes = atoi((*argv)[1]);
    if (num_nodes < 1) {
      fprintf (stderr, "GASNet: Invalid number of nodes: %s\n", (*argv)[1]);
      fprintf (stderr, "GASNet: Usage '%s <num_nodes> {program arguments}'\n", (*argv)[0]);
      exit (1);
    }

    /* remove the num_nodes argument */
    for (i = 1; i < (*argc)-1; i++) {
      (*argv)[i] = (*argv)[i+1];
    }
    (*argv)[(*argc)-1] = NULL;
    (*argc)--;

    /* get spawnfn */
    spawnfn = *gasneti_getenv_withdefault("GASNET_SPAWNFN", _STRINGIFY(GASNETC_DEFAULT_SPAWNFN));

    { /* ensure we pass the effective spawnfn to worker env */
      char spawnstr[2];
      spawnstr[0] = toupper(spawnfn);
      spawnstr[1] = '\0';
      gasneti_setenv("GASNET_SPAWNFN",spawnstr);
    }

    /* ensure reliable localhost operation by forcing use of 127.0.0.1
     * setting GASNET_MASTERIP to the empty string will prevent this */
    if (('L' == toupper(spawnfn)) && !gasneti_getenv("GASNET_MASTERIP")) {
      gasneti_setenv("GASNET_MASTERIP","127.0.0.1");
    }

    for (i=0; AMUDP_Spawnfn_Desc[i].abbrev; i++) {
      if (toupper(spawnfn) == toupper(AMUDP_Spawnfn_Desc[i].abbrev)) {
        fp = AMUDP_Spawnfn_Desc[i].fnptr;
        break;
      }
    }

    if (!fp) {
      fprintf (stderr, "GASNet: Invalid spawn function specified in GASNET_SPAWNFN\n");
      fprintf (stderr, "GASNet: The following mechanisms are available:\n");
      for (i=0; AMUDP_Spawnfn_Desc[i].abbrev; i++) {
        fprintf(stderr, "    '%c'  %s\n",  
              toupper(AMUDP_Spawnfn_Desc[i].abbrev), AMUDP_Spawnfn_Desc[i].desc);
      }
      exit(1);
    }

    #if GASNET_DEBUG_VERBOSE
      /* note - can't call trace macros during gasnet_init because trace system not yet initialized */
      fprintf(stderr,"gasnetc_init(): about to spawn...\n"); fflush(stderr);
    #endif

    retval = AMUDP_SPMDStartup(argc, argv, 
      num_nodes, 0, fp,
      NULL, &gasnetc_bundle, &gasnetc_endpoint);
    /* master startup should never return */
    gasneti_fatalerror("master AMUDP_SPMDStartup() failed");
  }

  /* --------- begin Worker code ------------ */
  AMLOCK();
    if (gasneti_init_done) 
      INITERR(NOT_INIT, "GASNet already initialized");

    gasneti_freezeForDebugger();

    AMUDP_VerboseErrors = gasneti_VerboseErrors;
    AMUDP_SPMDkillmyprocess = gasneti_killmyprocess;

    /*  perform job spawn */
    retval = AMUDP_SPMDStartup(argc, argv, 
      0, 0, NULL, /* dummies */
      &gasnetc_networkpid, &gasnetc_bundle, &gasnetc_endpoint);
    if (retval != AM_OK) INITERR(RESOURCE, "slave AMUDP_SPMDStartup() failed");
    gasneti_init_done = 1; /* enable early to allow tracing */

    gasneti_conduit_getenv = (/* cast drops const */ gasneti_getenv_fn_t*)&AMUDP_SPMDgetenvMaster;
    gasneti_mynode = AMUDP_SPMDMyProc();
    gasneti_nodes = AMUDP_SPMDNumProcs();

    /* enable tracing */
    gasneti_trace_init(argc, argv);
    GASNETI_AM_SAFE(AMUDP_SPMDSetExitCallback(gasnetc_traceoutput));

    /* for local spawn, assume we want to wait-block */
    if (gasneti_getenv("GASNET_SPAWNFN") && *gasneti_getenv("GASNET_SPAWNFN") == 'L') { 
      GASNETI_TRACE_PRINTF(C,("setting gasnet_set_waitmode(GASNET_WAIT_BLOCK) for localhost spawn"));
      gasnet_set_waitmode(GASNET_WAIT_BLOCK);
    }

    #if GASNET_DEBUG_VERBOSE
      fprintf(stderr,"gasnetc_init(): spawn successful - node %i/%i starting...\n", 
        gasneti_mynode, gasneti_nodes); fflush(stderr);
    #endif

    gasneti_nodemapInit(&gasnetc_bootstrapExchange, NULL, 0, 0);

    #if GASNET_PSHM
      gasneti_pshm_init(&gasnetc_bootstrapSNodeBroadcast, 0);
    #endif

    #if GASNET_SEGMENT_FAST || GASNET_SEGMENT_LARGE
    { uintptr_t limit;
      #if HAVE_MMAP
        limit = gasneti_mmapLimit((uintptr_t)-1, (uint64_t)-1,
                                  &gasnetc_bootstrapExchange,
                                  &gasnetc_bootstrapBarrier);
      #else
        limit = (intptr_t)-1;
      #endif
      gasneti_segmentInit(limit, &gasnetc_bootstrapExchange);
    }
    #elif GASNET_SEGMENT_EVERYTHING
      /* segment is everything - nothing to do */
    #else
      #error Bad segment config
    #endif

    #if GASNET_BLCR
      gasneti_checkpoint_guid = gasnetc_networkpid;
      gasneti_checkpoint_init(NULL);
    #endif

  AMUNLOCK();

  gasneti_auxseg_init(); /* adjust max seg values based on auxseg */

  gasneti_assert(retval == GASNET_OK);
  return retval;

done: /*  error return while locked */
  AMUNLOCK();
  GASNETI_RETURN(retval);
}
Ejemplo n.º 4
0
static int gasnetc_init(int *argc, char ***argv) {
  int retval = GASNET_OK;
  int networkdepth = 0;
  const char *pstr = NULL;
  const char *tmsgstr = NULL;

  AMLOCK();
    if (gasneti_init_done) 
      INITERR(NOT_INIT, "GASNet already initialized");
    gasneti_init_done = 1; /* enable early to allow tracing */

    /*  check system sanity */
    gasnetc_check_config();

    gasneti_freezeForDebugger();

    #if GASNET_DEBUG_VERBOSE
      /* note - can't call trace macros during gasnet_init because trace system not yet initialized */
      fprintf(stderr,"gasnetc_init(): about to spawn...\n"); fflush(stderr);
    #endif

    /*  choose network depth */
    networkdepth = gasnett_getenv_int_withdefault("GASNET_NETWORKDEPTH", GASNETC_DEFAULT_NETWORKDEPTH, 0);
    if (networkdepth <= 1) networkdepth = GASNETC_DEFAULT_NETWORKDEPTH;

    AMMPI_VerboseErrors = gasneti_VerboseErrors;
    AMMPI_SPMDkillmyprocess = gasneti_killmyprocess;
    #if !defined(GASNETI_DISABLE_MPI_INIT_THREAD) 
      { int res; 
      #if GASNETI_THREADS
        /* tell MPI to be thread-safe */
        res = AMMPI_SPMDSetThreadMode(1, &pstr, argc, argv);
      #else
        res = AMMPI_SPMDSetThreadMode(0, &pstr, argc, argv);
      #endif
        if (!res) { 
          #if GASNETI_THREADS
          { static char tmsg[255];
            snprintf(tmsg, sizeof(tmsg),
                          "*** WARNING: The pthreaded version of mpi-conduit requires an MPI implementation "
                          "which supports threading mode MPI_THREAD_SERIALIZED, "
                          "but this implementation reports it can only support %s\n", pstr);
            #if GASNET_DEBUG_VERBOSE
              /* only show this in verbose mode, because some versions of MPICH (eg Quadrics version)
                 lie and report THREAD_SINGLE, when in actuality MPI_THREAD_SERIALIZED works just fine */
              if (!gasneti_getenv_yesno_withdefault("GASNET_QUIET",0)) fprintf(stderr, "%s", tmsg);
            #else
              tmsgstr = tmsg;
            #endif
          }
          #else
            fprintf(stderr,"unknown failure in AMMPI_SPMDSetThreadMode() => %s\n",pstr);
          #endif
        }
      }
    #endif

    /*  perform job spawn */
    retval = AMMPI_SPMDStartup(argc, argv, networkdepth, NULL, &gasnetc_bundle, &gasnetc_endpoint);
    if (retval != AM_OK) INITERR(RESOURCE, "AMMPI_SPMDStartup() failed");

    gasneti_mynode = AMMPI_SPMDMyProc();
    gasneti_nodes = AMMPI_SPMDNumProcs();

    /* a number of MPI job spawners fail to propagate the environment to all compute nodes */
    /* do this before trace_init to make sure it gets right environment */
    gasneti_setupGlobalEnvironment(gasneti_nodes, gasneti_mynode, 
                                   gasnetc_bootstrapExchange, gasnetc_bootstrapBroadcast);

    /* enable tracing */
    gasneti_trace_init(argc, argv);
    GASNETI_AM_SAFE(AMMPI_SPMDSetExitCallback(gasnetc_traceoutput));
    if (pstr)    GASNETI_TRACE_PRINTF(C,("AMMPI_SPMDSetThreadMode/MPI_Init_thread()=>%s",pstr));
    if (tmsgstr) GASNETI_TRACE_PRINTF(I,("%s",tmsgstr));

    #if GASNET_DEBUG_VERBOSE
      fprintf(stderr,"gasnetc_init(): spawn successful - node %i/%i starting...\n", 
        gasneti_mynode, gasneti_nodes); fflush(stderr);
    #endif

    gasneti_nodemapInit(&gasnetc_bootstrapExchange, NULL, 0, 0);

    #if GASNET_PSHM
      gasneti_pshm_init(&gasnetc_bootstrapExchange, 0);
    #endif
 
    #if GASNET_SEGMENT_FAST || GASNET_SEGMENT_LARGE
    { uintptr_t limit;
      #if HAVE_MMAP
        limit = gasneti_mmapLimit((uintptr_t)-1, (uint64_t)-1,
                                  &gasnetc_bootstrapExchange,
                                  &gasnetc_bootstrapBarrier);
      #else
        limit = (intptr_t)-1;
      #endif
      gasneti_segmentInit(limit, &gasnetc_bootstrapExchange);
    }
    #elif GASNET_SEGMENT_EVERYTHING
      /* segment is everything - nothing to do */
    #else
      #error Bad segment config
    #endif

  AMUNLOCK();

  gasneti_auxseg_init(); /* adjust max seg values based on auxseg */

  gasneti_assert(retval == GASNET_OK);
  return retval;

done: /*  error return while locked */
  AMUNLOCK();
  GASNETI_RETURN(retval);
}
Ejemplo n.º 5
0
static int gasnetc_init(int *argc, char ***argv, gex_Flags_t flags) {
  int retval = GASNET_OK;

  /*  check system sanity */
  gasnetc_check_config();

  /* --------- begin Master code ------------ */
  if (!AMUDP_SPMDIsWorker(argv?*argv:NULL)) {
    /* assume we're an implicit master 
       (we don't currently support explicit workers spawned 
        without using the AMUDP SPMD API)   
     */
    int num_nodes;
    int i;
    char spawnfn;
    amudp_spawnfn_t fp = (amudp_spawnfn_t)NULL;

    if (!argv) {
      gasneti_fatalerror("implicit-master without argv not supported - use amudprun");
    }

    /* pretend we're node 0, for purposes of verbose env reporting */
    gasneti_init_done = 1;
    gasneti_mynode = 0;

    #if defined(GASNET_CSPAWN_CMD)
    { /* set configure default cspawn cmd */
      const char *cmd = gasneti_getenv_withdefault("GASNET_CSPAWN_CMD",GASNET_CSPAWN_CMD);
      gasneti_setenv("GASNET_CSPAWN_CMD",cmd);
    }
    #endif

    /* parse node count from command line */
    if (*argc < 2) {
      fprintf(stderr, "GASNet: Missing parallel node count\n");
      fprintf(stderr, "GASNet: Specify node count as first argument, or use upcrun/tcrun spawner script to start job\n");
      fprintf(stderr, "GASNet: Usage '%s <num_nodes> {program arguments}'\n", (*argv)[0]);
      exit(-1);
    }
    /*
     * argv[1] is number of nodes; argv[0] is program name; argv is
     * list of arguments including program name and number of nodes.
     * We need to remove argv[1] when the argument array is passed
     * to the tic_main().
     */
    num_nodes = atoi((*argv)[1]);
    if (num_nodes < 1) {
      fprintf (stderr, "GASNet: Invalid number of nodes: %s\n", (*argv)[1]);
      fprintf (stderr, "GASNet: Usage '%s <num_nodes> {program arguments}'\n", (*argv)[0]);
      exit (1);
    }

    /* remove the num_nodes argument */
    for (i = 1; i < (*argc)-1; i++) {
      (*argv)[i] = (*argv)[i+1];
    }
    (*argv)[(*argc)-1] = NULL;
    (*argc)--;

    /* get spawnfn */
    spawnfn = *gasneti_getenv_withdefault("GASNET_SPAWNFN", _STRINGIFY(GASNETC_DEFAULT_SPAWNFN));

    { /* ensure we pass the effective spawnfn to worker env */
      char spawnstr[2];
      spawnstr[0] = toupper(spawnfn);
      spawnstr[1] = '\0';
      gasneti_setenv("GASNET_SPAWNFN",spawnstr);
    }

    /* ensure reliable localhost operation by forcing use of 127.0.0.1
     * setting GASNET_MASTERIP to the empty string will prevent this */
    if (('L' == toupper(spawnfn)) && !gasneti_getenv("GASNET_MASTERIP")) {
      gasneti_setenv("GASNET_MASTERIP","127.0.0.1");
    }

    for (i=0; AMUDP_Spawnfn_Desc[i].abbrev; i++) {
      if (toupper(spawnfn) == toupper(AMUDP_Spawnfn_Desc[i].abbrev)) {
        fp = AMUDP_Spawnfn_Desc[i].fnptr;
        break;
      }
    }

    if (!fp) {
      fprintf (stderr, "GASNet: Invalid spawn function specified in GASNET_SPAWNFN\n");
      fprintf (stderr, "GASNet: The following mechanisms are available:\n");
      for (i=0; AMUDP_Spawnfn_Desc[i].abbrev; i++) {
        fprintf(stderr, "    '%c'  %s\n",  
              toupper(AMUDP_Spawnfn_Desc[i].abbrev), AMUDP_Spawnfn_Desc[i].desc);
      }
      exit(1);
    }

    #if GASNET_DEBUG_VERBOSE
      /* note - can't call trace macros during gasnet_init because trace system not yet initialized */
      fprintf(stderr,"gasnetc_init(): about to spawn...\n"); fflush(stderr);
    #endif

    retval = AMUDP_SPMDStartup(argc, argv, 
      num_nodes, 0, fp,
      NULL, &gasnetc_bundle, &gasnetc_endpoint);
    /* master startup should never return */
    gasneti_fatalerror("master AMUDP_SPMDStartup() failed");
  }

  /* --------- begin Worker code ------------ */
  AMLOCK();
    if (gasneti_init_done) 
      INITERR(NOT_INIT, "GASNet already initialized");

    gasneti_freezeForDebugger();

    AMX_VerboseErrors = gasneti_VerboseErrors;
    AMUDP_SPMDkillmyprocess = gasneti_killmyprocess;

#if GASNETI_CALIBRATE_TSC
    // Early x86*/Linux timer initialization before AMUDP_SPMDStartup()
    //
    // udp-conduit does not support user-provided values for GASNET_TSC_RATE*
    // (which fine-tune timer calibration on x86/Linux).  This is partially due
    // to a dependency cycle at startup with envvar propagation, but more
    // importantly because the retransmission algorithm (and hence all conduit
    // comms) rely on gasnet timers to be accurate (at least approximately), so
    // we don't allow the user to weaken or disable their calibration.
    gasneti_unsetenv("GASNET_TSC_RATE");
    gasneti_unsetenv("GASNET_TSC_RATE_TOLERANCE");
    gasneti_unsetenv("GASNET_TSC_RATE_HARD_TOLERANCE");
    GASNETI_TICKS_INIT();
#endif

    /*  perform job spawn */
    retval = AMUDP_SPMDStartup(argc, argv, 
      0, 0, NULL, /* dummies */
      &gasnetc_networkpid, &gasnetc_bundle, &gasnetc_endpoint);
    if (retval != AM_OK) INITERR(RESOURCE, "slave AMUDP_SPMDStartup() failed");
    gasneti_init_done = 1; /* enable early to allow tracing */

    gasneti_getenv_hook = (/* cast drops const */ gasneti_getenv_fn_t*)&AMUDP_SPMDgetenvMaster;
    gasneti_mynode = AMUDP_SPMDMyProc();
    gasneti_nodes = AMUDP_SPMDNumProcs();

#if !GASNETI_CALIBRATE_TSC
    /* Must init timers after global env, and preferably before tracing */
    GASNETI_TICKS_INIT();
#endif

    /* enable tracing */
    gasneti_trace_init(argc, argv);
    GASNETI_AM_SAFE(AMUDP_SPMDSetExitCallback(gasnetc_traceoutput));

    /* for local spawn, assume we want to wait-block */
    if (gasneti_getenv("GASNET_SPAWNFN") && *gasneti_getenv("GASNET_SPAWNFN") == 'L') { 
      GASNETI_TRACE_PRINTF(C,("setting gasnet_set_waitmode(GASNET_WAIT_BLOCK) for localhost spawn"));
      gasnet_set_waitmode(GASNET_WAIT_BLOCK);
    }

    #if GASNET_DEBUG_VERBOSE
      fprintf(stderr,"gasnetc_init(): spawn successful - node %i/%i starting...\n", 
        gasneti_mynode, gasneti_nodes); fflush(stderr);
    #endif

    gasneti_nodemapInit(&gasnetc_bootstrapExchange, NULL, 0, 0);

    #if GASNET_PSHM
      gasneti_pshm_init(&gasnetc_bootstrapSNodeBroadcast, 0);
    #endif

    uintptr_t mmap_limit;
    #if HAVE_MMAP
      mmap_limit = gasneti_mmapLimit((uintptr_t)-1, (uint64_t)-1,
                                  &gasnetc_bootstrapExchange,
                                  &gasnetc_bootstrapBarrier);
    #else
      // TODO-EX: we can at least look at rlimits but such logic belongs in conduit-indep code
      mmap_limit = (intptr_t)-1;
    #endif

    /* allocate and attach an aux segment */

    gasneti_auxsegAttach(mmap_limit, &gasnetc_bootstrapExchange);

    /* determine Max{Local,GLobal}SegmentSize */
    gasneti_segmentInit(mmap_limit, &gasnetc_bootstrapExchange, flags);

    #if GASNET_BLCR
      gasneti_checkpoint_guid = gasnetc_networkpid;
      gasneti_checkpoint_init(NULL);
    #endif

  AMUNLOCK();

  gasneti_assert(retval == GASNET_OK);
  return retval;

done: /*  error return while locked */
  AMUNLOCK();
  GASNETI_RETURN(retval);
}