Beispiel #1
0
/** Broadcast on a group. Collective.
  *
  * @param[in]    scope ARMCI scope
  * @param[inout] buf   Input on the root, output on all other processes
  * @param[in]    len   Number of bytes in the message
  * @param[in]    abs_root Absolute rank of the process at the root of the broadcast
  * @param[in]    group ARMCI group on which to perform communication
  */
void armci_msg_group_bcast_scope(int scope, void *buf_in, int len, int abs_root, ARMCI_Group *group) {
  int    grp_root;
  void **buf;

  if (scope == SCOPE_ALL || scope == SCOPE_MASTERS) {
    /* Is the buffer an input or an output? */
    if (ARMCI_GROUP_WORLD.rank == abs_root)
      ARMCII_Buf_prepare_read_vec(&buf_in, &buf, 1, len);
    else
      ARMCII_Buf_prepare_write_vec(&buf_in, &buf, 1, len);

    grp_root = ARMCII_Translate_absolute_to_group(group, abs_root);
    ARMCII_Assert(grp_root >= 0 && grp_root < group->size);

    MPI_Bcast(buf[0], len, MPI_BYTE, grp_root, group->comm);

    if (ARMCI_GROUP_WORLD.rank == abs_root)
      ARMCII_Buf_finish_read_vec(&buf_in, buf, 1, len);
    else
      ARMCII_Buf_finish_write_vec(&buf_in, buf, 1, len);
  } else /* SCOPE_NODE */ {
    grp_root = 0;

    /* This is a self-broadcast, which is a no-op. */
  }
}
Beispiel #2
0
/** Receive a two-sided message.
  *
  * @param[in]  tag    Message tag (must match on sender and receiver)
  * @param[in]  buf    Buffer containing the message
  * @param[in]  nbytes_buf Size of the buffer in bytes
  * @param[out] nbytes_msg Length of the message received in bytes (NULL to ignore)
  * @param[in]  src    Source process id
  */
void armci_msg_rcv(int tag, void *buf_out, int nbytes_buf, int *nbytes_msg, int src) {
  void     **buf;
  MPI_Status status;

  ARMCII_Buf_prepare_write_vec(&buf_out, &buf, 1, nbytes_buf);
  MPI_Recv(buf[0], nbytes_buf, MPI_BYTE, src, tag, ARMCI_GROUP_WORLD.comm, &status);
  ARMCII_Buf_finish_write_vec(&buf_out, buf, 1, nbytes_buf);

  if (nbytes_msg != NULL)
    MPI_Get_count(&status, MPI_BYTE, nbytes_msg);
}
Beispiel #3
0
/** Receive a two-sided message from any source.
  *
  * @param[in]  tag    Message tag (must match on sender and receiver)
  * @param[in]  buf    Buffer containing the message
  * @param[in]  nbytes_buf Size of the buffer in bytes
  * @param[out] nbytes_msg Length of the message received in bytes (NULL to ignore)
  * @return            Rank of the message source
  */
int armci_msg_rcvany(int tag, void *buf_out, int nbytes_buf, int *nbytes_msg) {
  void     **buf;
  MPI_Status status;

  ARMCII_Buf_prepare_write_vec(&buf_out, &buf, 1, nbytes_buf);
  MPI_Recv(buf[0], nbytes_buf, MPI_BYTE, MPI_ANY_SOURCE, tag, ARMCI_GROUP_WORLD.comm, &status);
  ARMCII_Buf_finish_write_vec(&buf_out, buf, 1, nbytes_buf);

  if (nbytes_msg != NULL)
    MPI_Get_count(&status, MPI_BYTE, nbytes_msg);

  return status.MPI_SOURCE;
}
Beispiel #4
0
/** Broadcast a message.  Collective.
  *
  * @param[in] buffer Source buffer on root, destination elsewhere.
  * @param[in] len    Length of the message in bytes.
  * @param[in] root   Rank of the root process.
  */
void armci_msg_bcast(void *buf_in, int len, int root) {
  void **buf;

  /* Is the buffer an input or an output? */
  if (ARMCI_GROUP_WORLD.rank == root)
    ARMCII_Buf_prepare_read_vec(&buf_in, &buf, 1, len);
  else
    ARMCII_Buf_prepare_write_vec(&buf_in, &buf, 1, len);

  MPI_Bcast(buf[0], len, MPI_BYTE, root, ARMCI_GROUP_WORLD.comm);

  if (ARMCI_GROUP_WORLD.rank == root)
    ARMCII_Buf_finish_read_vec(&buf_in, buf, 1, len);
  else
    ARMCII_Buf_finish_write_vec(&buf_in, buf, 1, len);
}
Beispiel #5
0
/** Collective index selection reduce operation (scoped).
  */
void armci_msg_sel_scope(int scope, void *x, int n, char* op, int type, int contribute) {
  MPI_Comm    sel_comm;
  sel_data_t *data_in, *data_out;
  void      **x_buf;

  /*
  printf("[%d] armci_msg_sel_scope(scope=%d, x=%p, n=%d, op=%s, type=%d, contribute=%d)\n",
      ARMCI_GROUP_WORLD.rank, scope, x, n, op, type, contribute);
  */

  /* Determine the scope of the collective operation */
  if (scope == SCOPE_ALL || scope == SCOPE_MASTERS)
    sel_comm = ARMCI_GROUP_WORLD.comm;
  else
    sel_comm = MPI_COMM_SELF;

  data_in  = malloc(sizeof(sel_data_t)+n-1);
  data_out = malloc(sizeof(sel_data_t)+n-1);

  ARMCII_Assert(data_in != NULL && data_out != NULL);

  ARMCII_Buf_prepare_read_vec(&x, &x_buf, 1, n);

  data_in->contribute = contribute;
  data_in->type       = type;

  if (contribute)
    ARMCI_Copy(x, data_in->data, n);

  if (strncmp(op, "min", 3) == 0) {
    MPI_Allreduce(data_in, data_out, sizeof(sel_data_t)+n-1, MPI_BYTE, ARMCI_MPI_SELMIN_OP, sel_comm);
  } else if (strncmp(op, "max", 3) == 0) {
    MPI_Allreduce(data_in, data_out, sizeof(sel_data_t)+n-1, MPI_BYTE, ARMCI_MPI_SELMAX_OP, sel_comm);
  } else {
      ARMCII_Error("Invalid operation (%s)", op);
  }

  ARMCI_Copy(data_out->data, x, n);

  ARMCII_Buf_finish_write_vec(&x, x_buf, 1, n);

  free(data_in);
  free(data_out);
}
Beispiel #6
0
/** Generalized I/O vector one-sided get.
  *
  * @param[in] iov      Vector of transfer information.
  * @param[in] iov_len  Length of iov.
  * @param[in] proc     Target process.
  * @return             Success 0, otherwise non-zero.
  */
int PARMCI_GetV(armci_giov_t *iov, int iov_len, int proc) {
  int v;

  for (v = 0; v < iov_len; v++) {
    void **dst_buf;
    int    overlapping, same_alloc;

    if (iov[v].ptr_array_len == 0) continue; // NOP //
    if (iov[v].bytes == 0) continue; // NOP //

    // overlapping = ARMCII_Iov_check_overlap(iov[v].src_ptr_array, iov[v].ptr_array_len, iov[v].bytes);
    overlapping = ARMCII_Iov_check_overlap(iov[v].dst_ptr_array, iov[v].ptr_array_len, iov[v].bytes);
    same_alloc  = ARMCII_Iov_check_same_allocation(iov[v].src_ptr_array, iov[v].ptr_array_len, proc);

    ARMCII_Buf_prepare_write_vec(iov[v].dst_ptr_array, &dst_buf, iov[v].ptr_array_len, iov[v].bytes);
    ARMCII_Iov_op_dispatch(ARMCII_OP_GET, iov[v].src_ptr_array, dst_buf, iov[v].ptr_array_len, iov[v].bytes, 0, overlapping, same_alloc, proc);
    ARMCII_Buf_finish_write_vec(iov[v].dst_ptr_array, dst_buf, iov[v].ptr_array_len, iov[v].bytes);
  }

  return 0;
}
Beispiel #7
0
/** General ARMCI global operation (reduction).  Collective on group.
  *
  * @param[in]    scope Scope in which to perform the GOP (only SCOPE_ALL is supported)
  * @param[inout] x     Vector of n data elements, contains input and will contain output.
  * @param[in]    n     Length of x
  * @param[in]    op    One of '+', '*', 'max', 'min', 'absmax', 'absmin'
  * @param[in]    type  Data type of x (e.g. ARMCI_INT, ...)
  * @param[in]    group Group on which to perform the GOP
  */
void armci_msg_group_gop_scope(int scope, void *x, int n, char *op, int type, ARMCI_Group *group) {
  void        *out, **x_buf;
  MPI_Op       mpi_op;
  MPI_Datatype mpi_type;
  MPI_Comm     comm;
  int          mpi_type_size;

  /* FIXME: scope argument presently ignored */
  if (scope == SCOPE_ALL || scope == SCOPE_MASTERS)
    comm = group->comm;
  else
    comm = MPI_COMM_SELF;

  if (op[0] == '+') {
    mpi_op = MPI_SUM;
  } else if (op[0] == '*') {
    mpi_op = MPI_PROD;
  } else if (strncmp(op, "max", 3) == 0) {
    mpi_op = MPI_MAX;
  } else if (strncmp(op, "min", 3) == 0) {
    mpi_op = MPI_MIN;
  } else if (strncmp(op, "or", 2) == 0) {
    mpi_op = MPI_BOR;
  } else if (strncmp(op, "absmax", 6) == 0) {
    mpi_op = MPI_ABSMAX_OP;
  } else if (strncmp(op, "absmin", 6) == 0) {
    mpi_op = MPI_ABSMIN_OP;
  } else {
    ARMCII_Error("unknown operation \'%s\'", op);
    return;
  }

  switch(type) {
    case ARMCI_INT:
      mpi_type = MPI_INT;
      break;
    case ARMCI_LONG:
      mpi_type = MPI_LONG;
      break;
    case ARMCI_LONG_LONG:
      mpi_type = MPI_LONG_LONG;
      break;
    case ARMCI_FLOAT:
      mpi_type = MPI_FLOAT;
      break;
    case ARMCI_DOUBLE:
      mpi_type = MPI_DOUBLE;
      break;
    default:
      ARMCII_Error("unknown type (%d)", type);
      return;
  }

  MPI_Type_size(mpi_type, &mpi_type_size);

  ARMCII_Buf_prepare_read_vec(&x, &x_buf, 1, n*mpi_type_size);

  // ABS MAX/MIN are unary as well as binary.  We need to also apply abs in the
  // single processor case when reduce would normally just be a no-op.
  if (group->size == 1 && (mpi_op == MPI_ABSMAX_OP || mpi_op == MPI_ABSMIN_OP)) {
    ARMCII_Absv_op(x_buf[0], x_buf[0], &n, &mpi_type);
  }

  else {
    out = malloc(n*mpi_type_size);
    ARMCII_Assert(out != NULL);

    MPI_Allreduce(x_buf[0], out, n, mpi_type, mpi_op, comm);

    ARMCI_Copy(out, x_buf[0], n*mpi_type_size);
    free(out);
  }

  ARMCII_Buf_finish_write_vec(&x, x_buf, 1, n*mpi_type_size);
}
Beispiel #8
0
/** General ARMCI global operation (reduction).  Collective on group.
  *
  * @param[in]    scope Scope in which to perform the GOP (only SCOPE_ALL is supported)
  * @param[inout] x     Vector of n data elements, contains input and will contain output.
  * @param[in]    n     Length of x
  * @param[in]    op    One of '+', '*', 'max', 'min', 'absmax', 'absmin'
  * @param[in]    type  Data type of x (e.g. ARMCI_INT, ...)
  * @param[in]    group Group on which to perform the GOP
  */
void armci_msg_group_gop_scope(int scope, void *x, int n, char *op, int type, ARMCI_Group *group) {
  void        *out, **x_buf;
  MPI_Op       mpi_op;
  MPI_Datatype mpi_type;
  MPI_Comm     comm;
  int          mpi_type_size;

  /* FIXME: scope argument presently ignored */
  if (scope == SCOPE_ALL || scope == SCOPE_MASTERS)
    comm = group->comm;
  else
    comm = MPI_COMM_SELF;

  if (op[0] == '+') {
    mpi_op = MPI_SUM;
  } else if (op[0] == '*') {
    mpi_op = MPI_PROD;
  } else if (strncmp(op, "max", 3) == 0) {
    mpi_op = MPI_MAX;
  } else if (strncmp(op, "min", 3) == 0) {
    mpi_op = MPI_MIN;
  } else if (strncmp(op, "or", 2) == 0) {
    mpi_op = MPI_BOR;
  } else if (strncmp(op, "absmax", 6) == 0) {
    mpi_op = ARMCI_MPI_ABSMAX_OP;
  } else if (strncmp(op, "absmin", 6) == 0) {
    mpi_op = ARMCI_MPI_ABSMIN_OP;
  /* The following were added ComEx/ARMCI in 2017. */
  /* https://github.com/GlobalArrays/ga/commit/14ef3cfa4ea3ffa7ee721c2a98685669359f7044 */
  /* && and || need to be tested before & and | to avoid the latter matching the former. */
  } else if ((strncmp(op, "land", 4) == 0) || (strncmp(op, "&&", 2) == 0)) {
    mpi_op = MPI_LAND;
  } else if ((strncmp(op, "lor", 3) == 0) || (strncmp(op, "||", 2) == 0)) {
    mpi_op = MPI_LOR;
  } else if ((strncmp(op, "band", 4) == 0) || (strncmp(op, "&", 1) == 0)) {
    mpi_op = MPI_BAND;
  } else if ((strncmp(op, "bor", 3) == 0) || (strncmp(op, "|", 1) == 0)) {
    mpi_op = MPI_BOR;
  } else {
    ARMCII_Error("unknown operation \'%s\'", op);
    return;
  }

  switch(type) {
    case ARMCI_INT:
      mpi_type = MPI_INT;
      break;
    case ARMCI_LONG:
      mpi_type = MPI_LONG;
      break;
    case ARMCI_LONG_LONG:
      mpi_type = MPI_LONG_LONG;
      break;
    case ARMCI_FLOAT:
      mpi_type = MPI_FLOAT;
      break;
    case ARMCI_DOUBLE:
      mpi_type = MPI_DOUBLE;
      break;
    default:
      ARMCII_Error("unknown type (%d)", type);
      return;
  }

  MPI_Type_size(mpi_type, &mpi_type_size);

  ARMCII_Buf_prepare_read_vec(&x, &x_buf, 1, n*mpi_type_size);

  // ABS MAX/MIN are unary as well as binary.  We need to also apply abs in the
  // single processor case when reduce would normally just be a no-op.
  if (group->size == 1 && (mpi_op == ARMCI_MPI_ABSMAX_OP || mpi_op == ARMCI_MPI_ABSMIN_OP)) {
    ARMCII_Absv_op(x_buf[0], x_buf[0], &n, &mpi_type);
  }

  else {
    out = malloc(n*mpi_type_size);
    ARMCII_Assert(out != NULL);

    MPI_Allreduce(x_buf[0], out, n, mpi_type, mpi_op, comm);

    ARMCI_Copy(out, x_buf[0], n*mpi_type_size);
    free(out);
  }

  ARMCII_Buf_finish_write_vec(&x, x_buf, 1, n*mpi_type_size);
}