/** Calculates G = inv(I + S' * S) * S' = S' * inv(I + S * S').
*/
void calc_G_denkf(int m, int p, double** S, int i, int j, double** G)
{
double** M;
double a = 1.0;
double b = 0.0;
int lapack_info;
if (p < m) {
M = alloc2d(p, p, sizeof(double));
/*
* M = inv(I + S * S')
*/
lapack_info = calc_M(p, m, 1, S, M);
if (lapack_info != 0)
enkf_quit("dpotrf() or dpotri(): lapack_info = %d at (i, j) = (%d, %d)", lapack_info, i, j);
/*
* G = S' * inv(I + S * S')
*/
dgemm_(&doT, &noT, &m, &p, &p, &a, S[0], &p, M[0], &p, &b, G[0], &m);
} else {
/*
* M = inv(I + S' * S)
*/
M = alloc2d(m, m, sizeof(double));
lapack_info = calc_M(p, m, 0, S, M);
if (lapack_info != 0)
enkf_quit("dpotrf() or dpotri(): lapack_info = %d at (i, j) = (%d, %d)", lapack_info, i, j);
/*
* G = inv(I + S * S') * S'
*/
dgemm_(&noT, &doT, &m, &p, &m, &a, M[0], &m, S[0], &p, &b, G[0], &m);
}
#if defined(CHECK_G)
{
int e, o;
/*
* check that columns of G sum up to 0
*/
for (o = 1; o < p; ++o) {
double* Go = G[o];
double sumG = 0.0;
double sumS = 0.0;
for (e = 0; e < m; ++e) {
sumG += Go[e];
sumS += S[e][o];
}
if (fabs(sumG) > EPS)
enkf_quit("inconsistency in G: column %d sums up to %.15f for (i, j) = (%d, %d); sum(S(%d,:) = %.15f)", o, sumG, i, j, o, sumS);
}
}
#endif
free(M);
}
void allocate_sf6 ( ) {
mn_sf6 = alloc3d(NZ,NXMEM,NYMEM);
sf6_init = alloc3d(NZ,NXMEM,NYMEM);
sf6_sat = alloc2d(NXMEM,NYMEM);
sf6_sol = alloc3d(NZ,NXMEM,NYMEM);
mn_sf6sat = alloc2d(NXMEM,NYMEM);
sf6_atmconc = alloc2d(NXMEM,NYMEM);
mn_psf6 = alloc3d(NZ,NXMEM,NYMEM);
psf6 = alloc3d(NZ,NXMEM,NYMEM);
}
/** Calculates inverse _and_ inverse square root of a square matrix via SVD.
* @param m - matrix size
* @param S - input: matrix; output: S^-1
* @param D - output: S^-1/2
* @return lapack_info from dgesvd_() (0 = success)
*/
static int invsqrtm2(int m, double alpha, double** S, double** D)
{
double** U = alloc2d(m, m, sizeof(double));
double** Us1 = alloc2d(m, m, sizeof(double));
double** Us2 = alloc2d(m, m, sizeof(double));
double* sigmas = malloc(m * sizeof(double));
int lwork = 10 * m;
double* work = malloc(lwork * sizeof(double));
char specU = 'A'; /* "all M columns of U are returned in array
* * U" */
char specV = 'N'; /* "no rows of V**T are computed" */
int lapack_info;
double a = 1.0;
double b = 0.0;
int i, j;
dgesvd_(&specU, &specV, &m, &m, S[0], &m, sigmas, U[0], &m, NULL, &m, work, &lwork, &lapack_info);
if (lapack_info != 0) {
free(U);
free(Us1);
free(Us2);
free(sigmas);
free(work);
return lapack_info;
}
for (i = 0; i < m; ++i) {
double* Ui = U[i];
double* Us1i = Us1[i];
double* Us2i = Us2[i];
double si = sigmas[i];
double si_sqrt = sqrt(1.0 - alpha + alpha * sigmas[i]);
for (j = 0; j < m; ++j) {
Us1i[j] = Ui[j] / si;
Us2i[j] = Ui[j] / si_sqrt;
}
}
dgemm_(&noT, &doT, &m, &m, &m, &a, Us1[0], &m, U[0], &m, &b, S[0], &m);
dgemm_(&noT, &doT, &m, &m, &m, &a, Us2[0], &m, U[0], &m, &b, D[0], &m);
free(U);
free(Us1);
free(Us2);
free(sigmas);
free(work);
return 0;
}
int maximalSquare(char** matrix, int rows, int cols) {
int i, j, r = 0;
int** res;
if (rows == 0 || cols == 0) return 0;
res = alloc2d(rows, cols);
for (i = 0; i < rows; i++) {
for (j = 0; j < cols; j++) {
if (matrix[i][j] == '1') {res[i][j] = 1; }
}
}
for (i = 1; i < rows; i++) {
for (j = 1; j < cols; j++) {
if (matrix[i][j] == '1') {
int t;
t = min(res[i-1][j-1], res[i-1][j]);
res[i][j] = 1 + min(t, res[i][j-1]);
} else {
res[i][j] = 0;
}
}
}
for (i = 0; i < rows; i++) {
for (j = 0; j < cols; j++) {
if (res[i][j] > r) r = res[i][j];
}
}
free2d(res, rows, cols);
return (r*r);
}
int minimumTotal(int** triangle, int triangleRowSize, int *triangleColSizes)
{
int maxCols = maximum(triangleColSizes, triangleRowSize);
struct Pair last[2];
int x, y;
int k, n, t;
int** cache = alloc2d(triangleRowSize, maxCols);
int i, j;
for (i = 0; i < triangleColSizes[0]; i++) {
cache[0][i] = triangle[0][i];
}
for (i = 1; i < triangleRowSize; i++) {
for (j = 0; j < triangleColSizes[i]; j++) {
n = prev(triangle, triangleRowSize, triangleColSizes, i, j, last);
t = maxBound;
for (k = 0; k < n; k++) {
x = last[k].x;
y = last[k].y;
if (cache[x][y] < t) {
t = cache[x][y];
}
}
cache[i][j] = triangle[i][j] + t;
}
}
t = minimum(cache[triangleRowSize-1], triangleColSizes[triangleRowSize-1]);
free2d(cache, triangleRowSize, maxCols);
return t;
}
DLL_PREFIX ANN *init_net(char *weightfile)
{
float ferr;
int il, szmax, itemp, nconn;
int nlayers, nl1, nhl;
FILE *wp;
ANN *net;
net = (ANN *) malloc(sizeof(ANN));
/* Initialize weights from file*/
if ((wp = fopen(weightfile, "r")) == (FILE *) NULL) {
free(net);
return (ANN *) NULL;
}
fread (&itemp, sizeof (int), 1, wp);
fread (&nlayers, sizeof (int), 1, wp);
net->nlayers = nlayers;
net->sz = (int *) malloc(nlayers * sizeof(int));
nl1 = nlayers - 1;
nhl = nl1 - 1;
fread (net->sz, sizeof (int), nlayers, wp);
szmax = 0;
nconn = 0;
for (il = 0; il < nl1; il++) {
if ((il > 0) && (net->sz[il] > szmax))
szmax = net->sz[il];
nconn += net->sz[il] * net->sz[il+1];
}
/*
* act = matrix of neural activities
* w = weight matrix
*/
if ((net->act = (float **) alloc2d (nhl, szmax, sizeof(float))) ==
(float **) NULL) {
free(net->sz);
free(net);
return (ANN *) NULL;
}
if ((net->w = (float *) malloc (nconn * sizeof(float))) == (float *) NULL) {
free2d((void **) (net->act));
free(net->sz);
free(net);
return (ANN *) NULL;
}
fread (net->w, sizeof(float), nconn, wp);
fread (&ferr, sizeof(float), 1, wp);
fclose (wp);
return(net);
}
/** Calculates G = inv(I + S' * S) * S' and T = (I + S' * S)^-1/2.
*/
void calc_G_etkf(int m, int p, double** S, double alpha, int ii, int jj, double** G, double** T)
{
double** M = alloc2d(m, m, sizeof(double));
int lapack_info;
int i;
/*
* dgemm stuff
*/
double a = 1.0;
double b = 0.0;
/*
* M = S' * S
*/
dgemm_(&doT, &noT, &m, &m, &p, &a, S[0], &p, S[0], &p, &b, M[0], &m);
/*
* M = I + S' * S
*/
for (i = 0; i < m; ++i)
M[i][i] += 1.0;
lapack_info = invsqrtm2(m, alpha, M, T); /* M = M^-1, T = M^-1/2 */
if (lapack_info != 0)
enkf_quit("dgesvd(): lapack_info = %d at (i, j) = (%d, %d)", lapack_info, ii, jj);
/*
* G = inv(I + S * S') * S'
*/
dgemm_(&noT, &doT, &m, &p, &m, &a, M[0], &m, S[0], &p, &b, G[0], &m);
#if defined(CHECK_G)
/*
* check that columns of G sum up to 0
*/
{
int e, o;
for (o = 1; o < p; ++o) {
double* Go = G[o];
double sumG = 0.0;
double sumS = 0.0;
for (e = 0; e < m; ++e) {
sumG += Go[e];
sumS += S[e][o];
}
if (fabs(sumG) > EPS)
enkf_quit("inconsistency in G: column %d sums up to %.15f for (i, j) = (%d, %d); sum(S(%d,:) = %.15f)", o, sumG, ii, jj, o, sumS);
}
}
#endif
free(M);
}
int
main(void) {
int **ptr = alloc2d(10,10);
for (int i=0;i<10;i++) {
for (int j=0;j<10;j++) {
printf("%d ", ptr[i][j]);
}
printf("\n");
}
}
int kmeans(double **data, double **centroids, int *membership, \
double *inertia, int rank, int size, int *ppp, mytimer *t, options opt) {
int i, iterations = 0;
double **temp_centroids = (double**) alloc2d(opt.n_centroids, opt.dimensions);
int *temp_membership = (int*) calloc(opt.local_rows, sizeof(int));
check(temp_membership);
double temp_inertia = DBL_MAX;
for(i = 0; i < opt.trials; i++){
// MPI_Barrier(MPI_COMM_WORLD);
if(opt.verbose > 1 && rank == 0) printf("\nTRIAL %d\n", i+1);
iterations += _kmeans(data, temp_centroids, temp_membership, &temp_inertia, rank, size, ppp, t, opt);
if(temp_inertia < *inertia) {
*inertia = temp_inertia;
memcpy(*centroids, *temp_centroids, opt.n_centroids * opt.dimensions * sizeof(double));
memcpy(membership, temp_membership, opt.local_rows * sizeof(int));
}
}
free(*temp_centroids);
free(temp_centroids);
free(temp_membership);
return iterations;
}
int can_equally_split(int arr[], int n) {
int sum = 0;
int i, s, ret = 0;
int **dp;
for (i = 0; i < n; ++i)
sum += arr[i];
if (sum % 2 != 0)
return FALSE;
/* initialize dp array */
dp = alloc2d(sum / 2 + 1, n + 1);
for (i = 0; i < n + 1; ++i)
dp[0][i] = TRUE;
for (i = 1; i < sum / 2 + 1; ++i)
dp[i][0] = FALSE;
/* build dp array
* dp[s][i] is true if set Ai: { arr[0], ..., arr[i-1] }
* have subset with sum = s. There are two cases:
* a) arr[i-1] is part of that subset, so exclude it
* from from Ai and take into account sum = s - arr[i-1]
* b) arr[i-1] is not part of that subset, so exclude it but still sum = s; */
for (s = 1; s <= sum / 2; s++) {
for (i = 1; i <= n; i++) {
dp[s][i] = dp[s][i - 1];
if (s >= arr[i - 1])
dp[s][i] = dp[s][i] || dp[s - arr[i - 1]][i - 1];
}
}
ret = dp[sum / 2][n];
free2d(dp);
return ret;
}
/** Updates ensemble observations by applying X5
*/
static void update_HE(dasystem* das)
{
model* m = das->m;
int ngrid = model_getngrid(m);
int gid;
observations* obs = das->obs;
int e, o;
float* HEi_f;
float* HEi_a;
char do_T = 'T';
float alpha = 1.0f;
float beta = 0.0f;
int inc = 1;
enkf_printf(" updating HE:\n");
assert(das->s_mode == S_MODE_HE_f);
HEi_f = malloc(das->nmem * sizeof(ENSOBSTYPE));
HEi_a = malloc(das->nmem * sizeof(ENSOBSTYPE));
/*
* the following code for interpolation of X5 essentially coincides with
* that in das_updatefields()
*/
for (gid = 0, o = 0; gid < ngrid && o < obs->nobs; ++gid) {
void* grid = model_getgridbyid(m, gid);
int periodic_i = grid_isperiodic_x(grid);
int periodic_j = grid_isperiodic_y(grid);
char fname_X5[MAXSTRLEN];
int ncid;
int varid;
int dimids[3];
size_t dimlens[3];
size_t start[3], count[3];
float** X5j = NULL;
float** X5jj = NULL;
float** X5jj1 = NULL;
float** X5jj2 = NULL;
int mni, mnj;
int* iiter;
int* jiter;
int i, j, ni, nj;
int jj, stepj, ii, stepi;
assert(obs->obstypes[obs->data[o].type].gridid == gid);
das_getfname_X5(das, grid, fname_X5);
ncw_open(fname_X5, NC_NOWRITE, &ncid);
ncw_inq_varid(fname_X5, ncid, "X5", &varid);
ncw_inq_vardimid(fname_X5, ncid, varid, dimids);
for (i = 0; i < 3; ++i)
ncw_inq_dimlen(fname_X5, ncid, dimids[i], &dimlens[i]);
ni = dimlens[1];
nj = dimlens[0];
assert((int) dimlens[2] == das->nmem * das->nmem);
jiter = malloc((nj + 1) * sizeof(int)); /* "+ 1" to handle periodic
* grids */
iiter = malloc((ni + 1) * sizeof(int));
for (j = 0, i = 0; j < nj; ++j, i += das->stride)
jiter[j] = i;
if (periodic_j)
jiter[nj] = jiter[nj - 1] + das->stride;
for (i = 0, j = 0; i < ni; ++i, j += das->stride)
iiter[i] = j;
if (periodic_i)
iiter[ni] = iiter[ni - 1] + das->stride;
grid_getdims(grid, &mni, &mnj, NULL);
start[0] = 0;
start[1] = 0;
start[2] = 0;
count[0] = 1;
count[1] = ni;
count[2] = das->nmem * das->nmem;
X5j = alloc2d(mni, das->nmem * das->nmem, sizeof(float));
if (das->stride > 1) {
X5jj = alloc2d(ni, das->nmem * das->nmem, sizeof(float));
X5jj1 = alloc2d(ni, das->nmem * das->nmem, sizeof(float));
X5jj2 = alloc2d(ni, das->nmem * das->nmem, sizeof(float));
ncw_get_vara_float(fname_X5, ncid, varid, start, count, X5jj2[0]);
}
/*
* jj, ii are the indices of the subsampled grid; i, j are the indices
* of the model grid
*/
for (jj = 0, j = 0; jj < nj; ++jj) {
for (stepj = 0; stepj < das->stride && j < mnj; ++stepj, ++j) {
if (das->stride == 1) {
/*
* no interpolation necessary; simply read the ETMs for the
* j-th row from disk
*/
start[0] = j;
ncw_get_vara_float(fname_X5, ncid, varid, start, count, X5j[0]);
} else {
/*
* the following code interpolates the ETM back to the
* original grid, first by j, and then by i
*/
if (stepj == 0) {
memcpy(X5jj[0], X5jj2[0], ni * das->nmem * das->nmem * sizeof(float));
memcpy(X5jj1[0], X5jj2[0], ni * das->nmem * das->nmem * sizeof(float));
if (jj < nj - 1 || periodic_j) {
start[0] = (jj + 1) % nj;
ncw_get_vara_float(fname_X5, ncid, varid, start, count, X5jj2[0]);
}
} else {
float weight2 = (float) stepj / das->stride;
float weight1 = (float) 1.0 - weight2;
for (ii = 0; ii < ni; ++ii) {
float* X5jjii = X5jj[ii];
float* X5jj1ii = X5jj1[ii];
float* X5jj2ii = X5jj2[ii];
for (e = 0; e < das->nmem * das->nmem; ++e)
X5jjii[e] = X5jj1ii[e] * weight1 + X5jj2ii[e] * weight2;
}
}
for (ii = 0, i = 0; ii < ni; ++ii) {
for (stepi = 0; stepi < das->stride && i < mni; ++stepi, ++i) {
if (stepi == 0)
memcpy(X5j[i], X5jj[ii], das->nmem * das->nmem * sizeof(float));
else {
float weight2 = (float) stepi / das->stride;
float weight1 = (float) 1.0 - weight2;
float* X5jjii1 = X5jj[ii];
float* X5ji = X5j[i];
float* X5jjii2;
if (ii < ni - 1)
X5jjii2 = X5jj[ii + 1];
else
X5jjii2 = X5jj[(periodic_i) ? (ii + 1) % ni : ii];
for (e = 0; e < das->nmem * das->nmem; ++e)
X5ji[e] = X5jjii1[e] * weight1 + X5jjii2[e] * weight2;
}
}
}
} /* stride != 1 */
/*
* (at this stage X5j should contain the array of X5 matrices
* for the j-th row of the grid)
*/
if (o >= obs->nobs)
break;
if ((int) (obs->data[o].fj) > j)
continue;
for (; o < obs->nobs && (int) (obs->data[o].fj) == j; ++o) {
float inflation = model_getvarinflation(m, obs->obstypes[obs->data[o].type].vid);
/*
* HE(i, :) = HE(i, :) * X5
*/
i = (int) (obs->data[o].fi);
for (e = 0; e < das->nmem; ++e)
HEi_f[e] = das->S[e][o];
sgemv_(&do_T, &das->nmem, &das->nmem, &alpha, X5j[i], &das->nmem, HEi_f, &inc, &beta, HEi_a, &inc);
/*
* applying inflation:
*/
if (fabsf(inflation - 1.0f) > EPSF) {
float v_av = 0.0f;
for (e = 0; e < das->nmem; ++e)
v_av += HEi_a[e];
v_av /= (float) das->nmem;
for (e = 0; e < das->nmem; ++e)
HEi_a[e] = (HEi_a[e] - v_av) * inflation + v_av;
}
for (e = 0; e < das->nmem; ++e)
das->S[e][o] = HEi_a[e];
}
} /* for stepj */
} /* for jj */
ncw_close(fname_X5, ncid);
free(iiter);
free(jiter);
free2d(X5j);
if (das->stride > 1) {
free2d(X5jj);
free2d(X5jj1);
free2d(X5jj2);
}
} /* for gid */
free(HEi_a);
free(HEi_f);
das->s_mode = S_MODE_HE_a;
} /* update_HE() */
void das_getHE(dasystem* das)
{
observations* obs = das->obs;
model* m = das->m;
ENSOBSTYPE* Hx = NULL;
int i, e;
das->s_mode = S_MODE_HE_f;
if (obs->nobs == 0)
return;
if (das->nmem <= 0)
das_getnmem(das);
enkf_printf(" ensemble size = %d\n", das->nmem);
assert(das->nmem > 0);
distribute_iterations(0, das->nmem - 1, nprocesses, rank, " ");
/*
* ensemble observation array to be filled
*/
assert(das->S == NULL);
das->S = alloc2d(das->nmem, obs->nobs, sizeof(ENSOBSTYPE));
if (das->mode == MODE_ENOI)
Hx = calloc(obs->nobs, sizeof(ENSOBSTYPE));
for (i = 0; i < obs->nobstypes; ++i) {
obstype* ot = &obs->obstypes[i];
float*** vvv = NULL;
float** vv = NULL;
H_fn H = NULL;
int mvid;
int ni, nj, nk;
int nobs;
int* obsids;
char fname[MAXSTRLEN];
enkf_printf(" %s ", ot->name);
fflush(stdout);
mvid = model_getvarid(m, obs->obstypes[i].varname);
if (mvid < 0)
enkf_quit("variable \"%s\" required for observation type \"%s\" is not defined", obs->obstypes[i].varname, ot->name);
if (ot->issurface) {
model_getvardims(m, mvid, &ni, &nj, NULL);
vv = alloc2d(nj, ni, sizeof(float));
} else {
model_getvardims(m, mvid, &ni, &nj, &nk);
vvv = alloc3d(nk, nj, ni, sizeof(float));
}
/*
* set H
*/
H = getH(ot->name, ot->hfunction);
if (ot->isasync) {
int t1 = get_tshift(ot->date_min, ot->async_tstep);
int t2 = get_tshift(ot->date_max, ot->async_tstep);
int t;
for (t = t1; t <= t2; ++t) {
enkf_printf("|");
obs_find_bytypeandtime(obs, i, t, &nobs, &obsids);
if (nobs == 0)
continue;
if (das->mode == MODE_ENKF || !enkf_fstatsonly) {
for (e = my_first_iteration; e <= my_last_iteration; ++e) {
int success = model_getmemberfname_async(m, das->ensdir, ot->varname, ot->name, e + 1, t, fname);
H(das, nobs, obsids, fname, e + 1, t, ot->varname, ot->varname2, (ot->issurface) ? (void*) vv : (void*) vvv, das->S[e]);
enkf_printf((success) ? "a" : "s");
fflush(stdout);
}
}
if (das->mode == MODE_ENOI) {
if (enkf_obstype == OBSTYPE_VALUE) {
int success = model_getbgfname_async(m, das->bgdir, ot->varname, ot->name, t, fname);
H(das, nobs, obsids, fname, -1, t, ot->varname, ot->varname2, (ot->issurface) ? (void*) vv : (void*) vvv, Hx);
enkf_printf((success) ? "A" : "S");
fflush(stdout);
} else if (enkf_obstype == OBSTYPE_INNOVATION) {
Hx[0] = 0;
enkf_printf("-");
fflush(stdout);
}
}
free(obsids);
}
} else {
obs_find_bytype(obs, i, &nobs, &obsids);
if (nobs == 0)
goto next;
if (das->mode == MODE_ENKF || !enkf_fstatsonly) {
for (e = my_first_iteration; e <= my_last_iteration; ++e) {
model_getmemberfname(m, das->ensdir, ot->varname, e + 1, fname);
H(das, nobs, obsids, fname, e + 1, MAXINT, ot->varname, ot->varname2, (ot->issurface) ? (void*) vv : (void*) vvv, das->S[e]);
enkf_printf(".");
fflush(stdout);
}
}
if (das->mode == MODE_ENOI) {
if (enkf_obstype == OBSTYPE_VALUE) {
model_getbgfname(m, das->bgdir, ot->varname, fname);
H(das, nobs, obsids, fname, -1, MAXINT, ot->varname, ot->varname2, (ot->issurface) ? (void*) vv : (void*) vvv, Hx);
enkf_printf("+");
fflush(stdout);
} else if (enkf_obstype == OBSTYPE_INNOVATION) {
Hx[0] = 0;
enkf_printf("-");
fflush(stdout);
}
}
free(obsids);
}
next:
if (ot->issurface)
free2d(vv);
else
free3d(vvv);
enkf_printf("\n");
} /* for i (over obstypes) */
#if defined(MPI)
if (das->mode == MODE_ENKF || !enkf_fstatsonly) {
#if !defined(HE_VIAFILE)
/*
* communicate HE via MPI
*/
int ierror, count;
/*
* Blocking communications can create a bottleneck in instances with
* large number of observations (e.g., 2e6 obs., 144 members, 48
* processes), but asynchronous send/receive seem to work well
*/
if (rank > 0) {
MPI_Request request;
/*
* send ensemble observations to master
*/
count = (my_last_iteration - my_first_iteration + 1) * obs->nobs;
ierror = MPI_Isend(das->S[my_first_iteration], count, MPIENSOBSTYPE, 0, rank, MPI_COMM_WORLD, &request);
assert(ierror == MPI_SUCCESS);
} else {
int r;
MPI_Request* requests = malloc((nprocesses - 1) * sizeof(MPI_Request));
/*
* collect ensemble observations from slaves
*/
for (r = 1; r < nprocesses; ++r) {
count = (last_iteration[r] - first_iteration[r] + 1) * obs->nobs;
ierror = MPI_Irecv(das->S[first_iteration[r]], count, MPIENSOBSTYPE, r, r, MPI_COMM_WORLD, &requests[r - 1]);
assert(ierror == MPI_SUCCESS);
}
ierror = MPI_Waitall(nprocesses - 1, requests, MPI_STATUS_IGNORE);
assert(ierror == MPI_SUCCESS);
free(requests);
}
/*
* now send the full set of ensemble observations to slaves
*/
count = das->nmem * obs->nobs;
ierror = MPI_Bcast(das->S[0], count, MPIENSOBSTYPE, 0, MPI_COMM_WORLD);
assert(ierror == MPI_SUCCESS);
#else
/*
* communicate HE via file
*/
{
int ncid;
int varid;
size_t start[2], count[2];
if (rank == 0) {
int dimids[2];
ncw_create(FNAME_HE, NC_CLOBBER | NC_64BIT_OFFSET, &ncid);
ncw_def_dim(FNAME_HE, ncid, "m", das->nmem, &dimids[0]);
ncw_def_dim(FNAME_HE, ncid, "p", obs->nobs, &dimids[1]);
ncw_def_var(FNAME_HE, ncid, "HE", NC_FLOAT, 2, dimids, &varid);
ncw_close(FNAME_HE, ncid);
}
MPI_Barrier(MPI_COMM_WORLD);
ncw_open(FNAME_HE, NC_WRITE, &ncid);
ncw_inq_varid(FNAME_HE, ncid, "HE", &varid);
start[0] = my_first_iteration;
start[1] = 0;
count[0] = my_last_iteration - my_first_iteration + 1;
count[1] = obs->nobs;
ncw_put_vara_float(FNAME_HE, ncid, varid, start, count, das->S[my_first_iteration]);
ncw_close(FNAME_HE, ncid);
MPI_Barrier(MPI_COMM_WORLD);
ncw_open(FNAME_HE, NC_NOWRITE, &ncid);
ncw_inq_varid(FNAME_HE, ncid, "HE", &varid);
ncw_get_var_float(FNAME_HE, ncid, varid, das->S[0]);
ncw_close(FNAME_HE, ncid);
}
#endif
}
#endif
if (das->mode == MODE_ENOI) {
/*
* subtract ensemble mean; add background
*/
if (!enkf_fstatsonly) {
double* ensmean = calloc(obs->nobs, sizeof(double));
for (e = 0; e < das->nmem; ++e) {
ENSOBSTYPE* Se = das->S[e];
for (i = 0; i < obs->nobs; ++i)
ensmean[i] += Se[i];
}
for (i = 0; i < obs->nobs; ++i)
ensmean[i] /= (double) das->nmem;
for (e = 0; e < das->nmem; ++e) {
ENSOBSTYPE* Se = das->S[e];
for (i = 0; i < obs->nobs; ++i)
Se[i] += Hx[i] - ensmean[i];
}
free(ensmean);
} else {
for (e = 0; e < das->nmem; ++e) {
ENSOBSTYPE* Se = das->S[e];
for (i = 0; i < obs->nobs; ++i)
Se[i] = Hx[i];
}
}
}
if (das->mode == MODE_ENOI)
free(Hx);
}
static void update_Hx(dasystem* das)
{
model* m = das->m;
int ngrid = model_getngrid(m);
int gid;
observations* obs = das->obs;
int e, o;
enkf_printf(" updating Hx:\n");
assert(das->s_mode == S_MODE_HE_f);
/*
* the following code for interpolation of X5 essentially coincides with
* that in das_updatefields()
*/
for (gid = 0, o = 0; gid < ngrid && o < obs->nobs; ++gid) {
void* grid = model_getgridbyid(m, gid);
int periodic_i = grid_isperiodic_x(grid);
int periodic_j = grid_isperiodic_y(grid);
char fname_w[MAXSTRLEN];
int ncid;
int varid;
int dimids[3];
size_t dimlens[3];
size_t start[3], count[3];
float** wj = NULL;
float** wjj = NULL;
float** wjj1 = NULL;
float** wjj2 = NULL;
int mni, mnj;
int* iiter;
int* jiter;
int i, j, ni, nj;
int jj, stepj, ii, stepi;
assert(obs->obstypes[obs->data[o].type].gridid == gid);
das_getfname_w(das, grid, fname_w);
ncw_open(fname_w, NC_NOWRITE, &ncid);
ncw_inq_varid(fname_w, ncid, "w", &varid);
ncw_inq_vardimid(fname_w, ncid, varid, dimids);
for (i = 0; i < 3; ++i)
ncw_inq_dimlen(fname_w, ncid, dimids[i], &dimlens[i]);
ni = dimlens[1];
nj = dimlens[0];
assert((int) dimlens[2] == das->nmem);
jiter = malloc((nj + 1) * sizeof(int)); /* "+ 1" to handle periodic
* grids */
iiter = malloc((ni + 1) * sizeof(int));
for (j = 0, i = 0; j < nj; ++j, i += das->stride)
jiter[j] = i;
if (periodic_j)
jiter[nj] = jiter[nj - 1] + das->stride;
for (i = 0, j = 0; i < ni; ++i, j += das->stride)
iiter[i] = j;
if (periodic_i)
iiter[ni] = iiter[ni - 1] + das->stride;
grid_getdims(grid, &mni, &mnj, NULL);
start[0] = 0;
start[1] = 0;
start[2] = 0;
count[0] = 1;
count[1] = ni;
count[2] = das->nmem;
wj = alloc2d(mni, das->nmem, sizeof(float));
if (das->stride > 1) {
wjj = alloc2d(ni, das->nmem, sizeof(float));
wjj1 = alloc2d(ni, das->nmem, sizeof(float));
wjj2 = alloc2d(ni, das->nmem, sizeof(float));
ncw_get_vara_float(fname_w, ncid, varid, start, count, wjj2[0]);
}
/*
* jj, ii are the indices of the subsampled grid; i, j are the indices
* of the model grid
*/
for (jj = 0, j = 0; jj < nj; ++jj) {
for (stepj = 0; stepj < das->stride && j < mnj; ++stepj, ++j) {
if (das->stride == 1) {
/*
* no interpolation necessary; simply read the ETMs for the
* j-th row from disk
*/
start[0] = j;
ncw_get_vara_float(fname_w, ncid, varid, start, count, wj[0]);
} else {
/*
* the following code interpolates the ETM back to the
* original grid, first by j, and then by i
*/
if (stepj == 0) {
memcpy(wjj[0], wjj2[0], ni * das->nmem * sizeof(float));
memcpy(wjj1[0], wjj2[0], ni * das->nmem * sizeof(float));
if (jj < nj - 1 || periodic_j) {
start[0] = (jj + 1) % nj;
ncw_get_vara_float(fname_w, ncid, varid, start, count, wjj2[0]);
}
} else {
float weight2 = (float) stepj / das->stride;
float weight1 = (float) 1.0 - weight2;
for (ii = 0; ii < ni; ++ii) {
float* wjjii = wjj[ii];
float* wjj1ii = wjj1[ii];
float* wjj2ii = wjj2[ii];
for (e = 0; e < das->nmem; ++e)
wjjii[e] = wjj1ii[e] * weight1 + wjj2ii[e] * weight2;
}
}
for (ii = 0, i = 0; ii < ni; ++ii) {
for (stepi = 0; stepi < das->stride && i < mni; ++stepi, ++i) {
if (stepi == 0)
memcpy(wj[i], wjj[ii], das->nmem * sizeof(float));
else {
float weight2 = (float) stepi / das->stride;
float weight1 = (float) 1.0 - weight2;
float* wjjii1 = wjj[ii];
float* wji = wj[i];
float* wjjii2;
if (ii < ni - 1)
wjjii2 = wjj[ii + 1];
else
wjjii2 = wjj[(periodic_i) ? (ii + 1) % ni : ii];
for (e = 0; e < das->nmem; ++e)
wji[e] = wjjii1[e] * weight1 + wjjii2[e] * weight2;
}
}
}
} /* stride != 1 */
/*
* (at this stage wj should contain the array of b vectors for
* the j-th row of the grid)
*/
if (o >= obs->nobs)
break;
if ((int) (obs->data[o].fj) > j)
continue;
for (; o < obs->nobs && (int) (obs->data[o].fj) == j; ++o) {
double dHx = 0.0;
double Hx = 0.0;
for (e = 0; e < das->nmem; ++e)
Hx += das->S[e][o];
Hx /= (double) das->nmem;
i = (int) (obs->data[o].fi);
/*
* HE(i, :) += HA(i, :) * b * 1'
*/
for (e = 0; e < das->nmem; ++e)
dHx += (das->S[e][o] - Hx) * wj[i][e];
for (e = 0; e < das->nmem; ++e)
das->S[e][o] += dHx;
}
} /* for stepj */
} /* for jj */
ncw_close(fname_w, ncid);
free(iiter);
free(jiter);
free2d(wj);
if (das->stride > 1) {
free2d(wjj);
free2d(wjj1);
free2d(wjj2);
}
} /* for gid */
das->s_mode = S_MODE_HE_a;
} /* update_Hx() */
int main( int argc, char **argv) {
srand(time(NULL));
mytimer timer;
timer.total_time = timer.init_time = timer.comp_time = timer.comm_time = 0.0;
timestamp_type time_s, time_e;
int mpi_rank, mpi_size;
int i, r, rows, cols, total_rows, err;
int *points_per_proc, *buffer;
MPI_File filename;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
options opt;
parse_command_line(argc, argv, &opt);
err = MPI_File_open(MPI_COMM_WORLD, opt.filename, MPI_MODE_RDONLY, MPI_INFO_NULL, &filename);
if (err) {
if (mpi_rank == 0) fprintf(stderr, "Couldn't open file %s\n", argv[1]);
MPI_Finalize();
exit(1);
}
double **data = mpi_read_data(&filename, &rows, &cols, mpi_rank, mpi_size, opt.overlap);
points_per_proc = (int*) calloc(mpi_size, sizeof(int));
check(points_per_proc);
buffer = (int*) calloc(mpi_size, sizeof(int));
check(buffer);
buffer[mpi_rank] = rows;
MPI_Barrier(MPI_COMM_WORLD);
MPI_Allreduce(buffer, points_per_proc, mpi_size, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
free(buffer);
MPI_Allreduce(&rows, &total_rows, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
opt.n_points = total_rows;
opt.dimensions = cols;
opt.local_rows = rows;
if(mpi_rank == 0 && opt.verbose > 1) {
printf("Total rows: %d\n", opt.n_points);
for(r = 0; r < mpi_size; r++)
printf("proc %d has %d\n", r, points_per_proc[r]);
}
for(r=0; r < mpi_size; r++) {
MPI_Barrier(MPI_COMM_WORLD);
if(mpi_rank == r && opt.verbose > 2) {
for(i=0; i < rows; i++) {
printf("proc %d: %d --- ", mpi_rank, i);
print_vec(data[i], cols);
}
}
}
// allocate centroids, everyone gets their own copy
double **centroids = (double**) alloc2d(opt.n_centroids, opt.dimensions);
// allocate cluster memberships
// only track the ones this process is responsible for
int *membership = (int*) malloc(opt.local_rows * sizeof(int));
check(membership);
double inertia = DBL_MAX;
int total_iterations = 0;
get_timestamp(&time_s);
total_iterations = kmeans(data, centroids, membership, &inertia, mpi_rank, mpi_size, points_per_proc, &timer, opt);
get_timestamp(&time_e);
timer.total_time = timestamp_diff_in_seconds(time_s, time_e);
if(mpi_rank == 0 && opt.verbose > 0) {
print_vecs(centroids, opt, "centroids");
}
if(mpi_rank == 0) {
printf("\nMPI K-MEANS\n");
printf("%dx%d data, %d clusters, %d trials, %d cores\n", opt.n_points, opt.dimensions, opt.n_centroids, opt.trials, mpi_size);
printf("Inertia: %f\n", inertia);
printf("Total Iterations: %d\n", total_iterations);
printf("Runtime: %fs\n", timer.total_time);
printf("Initialization time: %fs\n", timer.init_time);
printf("Computation time: %fs\n", timer.comp_time);
printf("Communication time: %fs\n", timer.comm_time);
}
MPI_File_close(&filename);
free(points_per_proc);
free(*data);
free(data);
free(*centroids);
free(centroids);
free(membership);
MPI_Finalize();
return 0;
}
int _kmeans(double **data, double **centroids, int *membership, \
double *inertia, int rank, int size, int *ppp, mytimer *t, options opt) {
timestamp_type time_is, time_ie;
timestamp_type time_cs, time_ce;
#ifdef TIME_ALL
timestamp_type comm_s, comm_e;
#endif
double dist, total_inertia, total_delta, delta = (double) opt.n_points;
int i, center, iters = 0;
// allocate for new centroids that will be computed
double **new_centers = (double**) alloc2d(opt.n_centroids, opt.dimensions);
memset(*new_centers, 0, opt.n_centroids * opt.dimensions * sizeof(double));
// allocate array to count points in each cluster, initialize to 0
int *count_centers = (int*) calloc(opt.n_centroids, sizeof(int));
check(count_centers);
int *new_count_centers = (int*) calloc(opt.n_centroids, sizeof(int));
check(new_count_centers);
// if a cluster has 0 points assigned use this for random reinitialization
double *point = (double*) malloc(opt.dimensions * sizeof(double));
check(point);
double *tofree = point;
get_timestamp(&time_is);
t->comm_time += initialize(data, centroids, ppp, rank, size, opt);
get_timestamp(&time_ie);
#ifdef TIME_ALL
get_timestamp(&comm_s);
#endif
MPI_Bcast(*centroids, opt.n_centroids*opt.dimensions, MPI_DOUBLE, 0, MPI_COMM_WORLD);
#ifdef TIME_ALL
get_timestamp(&comm_e);
t->comm_time += timestamp_diff_in_seconds(comm_s, comm_e);
#endif
get_timestamp(&time_cs);
while (delta / ((double) opt.n_points) > opt.tol && iters < opt.max_iter) {
// MPI_Barrier(MPI_COMM_WORLD);
delta = 0.0;
*inertia = 0.0;
for(i = 0; i < opt.local_rows; i++){
find_nearest_centroid(data[i], centroids, opt, ¢er, &dist);
*inertia += dist;
if (membership[i] != center) {
delta++;
membership[i] = center;
}
add(new_centers[center], data[i], opt);
new_count_centers[center]++;
}
#ifdef TIME_ALL
get_timestamp(&comm_s);
#endif
MPI_Allreduce(*new_centers, *centroids, opt.n_centroids * opt.dimensions,
MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
MPI_Allreduce(new_count_centers, count_centers, opt.n_centroids,
MPI_INT, MPI_SUM, MPI_COMM_WORLD);
#ifdef TIME_ALL
get_timestamp(&comm_e);
t->comm_time += timestamp_diff_in_seconds(comm_s, comm_e);
#endif
for(i = 0; i < opt.n_centroids; i++) {
if(count_centers[i] == 0) {
if(rank == 0){
add(centroids[i], data[randint(opt.local_rows)], opt);
}
// broadcast this new point to everyone
#ifdef TIME_ALL
get_timestamp(&comm_s);
#endif
MPI_Bcast(centroids[i], opt.dimensions, MPI_DOUBLE, 0, MPI_COMM_WORLD);
#ifdef TIME_ALL
get_timestamp(&comm_e);
t->comm_time += timestamp_diff_in_seconds(comm_s, comm_e);
#endif
// add to delta to ensure we dont stop after this
delta += opt.tol * opt.local_rows + 1.0;
}
// all good to divide, count is not 0
else {
// calculate the new center
div_by(centroids[i], count_centers[i], opt);
}
}
// sum up the number of cluster assignments that changed
#ifdef TIME_ALL
get_timestamp(&comm_s);
#endif
MPI_Allreduce(&delta, &total_delta, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
delta = total_delta;
// sum up the inertias
MPI_Allreduce(inertia, &total_inertia, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
*inertia = total_inertia;
#ifdef TIME_ALL
get_timestamp(&comm_e);
t->comm_time += timestamp_diff_in_seconds(comm_s, comm_e);
#endif
// zero out new_centers and count_centers
memset(*new_centers, 0, opt.n_centroids * opt.dimensions * sizeof(double));
memset(new_count_centers, 0, opt.n_centroids * sizeof(int));
memset(count_centers, 0, opt.n_centroids * sizeof(int));
iters++;
if(opt.verbose > 1 && rank == 0) {
printf("\n\titers: %d\n", iters);
printf("\tdelta: %d\n", (int) delta);
printf("\teps: %f\n", delta / ((double) opt.n_points));
printf("\tinertia: %f\n", *inertia);
}
}
get_timestamp(&time_ce);
t->init_time += timestamp_diff_in_seconds(time_is, time_ie);
t->comp_time += timestamp_diff_in_seconds(time_cs, time_ce);
free(*new_centers);
free(new_centers);
free(count_centers);
free(new_count_centers);
free(tofree);
if(iters == opt.max_iter && rank == 0 && opt.verbose > 0) {
printf("HIT MAX ITERS\n");
}
return iters;
}
void reader_mmt_standard(char* fname, int fid, obsmeta* meta, model* m, observations* obs)
{
int ncid;
int dimid_nprof, dimid_nz;
size_t nprof, nz;
int varid_lon, varid_lat, varid_z, varid_type;
int varid_v = -1;
double* lon;
double* lat;
double** z;
double** v;
double missval;
double validmin = DBL_MAX;
double validmax = -DBL_MAX;
char* type;
char buf[MAXSTRLEN];
int len;
int year, month, day;
double tunits_multiple, tunits_offset;
int mvid;
int p, i;
for (i = 0; i < meta->npars; ++i)
enkf_quit("unknown PARAMETER \"%s\"\n", meta->pars[i].name);
if (meta->nstds == 0)
enkf_quit("ERROR_STD is necessary but not specified for product \"%s\"", meta->product);
ncw_open(fname, NC_NOWRITE, &ncid);
ncw_inq_dimid(fname, ncid, "N_PROF", &dimid_nprof);
ncw_inq_dimlen(fname, ncid, dimid_nprof, &nprof);
ncw_inq_dimid(fname, ncid, "N_LEVELS", &dimid_nz);
ncw_inq_dimlen(fname, ncid, dimid_nz, &nz);
enkf_printf(" # profiles = %u\n", (unsigned int) nprof);
if (nprof == 0) {
ncw_close(fname, ncid);
return;
}
enkf_printf(" # z levels = %u\n", (unsigned int) nz);
ncw_inq_varid(fname, ncid, "LONGITUDE", &varid_lon);
lon = malloc(nprof * sizeof(double));
ncw_get_var_double(fname, ncid, varid_lon, lon);
ncw_inq_varid(fname, ncid, "LATITUDE", &varid_lat);
lat = malloc(nprof * sizeof(double));
ncw_get_var_double(fname, ncid, varid_lat, lat);
ncw_inq_varid(fname, ncid, "PRES_BLUELINK", &varid_z);
z = alloc2d(nprof, nz, sizeof(double));
ncw_get_var_double(fname, ncid, varid_z, z[0]);
if (strncmp(meta->type, "TEM", 3) == 0) {
validmin = -2.0;
validmax = 40.0;
ncw_inq_varid(fname, ncid, "TEMP_BLUELINK", &varid_v);
} else if (strncmp(meta->type, "SAL", 3) == 0) {
validmin = 0;
validmax = 50.0;
ncw_inq_varid(fname, ncid, "PSAL_BLUELINK", &varid_v);
} else
enkf_quit("observation type \"%s\" not handled for MMT product", meta->type);
v = alloc2d(nprof, nz, sizeof(double));
ncw_get_var_double(fname, ncid, varid_v, v[0]);
ncw_get_att_double(fname, ncid, varid_v, "_FillValue", &missval);
ncw_inq_varid(fname, ncid, "WMO_INST_TYPE", &varid_type);
type = malloc(nprof * WMO_INSTSIZE);
ncw_get_var_text(fname, ncid, varid_type, type);
ncw_close(fname, ncid);
strcpy(buf, fname);
len = strlen(buf);
buf[len - 10] = 0; /* _mmt_qc.nc */
if (!str2int(&buf[len - 12], &day))
enkf_quit("MMT reader: could not convert file name \"%s\" to date", fname);
buf[len - 12] = 0;
if (!str2int(&buf[len - 14], &month))
enkf_quit("MMT reader: could not convert file name \"%s\" to date", fname);
buf[len - 14] = 0;
if (!str2int(&buf[len - 18], &year))
enkf_quit("MMT reader: could not convert file name \"%s\" to date", fname);
snprintf(buf, MAXSTRLEN, "days since %4d-%02d-%02d", year, month, day);
tunits_convert(buf, &tunits_multiple, &tunits_offset);
mvid = model_getvarid(m, obs->obstypes[obstype_getid(obs->nobstypes, obs->obstypes, meta->type)].varname, 1);
for (p = 0; p < (int) nprof; ++p) {
char inststr[MAXSTRLEN];
snprintf(inststr, MAXSTRLEN, "WMO%04u", type[p * WMO_INSTSIZE]);
for (i = 0; i < (int) nz; ++i) {
observation* o;
obstype* ot;
if (fabs(v[p][i] - missval) < EPS || v[p][i] < validmin || v[p][i] > validmax)
continue;
if (z[p][i] < 0.0)
continue;
obs_checkalloc(obs);
o = &obs->data[obs->nobs];
o->product = st_findindexbystring(obs->products, meta->product);
assert(o->product >= 0);
o->type = obstype_getid(obs->nobstypes, obs->obstypes, meta->type);
assert(o->type >= 0);
ot = &obs->obstypes[o->type];
o->instrument = st_add_ifabscent(obs->instruments, inststr, -1);
o->id = obs->nobs;
o->fid = fid;
o->batch = p;
o->value = v[p][i];
o->std = 0.0;
o->lon = lon[p];
o->lat = lat[p];
o->depth = z[p][i];
o->status = model_xy2fij(m, mvid, o->lon, o->lat, &o->fi, &o->fj);
if (!obs->allobs && o->status == STATUS_OUTSIDEGRID)
break;
if (o->status == STATUS_OK)
o->status = model_z2fk(m, mvid, o->fi, o->fj, o->depth, &o->fk);
else
o->fk = NaN;
if ((o->status == STATUS_OK) && (o->lon <= ot->xmin || o->lon >= ot->xmax || o->lat <= ot->ymin || o->lat >= ot->ymax || o->depth <= ot->zmin || o->depth >= ot->zmax))
o->status = STATUS_OUTSIDEOBSDOMAIN;
o->date = tunits_offset + 0.5;
o->aux = -1;
obs->nobs++;
}
}
free(lon);
free(lat);
free2d(v);
free2d(z);
free(type);
}
int find_max_partition(int arr[], int n) {
int sum = 0;
int i, s, ret = 0;
int **dp;
int *a;
int cnt, p;
for (i = 0; i < n; ++i)
sum += arr[i];
if (sum % 2 != 0)
sum -= 1;
/* initialize dp array */
dp = alloc2d(sum + 1, n + 1);
for (i = 0; i < n + 1; ++i)
dp[0][i] = TRUE;
for (i = 1; i < sum + 1; ++i)
dp[i][0] = FALSE;
/* build dp array
* dp[s][i] is true if set Ai: { arr[0], ..., arr[i-1] }
* have subset with sum = s. There are two cases:
* a) arr[i-1] is part of that subset, so exclude it
* from from Ai and take into account sum = s - arr[i-1]
* b) arr[i-1] is not part of that subset, so exclude it but still sum = s; */
for (s = 1; s <= sum; s++) {
for (i = 1; i <= n; i++) {
dp[s][i] = dp[s][i - 1];
if (s >= arr[i - 1])
dp[s][i] = dp[s][i] || dp[s - arr[i - 1]][i - 1];
}
}
a = (int*) malloc(n * sizeof(int));
/* If set Arr: { arr[0], ..., arr[n-1] } has two
* disjoint subsets with the same sum, Arr has to have
* subset A1 with sum = x and A2 with sum = 2*x, and
* moreover A1 has to be subset of A2
* be careful, checking if dp[s][n] and dp[s/2][n] is TRUE is not enough (A1 and A2 could be disjoint)
* moreover, from dp one cannot conclude that A1 \subset A2, bcos there can be two subsets A1a and A1v
* with sum x, and unluckily dp algorithm could take wrong one into account, so
* A2 has to be checked if can be splited into two equal parts
*/
for (s = sum; s >= 2; s -= 2) {
if (dp[s][n] == FALSE && dp[s / 2][n] == FALSE) continue;
cnt = 0;
p = s;
while (p != 0)
for (i = n; i >= 1; --i) if (dp[p][i] == TRUE && dp[p][i - 1] == FALSE) {
a[cnt++] = arr[i - 1];
p -= arr[i - 1];
break;
}
if (can_equally_split(a, cnt) == TRUE) {
ret = s / 2;
break;
}
}
free(a);
free2d(dp);
return ret;
}
grid* grid_create(void* p, int id)
{
gridprm* prm = (gridprm*) p;
grid* g = calloc(1, sizeof(grid));
char* fname = prm->fname;
int ncid;
int dimid_x, dimid_y, dimid_z;
int varid_x, varid_y, varid_z;
int ndims_x, ndims_y, ndims_z;
size_t nx, ny, nz;
int varid_depth, varid_numlevels;
g->name = strdup(prm->name);
g->id = id;
g->vtype = gridprm_getvtype(prm);
ncw_open(fname, NC_NOWRITE, &ncid);
ncw_inq_dimid(fname, ncid, prm->xdimname, &dimid_x);
ncw_inq_dimid(fname, ncid, prm->ydimname, &dimid_y);
ncw_inq_dimid(fname, ncid, prm->zdimname, &dimid_z);
ncw_inq_dimlen(fname, ncid, dimid_x, &nx);
ncw_inq_dimlen(fname, ncid, dimid_y, &ny);
ncw_inq_dimlen(fname, ncid, dimid_z, &nz);
ncw_inq_varid(fname, ncid, prm->xvarname, &varid_x);
ncw_inq_varid(fname, ncid, prm->yvarname, &varid_y);
ncw_inq_varid(fname, ncid, prm->zvarname, &varid_z);
ncw_inq_varndims(fname, ncid, varid_x, &ndims_x);
ncw_inq_varndims(fname, ncid, varid_y, &ndims_y);
ncw_inq_varndims(fname, ncid, varid_z, &ndims_z);
if (ndims_x == 1 && ndims_y == 1) {
double* x;
double* y;
double* z;
int i;
double dx, dy;
int periodic_x;
x = malloc(nx * sizeof(double));
y = malloc(ny * sizeof(double));
z = malloc(nz * sizeof(double));
ncw_get_var_double(fname, ncid, varid_x, x);
ncw_get_var_double(fname, ncid, varid_y, y);
ncw_get_var_double(fname, ncid, varid_z, z);
periodic_x = fabs(fmod(2.0 * x[nx - 1] - x[nx - 2], 360.0) - x[0]) < EPS_LON;
dx = (x[nx - 1] - x[0]) / (double) (nx - 1);
for (i = 1; i < (int) nx; ++i)
if (fabs(x[i] - x[i - 1] - dx) / fabs(dx) > EPS_LON)
break;
if (i != (int) nx)
grid_setcoords(g, GRIDHTYPE_LATLON_IRREGULAR, NT_NONE, periodic_x, 0, nx, ny, nz, x, y, z);
else {
dy = (y[ny - 1] - y[0]) / (double) (ny - 1);
for (i = 1; i < (int) ny; ++i)
if (fabs(y[i] - y[i - 1] - dy) / fabs(dy) > EPS_LON)
break;
if (i != (int) ny)
grid_setcoords(g, GRIDHTYPE_LATLON_IRREGULAR, NT_NONE, periodic_x, 0, nx, ny, nz, x, y, z);
else
grid_setcoords(g, GRIDHTYPE_LATLON_REGULAR, NT_NONE, periodic_x, 0, nx, ny, nz, x, y, z);
}
}
#if !defined(NO_GRIDUTILS)
else if (ndims_x == 2 && ndims_y == 2) {
double** x;
double** y;
double* z;
x = alloc2d(ny, nx, sizeof(double));
y = alloc2d(ny, nx, sizeof(double));
z = malloc(nz * sizeof(double));
ncw_get_var_double(fname, ncid, varid_x, x[0]);
ncw_get_var_double(fname, ncid, varid_y, y[0]);
ncw_get_var_double(fname, ncid, varid_z, z);
grid_setcoords(g, GRIDHTYPE_CURVILINEAR, NT_COR, 0, 0, nx, ny, nz, x, y, z);
}
#endif
else
enkf_quit("%s: could not determine the grid type", fname);
if (prm->depthvarname != NULL) {
float** depth = alloc2d(ny, nx, sizeof(float));
ncw_inq_varid(fname, ncid, prm->depthvarname, &varid_depth);
ncw_get_var_float(fname, ncid, varid_depth, depth[0]);
g->depth = depth;
}
if (prm->levelvarname != NULL) {
g->numlevels = alloc2d(ny, nx, sizeof(int));
ncw_inq_varid(fname, ncid, prm->levelvarname, &varid_numlevels);
ncw_get_var_int(fname, ncid, varid_numlevels, g->numlevels[0]);
if (g->vtype == GRIDVTYPE_SIGMA) {
int i, j;
for (j = 0; j < ny; ++j)
for (i = 0; i < nx; ++i)
g->numlevels[j][i] *= nz;
}
}
ncw_close(fname, ncid);
if (g->numlevels == NULL && g->depth != NULL) {
g->numlevels = alloc2d(ny, nx, sizeof(int));
if (g->vtype == GRIDVTYPE_SIGMA) {
int i, j;
for (j = 0; j < ny; ++j)
for (i = 0; i < nx; ++i)
if (g->depth[j][i] > 0.0)
g->numlevels[j][i] = nz;
} else {
int i, j;
for (j = 0; j < ny; ++j) {
for (i = 0; i < nx; ++i) {
double depth = g->depth[j][i];
double fk = NaN;
if (depth > 0.0) {
z2fk(g, j, i, depth, &fk);
g->numlevels[j][i] = ceil(fk + 0.5);
}
}
}
}
}
gridprm_print(prm, " ");
grid_print(g, " ");
return g;
}
void das_getHE(dasystem* das)
{
observations* obs = das->obs;
model* m = das->m;
ENSOBSTYPE* Hx = NULL;
int i, e;
das->s_mode = S_MODE_HE_f;
if (obs->nobs == 0)
return;
if (das->nmem <= 0)
das_setnmem(das);
enkf_printf(" ensemble size = %d\n", das->nmem);
assert(das->nmem > 0);
distribute_iterations(0, das->nmem - 1, nprocesses, rank, " ");
/*
* ensemble observation array to be filled
*/
assert(das->S == NULL);
das->S = alloc2d(das->nmem, obs->nobs, sizeof(ENSOBSTYPE));
if (das->mode == MODE_ENOI)
Hx = calloc(obs->nobs, sizeof(ENSOBSTYPE));
for (i = 0; i < obs->nobstypes; ++i) {
obstype* ot = &obs->obstypes[i];
float*** vvv = NULL;
float** vv = NULL;
H_fn H = NULL;
int mvid;
int ni, nj, nk;
int nobs;
int* obsids;
char fname[MAXSTRLEN];
enkf_printf(" %s ", ot->name);
fflush(stdout);
mvid = model_getvarid(m, obs->obstypes[i].varnames[0], 1);
if (ot->issurface) {
model_getvardims(m, mvid, &ni, &nj, NULL);
vv = alloc2d(nj, ni, sizeof(float));
} else {
model_getvardims(m, mvid, &ni, &nj, &nk);
vvv = alloc3d(nk, nj, ni, sizeof(float));
}
/*
* set H
*/
H = getH(ot->name, ot->hfunction);
if (ot->isasync) {
int t1 = get_tshift(ot->date_min, ot->async_tstep);
int t2 = get_tshift(ot->date_max, ot->async_tstep);
int t;
for (t = t1; t <= t2; ++t) {
enkf_printf("|");
obs_find_bytypeandtime(obs, i, t, &nobs, &obsids);
if (nobs == 0)
continue;
/*
* for EnOI it is essential sometimes (e.g. in some bias
* correction cases) that the background is interpolated first
*/
if (das->mode == MODE_ENOI) {
if (enkf_obstype == OBSTYPE_VALUE) {
int success = model_getbgfname_async(m, das->bgdir, ot->varnames[0], ot->name, t, fname);
H(das, nobs, obsids, fname, -1, t, (ot->issurface) ? (void*) vv : (void*) vvv, Hx);
enkf_printf((success) ? "A" : "S");
fflush(stdout);
} else if (enkf_obstype == OBSTYPE_INNOVATION) {
Hx[0] = 0;
enkf_printf("-");
fflush(stdout);
}
}
if (das->mode == MODE_ENKF || !enkf_fstatsonly) {
for (e = my_first_iteration; e <= my_last_iteration; ++e) {
int success = model_getmemberfname_async(m, das->ensdir, ot->varnames[0], ot->name, e + 1, t, fname);
H(das, nobs, obsids, fname, e + 1, t, (ot->issurface) ? (void*) vv : (void*) vvv, das->S[e]);
enkf_printf((success) ? "a" : "s");
fflush(stdout);
}
}
free(obsids);
}
} else {
obs_find_bytype(obs, i, &nobs, &obsids);
if (nobs == 0)
goto next;
/*
* for EnOI it is essential sometimes (e.g. in some bias correction
* cases) that the background is interpolated first
*/
if (das->mode == MODE_ENOI) {
if (enkf_obstype == OBSTYPE_VALUE) {
model_getbgfname(m, das->bgdir, ot->varnames[0], fname);
H(das, nobs, obsids, fname, -1, INT_MAX, (ot->issurface) ? (void*) vv : (void*) vvv, Hx);
enkf_printf("+");
fflush(stdout);
} else if (enkf_obstype == OBSTYPE_INNOVATION) {
Hx[0] = 0;
enkf_printf("-");
fflush(stdout);
}
}
if (das->mode == MODE_ENKF || !enkf_fstatsonly) {
for (e = my_first_iteration; e <= my_last_iteration; ++e) {
model_getmemberfname(m, das->ensdir, ot->varnames[0], e + 1, fname);
H(das, nobs, obsids, fname, e + 1, INT_MAX, (ot->issurface) ? (void*) vv : (void*) vvv, das->S[e]);
enkf_printf(".");
fflush(stdout);
}
}
free(obsids);
}
next:
if (ot->issurface)
free(vv);
else
free(vvv);
enkf_printf("\n");
} /* for i (over obstypes) */
#if defined(MPI)
if (das->mode == MODE_ENKF || !enkf_fstatsonly) {
#if !defined(HE_VIAFILE)
/*
* communicate HE via MPI
*/
int ierror, sendcount, *recvcounts, *displs;
recvcounts = malloc(nprocesses * sizeof(int));
displs = malloc(nprocesses * sizeof(int));
sendcount = my_number_of_iterations * obs->nobs;
for (i = 0; i < nprocesses; ++i) {
recvcounts[i] = number_of_iterations[i] * obs->nobs;
displs[i] = first_iteration[i] * obs->nobs;
}
ierror = MPI_Allgatherv(das->S[my_first_iteration], sendcount, MPIENSOBSTYPE, das->S[0], recvcounts, displs, MPIENSOBSTYPE, MPI_COMM_WORLD);
assert(ierror == MPI_SUCCESS);
free(recvcounts);
free(displs);
#else
/*
* communicate HE via file
*/
{
int ncid;
int varid;
size_t start[2], count[2];
if (rank == 0) {
int dimids[2];
ncw_create(FNAME_HE, NC_CLOBBER | NETCDF_FORMAT, &ncid);
ncw_def_dim(FNAME_HE, ncid, "m", das->nmem, &dimids[0]);
ncw_def_dim(FNAME_HE, ncid, "p", obs->nobs, &dimids[1]);
ncw_def_var(FNAME_HE, ncid, "HE", NC_FLOAT, 2, dimids, &varid);
ncw_close(FNAME_HE, ncid);
}
MPI_Barrier(MPI_COMM_WORLD);
ncw_open(FNAME_HE, NC_WRITE, &ncid);
ncw_inq_varid(FNAME_HE, ncid, "HE", &varid);
start[0] = my_first_iteration;
start[1] = 0;
count[0] = my_last_iteration - my_first_iteration + 1;
count[1] = obs->nobs;
ncw_put_vara_float(FNAME_HE, ncid, varid, start, count, das->S[my_first_iteration]);
ncw_close(FNAME_HE, ncid);
MPI_Barrier(MPI_COMM_WORLD);
ncw_open(FNAME_HE, NC_NOWRITE, &ncid);
ncw_inq_varid(FNAME_HE, ncid, "HE", &varid);
ncw_get_var_float(FNAME_HE, ncid, varid, das->S[0]);
ncw_close(FNAME_HE, ncid);
}
#endif
}
#endif
if (das->mode == MODE_ENOI) {
/*
* subtract ensemble mean; add background
*/
if (!enkf_fstatsonly) {
double* ensmean = calloc(obs->nobs, sizeof(double));
for (e = 0; e < das->nmem; ++e) {
ENSOBSTYPE* Se = das->S[e];
for (i = 0; i < obs->nobs; ++i)
ensmean[i] += Se[i];
}
for (i = 0; i < obs->nobs; ++i)
ensmean[i] /= (double) das->nmem;
for (e = 0; e < das->nmem; ++e) {
ENSOBSTYPE* Se = das->S[e];
for (i = 0; i < obs->nobs; ++i)
Se[i] += Hx[i] - ensmean[i];
}
free(ensmean);
} else {
for (e = 0; e < das->nmem; ++e) {
ENSOBSTYPE* Se = das->S[e];
for (i = 0; i < obs->nobs; ++i)
Se[i] = Hx[i];
}
}
}
if (das->mode == MODE_ENOI)
free(Hx);
}
int main(int argc, char** argv)
{
int ndims = 0;
char** dims = NULL;
int nsrc = 0;
char** src = NULL;
char* dst = NULL;
int i;
int** dimids = NULL;
int nvars = 0;
int* varids = NULL;
size_t* size = NULL;
int nvars0;
if (argc == 1) {
usage();
description();
exit(0);
}
i = 1;
while (i < argc) {
if (argv[i][0] != '-') {
usage();
exit(1);
}
if (argv[i][1] == 'd') {
i++;
while (i < argc && argv[i][0] != '-') {
if (ndims % INC == 0)
dims = realloc(dims, (ndims + INC) * sizeof(char*));
dims[ndims] = argv[i];
ndims++;
i++;
}
} else if (argv[i][1] == 'i') {
i++;
while (i < argc && argv[i][0] != '-') {
if (nsrc % INC == 0)
src = realloc(src, (nsrc + INC) * sizeof(char*));
src[nsrc] = strdup(argv[i]);
nsrc++;
i++;
}
} else if (argv[i][1] == 'o') {
i++;
if (i < argc && argv[i][0] != '-') {
dst = argv[i];
i++;
}
} else if (argv[i][1] == 'v') {
verbose = 1;
i++;
} else {
usage();
exit(1);
}
}
if (argc == 2 && verbose) {
printf(" ncw version %s\n", ncw_version);
exit(1);
}
if (ndims == 0 || nsrc < 1 || dst == NULL) {
usage();
exit(1);
}
printf_v(" src:\n");
for (i = 0; i < nsrc; ++i)
printf_v(" \"%s\"\n", src[i]);
printf_v(" dst = \"%s\"\n", dst);
printf_v(" merged dimensions:\n");
for (i = 0; i < ndims; ++i)
printf_v(" %s\n", dims[i]);
dimids = alloc2d(nsrc, ndims, sizeof(int));
for (i = 0; i < nsrc; ++i) {
int ncid, j;
ncw_open(src[i], NC_NOWRITE, &ncid);
for (j = 0; j < ndims; ++j)
ncw_inq_dimid(ncid, dims[j], &dimids[i][j]);
ncw_close(ncid);
}
{
int ncid;
ncw_open(src[0], NC_NOWRITE, &ncid);
ncw_inq_nvars(ncid, &nvars0);
varids = malloc(nvars0 * sizeof(int));
printf_v(" merged variables:\n");
for (i = 0; i < nvars0; ++i) {
int dimids_now[NC_MAX_VAR_DIMS];
char varname[NC_MAX_NAME] = "";
int j;
ncw_inq_vardimid(ncid, i, dimids_now);
for (j = 0; j < ndims; ++j) {
if (dimids_now[0] == dimids[0][j]) {
varids[nvars] = i;
nvars++;
ncw_inq_varname(ncid, i, varname);
printf_v(" %s\n", varname);
break;
}
}
}
ncw_close(ncid);
}
printf_v(" creating dst:");
{
int ncid_dst, ncid0;
int ndims0;
ncw_create(dst, NC_CLOBBER | NC_64BIT_OFFSET, &ncid_dst);
ncw_open(src[0], NC_NOWRITE, &ncid0);
ncw_inq_ndims(ncid0, &ndims0);
for (i = 0; i < ndims0; ++i) {
char dimname[NC_MAX_NAME];
size_t dimlen;
int dimid;
int j;
ncw_inq_dim(ncid0, i, dimname, &dimlen);
for (j = 0; j < ndims; ++j) {
if (i == dimids[0][j]) {
int ncid_now, dimid_now;
size_t dimlen_now;
int s;
for (s = 1; s < nsrc; ++s) {
ncw_open(src[s], NC_NOWRITE, &ncid_now);
ncw_inq_dimid(ncid_now, dimname, &dimid_now);
ncw_inq_dimlen(ncid_now, dimid_now, &dimlen_now);
ncw_close(ncid_now);
dimlen += dimlen_now;
}
break;
}
}
ncw_def_dim(ncid_dst, dimname, dimlen, &dimid);
}
for (i = 0; i < nvars0; ++i) {
nc_type type;
int nvardims;
char varname[NC_MAX_NAME];
int dimids0[NC_MAX_DIMS], dimids_dst[NC_MAX_DIMS];
int natts;
int varid_dst;
int j;
ncw_inq_var(ncid0, i, varname, &type, &nvardims, dimids0, &natts);
for (j = 0; j < nvardims; ++j) {
char dimname[NC_MAX_NAME];
size_t dimlen;
ncw_inq_dim(ncid0, dimids0[j], dimname, &dimlen);
ncw_inq_dimid(ncid_dst, dimname, &dimids_dst[j]);
}
ncw_def_var(ncid_dst, varname, type, nvardims, dimids_dst, &varid_dst);
ncw_copy_atts(ncid0, i, ncid_dst, varid_dst);
}
ncw_close(ncid_dst);
ncw_close(ncid0);
}
printf_v("\n");
printf_v(" copying data:");
size = malloc(nsrc * sizeof(size_t));
{
int ncid_dst;
ncw_open(dst, NC_WRITE, &ncid_dst);
for (i = 0; i < nvars0; ++i) {
int j;
for (j = 0; j < nvars; ++j)
if (varids[j] == i)
break;
if (j == nvars) {
int ncid0;
ncw_open(src[0], NC_NOWRITE, &ncid0);
ncw_copy_vardata(ncid0, i, ncid_dst);
ncw_close(ncid0);
} else {
int ncid;
char varname[NC_MAX_NAME];
int s;
size_t size_total = 0;
size_t nbytes;
void* data = NULL;
void* data_now;
nc_type type;
int varid_dst;
ncw_open(src[0], NC_NOWRITE, &ncid);
ncw_inq_varname(ncid, i, varname);
ncw_close(ncid);
size_total = 0;
for (s = 0; s < nsrc; ++s) {
int ncid;
int varid;
int ndims;
int dimids[NC_MAX_DIMS];
size_t dimlens[NC_MAX_DIMS];
ncw_open(src[s], NC_NOWRITE, &ncid);
ncw_inq_varid(ncid, varname, &varid);
ncw_inq_var(ncid, varid, NULL, &type, &ndims, dimids, NULL);
for (j = 0; j < ndims; ++j)
ncw_inq_dimlen(ncid, dimids[j], &dimlens[j]);
ncw_close(ncid);
size[s] = 1;
for (j = 0; j < ndims; ++j)
size[s] *= dimlens[j];
size_total += size[s];
}
nbytes = size_total * ncw_sizeof(type);
if (nbytes > 4294967296)
quit("sizeof(%s) = %zu exceeds 4GB", varname, nbytes);
data = malloc(nbytes);
if (data == NULL)
quit("malloc(): could not allocate memory for variable \"%s\" (size = %zu)", varname, size_total * ncw_sizeof(type));
data_now = data;
for (s = 0; s < nsrc; ++s) {
int ncid;
int varid;
ncw_open(src[s], NC_NOWRITE, &ncid);
ncw_inq_varid(ncid, varname, &varid);
ncw_get_var(ncid, varid, data_now);
ncw_close(ncid);
data_now = &((char*) data_now)[size[s] * ncw_sizeof(type)];
}
ncw_inq_varid(ncid_dst, varname, &varid_dst);
ncw_put_var(ncid_dst, varid_dst, data);
free(data);
}
printf_v(".");
}
ncw_close(ncid_dst);
}
printf_v("\n");
/*
* clean up
*/
free(size);
free(varids);
free(dimids);
for (i = 0; i < nsrc; ++i)
free(src[i]);
free(src);
return 0;
}
void reader_cars_standard(char* fname, int fid, obsmeta* meta, grid* g, observations* obs)
{
int ncid;
int dimid_nprof, dimid_nz = -1;
size_t nprof, nz;
int varid_lon, varid_lat, varid_z, varid_type;
int varid_v = -1;
double* lon;
double* lat;
double** z;
double** v;
double missval, validmin, validmax;
int* type;
char buf[MAXSTRLEN];
int len;
int year, month, day;
double tunits_multiple, tunits_offset;
int p, i;
for (i = 0; i < meta->npars; ++i)
enkf_quit("unknown PARAMETER \"%s\"\n", meta->pars[i].name);
if (meta->nstds == 0)
enkf_quit("ERROR_STD is necessary but not specified for product \"%s\"", meta->product);
ncw_open(fname, NC_NOWRITE, &ncid);
ncw_inq_dimid(ncid, "nobs", &dimid_nprof);
ncw_inq_dimlen(ncid, dimid_nprof, &nprof);
enkf_printf(" # profiles = %u\n", (unsigned int) nprof);
if (nprof == 0) {
ncw_close(ncid);
return;
}
if (ncw_dim_exists(ncid, "zt"))
ncw_inq_dimid(ncid, "zt", &dimid_nz);
else if (ncw_dim_exists(ncid, "ztd"))
ncw_inq_dimid(ncid, "ztd", &dimid_nz);
else
enkf_quit("reader_cars_standard(): neither dimension \"zt\" ot \"ztd\" exist");
ncw_inq_dimlen(ncid, dimid_nz, &nz);
enkf_printf(" # z levels = %u\n", (unsigned int) nz);
ncw_inq_varid(ncid, "lon", &varid_lon);
lon = malloc(nprof * sizeof(double));
ncw_get_var_double(ncid, varid_lon, lon);
ncw_inq_varid(ncid, "lat", &varid_lat);
lat = malloc(nprof * sizeof(double));
ncw_get_var_double(ncid, varid_lat, lat);
ncw_inq_varid(ncid, "zt", &varid_z);
z = alloc2d(nprof, nz, sizeof(double));
ncw_get_var_double(ncid, varid_z, z[0]);
if (strncmp(meta->type, "TEM", 3) == 0)
ncw_inq_varid(ncid, "temp", &varid_v);
else if (strncmp(meta->type, "SAL", 3) == 0)
ncw_inq_varid(ncid, "salt", &varid_v);
else
enkf_quit("observation type \"%s\" not handled for CARS product", meta->type);
v = alloc2d(nprof, nz, sizeof(double));
ncw_get_var_double(ncid, varid_v, v[0]);
ncw_get_att_double(ncid, varid_v, "missing_value", &missval);
ncw_get_att_double(ncid, varid_v, "valid_min", &validmin);
ncw_get_att_double(ncid, varid_v, "valid_max", &validmax);
ncw_inq_varid(ncid, "type", &varid_type);
type = malloc(nprof * sizeof(int));
ncw_get_var_int(ncid, varid_type, type);
ncw_close(ncid);
strcpy(buf, fname);
len = strlen(buf);
buf[len - 3] = 0; /* .nc */
if (!str2int(&buf[len - 5], &day))
enkf_quit("CARS reader: could not convert file name \"%s\" to date", fname);
buf[len - 17] = 0;
if (!str2int(&buf[len - 19], &month))
enkf_quit("CARS reader: could not convert file name \"%s\" to date", fname);
buf[len - 21] = 0;
if (!str2int(&buf[len - 25], &year))
enkf_quit("CARS reader: could not convert file name \"%s\" to date", fname);
snprintf(buf, MAXSTRLEN, "days since %4d-%02d-%02d", year, month, day);
tunits_convert(buf, &tunits_multiple, &tunits_offset);
for (p = 0; p < (int) nprof; ++p) {
char inststr[MAXSTRLEN];
if (type[p] == 11)
strcpy(inststr, "ARGO");
else if (type[p] == 12)
strcpy(inststr, "TAO");
else if (type[p] == 61)
strcpy(inststr, "PIRATA");
else if (type[p] == 7 || type[p] == 9 || type[p] == 13 || type[p] == 35 || type[p] == 41)
strcpy(inststr, "CTD");
else if (type[p] == 8 || type[p] == 17)
strcpy(inststr, "XBT");
else
snprintf(inststr, MAXSTRLEN, "CARS%02u", type[p]);
for (i = 0; i < (int) nz; ++i) {
observation* o;
obstype* ot;
if (fabs(v[p][i] - missval) < EPS || v[p][i] < validmin || v[p][i] > validmax)
continue;
if (z[p][i] < 0.0)
continue;
obs_checkalloc(obs);
o = &obs->data[obs->nobs];
o->product = st_findindexbystring(obs->products, meta->product);
assert(o->product >= 0);
o->type = obstype_getid(obs->nobstypes, obs->obstypes, meta->type, 1);
ot = &obs->obstypes[o->type];
o->instrument = st_add_ifabsent(obs->instruments, inststr, -1);
o->id = obs->nobs;
o->fid = fid;
o->batch = p;
o->value = v[p][i];
o->std = 0.0;
o->lon = lon[p];
o->lat = lat[p];
o->depth = z[p][i];
o->status = grid_xy2fij(g, o->lon, o->lat, &o->fi, &o->fj);
if (!obs->allobs && o->status == STATUS_OUTSIDEGRID)
break;
if (o->status == STATUS_OK)
o->status = grid_z2fk(g, o->fi, o->fj, o->depth, &o->fk);
else
o->fk = NAN;
if ((o->status == STATUS_OK) && (o->lon <= ot->xmin || o->lon >= ot->xmax || o->lat <= ot->ymin || o->lat >= ot->ymax || o->depth <= ot->zmin || o->depth >= ot->zmax))
o->status = STATUS_OUTSIDEOBSDOMAIN;
o->model_depth = NAN; /* set in obs_add() */
o->date = tunits_offset + 0.5;
o->aux = -1;
obs->nobs++;
}
}
free(lon);
free(lat);
free(v);
free(z);
free(type);
}
grid* grid_create(void* p, int id)
{
gridprm* prm = (gridprm*) p;
grid* g = calloc(1, sizeof(grid));
char* fname = prm->fname;
int ncid;
int dimid_x, dimid_y, dimid_z;
int varid_x, varid_y, varid_z;
int ndims_x, ndims_y, ndims_z;
size_t nx, ny, nz;
int varid_depth, varid_numlevels;
g->name = strdup(prm->name);
g->id = id;
g->vtype = gridprm_getvtype(prm);
g->sfactor = prm->sfactor;
#if !defined(NO_GRIDUTILS)
#if !defined(GRIDMAP_TYPE_DEF)
#error("GRIDMAP_TYPE_DEF not defined; please update gridutils-c");
#endif
if (prm->maptype == 'b' || prm->maptype == 'B')
g->maptype = GRIDMAP_TYPE_BINARY;
else if (prm->maptype == 'k' || prm->maptype == 'K')
g->maptype = GRIDMAP_TYPE_KDTREE;
else
enkf_quit("unknown grid map type \"%c\"", prm->maptype);
#endif
ncw_open(fname, NC_NOWRITE, &ncid);
ncw_inq_dimid(ncid, prm->xdimname, &dimid_x);
ncw_inq_dimid(ncid, prm->ydimname, &dimid_y);
ncw_inq_dimid(ncid, prm->zdimname, &dimid_z);
ncw_inq_dimlen(ncid, dimid_x, &nx);
ncw_inq_dimlen(ncid, dimid_y, &ny);
ncw_inq_dimlen(ncid, dimid_z, &nz);
ncw_inq_varid(ncid, prm->xvarname, &varid_x);
ncw_inq_varid(ncid, prm->yvarname, &varid_y);
ncw_inq_varid(ncid, prm->zvarname, &varid_z);
ncw_inq_varndims(ncid, varid_x, &ndims_x);
ncw_inq_varndims(ncid, varid_y, &ndims_y);
ncw_inq_varndims(ncid, varid_z, &ndims_z);
if (ndims_x == 1 && ndims_y == 1) {
double* x;
double* y;
double* z;
x = malloc(nx * sizeof(double));
y = malloc(ny * sizeof(double));
z = malloc(nz * sizeof(double));
ncw_get_var_double(ncid, varid_x, x);
ncw_get_var_double(ncid, varid_y, y);
ncw_get_var_double(ncid, varid_z, z);
grid_setcoords(g, GRIDHTYPE_LATLON, NT_NONE, nx, ny, nz, x, y, z);
}
#if !defined(NO_GRIDUTILS)
else if (ndims_x == 2 && ndims_y == 2) {
double** x;
double** y;
double* z;
x = gu_alloc2d(ny, nx, sizeof(double));
y = gu_alloc2d(ny, nx, sizeof(double));
z = malloc(nz * sizeof(double));
ncw_get_var_double(ncid, varid_x, x[0]);
ncw_get_var_double(ncid, varid_y, y[0]);
ncw_get_var_double(ncid, varid_z, z);
grid_setcoords(g, GRIDHTYPE_CURVILINEAR, NT_COR, nx, ny, nz, x, y, z);
}
#endif
else
enkf_quit("%s: could not determine the grid type", fname);
if (prm->depthvarname != NULL) {
float** depth = alloc2d(ny, nx, sizeof(float));
ncw_inq_varid(ncid, prm->depthvarname, &varid_depth);
ncw_get_var_float(ncid, varid_depth, depth[0]);
g->depth = depth;
}
if (prm->levelvarname != NULL) {
g->numlevels = alloc2d(ny, nx, sizeof(int));
ncw_inq_varid(ncid, prm->levelvarname, &varid_numlevels);
ncw_get_var_int(ncid, varid_numlevels, g->numlevels[0]);
if (g->vtype == GRIDVTYPE_SIGMA) {
int i, j;
for (j = 0; j < ny; ++j)
for (i = 0; i < nx; ++i)
g->numlevels[j][i] *= nz;
}
}
ncw_close(ncid);
if (g->numlevels == NULL) {
g->numlevels = alloc2d(ny, nx, sizeof(int));
if (g->vtype == GRIDVTYPE_SIGMA) {
int i, j;
for (j = 0; j < ny; ++j)
for (i = 0; i < nx; ++i)
if (g->depth == NULL || g->depth[j][i] > 0.0)
g->numlevels[j][i] = nz;
} else {
int i, j;
assert(g->depth != NULL);
for (j = 0; j < ny; ++j) {
for (i = 0; i < nx; ++i) {
double depth = g->depth[j][i];
double fk = NaN;
if (depth > 0.0) {
z2fk(g, j, i, depth, &fk);
g->numlevels[j][i] = ceil(fk + 0.5);
}
}
}
}
}
gridprm_print(prm, " ");
grid_print(g, " ");
return g;
}