static void scan_partialities(RefList *reflections, RefList *compare, int *valid, long double *vals[3], int idx) { int i; Reflection *refl; RefListIterator *iter; i = 0; for ( refl = first_refl(reflections, &iter); refl != NULL; refl = next_refl(refl, iter) ) { signed int h, k, l; Reflection *refl2; get_indices(refl, &h, &k, &l); refl2 = find_refl(compare, h, k, l); if ( refl2 == NULL ) { valid[i] = 0; i++; continue; } vals[idx][i] = get_lorentz(refl2); i++; } }
static void scan_partialities(RefList *reflections, RefList *compare, int *valid, long double *vals[3], int idx, PartialityModel pmodel) { int i; Reflection *refl; RefListIterator *iter; i = 0; for ( refl = first_refl(reflections, &iter); refl != NULL; refl = next_refl(refl, iter) ) { signed int h, k, l; Reflection *refl2; double rlow, rhigh, p; get_indices(refl, &h, &k, &l); refl2 = find_refl(compare, h, k, l); if ( refl2 == NULL ) { valid[i] = 0; i++; continue; } get_partial(refl2, &rlow, &rhigh, &p); vals[idx][i] = p; if ( unlikely(p < 0.0) ) { ERROR("Negative partiality! %3i %3i %3i %f\n", h, k, l, p); } i++; } }
static void write_stream_reflections(FILE *fh, RefList *list) { Reflection *refl; RefListIterator *iter; fprintf(fh, " h k l I sigma(I) peak background" " fs/px ss/px\n"); for ( refl = first_refl(list, &iter); refl != NULL; refl = next_refl(refl, iter) ) { signed int h, k, l; double intensity, esd_i, bg, pk; double fs, ss; get_indices(refl, &h, &k, &l); get_detector_pos(refl, &fs, &ss); intensity = get_intensity(refl); esd_i = get_esd_intensity(refl); pk = get_peak(refl); bg = get_mean_bg(refl); /* Reflections with redundancy = 0 are not written */ if ( get_redundancy(refl) == 0 ) continue; fprintf(fh, "%4i %4i %4i %10.2f %10.2f %10.2f %10.2f %6.1f %6.1f\n", h, k, l, intensity, esd_i, pk, bg, fs, ss); } }
static int write_stream_reflections_2_2(FILE *fh, RefList *list, struct image *image) { Reflection *refl; RefListIterator *iter; fprintf(fh, " h k l I sigma(I) " "peak background fs/px ss/px\n"); for ( refl = first_refl(list, &iter); refl != NULL; refl = next_refl(refl, iter) ) { signed int h, k, l; double intensity, esd_i, bg, pk; double fs, ss; get_indices(refl, &h, &k, &l); get_detector_pos(refl, &fs, &ss); intensity = get_intensity(refl); esd_i = get_esd_intensity(refl); pk = get_peak(refl); bg = get_mean_bg(refl); /* Reflections with redundancy = 0 are not written */ if ( get_redundancy(refl) == 0 ) continue; if ( image->det != NULL ) { struct panel *p; double write_fs, write_ss; p = find_orig_panel(image->det, fs, ss); if ( p == NULL ) { ERROR("Panel not found\n"); return 1; } /* Convert coordinates to match arrangement of panels in HDF5 * file */ write_fs = fs - p->min_fs + p->orig_min_fs; write_ss = ss - p->min_ss + p->orig_min_ss; fprintf(fh, "%4i %4i %4i %10.2f %10.2f %10.2f %10.2f" " %6.1f %6.1f\n", h, k, l, intensity, esd_i, pk, bg, write_fs, write_ss); } else { fprintf(fh, "%4i %4i %4i %10.2f %10.2f %10.2f %10.2f" " %6.1f %6.1f\n", h, k, l, intensity, esd_i, pk, bg, fs, ss); } } return 0; }
static int num_integrated_reflections(RefList *list) { Reflection *refl; RefListIterator *iter; int n = 0; for ( refl = first_refl(list, &iter); refl != NULL; refl = next_refl(refl, iter) ) { if ( get_redundancy(refl) > 0 ) n++; } return n; }
static int write_stream_reflections_2_3(FILE *fh, RefList *list, struct image *image) { Reflection *refl; RefListIterator *iter; fprintf(fh, " h k l I sigma(I) " "peak background fs/px ss/px panel\n"); for ( refl = first_refl(list, &iter); refl != NULL; refl = next_refl(refl, iter) ) { signed int h, k, l; double intensity, esd_i, pk, bg; double fs, ss; double write_fs, write_ss; struct panel *p = NULL; get_indices(refl, &h, &k, &l); get_detector_pos(refl, &fs, &ss); intensity = get_intensity(refl); esd_i = get_esd_intensity(refl); pk = get_peak(refl); bg = get_mean_bg(refl); /* Reflections with redundancy = 0 are not written */ if ( get_redundancy(refl) == 0 ) continue; p = find_panel(image->det,fs,ss); if ( p == NULL ) { ERROR("Panel not found\n"); return 1; } write_fs = fs-p->min_fs+p->orig_min_fs; write_ss = ss-p->min_ss+p->orig_min_ss; fprintf(fh, "%4i %4i %4i %10.2f %10.2f %10.2f %10.2f " "%6.1f %6.1f %s\n", h, k, l, intensity, esd_i, pk, bg, write_fs, write_ss, p->name); } return 0; }
static double guide_dev(Crystal *cr, const RefList *full) { double dev = 0.0; /* For each reflection */ Reflection *refl; RefListIterator *iter; for ( refl = first_refl(crystal_get_reflections(cr), &iter); refl != NULL; refl = next_refl(refl, iter) ) { double G, p; signed int h, k, l; Reflection *full_version; double I_full, I_partial; if ( (get_intensity(refl) < 3.0*get_esd_intensity(refl)) || (get_partiality(refl) < MIN_PART_REFINE) ) continue; get_indices(refl, &h, &k, &l); assert((h!=0) || (k!=0) || (l!=0)); full_version = find_refl(full, h, k, l); if ( full_version == NULL ) continue; /* Some reflections may have recently become scalable, but * scale_intensities() might not yet have been called, so the * full version may not have been calculated yet. */ G = crystal_get_osf(cr); p = get_partiality(refl); I_partial = get_intensity(refl); I_full = get_intensity(full_version); //STATUS("%3i %3i %3i %5.2f %5.2f %5.2f %5.2f %5.2f\n", // h, k, l, G, p, I_partial, I_full, // I_partial - p*G*I_full); dev += pow(I_partial - p*G*I_full, 2.0); } return dev; }
static void write_stream_reflections_2_1(FILE *fh, RefList *list) { Reflection *refl; RefListIterator *iter; fprintf(fh, " h k l I phase sigma(I) " " counts fs/px ss/px\n"); for ( refl = first_refl(list, &iter); refl != NULL; refl = next_refl(refl, iter) ) { signed int h, k, l; double intensity, esd_i, ph; int red; double fs, ss; char phs[16]; int have_phase; get_indices(refl, &h, &k, &l); get_detector_pos(refl, &fs, &ss); intensity = get_intensity(refl); esd_i = get_esd_intensity(refl); red = get_redundancy(refl); ph = get_phase(refl, &have_phase); /* Reflections with redundancy = 0 are not written */ if ( red == 0 ) continue; if ( have_phase ) { snprintf(phs, 16, "%8.2f", rad2deg(ph)); } else { strncpy(phs, " -", 15); } fprintf(fh, "%3i %3i %3i %10.2f %s %10.2f %7i %6.1f %6.1f\n", h, k, l, intensity, phs, esd_i, red, fs, ss); } }
static double test_gradients(Crystal *cr, double incr_val, int refine, const char *str, const char *file, PartialityModel pmodel, int quiet, int plot) { Reflection *refl; RefListIterator *iter; long double *vals[3]; int i; int *valid; int nref; int n_good, n_invalid, n_small, n_nan, n_bad; RefList *reflections; FILE *fh = NULL; int ntot = 0; double total = 0.0; char tmp[32]; double *vec1; double *vec2; int n_line; double cc; reflections = find_intersections(crystal_get_image(cr), cr, pmodel); crystal_set_reflections(cr, reflections); nref = num_reflections(reflections); if ( nref < 10 ) { ERROR("Too few reflections found. Failing test by default.\n"); return 0.0; } vals[0] = malloc(nref*sizeof(long double)); vals[1] = malloc(nref*sizeof(long double)); vals[2] = malloc(nref*sizeof(long double)); if ( (vals[0] == NULL) || (vals[1] == NULL) || (vals[2] == NULL) ) { ERROR("Couldn't allocate memory.\n"); return 0.0; } valid = malloc(nref*sizeof(int)); if ( valid == NULL ) { ERROR("Couldn't allocate memory.\n"); return 0.0; } for ( i=0; i<nref; i++ ) valid[i] = 1; scan_partialities(reflections, reflections, valid, vals, 1, pmodel); calc_either_side(cr, incr_val, valid, vals, refine, pmodel); if ( plot ) { snprintf(tmp, 32, "gradient-test-%s.dat", file); fh = fopen(tmp, "w"); } vec1 = malloc(nref*sizeof(double)); vec2 = malloc(nref*sizeof(double)); if ( (vec1 == NULL) || (vec2 == NULL) ) { ERROR("Couldn't allocate memory.\n"); return 0.0; } n_invalid = 0; n_good = 0; n_nan = 0; n_small = 0; n_bad = 0; n_line = 0; i = 0; for ( refl = first_refl(reflections, &iter); refl != NULL; refl = next_refl(refl, iter) ) { long double grad1, grad2, grad; double cgrad; signed int h, k, l; get_indices(refl, &h, &k, &l); if ( !valid[i] ) { n_invalid++; i++; } else { double r1, r2, p; grad1 = (vals[1][i] - vals[0][i]) / incr_val; grad2 = (vals[2][i] - vals[1][i]) / incr_val; grad = (grad1 + grad2) / 2.0; i++; cgrad = p_gradient(cr, refine, refl, pmodel); get_partial(refl, &r1, &r2, &p); if ( isnan(cgrad) ) { n_nan++; continue; } if ( plot ) { fprintf(fh, "%e %Le\n", cgrad, grad); } vec1[n_line] = cgrad; vec2[n_line] = grad; n_line++; if ( (fabs(cgrad) < 5e-8) && (fabs(grad) < 5e-8) ) { n_small++; continue; } total += fabs(cgrad - grad); ntot++; if ( !within_tolerance(grad, cgrad, 5.0) || !within_tolerance(cgrad, grad, 5.0) ) { if ( !quiet ) { STATUS("!- %s %3i %3i %3i" " %10.2Le %10.2e ratio = %5.2Lf" " %10.2e %10.2e\n", str, h, k, l, grad, cgrad, cgrad/grad, r1, r2); } n_bad++; } else { //STATUS("OK %s %3i %3i %3i" // " %10.2Le %10.2e ratio = %5.2Lf" // " %10.2e %10.2e\n", // str, h, k, l, grad, cgrad, cgrad/grad, // r1, r2); n_good++; } } } STATUS("%3s: %3i within 5%%, %3i outside, %3i nan, %3i invalid, " "%3i small. ", str, n_good, n_bad, n_nan, n_invalid, n_small); if ( plot ) { fclose(fh); } cc = gsl_stats_correlation(vec1, 1, vec2, 1, n_line); STATUS("CC = %+f\n", cc); return cc; }
static struct flist *asymm_and_merge(RefList *in, const SymOpList *sym, UnitCell *cell, double rmin, double rmax, SymOpList *amb) { Reflection *refl; RefListIterator *iter; RefList *asym; struct flist *f; int n; asym = reflist_new(); if ( asym == NULL ) return NULL; for ( refl = first_refl(in, &iter); refl != NULL; refl = next_refl(refl, iter) ) { signed int h, k, l; signed int ha, ka, la; Reflection *cr; double res; get_indices(refl, &h, &k, &l); if ( cell == NULL ) { ERROR("Can't calculate resolution cutoff - no cell\n"); } else { res = 2.0*resolution(cell, h, k, l); if ( res < rmin ) continue; if ( res > rmax ) continue; } get_asymm(sym, h, k, l, &ha, &ka, &la); if ( amb != NULL ) { signed int hr, kr, lr; signed int hra, kra, lra; get_equiv(amb, NULL, 0, ha, ka, la, &hr, &kr, &lr); get_asymm(sym, hr, kr, lr, &hra, &kra, &lra); /* Skip twin-proof reflections */ if ( (ha==hra) && (ka==kra) && (la==lra) ) { //STATUS("%i %i %i is twin proof\n", h, k, l); continue; } } cr = find_refl(asym, ha, ka, la); if ( cr == NULL ) { cr = add_refl(asym, ha, ka, la); assert(cr != NULL); copy_data(cr, refl); } else { const double i = get_intensity(cr); const int r = get_redundancy(cr); set_intensity(cr, (r*i + get_intensity(refl))/(r+1)); set_redundancy(cr, r+1); } } f = malloc(sizeof(struct flist)); if ( f == NULL ) { ERROR("Failed to allocate flist\n"); return NULL; } n = num_reflections(asym); f->s = malloc(n*sizeof(unsigned int)); f->s_reidx = malloc(n*sizeof(unsigned int)); f->i = malloc(n*sizeof(float)); f->i_reidx = malloc(n*sizeof(float)); if ( (f->s == NULL) || (f->i == NULL) || (f->s_reidx == NULL) || (f->i_reidx == NULL) ) { ERROR("Failed to allocate flist\n"); return NULL; } f->n = 0; for ( refl = first_refl(asym, &iter); refl != NULL; refl = next_refl(refl, iter) ) { signed int h, k, l; get_indices(refl, &h, &k, &l); f->s[f->n] = SERIAL(h, k, l); f->i[f->n] = get_intensity(refl); f->n++; } assert(f->n == n); if ( amb != NULL ) { RefList *reidx = reflist_new(); if ( reidx == NULL ) return NULL; for ( refl = first_refl(asym, &iter); refl != NULL; refl = next_refl(refl, iter) ) { signed int h, k, l; signed int hr, kr, lr; signed int hra, kra, lra; Reflection *cr; get_indices(refl, &h, &k, &l); get_equiv(amb, NULL, 0, h, k, l, &hr, &kr, &lr); get_asymm(sym, hr, kr, lr, &hra, &kra, &lra); cr = add_refl(reidx, hra, kra, lra); copy_data(cr, refl); } n = 0; for ( refl = first_refl(reidx, &iter); refl != NULL; refl = next_refl(refl, iter) ) { signed int h, k, l; get_indices(refl, &h, &k, &l); f->s_reidx[n] = SERIAL(h, k, l); f->i_reidx[n++] = get_intensity(refl); } reflist_free(reidx); } reflist_free(asym); return f; }
static int write_stream_reflections_2_1(FILE *fh, RefList *list, struct image *image) { Reflection *refl; RefListIterator *iter; fprintf(fh, " h k l I phase sigma(I) " " counts fs/px ss/px\n"); for ( refl = first_refl(list, &iter); refl != NULL; refl = next_refl(refl, iter) ) { signed int h, k, l; double intensity, esd_i, ph; int red; double fs, ss; char phs[16]; int have_phase; get_indices(refl, &h, &k, &l); get_detector_pos(refl, &fs, &ss); intensity = get_intensity(refl); esd_i = get_esd_intensity(refl); red = get_redundancy(refl); ph = get_phase(refl, &have_phase); /* Reflections with redundancy = 0 are not written */ if ( red == 0 ) continue; if ( have_phase ) { snprintf(phs, 16, "%8.2f", rad2deg(ph)); } else { strncpy(phs, " -", 15); } if ( image->det != NULL ) { struct panel *p; double write_fs, write_ss; p = find_orig_panel(image->det, fs, ss); if ( p == NULL ) { ERROR("Panel not found\n"); return 1; } /* Convert coordinates to match arrangement of panels * in HDF5 file */ write_fs = fs - p->min_fs + p->orig_min_fs; write_ss = ss - p->min_ss + p->orig_min_ss; fprintf(fh, "%3i %3i %3i %10.2f %s %10.2f %7i " "%6.1f %6.1f\n", h, k, l, intensity, phs, esd_i, red, write_fs, write_ss); } else { fprintf(fh, "%3i %3i %3i %10.2f %s %10.2f %7i " "%6.1f %6.1f\n", h, k, l, intensity, phs, esd_i, red, fs, ss); } } return 0; }
static int test_lists(int num_items) { struct refltemp *check; RefList *list; int i; signed int h, k, l; Reflection *refl; RefListIterator *iter; check = malloc(num_items * sizeof(struct refltemp)); list = reflist_new(); h = RANDOM_INDEX; k = RANDOM_INDEX; l = RANDOM_INDEX; for ( i=0; i<num_items; i++ ) { int j; int num; if ( random() > RAND_MAX/2 ) { h = RANDOM_INDEX; k = RANDOM_INDEX; l = RANDOM_INDEX; } /* else use the same as last time */ /* Count the number of times this reflection appeared before */ num = 1; for ( j=0; j<i; j++ ) { if ( (check[j].h == h) && (check[j].k == k) && (check[j].l == l) ) { num++; } } /* Update all copies with this number */ for ( j=0; j<i; j++ ) { if ( (check[j].h == h) && (check[j].k == k) && (check[j].l == l) ) { check[j].num = num; } } add_refl(list, h, k, l); check[i].h = h; check[i].k = k; check[i].l = l; check[i].num = num; check[i].found = 0; } printf("Created %i items, num_reflections is %i, tree depth is %i\n", num_items, num_reflections(list), tree_depth(list)); /* Iterate over the list and check we find everything */ int count = 0; for ( refl = first_refl(list, &iter); refl != NULL; refl = next_refl(refl, iter) ) { signed int h, k, l; get_indices(refl, &h, &k, &l); for ( i=0; i<num_items; i++ ) { if ( (check[i].h == h) && (check[i].k == k) && (check[i].l == l) && (check[i].found == 0) ) { check[i].found = 1; break; } } count++; } if ( count != num_reflections(list) ) { fprintf(stderr, "num_reflections gave %i, iteration gave %i\n", num_reflections(list), count); return 1; } for ( i=0; i<num_items; i++ ) { if ( check[i].found == 0 ) { Reflection *test; fprintf(stderr, "Iteration didn't find %3i %3i %3i %i\n", check[i].h, check[i].k, check[i].l, i); test = find_refl(list, check[i].h, check[i].k, check[i].l); if ( test == NULL ) { fprintf(stderr, "Not in list\n"); } else { fprintf(stderr, "But found in list.\n"); } return 1; } } /* Check that all the reflections can be found */ for ( i=0; i<num_items; i++ ) { signed int h, k, l; Reflection *refl; h = check[i].h; k = check[i].k; l = check[i].l; refl = find_refl(list, h, k, l); if ( refl == NULL ) { fprintf(stderr, "Couldn't find %3i %3i %3i\n", h, k, l); return 1; } } reflist_free(list); free(check); return 0; }
/* Perform one cycle of post refinement on 'image' against 'full' */ static double pr_iterate(Crystal *cr, const RefList *full, PartialityModel pmodel, int *n_filtered) { gsl_matrix *M; gsl_vector *v; gsl_vector *shifts; int param; Reflection *refl; RefListIterator *iter; RefList *reflections; double max_shift; int nref = 0; const int verbose = 0; int num_params = 0; enum gparam rv[32]; *n_filtered = 0; /* If partiality model is anything other than "unity", refine all the * geometrical parameters */ if ( pmodel != PMODEL_UNITY ) { rv[num_params++] = GPARAM_ASX; rv[num_params++] = GPARAM_ASY; rv[num_params++] = GPARAM_ASZ; rv[num_params++] = GPARAM_BSX; rv[num_params++] = GPARAM_BSY; rv[num_params++] = GPARAM_BSZ; rv[num_params++] = GPARAM_CSX; rv[num_params++] = GPARAM_CSY; rv[num_params++] = GPARAM_CSZ; } STATUS("Refining %i parameters.\n", num_params); reflections = crystal_get_reflections(cr); M = gsl_matrix_calloc(num_params, num_params); v = gsl_vector_calloc(num_params); /* Construct the equations, one per reflection in this image */ for ( refl = first_refl(reflections, &iter); refl != NULL; refl = next_refl(refl, iter) ) { signed int ha, ka, la; double I_full, delta_I; double w; double I_partial; int k; double p, l; Reflection *match; double gradients[num_params]; /* Find the full version */ get_indices(refl, &ha, &ka, &la); match = find_refl(full, ha, ka, la); if ( match == NULL ) continue; if ( (get_intensity(refl) < 3.0*get_esd_intensity(refl)) || (get_partiality(refl) < MIN_PART_REFINE) || (get_redundancy(match) < 2) ) continue; I_full = get_intensity(match); /* Actual measurement of this reflection from this pattern? */ I_partial = get_intensity(refl) / crystal_get_osf(cr); p = get_partiality(refl); l = get_lorentz(refl); /* Calculate the weight for this reflection */ w = pow(get_esd_intensity(refl), 2.0); w += l * p * I_full * pow(get_esd_intensity(match), 2.0); w = pow(w, -1.0); /* Calculate all gradients for this reflection */ for ( k=0; k<num_params; k++ ) { gradients[k] = p_gradient(cr, rv[k], refl, pmodel) * l; } for ( k=0; k<num_params; k++ ) { int g; double v_c, v_curr; for ( g=0; g<num_params; g++ ) { double M_c, M_curr; /* Matrix is symmetric */ if ( g > k ) continue; M_c = gradients[g] * gradients[k]; M_c *= w * pow(I_full, 2.0); M_curr = gsl_matrix_get(M, k, g); gsl_matrix_set(M, k, g, M_curr + M_c); gsl_matrix_set(M, g, k, M_curr + M_c); } delta_I = I_partial - (l * p * I_full); v_c = w * delta_I * I_full * gradients[k]; v_curr = gsl_vector_get(v, k); gsl_vector_set(v, k, v_curr + v_c); } nref++; } if ( verbose ) { STATUS("The original equation:\n"); show_matrix_eqn(M, v); } //STATUS("%i reflections went into the equations.\n", nref); if ( nref == 0 ) { crystal_set_user_flag(cr, 2); gsl_matrix_free(M); gsl_vector_free(v); return 0.0; } max_shift = 0.0; shifts = solve_svd(v, M, n_filtered, verbose); if ( shifts != NULL ) { for ( param=0; param<num_params; param++ ) { double shift = gsl_vector_get(shifts, param); apply_shift(cr, rv[param], shift); //STATUS("Shift %i: %e\n", param, shift); if ( fabs(shift) > max_shift ) max_shift = fabs(shift); } } else { crystal_set_user_flag(cr, 3); } gsl_matrix_free(M); gsl_vector_free(v); gsl_vector_free(shifts); return max_shift; }