static void scan_partialities(RefList *reflections, RefList *compare,
int *valid, long double *vals[3], int idx)
{
int i;
Reflection *refl;
RefListIterator *iter;
i = 0;
for ( refl = first_refl(reflections, &iter);
refl != NULL;
refl = next_refl(refl, iter) )
{
signed int h, k, l;
Reflection *refl2;
get_indices(refl, &h, &k, &l);
refl2 = find_refl(compare, h, k, l);
if ( refl2 == NULL ) {
valid[i] = 0;
i++;
continue;
}
vals[idx][i] = get_lorentz(refl2);
i++;
}
}
static void scan_partialities(RefList *reflections, RefList *compare,
int *valid, long double *vals[3], int idx,
PartialityModel pmodel)
{
int i;
Reflection *refl;
RefListIterator *iter;
i = 0;
for ( refl = first_refl(reflections, &iter);
refl != NULL;
refl = next_refl(refl, iter) )
{
signed int h, k, l;
Reflection *refl2;
double rlow, rhigh, p;
get_indices(refl, &h, &k, &l);
refl2 = find_refl(compare, h, k, l);
if ( refl2 == NULL ) {
valid[i] = 0;
i++;
continue;
}
get_partial(refl2, &rlow, &rhigh, &p);
vals[idx][i] = p;
if ( unlikely(p < 0.0) ) {
ERROR("Negative partiality! %3i %3i %3i %f\n",
h, k, l, p);
}
i++;
}
}
static void write_stream_reflections(FILE *fh, RefList *list)
{
Reflection *refl;
RefListIterator *iter;
fprintf(fh, " h k l I sigma(I) peak background"
" fs/px ss/px\n");
for ( refl = first_refl(list, &iter);
refl != NULL;
refl = next_refl(refl, iter) )
{
signed int h, k, l;
double intensity, esd_i, bg, pk;
double fs, ss;
get_indices(refl, &h, &k, &l);
get_detector_pos(refl, &fs, &ss);
intensity = get_intensity(refl);
esd_i = get_esd_intensity(refl);
pk = get_peak(refl);
bg = get_mean_bg(refl);
/* Reflections with redundancy = 0 are not written */
if ( get_redundancy(refl) == 0 ) continue;
fprintf(fh,
"%4i %4i %4i %10.2f %10.2f %10.2f %10.2f %6.1f %6.1f\n",
h, k, l, intensity, esd_i, pk, bg, fs, ss);
}
}
static int write_stream_reflections_2_2(FILE *fh, RefList *list,
struct image *image)
{
Reflection *refl;
RefListIterator *iter;
fprintf(fh, " h k l I sigma(I) "
"peak background fs/px ss/px\n");
for ( refl = first_refl(list, &iter);
refl != NULL;
refl = next_refl(refl, iter) )
{
signed int h, k, l;
double intensity, esd_i, bg, pk;
double fs, ss;
get_indices(refl, &h, &k, &l);
get_detector_pos(refl, &fs, &ss);
intensity = get_intensity(refl);
esd_i = get_esd_intensity(refl);
pk = get_peak(refl);
bg = get_mean_bg(refl);
/* Reflections with redundancy = 0 are not written */
if ( get_redundancy(refl) == 0 ) continue;
if ( image->det != NULL ) {
struct panel *p;
double write_fs, write_ss;
p = find_orig_panel(image->det, fs, ss);
if ( p == NULL ) {
ERROR("Panel not found\n");
return 1;
}
/* Convert coordinates to match arrangement of panels in HDF5
* file */
write_fs = fs - p->min_fs + p->orig_min_fs;
write_ss = ss - p->min_ss + p->orig_min_ss;
fprintf(fh, "%4i %4i %4i %10.2f %10.2f %10.2f %10.2f"
" %6.1f %6.1f\n",
h, k, l, intensity, esd_i, pk, bg, write_fs,
write_ss);
} else {
fprintf(fh, "%4i %4i %4i %10.2f %10.2f %10.2f %10.2f"
" %6.1f %6.1f\n",
h, k, l, intensity, esd_i, pk, bg, fs, ss);
}
}
return 0;
}
static int num_integrated_reflections(RefList *list)
{
Reflection *refl;
RefListIterator *iter;
int n = 0;
for ( refl = first_refl(list, &iter);
refl != NULL;
refl = next_refl(refl, iter) )
{
if ( get_redundancy(refl) > 0 ) n++;
}
return n;
}
static int write_stream_reflections_2_3(FILE *fh, RefList *list,
struct image *image)
{
Reflection *refl;
RefListIterator *iter;
fprintf(fh, " h k l I sigma(I) "
"peak background fs/px ss/px panel\n");
for ( refl = first_refl(list, &iter);
refl != NULL;
refl = next_refl(refl, iter) )
{
signed int h, k, l;
double intensity, esd_i, pk, bg;
double fs, ss;
double write_fs, write_ss;
struct panel *p = NULL;
get_indices(refl, &h, &k, &l);
get_detector_pos(refl, &fs, &ss);
intensity = get_intensity(refl);
esd_i = get_esd_intensity(refl);
pk = get_peak(refl);
bg = get_mean_bg(refl);
/* Reflections with redundancy = 0 are not written */
if ( get_redundancy(refl) == 0 ) continue;
p = find_panel(image->det,fs,ss);
if ( p == NULL ) {
ERROR("Panel not found\n");
return 1;
}
write_fs = fs-p->min_fs+p->orig_min_fs;
write_ss = ss-p->min_ss+p->orig_min_ss;
fprintf(fh,
"%4i %4i %4i %10.2f %10.2f %10.2f %10.2f "
"%6.1f %6.1f %s\n",
h, k, l, intensity, esd_i, pk, bg,
write_fs, write_ss, p->name);
}
return 0;
}
static double guide_dev(Crystal *cr, const RefList *full)
{
double dev = 0.0;
/* For each reflection */
Reflection *refl;
RefListIterator *iter;
for ( refl = first_refl(crystal_get_reflections(cr), &iter);
refl != NULL;
refl = next_refl(refl, iter) ) {
double G, p;
signed int h, k, l;
Reflection *full_version;
double I_full, I_partial;
if ( (get_intensity(refl) < 3.0*get_esd_intensity(refl))
|| (get_partiality(refl) < MIN_PART_REFINE) ) continue;
get_indices(refl, &h, &k, &l);
assert((h!=0) || (k!=0) || (l!=0));
full_version = find_refl(full, h, k, l);
if ( full_version == NULL ) continue;
/* Some reflections may have recently become scalable, but
* scale_intensities() might not yet have been called, so the
* full version may not have been calculated yet. */
G = crystal_get_osf(cr);
p = get_partiality(refl);
I_partial = get_intensity(refl);
I_full = get_intensity(full_version);
//STATUS("%3i %3i %3i %5.2f %5.2f %5.2f %5.2f %5.2f\n",
// h, k, l, G, p, I_partial, I_full,
// I_partial - p*G*I_full);
dev += pow(I_partial - p*G*I_full, 2.0);
}
return dev;
}
static void write_stream_reflections_2_1(FILE *fh, RefList *list)
{
Reflection *refl;
RefListIterator *iter;
fprintf(fh, " h k l I phase sigma(I) "
" counts fs/px ss/px\n");
for ( refl = first_refl(list, &iter);
refl != NULL;
refl = next_refl(refl, iter) )
{
signed int h, k, l;
double intensity, esd_i, ph;
int red;
double fs, ss;
char phs[16];
int have_phase;
get_indices(refl, &h, &k, &l);
get_detector_pos(refl, &fs, &ss);
intensity = get_intensity(refl);
esd_i = get_esd_intensity(refl);
red = get_redundancy(refl);
ph = get_phase(refl, &have_phase);
/* Reflections with redundancy = 0 are not written */
if ( red == 0 ) continue;
if ( have_phase ) {
snprintf(phs, 16, "%8.2f", rad2deg(ph));
} else {
strncpy(phs, " -", 15);
}
fprintf(fh,
"%3i %3i %3i %10.2f %s %10.2f %7i %6.1f %6.1f\n",
h, k, l, intensity, phs, esd_i, red, fs, ss);
}
}
static double test_gradients(Crystal *cr, double incr_val, int refine,
const char *str, const char *file,
PartialityModel pmodel, int quiet, int plot)
{
Reflection *refl;
RefListIterator *iter;
long double *vals[3];
int i;
int *valid;
int nref;
int n_good, n_invalid, n_small, n_nan, n_bad;
RefList *reflections;
FILE *fh = NULL;
int ntot = 0;
double total = 0.0;
char tmp[32];
double *vec1;
double *vec2;
int n_line;
double cc;
reflections = find_intersections(crystal_get_image(cr), cr, pmodel);
crystal_set_reflections(cr, reflections);
nref = num_reflections(reflections);
if ( nref < 10 ) {
ERROR("Too few reflections found. Failing test by default.\n");
return 0.0;
}
vals[0] = malloc(nref*sizeof(long double));
vals[1] = malloc(nref*sizeof(long double));
vals[2] = malloc(nref*sizeof(long double));
if ( (vals[0] == NULL) || (vals[1] == NULL) || (vals[2] == NULL) ) {
ERROR("Couldn't allocate memory.\n");
return 0.0;
}
valid = malloc(nref*sizeof(int));
if ( valid == NULL ) {
ERROR("Couldn't allocate memory.\n");
return 0.0;
}
for ( i=0; i<nref; i++ ) valid[i] = 1;
scan_partialities(reflections, reflections, valid, vals, 1, pmodel);
calc_either_side(cr, incr_val, valid, vals, refine, pmodel);
if ( plot ) {
snprintf(tmp, 32, "gradient-test-%s.dat", file);
fh = fopen(tmp, "w");
}
vec1 = malloc(nref*sizeof(double));
vec2 = malloc(nref*sizeof(double));
if ( (vec1 == NULL) || (vec2 == NULL) ) {
ERROR("Couldn't allocate memory.\n");
return 0.0;
}
n_invalid = 0; n_good = 0;
n_nan = 0; n_small = 0; n_bad = 0; n_line = 0;
i = 0;
for ( refl = first_refl(reflections, &iter);
refl != NULL;
refl = next_refl(refl, iter) )
{
long double grad1, grad2, grad;
double cgrad;
signed int h, k, l;
get_indices(refl, &h, &k, &l);
if ( !valid[i] ) {
n_invalid++;
i++;
} else {
double r1, r2, p;
grad1 = (vals[1][i] - vals[0][i]) / incr_val;
grad2 = (vals[2][i] - vals[1][i]) / incr_val;
grad = (grad1 + grad2) / 2.0;
i++;
cgrad = p_gradient(cr, refine, refl, pmodel);
get_partial(refl, &r1, &r2, &p);
if ( isnan(cgrad) ) {
n_nan++;
continue;
}
if ( plot ) {
fprintf(fh, "%e %Le\n", cgrad, grad);
}
vec1[n_line] = cgrad;
vec2[n_line] = grad;
n_line++;
if ( (fabs(cgrad) < 5e-8) && (fabs(grad) < 5e-8) ) {
n_small++;
continue;
}
total += fabs(cgrad - grad);
ntot++;
if ( !within_tolerance(grad, cgrad, 5.0)
|| !within_tolerance(cgrad, grad, 5.0) )
{
if ( !quiet ) {
STATUS("!- %s %3i %3i %3i"
" %10.2Le %10.2e ratio = %5.2Lf"
" %10.2e %10.2e\n",
str, h, k, l, grad, cgrad,
cgrad/grad, r1, r2);
}
n_bad++;
} else {
//STATUS("OK %s %3i %3i %3i"
// " %10.2Le %10.2e ratio = %5.2Lf"
// " %10.2e %10.2e\n",
// str, h, k, l, grad, cgrad, cgrad/grad,
// r1, r2);
n_good++;
}
}
}
STATUS("%3s: %3i within 5%%, %3i outside, %3i nan, %3i invalid, "
"%3i small. ", str, n_good, n_bad, n_nan, n_invalid, n_small);
if ( plot ) {
fclose(fh);
}
cc = gsl_stats_correlation(vec1, 1, vec2, 1, n_line);
STATUS("CC = %+f\n", cc);
return cc;
}
static struct flist *asymm_and_merge(RefList *in, const SymOpList *sym,
UnitCell *cell, double rmin, double rmax,
SymOpList *amb)
{
Reflection *refl;
RefListIterator *iter;
RefList *asym;
struct flist *f;
int n;
asym = reflist_new();
if ( asym == NULL ) return NULL;
for ( refl = first_refl(in, &iter);
refl != NULL;
refl = next_refl(refl, iter) )
{
signed int h, k, l;
signed int ha, ka, la;
Reflection *cr;
double res;
get_indices(refl, &h, &k, &l);
if ( cell == NULL ) {
ERROR("Can't calculate resolution cutoff - no cell\n");
} else {
res = 2.0*resolution(cell, h, k, l);
if ( res < rmin ) continue;
if ( res > rmax ) continue;
}
get_asymm(sym, h, k, l, &ha, &ka, &la);
if ( amb != NULL ) {
signed int hr, kr, lr;
signed int hra, kra, lra;
get_equiv(amb, NULL, 0, ha, ka, la, &hr, &kr, &lr);
get_asymm(sym, hr, kr, lr, &hra, &kra, &lra);
/* Skip twin-proof reflections */
if ( (ha==hra) && (ka==kra) && (la==lra) ) {
//STATUS("%i %i %i is twin proof\n", h, k, l);
continue;
}
}
cr = find_refl(asym, ha, ka, la);
if ( cr == NULL ) {
cr = add_refl(asym, ha, ka, la);
assert(cr != NULL);
copy_data(cr, refl);
} else {
const double i = get_intensity(cr);
const int r = get_redundancy(cr);
set_intensity(cr, (r*i + get_intensity(refl))/(r+1));
set_redundancy(cr, r+1);
}
}
f = malloc(sizeof(struct flist));
if ( f == NULL ) {
ERROR("Failed to allocate flist\n");
return NULL;
}
n = num_reflections(asym);
f->s = malloc(n*sizeof(unsigned int));
f->s_reidx = malloc(n*sizeof(unsigned int));
f->i = malloc(n*sizeof(float));
f->i_reidx = malloc(n*sizeof(float));
if ( (f->s == NULL) || (f->i == NULL)
|| (f->s_reidx == NULL) || (f->i_reidx == NULL) ) {
ERROR("Failed to allocate flist\n");
return NULL;
}
f->n = 0;
for ( refl = first_refl(asym, &iter);
refl != NULL;
refl = next_refl(refl, iter) )
{
signed int h, k, l;
get_indices(refl, &h, &k, &l);
f->s[f->n] = SERIAL(h, k, l);
f->i[f->n] = get_intensity(refl);
f->n++;
}
assert(f->n == n);
if ( amb != NULL ) {
RefList *reidx = reflist_new();
if ( reidx == NULL ) return NULL;
for ( refl = first_refl(asym, &iter);
refl != NULL;
refl = next_refl(refl, iter) )
{
signed int h, k, l;
signed int hr, kr, lr;
signed int hra, kra, lra;
Reflection *cr;
get_indices(refl, &h, &k, &l);
get_equiv(amb, NULL, 0, h, k, l, &hr, &kr, &lr);
get_asymm(sym, hr, kr, lr, &hra, &kra, &lra);
cr = add_refl(reidx, hra, kra, lra);
copy_data(cr, refl);
}
n = 0;
for ( refl = first_refl(reidx, &iter);
refl != NULL;
refl = next_refl(refl, iter) )
{
signed int h, k, l;
get_indices(refl, &h, &k, &l);
f->s_reidx[n] = SERIAL(h, k, l);
f->i_reidx[n++] = get_intensity(refl);
}
reflist_free(reidx);
}
reflist_free(asym);
return f;
}
static int write_stream_reflections_2_1(FILE *fh, RefList *list,
struct image *image)
{
Reflection *refl;
RefListIterator *iter;
fprintf(fh, " h k l I phase sigma(I) "
" counts fs/px ss/px\n");
for ( refl = first_refl(list, &iter);
refl != NULL;
refl = next_refl(refl, iter) )
{
signed int h, k, l;
double intensity, esd_i, ph;
int red;
double fs, ss;
char phs[16];
int have_phase;
get_indices(refl, &h, &k, &l);
get_detector_pos(refl, &fs, &ss);
intensity = get_intensity(refl);
esd_i = get_esd_intensity(refl);
red = get_redundancy(refl);
ph = get_phase(refl, &have_phase);
/* Reflections with redundancy = 0 are not written */
if ( red == 0 ) continue;
if ( have_phase ) {
snprintf(phs, 16, "%8.2f", rad2deg(ph));
} else {
strncpy(phs, " -", 15);
}
if ( image->det != NULL ) {
struct panel *p;
double write_fs, write_ss;
p = find_orig_panel(image->det, fs, ss);
if ( p == NULL ) {
ERROR("Panel not found\n");
return 1;
}
/* Convert coordinates to match arrangement of panels
* in HDF5 file */
write_fs = fs - p->min_fs + p->orig_min_fs;
write_ss = ss - p->min_ss + p->orig_min_ss;
fprintf(fh, "%3i %3i %3i %10.2f %s %10.2f %7i "
"%6.1f %6.1f\n",
h, k, l, intensity, phs, esd_i, red,
write_fs, write_ss);
} else {
fprintf(fh, "%3i %3i %3i %10.2f %s %10.2f %7i "
"%6.1f %6.1f\n",
h, k, l, intensity, phs, esd_i, red,
fs, ss);
}
}
return 0;
}
static int test_lists(int num_items)
{
struct refltemp *check;
RefList *list;
int i;
signed int h, k, l;
Reflection *refl;
RefListIterator *iter;
check = malloc(num_items * sizeof(struct refltemp));
list = reflist_new();
h = RANDOM_INDEX;
k = RANDOM_INDEX;
l = RANDOM_INDEX;
for ( i=0; i<num_items; i++ ) {
int j;
int num;
if ( random() > RAND_MAX/2 ) {
h = RANDOM_INDEX;
k = RANDOM_INDEX;
l = RANDOM_INDEX;
} /* else use the same as last time */
/* Count the number of times this reflection appeared before */
num = 1;
for ( j=0; j<i; j++ ) {
if ( (check[j].h == h)
&& (check[j].k == k)
&& (check[j].l == l) ) {
num++;
}
}
/* Update all copies with this number */
for ( j=0; j<i; j++ ) {
if ( (check[j].h == h)
&& (check[j].k == k)
&& (check[j].l == l) ) {
check[j].num = num;
}
}
add_refl(list, h, k, l);
check[i].h = h;
check[i].k = k;
check[i].l = l;
check[i].num = num;
check[i].found = 0;
}
printf("Created %i items, num_reflections is %i, tree depth is %i\n",
num_items, num_reflections(list), tree_depth(list));
/* Iterate over the list and check we find everything */
int count = 0;
for ( refl = first_refl(list, &iter);
refl != NULL;
refl = next_refl(refl, iter) ) {
signed int h, k, l;
get_indices(refl, &h, &k, &l);
for ( i=0; i<num_items; i++ ) {
if ( (check[i].h == h)
&& (check[i].k == k)
&& (check[i].l == l)
&& (check[i].found == 0) ) {
check[i].found = 1;
break;
}
}
count++;
}
if ( count != num_reflections(list) ) {
fprintf(stderr, "num_reflections gave %i, iteration gave %i\n",
num_reflections(list), count);
return 1;
}
for ( i=0; i<num_items; i++ ) {
if ( check[i].found == 0 ) {
Reflection *test;
fprintf(stderr, "Iteration didn't find %3i %3i %3i %i\n",
check[i].h, check[i].k, check[i].l, i);
test = find_refl(list, check[i].h, check[i].k,
check[i].l);
if ( test == NULL ) {
fprintf(stderr, "Not in list\n");
} else {
fprintf(stderr, "But found in list.\n");
}
return 1;
}
}
/* Check that all the reflections can be found */
for ( i=0; i<num_items; i++ ) {
signed int h, k, l;
Reflection *refl;
h = check[i].h;
k = check[i].k;
l = check[i].l;
refl = find_refl(list, h, k, l);
if ( refl == NULL ) {
fprintf(stderr, "Couldn't find %3i %3i %3i\n", h, k, l);
return 1;
}
}
reflist_free(list);
free(check);
return 0;
}
/* Perform one cycle of post refinement on 'image' against 'full' */
static double pr_iterate(Crystal *cr, const RefList *full,
PartialityModel pmodel, int *n_filtered)
{
gsl_matrix *M;
gsl_vector *v;
gsl_vector *shifts;
int param;
Reflection *refl;
RefListIterator *iter;
RefList *reflections;
double max_shift;
int nref = 0;
const int verbose = 0;
int num_params = 0;
enum gparam rv[32];
*n_filtered = 0;
/* If partiality model is anything other than "unity", refine all the
* geometrical parameters */
if ( pmodel != PMODEL_UNITY ) {
rv[num_params++] = GPARAM_ASX;
rv[num_params++] = GPARAM_ASY;
rv[num_params++] = GPARAM_ASZ;
rv[num_params++] = GPARAM_BSX;
rv[num_params++] = GPARAM_BSY;
rv[num_params++] = GPARAM_BSZ;
rv[num_params++] = GPARAM_CSX;
rv[num_params++] = GPARAM_CSY;
rv[num_params++] = GPARAM_CSZ;
}
STATUS("Refining %i parameters.\n", num_params);
reflections = crystal_get_reflections(cr);
M = gsl_matrix_calloc(num_params, num_params);
v = gsl_vector_calloc(num_params);
/* Construct the equations, one per reflection in this image */
for ( refl = first_refl(reflections, &iter);
refl != NULL;
refl = next_refl(refl, iter) )
{
signed int ha, ka, la;
double I_full, delta_I;
double w;
double I_partial;
int k;
double p, l;
Reflection *match;
double gradients[num_params];
/* Find the full version */
get_indices(refl, &ha, &ka, &la);
match = find_refl(full, ha, ka, la);
if ( match == NULL ) continue;
if ( (get_intensity(refl) < 3.0*get_esd_intensity(refl))
|| (get_partiality(refl) < MIN_PART_REFINE)
|| (get_redundancy(match) < 2) ) continue;
I_full = get_intensity(match);
/* Actual measurement of this reflection from this pattern? */
I_partial = get_intensity(refl) / crystal_get_osf(cr);
p = get_partiality(refl);
l = get_lorentz(refl);
/* Calculate the weight for this reflection */
w = pow(get_esd_intensity(refl), 2.0);
w += l * p * I_full * pow(get_esd_intensity(match), 2.0);
w = pow(w, -1.0);
/* Calculate all gradients for this reflection */
for ( k=0; k<num_params; k++ ) {
gradients[k] = p_gradient(cr, rv[k], refl, pmodel) * l;
}
for ( k=0; k<num_params; k++ ) {
int g;
double v_c, v_curr;
for ( g=0; g<num_params; g++ ) {
double M_c, M_curr;
/* Matrix is symmetric */
if ( g > k ) continue;
M_c = gradients[g] * gradients[k];
M_c *= w * pow(I_full, 2.0);
M_curr = gsl_matrix_get(M, k, g);
gsl_matrix_set(M, k, g, M_curr + M_c);
gsl_matrix_set(M, g, k, M_curr + M_c);
}
delta_I = I_partial - (l * p * I_full);
v_c = w * delta_I * I_full * gradients[k];
v_curr = gsl_vector_get(v, k);
gsl_vector_set(v, k, v_curr + v_c);
}
nref++;
}
if ( verbose ) {
STATUS("The original equation:\n");
show_matrix_eqn(M, v);
}
//STATUS("%i reflections went into the equations.\n", nref);
if ( nref == 0 ) {
crystal_set_user_flag(cr, 2);
gsl_matrix_free(M);
gsl_vector_free(v);
return 0.0;
}
max_shift = 0.0;
shifts = solve_svd(v, M, n_filtered, verbose);
if ( shifts != NULL ) {
for ( param=0; param<num_params; param++ ) {
double shift = gsl_vector_get(shifts, param);
apply_shift(cr, rv[param], shift);
//STATUS("Shift %i: %e\n", param, shift);
if ( fabs(shift) > max_shift ) max_shift = fabs(shift);
}
} else {
crystal_set_user_flag(cr, 3);
}
gsl_matrix_free(M);
gsl_vector_free(v);
gsl_vector_free(shifts);
return max_shift;
}
