/* Opens the file designated by file handle FH for reading as a
data file. Providing fh_inline_file() for FH designates the
"inline file", that is, data included inline in the command
file between BEGIN FILE and END FILE. Returns a reader if
successful, or a null pointer otherwise. */
struct dfm_reader *
dfm_open_reader (struct file_handle *fh, struct lexer *lexer)
{
struct dfm_reader *r;
struct fh_lock *lock;
/* TRANSLATORS: this fragment will be interpolated into
messages in fh_lock() that identify types of files. */
lock = fh_lock (fh, FH_REF_FILE | FH_REF_INLINE, N_("data file"),
FH_ACC_READ, false);
if (lock == NULL)
return NULL;
r = fh_lock_get_aux (lock);
if (r != NULL)
return r;
r = xmalloc (sizeof *r);
r->fh = fh_ref (fh);
r->lock = lock;
r->lexer = lexer;
ds_init_empty (&r->line);
ds_init_empty (&r->scratch);
r->flags = DFM_ADVANCE;
r->eof_cnt = 0;
r->block_left = 0;
if (fh_get_referent (fh) != FH_REF_INLINE)
{
struct stat s;
r->line_number = 0;
r->file = fn_open (fh_get_file_name (fh), "rb");
if (r->file == NULL)
{
msg (ME, _("Could not open `%s' for reading as a data file: %s."),
fh_get_file_name (r->fh), strerror (errno));
fh_unlock (r->lock);
fh_unref (fh);
free (r);
return NULL;
}
r->file_size = fstat (fileno (r->file), &s) == 0 ? s.st_size : -1;
}
else
r->file_size = -1;
fh_lock_set_aux (lock, r);
return r;
}
static char *
generate_syntax (const struct comment_dialog *cd)
{
gint i;
GString *str;
gchar *text;
GtkWidget *tv = get_widget_assert (cd->xml, "comments-textview1");
GtkWidget *check = get_widget_assert (cd->xml, "comments-checkbutton1");
GtkTextBuffer *buffer = gtk_text_view_get_buffer (GTK_TEXT_VIEW (tv));
str = g_string_new ("\n* Data File Comments.\n\n");
if (dict_get_documents (cd->dict->dict) != NULL)
g_string_append (str, "DROP DOCUMENTS.\n");
g_string_append (str, "ADD DOCUMENT\n");
for (i = 0 ; i < gtk_text_buffer_get_line_count (buffer) ; ++i )
{
struct string tmp;
GtkTextIter start;
char *line;
gtk_text_buffer_get_iter_at_line (buffer, &start, i);
if (gtk_text_iter_ends_line (&start))
line = g_strdup ("");
else
{
GtkTextIter end = start;
gtk_text_iter_forward_to_line_end (&end);
line = gtk_text_buffer_get_text (buffer, &start, &end, FALSE);
}
ds_init_empty (&tmp);
syntax_gen_string (&tmp, ss_cstr (line));
g_free (line);
g_string_append_printf (str, " %s\n", ds_cstr (&tmp));
ds_destroy (&tmp);
}
g_string_append (str, " .\n");
if ( gtk_toggle_button_get_active (GTK_TOGGLE_BUTTON (check)))
g_string_append (str, "DISPLAY DOCUMENTS.\n");
text = str->str;
g_string_free (str, FALSE);
return text;
}
static void
acc (struct statistic *s, const struct ccase *cx,
double c UNUSED, double cc UNUSED, double y)
{
struct box_whisker *bw = UP_CAST (s, struct box_whisker, parent.parent);
bool extreme;
struct outlier *o;
if ( y > bw->hinges[2] + bw->step) /* Upper outlier */
{
extreme = (y > bw->hinges[2] + 2 * bw->step) ;
}
else if (y < bw->hinges[0] - bw->step) /* Lower outlier */
{
extreme = (y < bw->hinges[0] - 2 * bw->step) ;
}
else /* Not an outlier */
{
if (bw->whiskers[0] == SYSMIS)
bw->whiskers[0] = y;
if (y > bw->whiskers[1])
bw->whiskers[1] = y;
return;
}
/* y is an outlier */
o = xzalloc (sizeof *o) ;
o->value = y;
o->extreme = extreme;
ds_init_empty (&o->label);
if (bw->id_var)
{
char *s = data_out (case_data_idx (cx, bw->id_idx),
var_get_encoding (bw->id_var),
var_get_print_format (bw->id_var));
ds_put_cstr (&o->label, s);
free (s);
}
else
{
ds_put_format (&o->label,
"%ld",
(casenumber) case_data_idx (cx, bw->id_idx)->f);
}
ll_push_head (&bw->outliers, &o->ll);
}
/* Create a table which can be populated with the encodings for
the covariance matrix COV */
struct tab_table *
covariance_dump_enc_header (const struct covariance *cov, int length)
{
struct tab_table *t = tab_create (cov->dim, length);
int n;
int i;
tab_title (t, "Covariance Encoding");
tab_box (t,
TAL_2, TAL_2, 0, 0,
0, 0, tab_nc (t) - 1, tab_nr (t) - 1);
tab_hline (t, TAL_2, 0, tab_nc (t) - 1, 1);
for (i = 0 ; i < cov->n_vars; ++i)
{
tab_text (t, i, 0, TAT_TITLE, var_get_name (cov->vars[i]));
tab_vline (t, TAL_1, i + 1, 0, tab_nr (t) - 1);
}
n = 0;
while (i < cov->dim)
{
struct string str;
int idx = i - cov->n_vars;
const struct interaction *iact =
categoricals_get_interaction_by_subscript (cov->categoricals, idx);
int df;
ds_init_empty (&str);
interaction_to_string (iact, &str);
df = categoricals_df (cov->categoricals, n);
tab_joint_text (t,
i, 0,
i + df - 1, 0,
TAT_TITLE, ds_cstr (&str));
if (i + df < tab_nr (t) - 1)
tab_vline (t, TAL_1, i + df, 0, tab_nr (t) - 1);
i += df;
n++;
ds_destroy (&str);
}
return t;
}
/* Returns a single string that consists of each of the strings in SA
concatenated, separated from each other with SEPARATOR.
The caller is responsible for freeing the returned string with free(). */
char *
string_array_join (const struct string_array *sa, const char *separator)
{
struct string dst;
const char *s;
size_t i;
ds_init_empty (&dst);
STRING_ARRAY_FOR_EACH (s, i, sa)
{
if (i > 0)
ds_put_cstr (&dst, separator);
ds_put_cstr (&dst, s);
}
return ds_steal_cstr (&dst);
}
static char *
generate_syntax (PsppireDialogAction *a)
{
PsppireDialogActionIndepSamps *act = PSPPIRE_DIALOG_ACTION_INDEP_SAMPS (a);
gchar *text;
GString *str = g_string_new ("T-TEST /VARIABLES=");
psppire_var_view_append_names (PSPPIRE_VAR_VIEW (act->test_vars_tv), 0, str);
g_string_append (str, "\n\t/GROUPS=");
g_string_append (str, var_get_name (act->grp_var));
if (act->group_defn != GROUPS_UNDEF)
{
g_string_append (str, "(");
{
const union value *val =
(act->group_defn == GROUPS_VALUES) ?
&act->grp_val[0] :
&act->cut_point;
struct string strx;
ds_init_empty (&strx);
syntax_gen_value (&strx, val, var_get_width (act->grp_var),
var_get_print_format (act->grp_var));
g_string_append (str, ds_cstr (&strx));
ds_destroy (&strx);
}
if (act->group_defn == GROUPS_VALUES)
{
g_string_append (str, ",");
{
struct string strx;
ds_init_empty (&strx);
syntax_gen_value (&strx, &act->grp_val[1], var_get_width (act->grp_var),
var_get_print_format (act->grp_var));
g_string_append (str, ds_cstr (&strx));
ds_destroy (&strx);
}
}
g_string_append (str, ")");
}
tt_options_dialog_append_syntax (act->opts, str);
g_string_append (str, ".\n");
text = str->str;
g_string_free (str, FALSE);
return text;
}
/* Checks that test state TS matches the test model TM and
reports any mismatches via mc_error. Then, adds SX to MC as a
new state. */
static void
check_state (struct mc *mc, struct test_state *ts, const struct test_model *tm)
{
const struct test_params *params = mc_get_aux (mc);
int n_columns = params->n_columns;
unsigned int hash;
int i;
for (i = 0; i < params->n_xarrays; i++)
{
const struct xarray_model *model = &tm->models[i];
const struct sparse_xarray *sx = ts->xarrays[i];
bool difference;
int row, col;
int n_rows;
assert (n_columns < MAX_COLS);
/* Check row count. */
n_rows = 0;
for (row = 0; row < params->max_rows; row++)
if (model->contains_row[row])
n_rows = row + 1;
if (n_rows != sparse_xarray_get_n_rows (sx))
mc_error (mc, "xarray %d: row count (%zu) does not match expected "
"(%d)", i, sparse_xarray_get_n_rows (sx), n_rows);
/* Check row containment. */
for (row = 0; row < params->max_rows; row++)
{
bool contains = sparse_xarray_contains_row (sx, row);
if (contains && !model->contains_row[row])
mc_error (mc, "xarray %d: row %d is contained by sparse_xarray "
"but should not be", i, row);
else if (!contains && model->contains_row[row])
mc_error (mc, "xarray %d: row %d is not contained by "
"sparse_xarray but should be", i, row);
}
/* Check contents. */
difference = false;
for (row = 0; row < params->max_rows; row++)
{
unsigned char data[MAX_COLS];
if (!sparse_xarray_read (sx, row, 0, n_columns, data))
NOT_REACHED ();
for (col = 0; col < params->n_columns; col++)
if (data[col] != model->data[row][col])
{
mc_error (mc, "xarray %d: element %d,%d (of %d,%d) "
"differs: %d should be %d",
i, row, col, n_rows, n_columns, data[col],
model->data[row][col]);
difference = true;
}
}
if (difference)
{
struct string ds;
mc_error (mc, "xarray %d: expected:", i);
ds_init_empty (&ds);
for (row = 0; row < params->max_rows; row++)
{
ds_clear (&ds);
for (col = 0; col < n_columns; col++)
ds_put_format (&ds, " %d", model->data[row][col]);
mc_error (mc, "xarray %d: row %d:%s", i, row, ds_cstr (&ds));
}
mc_error (mc, "xarray %d: actual:", i);
ds_init_empty (&ds);
for (row = 0; row < params->max_rows; row++)
{
unsigned char data[MAX_COLS];
if (!sparse_xarray_read (sx, row, 0, n_columns, data))
NOT_REACHED ();
ds_clear (&ds);
for (col = 0; col < n_columns; col++)
ds_put_format (&ds, " %d", data[col]);
mc_error (mc, "xarray %d: row %d:%s", i, row, ds_cstr (&ds));
}
ds_destroy (&ds);
}
}
hash = 0;
for (i = 0; i < params->n_xarrays; i++)
hash = sparse_xarray_model_checker_hash (ts->xarrays[i], hash);
if (mc_discard_dup_state (mc, hash))
test_state_destroy (params, ts);
else
mc_add_state (mc, ts);
}
static char *
generate_syntax (const struct recode_dialog *rd)
{
gboolean ok;
GtkTreeIter iter;
gchar *text;
GString *str = g_string_sized_new (100);
/* Declare new string variables if applicable */
if ( rd->different &&
gtk_toggle_button_get_active (GTK_TOGGLE_BUTTON (rd->string_button)))
{
GHashTableIter iter;
struct variable *var = NULL;
struct nlp *nlp = NULL;
g_hash_table_iter_init (&iter, rd->varmap);
while (g_hash_table_iter_next (&iter, (void**) &var, (void**) &nlp))
{
g_string_append (str, "\nSTRING ");
g_string_append (str, nlp->name);
g_string_append_printf (str, " (A%d).",
(int)
gtk_spin_button_get_value (GTK_SPIN_BUTTON (rd->width_entry) )
);
}
}
g_string_append (str, "\nRECODE ");
psppire_var_view_append_names (PSPPIRE_VAR_VIEW (rd->variable_treeview), 0, str);
g_string_append (str, "\n\t");
if ( gtk_toggle_button_get_active (GTK_TOGGLE_BUTTON (rd->convert_button)))
{
g_string_append (str, "(CONVERT) ");
}
for (ok = gtk_tree_model_get_iter_first (GTK_TREE_MODEL (rd->value_map),
&iter);
ok;
ok = gtk_tree_model_iter_next (GTK_TREE_MODEL (rd->value_map), &iter))
{
GValue ov_value = {0};
GValue nv_value = {0};
struct old_value *ov;
struct new_value *nv;
gtk_tree_model_get_value (GTK_TREE_MODEL (rd->value_map), &iter,
COL_VALUE_OLD, &ov_value);
gtk_tree_model_get_value (GTK_TREE_MODEL (rd->value_map), &iter,
COL_VALUE_NEW, &nv_value);
ov = g_value_get_boxed (&ov_value);
nv = g_value_get_boxed (&nv_value);
g_string_append (str, "(");
old_value_append_syntax (str, ov);
g_string_append (str, " = ");
new_value_append_syntax (str, nv);
g_string_append (str, ") ");
g_value_unset (&ov_value);
g_value_unset (&nv_value);
}
if ( rd->different )
{
GtkTreeIter iter;
g_string_append (str, "\n\tINTO ");
for (ok = psppire_var_view_get_iter_first (PSPPIRE_VAR_VIEW (rd->variable_treeview), &iter);
ok;
ok = psppire_var_view_get_iter_next (PSPPIRE_VAR_VIEW (rd->variable_treeview), &iter))
{
struct nlp *nlp = NULL;
const struct variable *var = psppire_var_view_get_variable (PSPPIRE_VAR_VIEW (rd->variable_treeview), 0, &iter);
nlp = g_hash_table_lookup (rd->varmap, var);
g_string_append (str, nlp->name);
g_string_append (str, " ");
}
}
g_string_append (str, ".");
/* If applicable, set labels for the new variables. */
if ( rd->different )
{
GHashTableIter iter;
struct variable *var = NULL;
struct nlp *nlp = NULL;
g_hash_table_iter_init (&iter, rd->varmap);
while (g_hash_table_iter_next (&iter, (void**) &var, (void**) &nlp))
{
if (nlp->label)
{
struct string sl;
ds_init_empty (&sl);
syntax_gen_string (&sl, ss_cstr (nlp->label));
g_string_append_printf (str, "\nVARIABLE LABELS %s %s.",
nlp->name, ds_cstr (&sl));
ds_destroy (&sl);
}
}
}
g_string_append (str, "\nEXECUTE.\n");
text = str->str;
g_string_free (str, FALSE);
return text;
}
static bool
parse_commands (struct lexer *lexer, struct hmap *dummies)
{
enum lex_syntax_mode syntax_mode;
enum segmenter_mode mode;
struct string *outputs;
struct string input;
size_t input_len;
size_t n_values;
char *file_name;
int line_number;
bool ok;
size_t i;
if (lex_get_file_name (lexer) != NULL)
file_name = xstrdup (lex_get_file_name (lexer));
else
file_name = NULL;
line_number = lex_get_first_line_number (lexer, 0);
ds_init_empty (&input);
while (lex_is_string (lexer))
{
ds_put_substring (&input, lex_tokss (lexer));
ds_put_byte (&input, '\n');
lex_get (lexer);
}
if (ds_is_empty (&input))
ds_put_byte (&input, '\n');
ds_put_byte (&input, '\0');
input_len = ds_length (&input);
n_values = count_values (dummies);
outputs = xmalloc (n_values * sizeof *outputs);
for (i = 0; i < n_values; i++)
ds_init_empty (&outputs[i]);
syntax_mode = lex_get_syntax_mode (lexer);
if (syntax_mode == LEX_SYNTAX_AUTO)
mode = SEG_MODE_AUTO;
else if (syntax_mode == LEX_SYNTAX_INTERACTIVE)
mode = SEG_MODE_INTERACTIVE;
else if (syntax_mode == LEX_SYNTAX_BATCH)
mode = SEG_MODE_BATCH;
else
NOT_REACHED ();
do_parse_commands (ds_ss (&input), mode, dummies, outputs, n_values);
ds_destroy (&input);
while (lex_match (lexer, T_ENDCMD))
continue;
ok = (lex_force_match_id (lexer, "END")
&& lex_force_match_id (lexer, "REPEAT"));
if (ok)
lex_match_id (lexer, "PRINT"); /* XXX */
lex_discard_rest_of_command (lexer);
for (i = 0; i < n_values; i++)
{
struct string *output = &outputs[n_values - i - 1];
struct lex_reader *reader;
reader = lex_reader_for_substring_nocopy (ds_ss (output));
lex_reader_set_file_name (reader, file_name);
reader->line_number = line_number;
lex_include (lexer, reader);
}
free (file_name);
return ok;
}
/* Parse all the aggregate functions. */
static bool
parse_aggregate_functions (struct lexer *lexer, const struct dictionary *dict,
struct agr_proc *agr)
{
struct agr_var *tail; /* Tail of linked list starting at agr->vars. */
/* Parse everything. */
tail = NULL;
for (;;)
{
char **dest;
char **dest_label;
size_t n_dest;
struct string function_name;
enum mv_class exclude;
const struct agr_func *function;
int func_index;
union agr_argument arg[2];
const struct variable **src;
size_t n_src;
size_t i;
dest = NULL;
dest_label = NULL;
n_dest = 0;
src = NULL;
function = NULL;
n_src = 0;
arg[0].c = NULL;
arg[1].c = NULL;
ds_init_empty (&function_name);
/* Parse the list of target variables. */
while (!lex_match (lexer, T_EQUALS))
{
size_t n_dest_prev = n_dest;
if (!parse_DATA_LIST_vars (lexer, dict, &dest, &n_dest,
(PV_APPEND | PV_SINGLE | PV_NO_SCRATCH
| PV_NO_DUPLICATE)))
goto error;
/* Assign empty labels. */
{
int j;
dest_label = xnrealloc (dest_label, n_dest, sizeof *dest_label);
for (j = n_dest_prev; j < n_dest; j++)
dest_label[j] = NULL;
}
if (lex_is_string (lexer))
{
dest_label[n_dest - 1] = xstrdup (lex_tokcstr (lexer));
lex_get (lexer);
}
}
/* Get the name of the aggregation function. */
if (lex_token (lexer) != T_ID)
{
lex_error (lexer, _("expecting aggregation function"));
goto error;
}
ds_assign_substring (&function_name, lex_tokss (lexer));
exclude = ds_chomp_byte (&function_name, '.') ? MV_SYSTEM : MV_ANY;
for (function = agr_func_tab; function->name; function++)
if (!c_strcasecmp (function->name, ds_cstr (&function_name)))
break;
if (NULL == function->name)
{
msg (SE, _("Unknown aggregation function %s."),
ds_cstr (&function_name));
goto error;
}
ds_destroy (&function_name);
func_index = function - agr_func_tab;
lex_get (lexer);
/* Check for leading lparen. */
if (!lex_match (lexer, T_LPAREN))
{
if (function->src_vars == AGR_SV_YES)
{
lex_force_match (lexer, T_LPAREN);
goto error;
}
}
else
{
/* Parse list of source variables. */
{
int pv_opts = PV_NO_SCRATCH;
if (func_index == SUM || func_index == MEAN || func_index == SD)
pv_opts |= PV_NUMERIC;
else if (function->n_args)
pv_opts |= PV_SAME_TYPE;
if (!parse_variables_const (lexer, dict, &src, &n_src, pv_opts))
goto error;
}
/* Parse function arguments, for those functions that
require arguments. */
if (function->n_args != 0)
for (i = 0; i < function->n_args; i++)
{
int type;
lex_match (lexer, T_COMMA);
if (lex_is_string (lexer))
{
arg[i].c = recode_string (dict_get_encoding (agr->dict),
"UTF-8", lex_tokcstr (lexer),
-1);
type = VAL_STRING;
}
else if (lex_is_number (lexer))
{
arg[i].f = lex_tokval (lexer);
type = VAL_NUMERIC;
}
else
{
msg (SE, _("Missing argument %zu to %s."),
i + 1, function->name);
goto error;
}
lex_get (lexer);
if (type != var_get_type (src[0]))
{
msg (SE, _("Arguments to %s must be of same type as "
"source variables."),
function->name);
goto error;
}
}
/* Trailing rparen. */
if (!lex_force_match (lexer, T_RPAREN))
goto error;
/* Now check that the number of source variables match
the number of target variables. If we check earlier
than this, the user can get very misleading error
message, i.e. `AGGREGATE x=SUM(y t).' will get this
error message when a proper message would be more
like `unknown variable t'. */
if (n_src != n_dest)
{
msg (SE, _("Number of source variables (%zu) does not match "
"number of target variables (%zu)."),
n_src, n_dest);
goto error;
}
if ((func_index == PIN || func_index == POUT
|| func_index == FIN || func_index == FOUT)
&& (var_is_numeric (src[0])
? arg[0].f > arg[1].f
: str_compare_rpad (arg[0].c, arg[1].c) > 0))
{
union agr_argument t = arg[0];
arg[0] = arg[1];
arg[1] = t;
msg (SW, _("The value arguments passed to the %s function "
"are out-of-order. They will be treated as if "
"they had been specified in the correct order."),
function->name);
}
}
/* Finally add these to the linked list of aggregation
variables. */
for (i = 0; i < n_dest; i++)
{
struct agr_var *v = xzalloc (sizeof *v);
/* Add variable to chain. */
if (agr->agr_vars != NULL)
tail->next = v;
else
agr->agr_vars = v;
tail = v;
tail->next = NULL;
v->moments = NULL;
/* Create the target variable in the aggregate
dictionary. */
{
struct variable *destvar;
v->function = func_index;
if (src)
{
v->src = src[i];
if (var_is_alpha (src[i]))
{
v->function |= FSTRING;
v->string = xmalloc (var_get_width (src[i]));
}
if (function->alpha_type == VAL_STRING)
destvar = dict_clone_var_as (agr->dict, v->src, dest[i]);
else
{
assert (var_is_numeric (v->src)
|| function->alpha_type == VAL_NUMERIC);
destvar = dict_create_var (agr->dict, dest[i], 0);
if (destvar != NULL)
{
struct fmt_spec f;
if ((func_index == N || func_index == NMISS)
&& dict_get_weight (dict) != NULL)
f = fmt_for_output (FMT_F, 8, 2);
else
f = function->format;
var_set_both_formats (destvar, &f);
}
}
} else {
struct fmt_spec f;
v->src = NULL;
destvar = dict_create_var (agr->dict, dest[i], 0);
if (destvar != NULL)
{
if ((func_index == N || func_index == NMISS)
&& dict_get_weight (dict) != NULL)
f = fmt_for_output (FMT_F, 8, 2);
else
f = function->format;
var_set_both_formats (destvar, &f);
}
}
if (!destvar)
{
msg (SE, _("Variable name %s is not unique within the "
"aggregate file dictionary, which contains "
"the aggregate variables and the break "
"variables."),
dest[i]);
goto error;
}
free (dest[i]);
if (dest_label[i])
var_set_label (destvar, dest_label[i]);
v->dest = destvar;
}
v->exclude = exclude;
if (v->src != NULL)
{
int j;
if (var_is_numeric (v->src))
for (j = 0; j < function->n_args; j++)
v->arg[j].f = arg[j].f;
else
for (j = 0; j < function->n_args; j++)
v->arg[j].c = xstrdup (arg[j].c);
}
}
if (src != NULL && var_is_alpha (src[0]))
for (i = 0; i < function->n_args; i++)
{
free (arg[i].c);
arg[i].c = NULL;
}
free (src);
free (dest);
free (dest_label);
if (!lex_match (lexer, T_SLASH))
{
if (lex_token (lexer) == T_ENDCMD)
return true;
lex_error (lexer, "expecting end of command");
return false;
}
continue;
error:
ds_destroy (&function_name);
for (i = 0; i < n_dest; i++)
{
free (dest[i]);
free (dest_label[i]);
}
free (dest);
free (dest_label);
free (arg[0].c);
free (arg[1].c);
if (src && n_src && var_is_alpha (src[0]))
for (i = 0; i < function->n_args; i++)
{
free (arg[i].c);
arg[i].c = NULL;
}
free (src);
return false;
}
}
