int
cmd_variable_labels (struct lexer *lexer, struct dataset *ds)
{
struct dictionary *dict = dataset_dict (ds);
do
{
struct variable **v;
size_t nv;
size_t i;
if (!parse_variables (lexer, dict, &v, &nv, PV_NONE))
return CMD_FAILURE;
if (!lex_force_string (lexer))
{
free (v);
return CMD_FAILURE;
}
for (i = 0; i < nv; i++)
var_set_label (v[i], lex_tokcstr (lexer));
lex_get (lexer);
while (lex_token (lexer) == T_SLASH)
lex_get (lexer);
free (v);
}
while (lex_token (lexer) != T_ENDCMD);
return CMD_SUCCESS;
}
int
cmd_sample (struct lexer *lexer, struct dataset *ds)
{
struct sample_trns *trns;
int type;
int a, b;
unsigned frac;
if (!lex_force_num (lexer))
return CMD_FAILURE;
if (!lex_is_integer (lexer))
{
unsigned long min = gsl_rng_min (get_rng ());
unsigned long max = gsl_rng_max (get_rng ());
type = TYPE_FRACTION;
if (lex_tokval (lexer) <= 0 || lex_tokval (lexer) >= 1)
{
msg (SE, _("The sampling factor must be between 0 and 1 "
"exclusive."));
return CMD_FAILURE;
}
frac = lex_tokval (lexer) * (max - min) + min;
a = b = 0;
}
else
{
type = TYPE_A_FROM_B;
a = lex_integer (lexer);
lex_get (lexer);
if (!lex_force_match_id (lexer, "FROM"))
return CMD_FAILURE;
if (!lex_force_int (lexer))
return CMD_FAILURE;
b = lex_integer (lexer);
if (a >= b)
{
msg (SE, _("Cannot sample %d observations from a population of "
"%d."),
a, b);
return CMD_FAILURE;
}
frac = 0;
}
lex_get (lexer);
trns = xmalloc (sizeof *trns);
trns->type = type;
trns->n = a;
trns->N = b;
trns->m = trns->t = 0;
trns->frac = frac;
add_transformation (ds, sample_trns_proc, sample_trns_free, trns);
return CMD_SUCCESS;
}
/* Parse all the labels for the VAR_CNT variables in VARS and add
the specified labels to those variables. */
static int
get_label (struct lexer *lexer, struct variable **vars, size_t var_cnt,
const char *dict_encoding)
{
/* Parse all the labels and add them to the variables. */
do
{
enum { MAX_LABEL_LEN = 255 };
int width = var_get_width (vars[0]);
union value value;
struct string label;
size_t trunc_len;
size_t i;
/* Set value. */
value_init (&value, width);
if (!parse_value (lexer, &value, vars[0]))
{
value_destroy (&value, width);
return 0;
}
lex_match (lexer, T_COMMA);
/* Set label. */
if (lex_token (lexer) != T_ID && !lex_force_string (lexer))
{
value_destroy (&value, width);
return 0;
}
ds_init_substring (&label, lex_tokss (lexer));
trunc_len = utf8_encoding_trunc_len (ds_cstr (&label), dict_encoding,
MAX_LABEL_LEN);
if (ds_length (&label) > trunc_len)
{
msg (SW, _("Truncating value label to %d bytes."), MAX_LABEL_LEN);
ds_truncate (&label, trunc_len);
}
for (i = 0; i < var_cnt; i++)
var_replace_value_label (vars[i], &value, ds_cstr (&label));
ds_destroy (&label);
value_destroy (&value, width);
lex_get (lexer);
lex_match (lexer, T_COMMA);
}
while (lex_token (lexer) != T_SLASH && lex_token (lexer) != T_ENDCMD);
return 1;
}
/* Reads a record from the inline file into R.
Returns true if successful, false on failure. */
static bool
read_inline_record (struct dfm_reader *r)
{
if ((r->flags & DFM_SAW_BEGIN_DATA) == 0)
{
r->flags |= DFM_SAW_BEGIN_DATA;
r->flags &= ~DFM_CONSUME;
while (lex_token (r->lexer) == T_ENDCMD)
lex_get (r->lexer);
if (!lex_force_match_id (r->lexer, "BEGIN")
|| !lex_force_match_id (r->lexer, "DATA"))
return false;
lex_match (r->lexer, T_ENDCMD);
}
if (r->flags & DFM_CONSUME)
lex_get (r->lexer);
if (!lex_is_string (r->lexer))
{
if (!lex_match_id (r->lexer, "END") || !lex_match_id (r->lexer, "DATA"))
{
msg (SE, _("Missing END DATA while reading inline data. "
"This probably indicates a missing or incorrectly "
"formatted END DATA command. END DATA must appear "
"by itself on a single line with exactly one space "
"between words."));
lex_discard_rest_of_command (r->lexer);
}
return false;
}
ds_assign_substring (&r->line, lex_tokss (r->lexer));
r->flags |= DFM_CONSUME;
return true;
}
static int
do_value_labels (struct lexer *lexer, const struct dictionary *dict, bool erase)
{
struct variable **vars; /* Variable list. */
size_t var_cnt; /* Number of variables. */
int parse_err=0; /* true if error parsing variables */
lex_match (lexer, T_SLASH);
while (lex_token (lexer) != T_ENDCMD)
{
parse_err = !parse_variables (lexer, dict, &vars, &var_cnt,
PV_SAME_WIDTH);
if (var_cnt < 1)
{
free(vars);
return CMD_FAILURE;
}
if (erase)
erase_labels (vars, var_cnt);
while (lex_token (lexer) != T_SLASH && lex_token (lexer) != T_ENDCMD)
if (!get_label (lexer, vars, var_cnt, dict_get_encoding (dict)))
goto lossage;
if (lex_token (lexer) != T_SLASH)
{
free (vars);
break;
}
lex_get (lexer);
free (vars);
}
return parse_err ? CMD_FAILURE : CMD_SUCCESS;
lossage:
free (vars);
return CMD_FAILURE;
}
int
cmd_missing_values (struct lexer *lexer, struct dataset *ds)
{
struct dictionary *dict = dataset_dict (ds);
struct variable **v = NULL;
size_t nv;
bool ok = true;
while (lex_token (lexer) != T_ENDCMD)
{
size_t i;
if (!parse_variables (lexer, dict, &v, &nv, PV_NONE))
goto error;
if (!lex_force_match (lexer, T_LPAREN))
goto error;
for (i = 0; i < nv; i++)
var_clear_missing_values (v[i]);
if (!lex_match (lexer, T_RPAREN))
{
struct missing_values mv;
for (i = 0; i < nv; i++)
if (var_get_type (v[i]) != var_get_type (v[0]))
{
const struct variable *n = var_is_numeric (v[0]) ? v[0] : v[i];
const struct variable *s = var_is_numeric (v[0]) ? v[i] : v[0];
msg (SE, _("Cannot mix numeric variables (e.g. %s) and "
"string variables (e.g. %s) within a single list."),
var_get_name (n), var_get_name (s));
goto error;
}
if (var_is_numeric (v[0]))
{
mv_init (&mv, 0);
while (!lex_match (lexer, T_RPAREN))
{
enum fmt_type type = var_get_print_format (v[0])->type;
double x, y;
bool ok;
if (!parse_num_range (lexer, &x, &y, &type))
goto error;
ok = (x == y
? mv_add_num (&mv, x)
: mv_add_range (&mv, x, y));
if (!ok)
ok = false;
lex_match (lexer, T_COMMA);
}
}
else
{
mv_init (&mv, MV_MAX_STRING);
while (!lex_match (lexer, T_RPAREN))
{
uint8_t value[MV_MAX_STRING];
char *dict_mv;
size_t length;
if (!lex_force_string (lexer))
{
ok = false;
break;
}
dict_mv = recode_string (dict_get_encoding (dict), "UTF-8",
lex_tokcstr (lexer),
ss_length (lex_tokss (lexer)));
length = strlen (dict_mv);
if (length > MV_MAX_STRING)
{
/* XXX truncate graphemes not bytes */
msg (SE, _("Truncating missing value to maximum "
"acceptable length (%d bytes)."),
MV_MAX_STRING);
length = MV_MAX_STRING;
}
memset (value, ' ', MV_MAX_STRING);
memcpy (value, dict_mv, length);
free (dict_mv);
if (!mv_add_str (&mv, value))
ok = false;
lex_get (lexer);
lex_match (lexer, T_COMMA);
}
}
for (i = 0; i < nv; i++)
{
if (mv_is_resizable (&mv, var_get_width (v[i])))
var_set_missing_values (v[i], &mv);
else
{
msg (SE, _("Missing values provided are too long to assign "
"to variable of width %d."),
var_get_width (v[i]));
ok = false;
}
}
mv_destroy (&mv);
}
lex_match (lexer, T_SLASH);
free (v);
v = NULL;
}
free (v);
return ok ? CMD_SUCCESS : CMD_FAILURE;
error:
free (v);
return CMD_FAILURE;
}
int
cmd_t_test (struct lexer *lexer, struct dataset *ds)
{
bool ok;
const struct dictionary *dict = dataset_dict (ds);
struct tt tt;
int mode_count = 0;
/* Variables pertaining to the paired mode */
const struct variable **v1 = NULL;
size_t n_v1;
const struct variable **v2 = NULL;
size_t n_v2;
size_t n_pairs = 0;
vp *pairs = NULL;
/* One sample mode */
double testval = SYSMIS;
/* Independent samples mode */
const struct variable *gvar;
union value gval0;
union value gval1;
bool cut = false;
tt.wv = dict_get_weight (dict);
tt.dict = dict;
tt.confidence = 0.95;
tt.exclude = MV_ANY;
tt.missing_type = MISS_ANALYSIS;
tt.n_vars = 0;
tt.vars = NULL;
tt.mode = MODE_undef;
lex_match (lexer, T_EQUALS);
for (; lex_token (lexer) != T_ENDCMD; )
{
lex_match (lexer, T_SLASH);
if (lex_match_id (lexer, "TESTVAL"))
{
mode_count++;
tt.mode = MODE_SINGLE;
lex_match (lexer, T_EQUALS);
lex_force_num (lexer);
testval = lex_number (lexer);
lex_get (lexer);
}
else if (lex_match_id (lexer, "GROUPS"))
{
mode_count++;
cut = false;
tt.mode = MODE_INDEP;
lex_match (lexer, T_EQUALS);
if (NULL == (gvar = parse_variable (lexer, dict)))
goto parse_failed;
if (lex_match (lexer, T_LPAREN))
{
value_init (&gval0, var_get_width (gvar));
parse_value (lexer, &gval0, gvar);
cut = true;
if (lex_match (lexer, T_COMMA))
{
value_init (&gval1, var_get_width (gvar));
parse_value (lexer, &gval1, gvar);
cut = false;
}
lex_force_match (lexer, T_RPAREN);
}
else
{
value_init (&gval0, 0);
value_init (&gval1, 0);
gval0.f = 1.0;
gval1.f = 2.0;
cut = false;
}
if ( cut == true && var_is_alpha (gvar))
{
msg (SE, _("When applying GROUPS to a string variable, two "
"values must be specified."));
goto parse_failed;
}
}
else if (lex_match_id (lexer, "PAIRS"))
{
bool with = false;
bool paired = false;
if (tt.n_vars > 0)
{
msg (SE, _("VARIABLES subcommand may not be used with PAIRS."));
goto parse_failed;
}
mode_count++;
tt.mode = MODE_PAIRED;
lex_match (lexer, T_EQUALS);
if (!parse_variables_const (lexer, dict,
&v1, &n_v1,
PV_NO_DUPLICATE | PV_NUMERIC))
goto parse_failed;
if ( lex_match (lexer, T_WITH))
{
with = true;
if (!parse_variables_const (lexer, dict,
&v2, &n_v2,
PV_NO_DUPLICATE | PV_NUMERIC))
goto parse_failed;
if (lex_match (lexer, T_LPAREN)
&& lex_match_id (lexer, "PAIRED")
&& lex_match (lexer, T_RPAREN))
{
paired = true;
if (n_v1 != n_v2)
{
msg (SE, _("PAIRED was specified but the number of variables "
"preceding WITH (%zu) did not match the number "
"following (%zu)."),
n_v1, n_v2);
goto parse_failed;
}
}
}
{
int i;
if ( !with )
n_pairs = (n_v1 * (n_v1 - 1)) / 2.0;
else if ( paired )
n_pairs = n_v1;
else
n_pairs = n_v1 * n_v2;
pairs = xcalloc (n_pairs, sizeof *pairs);
if ( with)
{
int x = 0;
if (paired)
{
for (i = 0 ; i < n_v1; ++i)
{
vp *pair = &pairs[i];
(*pair)[0] = v1[i];
(*pair)[1] = v2[i];
}
}
else
{
for (i = 0 ; i < n_v1; ++i)
{
int j;
for (j = 0 ; j < n_v2; ++j)
{
vp *pair = &pairs[x++];
(*pair)[0] = v1[i];
(*pair)[1] = v2[j];
}
}
}
}
else
{
int x = 0;
for (i = 0 ; i < n_v1; ++i)
{
int j;
for (j = i + 1 ; j < n_v1; ++j)
{
vp *pair = &pairs[x++];
(*pair)[0] = v1[i];
(*pair)[1] = v1[j];
}
}
}
}
}
else if (lex_match_id (lexer, "VARIABLES"))
{
if ( tt.mode == MODE_PAIRED)
{
msg (SE, _("VARIABLES subcommand may not be used with PAIRS."));
goto parse_failed;
}
lex_match (lexer, T_EQUALS);
if (!parse_variables_const (lexer, dict,
&tt.vars,
&tt.n_vars,
PV_NO_DUPLICATE | PV_NUMERIC))
goto parse_failed;
}
else if ( lex_match_id (lexer, "MISSING"))
{
lex_match (lexer, T_EQUALS);
while (lex_token (lexer) != T_ENDCMD && lex_token (lexer) != T_SLASH)
{
if (lex_match_id (lexer, "INCLUDE"))
{
tt.exclude = MV_SYSTEM;
}
else if (lex_match_id (lexer, "EXCLUDE"))
{
tt.exclude = MV_ANY;
}
else if (lex_match_id (lexer, "LISTWISE"))
{
tt.missing_type = MISS_LISTWISE;
}
else if (lex_match_id (lexer, "ANALYSIS"))
{
tt.missing_type = MISS_ANALYSIS;
}
else
{
lex_error (lexer, NULL);
goto parse_failed;
}
lex_match (lexer, T_COMMA);
}
}
else if (lex_match_id (lexer, "CRITERIA"))
{
lex_match (lexer, T_EQUALS);
if ( lex_force_match_id (lexer, "CIN"))
if ( lex_force_match (lexer, T_LPAREN))
{
lex_force_num (lexer);
tt.confidence = lex_number (lexer);
lex_get (lexer);
lex_force_match (lexer, T_RPAREN);
}
}
else
{
lex_error (lexer, NULL);
goto parse_failed;
}
}
if ( mode_count != 1)
{
msg (SE, _("Exactly one of TESTVAL, GROUPS and PAIRS subcommands "
"must be specified."));
goto parse_failed;
}
if (tt.n_vars == 0 && tt.mode != MODE_PAIRED)
{
lex_sbc_missing ("VARIABLES");
goto parse_failed;
}
/* Deal with splits etc */
{
struct casereader *group;
struct casegrouper *grouper = casegrouper_create_splits (proc_open (ds), dict);
while (casegrouper_get_next_group (grouper, &group))
{
if ( tt.mode == MODE_SINGLE)
{
if ( tt.missing_type == MISS_LISTWISE )
group = casereader_create_filter_missing (group,
tt.vars, tt.n_vars,
tt.exclude,
NULL, NULL);
one_sample_run (&tt, testval, group);
}
else if ( tt.mode == MODE_PAIRED)
{
if ( tt.missing_type == MISS_LISTWISE )
{
group = casereader_create_filter_missing (group,
v1, n_v1,
tt.exclude,
NULL, NULL);
group = casereader_create_filter_missing (group,
v2, n_v2,
tt.exclude,
NULL, NULL);
}
paired_run (&tt, n_pairs, pairs, group);
}
else /* tt.mode == MODE_INDEP */
{
if ( tt.missing_type == MISS_LISTWISE )
{
group = casereader_create_filter_missing (group,
tt.vars, tt.n_vars,
tt.exclude,
NULL, NULL);
group = casereader_create_filter_missing (group,
&gvar, 1,
tt.exclude,
NULL, NULL);
}
indep_run (&tt, gvar, cut, &gval0, &gval1, group);
}
}
ok = casegrouper_destroy (grouper);
ok = proc_commit (ds) && ok;
}
free (pairs);
free (v1);
free (v2);
free (tt.vars);
return ok ? CMD_SUCCESS : CMD_FAILURE;
parse_failed:
return CMD_FAILURE;
}
int
cmd_reliability (struct lexer *lexer, struct dataset *ds)
{
const struct dictionary *dict = dataset_dict (ds);
struct reliability reliability;
reliability.n_variables = 0;
reliability.variables = NULL;
reliability.model = MODEL_ALPHA;
reliability.exclude = MV_ANY;
reliability.summary = 0;
reliability.wv = dict_get_weight (dict);
reliability.total_start = 0;
lex_match (lexer, T_SLASH);
if (!lex_force_match_id (lexer, "VARIABLES"))
{
goto error;
}
lex_match (lexer, T_EQUALS);
if (!parse_variables_const (lexer, dict, &reliability.variables, &reliability.n_variables,
PV_NO_DUPLICATE | PV_NUMERIC))
goto error;
if (reliability.n_variables < 2)
msg (MW, _("Reliability on a single variable is not useful."));
{
int i;
struct cronbach *c;
/* Create a default Scale */
reliability.n_sc = 1;
reliability.sc = xzalloc (sizeof (struct cronbach) * reliability.n_sc);
ds_init_cstr (&reliability.scale_name, "ANY");
c = &reliability.sc[0];
c->n_items = reliability.n_variables;
c->items = xzalloc (sizeof (struct variable*) * c->n_items);
for (i = 0 ; i < c->n_items ; ++i)
c->items[i] = reliability.variables[i];
}
while (lex_token (lexer) != T_ENDCMD)
{
lex_match (lexer, T_SLASH);
if (lex_match_id (lexer, "SCALE"))
{
struct const_var_set *vs;
if ( ! lex_force_match (lexer, T_LPAREN))
goto error;
if ( ! lex_force_string (lexer) )
goto error;
ds_init_substring (&reliability.scale_name, lex_tokss (lexer));
lex_get (lexer);
if ( ! lex_force_match (lexer, T_RPAREN))
goto error;
lex_match (lexer, T_EQUALS);
vs = const_var_set_create_from_array (reliability.variables, reliability.n_variables);
if (!parse_const_var_set_vars (lexer, vs, &reliability.sc->items, &reliability.sc->n_items, 0))
{
const_var_set_destroy (vs);
goto error;
}
const_var_set_destroy (vs);
}
else if (lex_match_id (lexer, "MODEL"))
{
lex_match (lexer, T_EQUALS);
if (lex_match_id (lexer, "ALPHA"))
{
reliability.model = MODEL_ALPHA;
}
else if (lex_match_id (lexer, "SPLIT"))
{
reliability.model = MODEL_SPLIT;
reliability.split_point = -1;
if ( lex_match (lexer, T_LPAREN))
{
lex_force_num (lexer);
reliability.split_point = lex_number (lexer);
lex_get (lexer);
lex_force_match (lexer, T_RPAREN);
}
}
else
goto error;
}
else if (lex_match_id (lexer, "SUMMARY"))
{
lex_match (lexer, T_EQUALS);
if (lex_match_id (lexer, "TOTAL"))
{
reliability.summary |= SUMMARY_TOTAL;
}
else if (lex_match (lexer, T_ALL))
{
reliability.summary = 0xFFFF;
}
else
goto error;
}
else if (lex_match_id (lexer, "MISSING"))
{
lex_match (lexer, T_EQUALS);
while (lex_token (lexer) != T_ENDCMD && lex_token (lexer) != T_SLASH)
{
if (lex_match_id (lexer, "INCLUDE"))
{
reliability.exclude = MV_SYSTEM;
}
else if (lex_match_id (lexer, "EXCLUDE"))
{
reliability.exclude = MV_ANY;
}
else
{
lex_error (lexer, NULL);
goto error;
}
}
}
else
{
lex_error (lexer, NULL);
goto error;
}
}
if ( reliability.model == MODEL_SPLIT)
{
int i;
const struct cronbach *s;
if ( reliability.split_point >= reliability.n_variables)
{
msg (ME, _("The split point must be less than the number of variables"));
goto error;
}
reliability.n_sc += 2 ;
reliability.sc = xrealloc (reliability.sc, sizeof (struct cronbach) * reliability.n_sc);
s = &reliability.sc[0];
reliability.sc[1].n_items =
(reliability.split_point == -1) ? s->n_items / 2 : reliability.split_point;
reliability.sc[2].n_items = s->n_items - reliability.sc[1].n_items;
reliability.sc[1].items = xzalloc (sizeof (struct variable *)
* reliability.sc[1].n_items);
reliability.sc[2].items = xzalloc (sizeof (struct variable *) *
reliability.sc[2].n_items);
for (i = 0; i < reliability.sc[1].n_items ; ++i)
reliability.sc[1].items[i] = s->items[i];
while (i < s->n_items)
{
reliability.sc[2].items[i - reliability.sc[1].n_items] = s->items[i];
i++;
}
}
if ( reliability.summary & SUMMARY_TOTAL)
{
int i;
const int base_sc = reliability.n_sc;
reliability.total_start = base_sc;
reliability.n_sc += reliability.sc[0].n_items ;
reliability.sc = xrealloc (reliability.sc, sizeof (struct cronbach) * reliability.n_sc);
for (i = 0 ; i < reliability.sc[0].n_items; ++i )
{
int v_src;
int v_dest = 0;
struct cronbach *s = &reliability.sc[i + base_sc];
s->n_items = reliability.sc[0].n_items - 1;
s->items = xzalloc (sizeof (struct variable *) * s->n_items);
for (v_src = 0 ; v_src < reliability.sc[0].n_items ; ++v_src)
{
if ( v_src != i)
s->items[v_dest++] = reliability.sc[0].items[v_src];
}
}
}
if ( ! run_reliability (ds, &reliability))
goto error;
free (reliability.variables);
return CMD_SUCCESS;
error:
free (reliability.variables);
return CMD_FAILURE;
}
static bool
parse_commands (struct lexer *lexer, struct hmap *dummies)
{
enum lex_syntax_mode syntax_mode;
enum segmenter_mode mode;
struct string *outputs;
struct string input;
size_t input_len;
size_t n_values;
char *file_name;
int line_number;
bool ok;
size_t i;
if (lex_get_file_name (lexer) != NULL)
file_name = xstrdup (lex_get_file_name (lexer));
else
file_name = NULL;
line_number = lex_get_first_line_number (lexer, 0);
ds_init_empty (&input);
while (lex_is_string (lexer))
{
ds_put_substring (&input, lex_tokss (lexer));
ds_put_byte (&input, '\n');
lex_get (lexer);
}
if (ds_is_empty (&input))
ds_put_byte (&input, '\n');
ds_put_byte (&input, '\0');
input_len = ds_length (&input);
n_values = count_values (dummies);
outputs = xmalloc (n_values * sizeof *outputs);
for (i = 0; i < n_values; i++)
ds_init_empty (&outputs[i]);
syntax_mode = lex_get_syntax_mode (lexer);
if (syntax_mode == LEX_SYNTAX_AUTO)
mode = SEG_MODE_AUTO;
else if (syntax_mode == LEX_SYNTAX_INTERACTIVE)
mode = SEG_MODE_INTERACTIVE;
else if (syntax_mode == LEX_SYNTAX_BATCH)
mode = SEG_MODE_BATCH;
else
NOT_REACHED ();
do_parse_commands (ds_ss (&input), mode, dummies, outputs, n_values);
ds_destroy (&input);
while (lex_match (lexer, T_ENDCMD))
continue;
ok = (lex_force_match_id (lexer, "END")
&& lex_force_match_id (lexer, "REPEAT"));
if (ok)
lex_match_id (lexer, "PRINT"); /* XXX */
lex_discard_rest_of_command (lexer);
for (i = 0; i < n_values; i++)
{
struct string *output = &outputs[n_values - i - 1];
struct lex_reader *reader;
reader = lex_reader_for_substring_nocopy (ds_ss (output));
lex_reader_set_file_name (reader, file_name);
reader->line_number = line_number;
lex_include (lexer, reader);
}
free (file_name);
return ok;
}
static int lex_scan_number(lex_t *lex, char c, json_error_t *error)
{
const char *saved_text;
char *end;
double value;
lex->token = TOKEN_INVALID;
if(c == '-')
c = lex_get_save(lex, error);
if(c == '0') {
c = lex_get_save(lex, error);
if(isdigit(c)) {
lex_unget_unsave(lex, c);
goto out;
}
}
else if(isdigit(c)) {
c = lex_get_save(lex, error);
while(isdigit(c))
c = lex_get_save(lex, error);
}
else {
lex_unget_unsave(lex, c);
goto out;
}
if(c != '.' && c != 'E' && c != 'e') {
long value;
lex_unget_unsave(lex, c);
saved_text = strbuffer_value(&lex->saved_text);
value = strtol(saved_text, &end, 10);
assert(end == saved_text + lex->saved_text.length);
if((value == LONG_MAX && errno == ERANGE) || value > INT_MAX) {
error_set(error, lex, "too big integer");
goto out;
}
else if((value == LONG_MIN && errno == ERANGE) || value < INT_MIN) {
error_set(error, lex, "too big negative integer");
goto out;
}
lex->token = TOKEN_INTEGER;
lex->value.integer = (int)value;
return 0;
}
if(c == '.') {
c = lex_get(lex, error);
if(!isdigit(c))
goto out;
lex_save(lex, c);
c = lex_get_save(lex, error);
while(isdigit(c))
c = lex_get_save(lex, error);
}
if(c == 'E' || c == 'e') {
c = lex_get_save(lex, error);
if(c == '+' || c == '-')
c = lex_get_save(lex, error);
if(!isdigit(c)) {
lex_unget_unsave(lex, c);
goto out;
}
c = lex_get_save(lex, error);
while(isdigit(c))
c = lex_get_save(lex, error);
}
lex_unget_unsave(lex, c);
saved_text = strbuffer_value(&lex->saved_text);
value = strtod(saved_text, &end);
assert(end == saved_text + lex->saved_text.length);
if(value == 0 && errno == ERANGE) {
error_set(error, lex, "real number underflow");
goto out;
}
/* Cannot test for +/-HUGE_VAL because the HUGE_VAL constant is
only defined in C99 mode. So let's trust in sole errno. */
else if(errno == ERANGE) {
error_set(error, lex, "real number overflow");
goto out;
}
lex->token = TOKEN_REAL;
lex->value.real = value;
return 0;
out:
return -1;
}
int
cmd_save_translate (struct lexer *lexer, struct dataset *ds)
{
enum { CSV_FILE = 1, TAB_FILE } type;
struct dictionary *dict;
struct case_map *map;
struct casewriter *writer;
struct file_handle *handle;
struct csv_writer_options csv_opts;
bool replace;
bool retain_unselected;
bool recode_user_missing;
bool include_var_names;
bool use_value_labels;
bool use_print_formats;
char decimal;
char delimiter;
char qualifier;
bool ok;
type = 0;
dict = dict_clone (dataset_dict (ds));
map = NULL;
handle = NULL;
replace = false;
retain_unselected = true;
recode_user_missing = false;
include_var_names = false;
use_value_labels = false;
use_print_formats = false;
decimal = settings_get_decimal_char (FMT_F);
delimiter = 0;
qualifier = '"';
case_map_prepare_dict (dict);
dict_delete_scratch_vars (dict);
while (lex_match (lexer, T_SLASH))
{
if (lex_match_id (lexer, "OUTFILE"))
{
if (handle != NULL)
{
lex_sbc_only_once ("OUTFILE");
goto error;
}
lex_match (lexer, T_EQUALS);
handle = fh_parse (lexer, FH_REF_FILE, NULL);
if (handle == NULL)
goto error;
}
else if (lex_match_id (lexer, "TYPE"))
{
if (type != 0)
{
lex_sbc_only_once ("TYPE");
goto error;
}
lex_match (lexer, T_EQUALS);
if (lex_match_id (lexer, "CSV"))
type = CSV_FILE;
else if (lex_match_id (lexer, "TAB"))
type = TAB_FILE;
else
{
lex_error_expecting (lexer, "CSV", "TAB", NULL_SENTINEL);
goto error;
}
}
else if (lex_match_id (lexer, "REPLACE"))
replace = true;
else if (lex_match_id (lexer, "FIELDNAMES"))
include_var_names = true;
else if (lex_match_id (lexer, "MISSING"))
{
lex_match (lexer, T_EQUALS);
if (lex_match_id (lexer, "IGNORE"))
recode_user_missing = false;
else if (lex_match_id (lexer, "RECODE"))
recode_user_missing = true;
else
{
lex_error_expecting (lexer, "IGNORE", "RECODE", NULL_SENTINEL);
goto error;
}
}
else if (lex_match_id (lexer, "CELLS"))
{
lex_match (lexer, T_EQUALS);
if (lex_match_id (lexer, "VALUES"))
use_value_labels = false;
else if (lex_match_id (lexer, "LABELS"))
use_value_labels = true;
else
{
lex_error_expecting (lexer, "VALUES", "LABELS", NULL_SENTINEL);
goto error;
}
}
else if (lex_match_id (lexer, "TEXTOPTIONS"))
{
lex_match (lexer, T_EQUALS);
for (;;)
{
if (lex_match_id (lexer, "DELIMITER"))
{
lex_match (lexer, T_EQUALS);
if (!lex_force_string (lexer))
goto error;
/* XXX should support multibyte UTF-8 delimiters */
if (ss_length (lex_tokss (lexer)) != 1)
{
msg (SE, _("The %s string must contain exactly one "
"character."), "DELIMITER");
goto error;
}
delimiter = ss_first (lex_tokss (lexer));
lex_get (lexer);
}
else if (lex_match_id (lexer, "QUALIFIER"))
{
lex_match (lexer, T_EQUALS);
if (!lex_force_string (lexer))
goto error;
/* XXX should support multibyte UTF-8 qualifiers */
if (ss_length (lex_tokss (lexer)) != 1)
{
msg (SE, _("The %s string must contain exactly one "
"character."), "QUALIFIER");
goto error;
}
qualifier = ss_first (lex_tokss (lexer));
lex_get (lexer);
}
else if (lex_match_id (lexer, "DECIMAL"))
{
lex_match (lexer, T_EQUALS);
if (lex_match_id (lexer, "DOT"))
decimal = '.';
else if (lex_match_id (lexer, "COMMA"))
decimal = ',';
else
{
lex_error_expecting (lexer, "DOT", "COMMA",
NULL_SENTINEL);
goto error;
}
}
else if (lex_match_id (lexer, "FORMAT"))
{
lex_match (lexer, T_EQUALS);
if (lex_match_id (lexer, "PLAIN"))
use_print_formats = false;
else if (lex_match_id (lexer, "VARIABLE"))
use_print_formats = true;
else
{
lex_error_expecting (lexer, "PLAIN", "VARIABLE",
NULL_SENTINEL);
goto error;
}
}
else
break;
}
}
else if (lex_match_id (lexer, "UNSELECTED"))
{
lex_match (lexer, T_EQUALS);
if (lex_match_id (lexer, "RETAIN"))
retain_unselected = true;
else if (lex_match_id (lexer, "DELETE"))
retain_unselected = false;
else
{
lex_error_expecting (lexer, "RETAIN", "DELETE", NULL_SENTINEL);
goto error;
}
}
else if (!parse_dict_trim (lexer, dict))
goto error;
}
if (lex_end_of_command (lexer) != CMD_SUCCESS)
goto error;
if (type == 0)
{
lex_sbc_missing ("TYPE");
goto error;
}
else if (handle == NULL)
{
lex_sbc_missing ("OUTFILE");
goto error;
}
else if (!replace && fn_exists (fh_get_file_name (handle)))
{
msg (SE, _("Output file `%s' exists but REPLACE was not specified."),
fh_get_file_name (handle));
goto error;
}
dict_delete_scratch_vars (dict);
dict_compact_values (dict);
csv_opts.recode_user_missing = recode_user_missing;
csv_opts.include_var_names = include_var_names;
csv_opts.use_value_labels = use_value_labels;
csv_opts.use_print_formats = use_print_formats;
csv_opts.decimal = decimal;
csv_opts.delimiter = (delimiter ? delimiter
: type == TAB_FILE ? '\t'
: decimal == '.' ? ','
: ';');
csv_opts.qualifier = qualifier;
writer = csv_writer_open (handle, dict, &csv_opts);
if (writer == NULL)
goto error;
fh_unref (handle);
map = case_map_from_dict (dict);
if (map != NULL)
writer = case_map_create_output_translator (map, writer);
dict_destroy (dict);
casereader_transfer (proc_open_filtering (ds, !retain_unselected), writer);
ok = casewriter_destroy (writer);
ok = proc_commit (ds) && ok;
return ok ? CMD_SUCCESS : CMD_CASCADING_FAILURE;
error:
fh_unref (handle);
dict_destroy (dict);
case_map_destroy (map);
return CMD_FAILURE;
}
static int
combine_files (enum comb_command_type command,
struct lexer *lexer, struct dataset *ds)
{
struct comb_proc proc;
bool saw_by = false;
bool saw_sort = false;
struct casereader *active_file = NULL;
char *first_name = NULL;
char *last_name = NULL;
struct taint *taint = NULL;
size_t n_tables = 0;
size_t allocated_files = 0;
size_t i;
proc.files = NULL;
proc.n_files = 0;
proc.dict = dict_create (get_default_encoding ());
proc.output = NULL;
proc.matcher = NULL;
subcase_init_empty (&proc.by_vars);
proc.first = NULL;
proc.last = NULL;
proc.buffered_case = NULL;
proc.prev_BY = NULL;
dict_set_case_limit (proc.dict, dict_get_case_limit (dataset_dict (ds)));
lex_match (lexer, T_SLASH);
for (;;)
{
struct comb_file *file;
enum comb_file_type type;
if (lex_match_id (lexer, "FILE"))
type = COMB_FILE;
else if (command == COMB_MATCH && lex_match_id (lexer, "TABLE"))
{
type = COMB_TABLE;
n_tables++;
}
else
break;
lex_match (lexer, T_EQUALS);
if (proc.n_files >= allocated_files)
proc.files = x2nrealloc (proc.files, &allocated_files,
sizeof *proc.files);
file = &proc.files[proc.n_files++];
file->type = type;
subcase_init_empty (&file->by_vars);
subcase_init_empty (&file->src);
subcase_init_empty (&file->dst);
file->mv = NULL;
file->handle = NULL;
file->dict = NULL;
file->reader = NULL;
file->data = NULL;
file->is_sorted = true;
file->in_name = NULL;
file->in_var = NULL;
if (lex_match (lexer, T_ASTERISK))
{
if (!dataset_has_source (ds))
{
msg (SE, _("Cannot specify the active dataset since none "
"has been defined."));
goto error;
}
if (proc_make_temporary_transformations_permanent (ds))
msg (SE, _("This command may not be used after TEMPORARY when "
"the active dataset is an input source. "
"Temporary transformations will be made permanent."));
file->dict = dict_clone (dataset_dict (ds));
}
else
{
file->handle = fh_parse (lexer, FH_REF_FILE, dataset_session (ds));
if (file->handle == NULL)
goto error;
file->reader = any_reader_open (file->handle, NULL, &file->dict);
if (file->reader == NULL)
goto error;
}
while (lex_match (lexer, T_SLASH))
if (lex_match_id (lexer, "RENAME"))
{
if (!parse_dict_rename (lexer, file->dict))
goto error;
}
else if (lex_match_id (lexer, "IN"))
{
lex_match (lexer, T_EQUALS);
if (lex_token (lexer) != T_ID)
{
lex_error (lexer, NULL);
goto error;
}
if (file->in_name)
{
msg (SE, _("Multiple IN subcommands for a single FILE or "
"TABLE."));
goto error;
}
file->in_name = xstrdup (lex_tokcstr (lexer));
lex_get (lexer);
}
else if (lex_match_id (lexer, "SORT"))
{
file->is_sorted = false;
saw_sort = true;
}
if (!merge_dictionary (proc.dict, file))
goto error;
}
while (lex_token (lexer) != T_ENDCMD)
{
if (lex_match (lexer, T_BY))
{
const struct variable **by_vars;
size_t i;
bool ok;
if (saw_by)
{
lex_sbc_only_once ("BY");
goto error;
}
saw_by = true;
lex_match (lexer, T_EQUALS);
if (!parse_sort_criteria (lexer, proc.dict, &proc.by_vars,
&by_vars, NULL))
goto error;
ok = true;
for (i = 0; i < proc.n_files; i++)
{
struct comb_file *file = &proc.files[i];
size_t j;
for (j = 0; j < subcase_get_n_fields (&proc.by_vars); j++)
{
const char *name = var_get_name (by_vars[j]);
struct variable *var = dict_lookup_var (file->dict, name);
if (var != NULL)
subcase_add_var (&file->by_vars, var,
subcase_get_direction (&proc.by_vars, j));
else
{
if (file->handle != NULL)
msg (SE, _("File %s lacks BY variable %s."),
fh_get_name (file->handle), name);
else
msg (SE, _("Active dataset lacks BY variable %s."),
name);
ok = false;
}
}
assert (!ok || subcase_conformable (&file->by_vars,
&proc.files[0].by_vars));
}
free (by_vars);
if (!ok)
goto error;
}
else if (command != COMB_UPDATE && lex_match_id (lexer, "FIRST"))
{
if (first_name != NULL)
{
lex_sbc_only_once ("FIRST");
goto error;
}
lex_match (lexer, T_EQUALS);
if (!lex_force_id (lexer))
goto error;
first_name = xstrdup (lex_tokcstr (lexer));
lex_get (lexer);
}
else if (command != COMB_UPDATE && lex_match_id (lexer, "LAST"))
{
if (last_name != NULL)
{
lex_sbc_only_once ("LAST");
goto error;
}
lex_match (lexer, T_EQUALS);
if (!lex_force_id (lexer))
goto error;
last_name = xstrdup (lex_tokcstr (lexer));
lex_get (lexer);
}
else if (lex_match_id (lexer, "MAP"))
{
/* FIXME. */
}
else if (lex_match_id (lexer, "DROP"))
{
if (!parse_dict_drop (lexer, proc.dict))
goto error;
}
else if (lex_match_id (lexer, "KEEP"))
{
if (!parse_dict_keep (lexer, proc.dict))
goto error;
}
else
{
lex_error (lexer, NULL);
goto error;
}
if (!lex_match (lexer, T_SLASH) && lex_token (lexer) != T_ENDCMD)
{
lex_end_of_command (lexer);
goto error;
}
}
if (!saw_by)
{
if (command == COMB_UPDATE)
{
lex_sbc_missing ("BY");
goto error;
}
if (n_tables)
{
msg (SE, _("BY is required when %s is specified."), "TABLE");
goto error;
}
if (saw_sort)
{
msg (SE, _("BY is required when %s is specified."), "SORT");
goto error;
}
}
/* Add IN, FIRST, and LAST variables to master dictionary. */
for (i = 0; i < proc.n_files; i++)
{
struct comb_file *file = &proc.files[i];
if (!create_flag_var ("IN", file->in_name, proc.dict, &file->in_var))
goto error;
}
if (!create_flag_var ("FIRST", first_name, proc.dict, &proc.first)
|| !create_flag_var ("LAST", last_name, proc.dict, &proc.last))
goto error;
dict_delete_scratch_vars (proc.dict);
dict_compact_values (proc.dict);
/* Set up mapping from each file's variables to master
variables. */
for (i = 0; i < proc.n_files; i++)
{
struct comb_file *file = &proc.files[i];
size_t src_var_cnt = dict_get_var_cnt (file->dict);
size_t j;
file->mv = xnmalloc (src_var_cnt, sizeof *file->mv);
for (j = 0; j < src_var_cnt; j++)
{
struct variable *src_var = dict_get_var (file->dict, j);
struct variable *dst_var = dict_lookup_var (proc.dict,
var_get_name (src_var));
if (dst_var != NULL)
{
size_t n = subcase_get_n_fields (&file->src);
file->mv[n] = var_get_missing_values (src_var);
subcase_add_var (&file->src, src_var, SC_ASCEND);
subcase_add_var (&file->dst, dst_var, SC_ASCEND);
}
}
}
proc.output = autopaging_writer_create (dict_get_proto (proc.dict));
taint = taint_clone (casewriter_get_taint (proc.output));
/* Set up case matcher. */
proc.matcher = case_matcher_create ();
for (i = 0; i < proc.n_files; i++)
{
struct comb_file *file = &proc.files[i];
if (file->reader == NULL)
{
if (active_file == NULL)
{
proc_discard_output (ds);
file->reader = active_file = proc_open_filtering (ds, false);
}
else
file->reader = casereader_clone (active_file);
}
if (!file->is_sorted)
file->reader = sort_execute (file->reader, &file->by_vars);
taint_propagate (casereader_get_taint (file->reader), taint);
file->data = casereader_read (file->reader);
if (file->type == COMB_FILE)
case_matcher_add_input (proc.matcher, &file->by_vars,
&file->data, &file->is_minimal);
}
if (command == COMB_ADD)
execute_add_files (&proc);
else if (command == COMB_MATCH)
execute_match_files (&proc);
else if (command == COMB_UPDATE)
execute_update (&proc);
else
NOT_REACHED ();
case_matcher_destroy (proc.matcher);
proc.matcher = NULL;
close_all_comb_files (&proc);
if (active_file != NULL)
proc_commit (ds);
dataset_set_dict (ds, proc.dict);
dataset_set_source (ds, casewriter_make_reader (proc.output));
proc.dict = NULL;
proc.output = NULL;
free_comb_proc (&proc);
free (first_name);
free (last_name);
return taint_destroy (taint) ? CMD_SUCCESS : CMD_CASCADING_FAILURE;
error:
if (active_file != NULL)
proc_commit (ds);
free_comb_proc (&proc);
taint_destroy (taint);
free (first_name);
free (last_name);
return CMD_CASCADING_FAILURE;
}
static int lex_scan_number(lex_t *lex, char c, json_error_t *error)
{
const char *saved_text;
char *end;
double value;
lex->token = TOKEN_INVALID;
if(c == '-')
c = lex_get_save(lex, error);
if(c == '0') {
c = lex_get_save(lex, error);
if(isdigit(c)) {
lex_unget_unsave(lex, c);
goto out;
}
}
else if(isdigit(c)) {
c = lex_get_save(lex, error);
while(isdigit(c))
c = lex_get_save(lex, error);
}
else {
lex_unget_unsave(lex, c);
goto out;
}
if(c != '.' && c != 'E' && c != 'e') {
long value;
lex_unget_unsave(lex, c);
saved_text = strbuffer_value(&lex->saved_text);
value = strtol(saved_text, &end, 10);
assert(end == saved_text + lex->saved_text.length);
if((value == LONG_MAX && errno == ERANGE) || value > INT_MAX)
{
unsigned long uvalue = strtoul(saved_text, &end, 10);
if( !(uvalue>value) )
{
error_set(error, lex, "too big integer");
goto out;
}
else
{
value = uvalue;
}
}
else if((value == LONG_MIN && errno == ERANGE) || value < INT_MIN) {
error_set(error, lex, "too big negative integer");
goto out;
}
lex->token = TOKEN_INTEGER;
lex->value.integer = (int)value;
return 0;
}
if(c == '.') {
c = lex_get(lex, error);
if(!isdigit(c))
goto out;
lex_save(lex, c);
c = lex_get_save(lex, error);
while(isdigit(c))
c = lex_get_save(lex, error);
}
if(c == 'E' || c == 'e') {
c = lex_get_save(lex, error);
if(c == '+' || c == '-')
c = lex_get_save(lex, error);
if(!isdigit(c)) {
lex_unget_unsave(lex, c);
goto out;
}
c = lex_get_save(lex, error);
while(isdigit(c))
c = lex_get_save(lex, error);
}
lex_unget_unsave(lex, c);
saved_text = strbuffer_value(&lex->saved_text);
value = strtod(saved_text, &end);
assert(end == saved_text + lex->saved_text.length);
if(errno == ERANGE && value != 0) {
error_set(error, lex, "real number overflow");
goto out;
}
lex->token = TOKEN_REAL;
lex->value.real = value;
return 0;
out:
return -1;
}
/* Parse all the aggregate functions. */
static bool
parse_aggregate_functions (struct lexer *lexer, const struct dictionary *dict,
struct agr_proc *agr)
{
struct agr_var *tail; /* Tail of linked list starting at agr->vars. */
/* Parse everything. */
tail = NULL;
for (;;)
{
char **dest;
char **dest_label;
size_t n_dest;
struct string function_name;
enum mv_class exclude;
const struct agr_func *function;
int func_index;
union agr_argument arg[2];
const struct variable **src;
size_t n_src;
size_t i;
dest = NULL;
dest_label = NULL;
n_dest = 0;
src = NULL;
function = NULL;
n_src = 0;
arg[0].c = NULL;
arg[1].c = NULL;
ds_init_empty (&function_name);
/* Parse the list of target variables. */
while (!lex_match (lexer, T_EQUALS))
{
size_t n_dest_prev = n_dest;
if (!parse_DATA_LIST_vars (lexer, dict, &dest, &n_dest,
(PV_APPEND | PV_SINGLE | PV_NO_SCRATCH
| PV_NO_DUPLICATE)))
goto error;
/* Assign empty labels. */
{
int j;
dest_label = xnrealloc (dest_label, n_dest, sizeof *dest_label);
for (j = n_dest_prev; j < n_dest; j++)
dest_label[j] = NULL;
}
if (lex_is_string (lexer))
{
dest_label[n_dest - 1] = xstrdup (lex_tokcstr (lexer));
lex_get (lexer);
}
}
/* Get the name of the aggregation function. */
if (lex_token (lexer) != T_ID)
{
lex_error (lexer, _("expecting aggregation function"));
goto error;
}
ds_assign_substring (&function_name, lex_tokss (lexer));
exclude = ds_chomp_byte (&function_name, '.') ? MV_SYSTEM : MV_ANY;
for (function = agr_func_tab; function->name; function++)
if (!c_strcasecmp (function->name, ds_cstr (&function_name)))
break;
if (NULL == function->name)
{
msg (SE, _("Unknown aggregation function %s."),
ds_cstr (&function_name));
goto error;
}
ds_destroy (&function_name);
func_index = function - agr_func_tab;
lex_get (lexer);
/* Check for leading lparen. */
if (!lex_match (lexer, T_LPAREN))
{
if (function->src_vars == AGR_SV_YES)
{
lex_force_match (lexer, T_LPAREN);
goto error;
}
}
else
{
/* Parse list of source variables. */
{
int pv_opts = PV_NO_SCRATCH;
if (func_index == SUM || func_index == MEAN || func_index == SD)
pv_opts |= PV_NUMERIC;
else if (function->n_args)
pv_opts |= PV_SAME_TYPE;
if (!parse_variables_const (lexer, dict, &src, &n_src, pv_opts))
goto error;
}
/* Parse function arguments, for those functions that
require arguments. */
if (function->n_args != 0)
for (i = 0; i < function->n_args; i++)
{
int type;
lex_match (lexer, T_COMMA);
if (lex_is_string (lexer))
{
arg[i].c = recode_string (dict_get_encoding (agr->dict),
"UTF-8", lex_tokcstr (lexer),
-1);
type = VAL_STRING;
}
else if (lex_is_number (lexer))
{
arg[i].f = lex_tokval (lexer);
type = VAL_NUMERIC;
}
else
{
msg (SE, _("Missing argument %zu to %s."),
i + 1, function->name);
goto error;
}
lex_get (lexer);
if (type != var_get_type (src[0]))
{
msg (SE, _("Arguments to %s must be of same type as "
"source variables."),
function->name);
goto error;
}
}
/* Trailing rparen. */
if (!lex_force_match (lexer, T_RPAREN))
goto error;
/* Now check that the number of source variables match
the number of target variables. If we check earlier
than this, the user can get very misleading error
message, i.e. `AGGREGATE x=SUM(y t).' will get this
error message when a proper message would be more
like `unknown variable t'. */
if (n_src != n_dest)
{
msg (SE, _("Number of source variables (%zu) does not match "
"number of target variables (%zu)."),
n_src, n_dest);
goto error;
}
if ((func_index == PIN || func_index == POUT
|| func_index == FIN || func_index == FOUT)
&& (var_is_numeric (src[0])
? arg[0].f > arg[1].f
: str_compare_rpad (arg[0].c, arg[1].c) > 0))
{
union agr_argument t = arg[0];
arg[0] = arg[1];
arg[1] = t;
msg (SW, _("The value arguments passed to the %s function "
"are out-of-order. They will be treated as if "
"they had been specified in the correct order."),
function->name);
}
}
/* Finally add these to the linked list of aggregation
variables. */
for (i = 0; i < n_dest; i++)
{
struct agr_var *v = xzalloc (sizeof *v);
/* Add variable to chain. */
if (agr->agr_vars != NULL)
tail->next = v;
else
agr->agr_vars = v;
tail = v;
tail->next = NULL;
v->moments = NULL;
/* Create the target variable in the aggregate
dictionary. */
{
struct variable *destvar;
v->function = func_index;
if (src)
{
v->src = src[i];
if (var_is_alpha (src[i]))
{
v->function |= FSTRING;
v->string = xmalloc (var_get_width (src[i]));
}
if (function->alpha_type == VAL_STRING)
destvar = dict_clone_var_as (agr->dict, v->src, dest[i]);
else
{
assert (var_is_numeric (v->src)
|| function->alpha_type == VAL_NUMERIC);
destvar = dict_create_var (agr->dict, dest[i], 0);
if (destvar != NULL)
{
struct fmt_spec f;
if ((func_index == N || func_index == NMISS)
&& dict_get_weight (dict) != NULL)
f = fmt_for_output (FMT_F, 8, 2);
else
f = function->format;
var_set_both_formats (destvar, &f);
}
}
} else {
struct fmt_spec f;
v->src = NULL;
destvar = dict_create_var (agr->dict, dest[i], 0);
if (destvar != NULL)
{
if ((func_index == N || func_index == NMISS)
&& dict_get_weight (dict) != NULL)
f = fmt_for_output (FMT_F, 8, 2);
else
f = function->format;
var_set_both_formats (destvar, &f);
}
}
if (!destvar)
{
msg (SE, _("Variable name %s is not unique within the "
"aggregate file dictionary, which contains "
"the aggregate variables and the break "
"variables."),
dest[i]);
goto error;
}
free (dest[i]);
if (dest_label[i])
var_set_label (destvar, dest_label[i]);
v->dest = destvar;
}
v->exclude = exclude;
if (v->src != NULL)
{
int j;
if (var_is_numeric (v->src))
for (j = 0; j < function->n_args; j++)
v->arg[j].f = arg[j].f;
else
for (j = 0; j < function->n_args; j++)
v->arg[j].c = xstrdup (arg[j].c);
}
}
if (src != NULL && var_is_alpha (src[0]))
for (i = 0; i < function->n_args; i++)
{
free (arg[i].c);
arg[i].c = NULL;
}
free (src);
free (dest);
free (dest_label);
if (!lex_match (lexer, T_SLASH))
{
if (lex_token (lexer) == T_ENDCMD)
return true;
lex_error (lexer, "expecting end of command");
return false;
}
continue;
error:
ds_destroy (&function_name);
for (i = 0; i < n_dest; i++)
{
free (dest[i]);
free (dest_label[i]);
}
free (dest);
free (dest_label);
free (arg[0].c);
free (arg[1].c);
if (src && n_src && var_is_alpha (src[0]))
for (i = 0; i < function->n_args; i++)
{
free (arg[i].c);
arg[i].c = NULL;
}
free (src);
return false;
}
}
static int lex_scan_number(lex_t *lex, int c, json_error_t *error)
{
const char *saved_text;
char *end;
double value;
lex->token = TOKEN_INVALID;
if(c == '-')
c = lex_get_save(lex, error);
if(c == '0') {
c = lex_get_save(lex, error);
if(l_isdigit(c)) {
lex_unget_unsave(lex, c);
goto out;
}
}
else if(l_isdigit(c)) {
c = lex_get_save(lex, error);
while(l_isdigit(c))
c = lex_get_save(lex, error);
}
else {
lex_unget_unsave(lex, c);
goto out;
}
if(c != '.' && c != 'E' && c != 'e') {
json_int_t value;
lex_unget_unsave(lex, c);
saved_text = strbuffer_value(&lex->saved_text);
errno = 0;
value = json_strtoint(saved_text, &end, 10);
if(errno == ERANGE) {
if(value < 0)
error_set(error, lex, "too big negative integer");
else
error_set(error, lex, "too big integer");
goto out;
}
assert(end == saved_text + lex->saved_text.length);
lex->token = TOKEN_INTEGER;
lex->value.integer = value;
return 0;
}
if(c == '.') {
c = lex_get(lex, error);
if(!l_isdigit(c)) {
lex_unget(lex, c);
goto out;
}
lex_save(lex, c);
c = lex_get_save(lex, error);
while(l_isdigit(c))
c = lex_get_save(lex, error);
}
if(c == 'E' || c == 'e') {
c = lex_get_save(lex, error);
if(c == '+' || c == '-')
c = lex_get_save(lex, error);
if(!l_isdigit(c)) {
lex_unget_unsave(lex, c);
goto out;
}
c = lex_get_save(lex, error);
while(l_isdigit(c))
c = lex_get_save(lex, error);
}
lex_unget_unsave(lex, c);
if(jsonp_strtod(&lex->saved_text, &value)) {
error_set(error, lex, "real number overflow");
goto out;
}
lex->token = TOKEN_REAL;
lex->value.real = value;
return 0;
out:
return -1;
}
/* Parses the whole DO REPEAT command specification.
Returns success. */
static bool
parse_specification (struct lexer *lexer, struct dictionary *dict,
struct hmap *dummies)
{
struct dummy_var *first_dv = NULL;
hmap_init (dummies);
do
{
struct dummy_var *dv;
const char *name;
bool ok;
/* Get a stand-in variable name and make sure it's unique. */
if (!lex_force_id (lexer))
goto error;
name = lex_tokcstr (lexer);
if (dict_lookup_var (dict, name))
msg (SW, _("Dummy variable name `%s' hides dictionary variable `%s'."),
name, name);
if (find_dummy_var (dummies, name, strlen (name)))
{
msg (SE, _("Dummy variable name `%s' is given twice."), name);
goto error;
}
/* Make a new macro. */
dv = xmalloc (sizeof *dv);
dv->name = xstrdup (name);
dv->values = NULL;
dv->n_values = 0;
hmap_insert (dummies, &dv->hmap_node, hash_dummy (name, strlen (name)));
/* Skip equals sign. */
lex_get (lexer);
if (!lex_force_match (lexer, T_EQUALS))
goto error;
/* Get the details of the variable's possible values. */
if (lex_token (lexer) == T_ID || lex_token (lexer) == T_ALL)
ok = parse_ids (lexer, dict, dv);
else if (lex_is_number (lexer))
ok = parse_numbers (lexer, dv);
else if (lex_is_string (lexer))
ok = parse_strings (lexer, dv);
else
{
lex_error (lexer, NULL);
goto error;
}
if (!ok)
goto error;
assert (dv->n_values > 0);
if (lex_token (lexer) != T_SLASH && lex_token (lexer) != T_ENDCMD)
{
lex_error (lexer, NULL);
goto error;
}
/* If this is the first variable then it defines how many replacements
there must be; otherwise enforce this number of replacements. */
if (first_dv == NULL)
first_dv = dv;
else if (first_dv->n_values != dv->n_values)
{
msg (SE, _("Dummy variable `%s' had %zu substitutions, so `%s' must "
"also, but %zu were specified."),
first_dv->name, first_dv->n_values,
dv->name, dv->n_values);
goto error;
}
lex_match (lexer, T_SLASH);
}
while (!lex_match (lexer, T_ENDCMD));
while (lex_match (lexer, T_ENDCMD))
continue;
return true;
error:
destroy_dummies (dummies);
return false;
}
static int lex_scan(lex_t *lex, json_error_t *error)
{
int c;
strbuffer_clear(&lex->saved_text);
if(lex->token == TOKEN_STRING) {
jsonp_free(lex->value.string);
lex->value.string = NULL;
}
c = lex_get(lex, error);
while(c == ' ' || c == '\t' || c == '\n' || c == '\r')
c = lex_get(lex, error);
if(c == STREAM_STATE_EOF) {
lex->token = TOKEN_EOF;
goto out;
}
if(c == STREAM_STATE_ERROR) {
lex->token = TOKEN_INVALID;
goto out;
}
lex_save(lex, c);
if(c == '{' || c == '}' || c == '[' || c == ']' || c == ':' || c == ',')
lex->token = c;
else if(c == '"')
lex_scan_string(lex, error);
else if(l_isdigit(c) || c == '-') {
if(lex_scan_number(lex, c, error))
goto out;
}
else if(l_isalpha(c)) {
/* eat up the whole identifier for clearer error messages */
const char *saved_text;
c = lex_get_save(lex, error);
while(l_isalpha(c))
c = lex_get_save(lex, error);
lex_unget_unsave(lex, c);
saved_text = strbuffer_value(&lex->saved_text);
if(strcmp(saved_text, "true") == 0)
lex->token = TOKEN_TRUE;
else if(strcmp(saved_text, "false") == 0)
lex->token = TOKEN_FALSE;
else if(strcmp(saved_text, "null") == 0)
lex->token = TOKEN_NULL;
else
lex->token = TOKEN_INVALID;
}
else {
/* save the rest of the input UTF-8 sequence to get an error
message of valid UTF-8 */
lex_save_cached(lex);
lex->token = TOKEN_INVALID;
}
out:
return lex->token;
}
/* Parses SAVE or XSAVE or EXPORT or XEXPORT command.
WRITER_TYPE identifies the type of file to write,
and COMMAND_TYPE identifies the type of command.
On success, returns a writer.
For procedures only, sets *RETAIN_UNSELECTED to true if cases
that would otherwise be excluded by FILTER or USE should be
included.
On failure, returns a null pointer. */
static struct casewriter *
parse_write_command (struct lexer *lexer, struct dataset *ds,
enum writer_type writer_type,
enum command_type command_type,
bool *retain_unselected)
{
/* Common data. */
struct file_handle *handle; /* Output file. */
struct dictionary *dict; /* Dictionary for output file. */
struct casewriter *writer; /* Writer. */
struct case_map *map; /* Map from input data to data for writer. */
/* Common options. */
bool print_map; /* Print map? TODO. */
bool print_short_names; /* Print long-to-short name map. TODO. */
struct sfm_write_options sysfile_opts;
struct pfm_write_options porfile_opts;
assert (writer_type == SYSFILE_WRITER || writer_type == PORFILE_WRITER);
assert (command_type == XFORM_CMD || command_type == PROC_CMD);
assert ((retain_unselected != NULL) == (command_type == PROC_CMD));
if (command_type == PROC_CMD)
*retain_unselected = true;
handle = NULL;
dict = dict_clone (dataset_dict (ds));
writer = NULL;
map = NULL;
print_map = false;
print_short_names = false;
sysfile_opts = sfm_writer_default_options ();
porfile_opts = pfm_writer_default_options ();
case_map_prepare_dict (dict);
dict_delete_scratch_vars (dict);
lex_match (lexer, T_SLASH);
for (;;)
{
if (lex_match_id (lexer, "OUTFILE"))
{
if (handle != NULL)
{
lex_sbc_only_once ("OUTFILE");
goto error;
}
lex_match (lexer, T_EQUALS);
handle = fh_parse (lexer, FH_REF_FILE, NULL);
if (handle == NULL)
goto error;
}
else if (lex_match_id (lexer, "NAMES"))
print_short_names = true;
else if (lex_match_id (lexer, "PERMISSIONS"))
{
bool cw;
lex_match (lexer, T_EQUALS);
if (lex_match_id (lexer, "READONLY"))
cw = false;
else if (lex_match_id (lexer, "WRITEABLE"))
cw = true;
else
{
lex_error_expecting (lexer, "READONLY", "WRITEABLE",
NULL_SENTINEL);
goto error;
}
sysfile_opts.create_writeable = porfile_opts.create_writeable = cw;
}
else if (command_type == PROC_CMD && lex_match_id (lexer, "UNSELECTED"))
{
lex_match (lexer, T_EQUALS);
if (lex_match_id (lexer, "RETAIN"))
*retain_unselected = true;
else if (lex_match_id (lexer, "DELETE"))
*retain_unselected = false;
else
{
lex_error_expecting (lexer, "RETAIN", "DELETE", NULL_SENTINEL);
goto error;
}
}
else if (writer_type == SYSFILE_WRITER
&& lex_match_id (lexer, "COMPRESSED"))
sysfile_opts.compress = true;
else if (writer_type == SYSFILE_WRITER
&& lex_match_id (lexer, "UNCOMPRESSED"))
sysfile_opts.compress = false;
else if (writer_type == SYSFILE_WRITER
&& lex_match_id (lexer, "VERSION"))
{
lex_match (lexer, T_EQUALS);
if (!lex_force_int (lexer))
goto error;
sysfile_opts.version = lex_integer (lexer);
lex_get (lexer);
}
else if (writer_type == PORFILE_WRITER && lex_match_id (lexer, "TYPE"))
{
lex_match (lexer, T_EQUALS);
if (lex_match_id (lexer, "COMMUNICATIONS"))
porfile_opts.type = PFM_COMM;
else if (lex_match_id (lexer, "TAPE"))
porfile_opts.type = PFM_TAPE;
else
{
lex_error_expecting (lexer, "COMM", "TAPE", NULL_SENTINEL);
goto error;
}
}
else if (writer_type == PORFILE_WRITER && lex_match_id (lexer, "DIGITS"))
{
lex_match (lexer, T_EQUALS);
if (!lex_force_int (lexer))
goto error;
porfile_opts.digits = lex_integer (lexer);
lex_get (lexer);
}
else if (!parse_dict_trim (lexer, dict))
goto error;
if (!lex_match (lexer, T_SLASH))
break;
}
if (lex_end_of_command (lexer) != CMD_SUCCESS)
goto error;
if (handle == NULL)
{
lex_sbc_missing ("OUTFILE");
goto error;
}
dict_delete_scratch_vars (dict);
dict_compact_values (dict);
if (fh_get_referent (handle) == FH_REF_FILE)
{
switch (writer_type)
{
case SYSFILE_WRITER:
writer = sfm_open_writer (handle, dict, sysfile_opts);
break;
case PORFILE_WRITER:
writer = pfm_open_writer (handle, dict, porfile_opts);
break;
}
}
else
writer = any_writer_open (handle, dict);
if (writer == NULL)
goto error;
map = case_map_from_dict (dict);
if (map != NULL)
writer = case_map_create_output_translator (map, writer);
dict_destroy (dict);
fh_unref (handle);
return writer;
error:
fh_unref (handle);
casewriter_destroy (writer);
dict_destroy (dict);
case_map_destroy (map);
return NULL;
}
