/*
Update IA according to the contents of DICT and CREADER.
CREADER will be destroyed by this function.
*/
void
update_assistant (struct import_assistant *ia)
{
struct sheet_spec_page *ssp = ia->sheet_spec;
int rows = 0;
if (ssp->dict)
{
struct ccase *c;
int col;
ia->column_cnt = dict_get_var_cnt (ssp->dict);
ia->columns = xcalloc (ia->column_cnt, sizeof (*ia->columns));
for (col = 0; col < ia->column_cnt ; ++col)
{
const struct variable *var = dict_get_var (ssp->dict, col);
ia->columns[col].name = xstrdup (var_get_name (var));
ia->columns[col].contents = NULL;
}
for (; (c = casereader_read (ssp->reader)) != NULL; case_unref (c))
{
rows++;
for (col = 0; col < ia->column_cnt ; ++col)
{
char *ss;
const struct variable *var = dict_get_var (ssp->dict, col);
ia->columns[col].contents = xrealloc (ia->columns[col].contents,
sizeof (struct substring) * rows);
ss = data_out (case_data (c, var), dict_get_encoding (ssp->dict),
var_get_print_format (var));
ia->columns[col].contents[rows - 1] = ss_cstr (ss);
}
if (rows > MAX_PREVIEW_LINES)
{
case_unref (c);
break;
}
}
}
ia->file.line_cnt = rows;
}
static void
write_var_names (struct csv_writer *w, const struct dictionary *d)
{
size_t i;
for (i = 0; i < w->n_csv_vars; i++)
{
if (i > 0)
putc (w->opts.delimiter, w->file);
csv_output_string (w, var_get_name (dict_get_var (d, i)));
}
putc ('\n', w->file);
}
static GString *
clip_to_html (void)
{
casenumber r;
GString *string;
const size_t val_cnt = caseproto_get_n_widths (casereader_get_proto (clip_datasheet));
const casenumber case_cnt = casereader_get_case_cnt (clip_datasheet);
const size_t var_cnt = dict_get_var_cnt (clip_dict);
/* Guestimate the size needed */
string = g_string_sized_new (80 + 20 * val_cnt * case_cnt);
g_string_append (string,
"<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\">\n");
g_string_append (string, "<table>\n");
for (r = 0 ; r < case_cnt ; ++r )
{
int c;
struct ccase *cc = casereader_peek (clip_datasheet, r);
if (cc == NULL)
{
g_warning ("Clipboard seems to have inexplicably shrunk");
break;
}
g_string_append (string, "<tr>\n");
for (c = 0 ; c < var_cnt ; ++c)
{
const struct variable *v = dict_get_var (clip_dict, c);
g_string_append (string, "<td>");
data_out_g_string (string, v, cc);
g_string_append (string, "</td>\n");
}
g_string_append (string, "</tr>\n");
case_unref (cc);
}
g_string_append (string, "</table>\n");
return string;
}
/* Creates and returns an output case for PROC, initializing each
of its values to system-missing or blanks, except that the
values of IN variables are set to 0. */
static struct ccase *
create_output_case (const struct comb_proc *proc)
{
size_t n_vars = dict_get_var_cnt (proc->dict);
struct ccase *output;
size_t i;
output = case_create (dict_get_proto (proc->dict));
for (i = 0; i < n_vars; i++)
{
struct variable *v = dict_get_var (proc->dict, i);
value_set_missing (case_data_rw (output, v), var_get_width (v));
}
for (i = 0; i < proc->n_files; i++)
{
struct comb_file *file = &proc->files[i];
if (file->in_var != NULL)
case_data_rw (output, file->in_var)->f = false;
}
return output;
}
static GString *
clip_to_text (void)
{
casenumber r;
GString *string;
const size_t val_cnt = caseproto_get_n_widths (casereader_get_proto (clip_datasheet));
const casenumber case_cnt = casereader_get_case_cnt (clip_datasheet);
const size_t var_cnt = dict_get_var_cnt (clip_dict);
string = g_string_sized_new (10 * val_cnt * case_cnt);
for (r = 0 ; r < case_cnt ; ++r )
{
int c;
struct ccase *cc;
cc = casereader_peek (clip_datasheet, r);
if (cc == NULL)
{
g_warning ("Clipboard seems to have inexplicably shrunk");
break;
}
for (c = 0 ; c < var_cnt ; ++c)
{
const struct variable *v = dict_get_var (clip_dict, c);
data_out_g_string (string, v, cc);
if ( c < val_cnt - 1 )
g_string_append (string, "\t");
}
if ( r < case_cnt)
g_string_append (string, "\n");
case_unref (cc);
}
return string;
}
/* Set the clip according to the currently
selected range in the data sheet */
static void
data_sheet_set_clip (PsppireSheet *sheet)
{
int i;
struct casewriter *writer ;
PsppireSheetRange range;
PsppireDataStore *ds;
struct case_map *map = NULL;
casenumber max_rows;
size_t max_columns;
gint row0, rowi;
gint col0, coli;
ds = PSPPIRE_DATA_STORE (psppire_sheet_get_model (sheet));
psppire_sheet_get_selected_range (sheet, &range);
col0 = MIN (range.col0, range.coli);
coli = MAX (range.col0, range.coli);
row0 = MIN (range.row0, range.rowi);
rowi = MAX (range.row0, range.rowi);
/* If nothing selected, then use active cell */
if ( row0 < 0 || col0 < 0 )
{
gint row, col;
psppire_sheet_get_active_cell (sheet, &row, &col);
row0 = rowi = row;
col0 = coli = col;
}
/* The sheet range can include cells that do not include data.
Exclude them from the range. */
max_rows = psppire_data_store_get_case_count (ds);
if (rowi >= max_rows)
{
if (max_rows == 0)
return;
rowi = max_rows - 1;
}
max_columns = dict_get_var_cnt (ds->dict->dict);
if (coli >= max_columns)
{
if (max_columns == 0)
return;
coli = max_columns - 1;
}
/* Destroy any existing clip */
if ( clip_datasheet )
{
casereader_destroy (clip_datasheet);
clip_datasheet = NULL;
}
if ( clip_dict )
{
dict_destroy (clip_dict);
clip_dict = NULL;
}
/* Construct clip dictionary. */
clip_dict = dict_create (dict_get_encoding (ds->dict->dict));
for (i = col0; i <= coli; i++)
dict_clone_var_assert (clip_dict, dict_get_var (ds->dict->dict, i));
/* Construct clip data. */
map = case_map_by_name (ds->dict->dict, clip_dict);
writer = autopaging_writer_create (dict_get_proto (clip_dict));
for (i = row0; i <= rowi ; ++i )
{
struct ccase *old = psppire_data_store_get_case (ds, i);
if (old != NULL)
casewriter_write (writer, case_map_execute (map, old));
else
casewriter_force_error (writer);
}
case_map_destroy (map);
clip_datasheet = casewriter_make_reader (writer);
data_sheet_update_clipboard (sheet);
}
/* Merge the dictionary for file F into master dictionary M. */
static bool
merge_dictionary (struct dictionary *const m, struct comb_file *f)
{
struct dictionary *d = f->dict;
const struct string_array *d_docs, *m_docs;
int i;
if (dict_get_label (m) == NULL)
dict_set_label (m, dict_get_label (d));
d_docs = dict_get_documents (d);
m_docs = dict_get_documents (m);
/* FIXME: If the input files have different encodings, then
the result is undefined.
The correct thing to do would be to convert to an encoding
which can cope with all the input files (eg UTF-8).
*/
if ( 0 != strcmp (dict_get_encoding (f->dict), dict_get_encoding (m)))
msg (MW, _("Combining files with incompatible encodings. String data may "
"not be represented correctly."));
if (d_docs != NULL)
{
if (m_docs == NULL)
dict_set_documents (m, d_docs);
else
{
struct string_array new_docs;
size_t i;
new_docs.n = m_docs->n + d_docs->n;
new_docs.strings = xmalloc (new_docs.n * sizeof *new_docs.strings);
for (i = 0; i < m_docs->n; i++)
new_docs.strings[i] = m_docs->strings[i];
for (i = 0; i < d_docs->n; i++)
new_docs.strings[m_docs->n + i] = d_docs->strings[i];
dict_set_documents (m, &new_docs);
free (new_docs.strings);
}
}
for (i = 0; i < dict_get_var_cnt (d); i++)
{
struct variable *dv = dict_get_var (d, i);
struct variable *mv = dict_lookup_var (m, var_get_name (dv));
if (dict_class_from_id (var_get_name (dv)) == DC_SCRATCH)
continue;
if (mv != NULL)
{
if (var_get_width (mv) != var_get_width (dv))
{
const char *var_name = var_get_name (dv);
struct string s = DS_EMPTY_INITIALIZER;
const char *file_name;
file_name = f->handle ? fh_get_name (f->handle) : "*";
ds_put_format (&s,
_("Variable %s in file %s has different "
"type or width from the same variable in "
"earlier file."),
var_name, file_name);
ds_put_cstr (&s, " ");
if (var_is_numeric (dv))
ds_put_format (&s, _("In file %s, %s is numeric."),
file_name, var_name);
else
ds_put_format (&s, _("In file %s, %s is a string variable "
"with width %d."),
file_name, var_name, var_get_width (dv));
ds_put_cstr (&s, " ");
if (var_is_numeric (mv))
ds_put_format (&s, _("In an earlier file, %s was numeric."),
var_name);
else
ds_put_format (&s, _("In an earlier file, %s was a string "
"variable with width %d."),
var_name, var_get_width (mv));
msg (SE, "%s", ds_cstr (&s));
ds_destroy (&s);
return false;
}
if (var_has_value_labels (dv) && !var_has_value_labels (mv))
var_set_value_labels (mv, var_get_value_labels (dv));
if (var_has_missing_values (dv) && !var_has_missing_values (mv))
var_set_missing_values (mv, var_get_missing_values (dv));
if (var_get_label (dv) && !var_get_label (mv))
var_set_label (mv, var_get_label (dv));
}
else
mv = dict_clone_var_assert (m, dv);
}
return true;
}
static int
combine_files (enum comb_command_type command,
struct lexer *lexer, struct dataset *ds)
{
struct comb_proc proc;
bool saw_by = false;
bool saw_sort = false;
struct casereader *active_file = NULL;
char *first_name = NULL;
char *last_name = NULL;
struct taint *taint = NULL;
size_t n_tables = 0;
size_t allocated_files = 0;
size_t i;
proc.files = NULL;
proc.n_files = 0;
proc.dict = dict_create (get_default_encoding ());
proc.output = NULL;
proc.matcher = NULL;
subcase_init_empty (&proc.by_vars);
proc.first = NULL;
proc.last = NULL;
proc.buffered_case = NULL;
proc.prev_BY = NULL;
dict_set_case_limit (proc.dict, dict_get_case_limit (dataset_dict (ds)));
lex_match (lexer, T_SLASH);
for (;;)
{
struct comb_file *file;
enum comb_file_type type;
if (lex_match_id (lexer, "FILE"))
type = COMB_FILE;
else if (command == COMB_MATCH && lex_match_id (lexer, "TABLE"))
{
type = COMB_TABLE;
n_tables++;
}
else
break;
lex_match (lexer, T_EQUALS);
if (proc.n_files >= allocated_files)
proc.files = x2nrealloc (proc.files, &allocated_files,
sizeof *proc.files);
file = &proc.files[proc.n_files++];
file->type = type;
subcase_init_empty (&file->by_vars);
subcase_init_empty (&file->src);
subcase_init_empty (&file->dst);
file->mv = NULL;
file->handle = NULL;
file->dict = NULL;
file->reader = NULL;
file->data = NULL;
file->is_sorted = true;
file->in_name = NULL;
file->in_var = NULL;
if (lex_match (lexer, T_ASTERISK))
{
if (!dataset_has_source (ds))
{
msg (SE, _("Cannot specify the active dataset since none "
"has been defined."));
goto error;
}
if (proc_make_temporary_transformations_permanent (ds))
msg (SE, _("This command may not be used after TEMPORARY when "
"the active dataset is an input source. "
"Temporary transformations will be made permanent."));
file->dict = dict_clone (dataset_dict (ds));
}
else
{
file->handle = fh_parse (lexer, FH_REF_FILE, dataset_session (ds));
if (file->handle == NULL)
goto error;
file->reader = any_reader_open (file->handle, NULL, &file->dict);
if (file->reader == NULL)
goto error;
}
while (lex_match (lexer, T_SLASH))
if (lex_match_id (lexer, "RENAME"))
{
if (!parse_dict_rename (lexer, file->dict))
goto error;
}
else if (lex_match_id (lexer, "IN"))
{
lex_match (lexer, T_EQUALS);
if (lex_token (lexer) != T_ID)
{
lex_error (lexer, NULL);
goto error;
}
if (file->in_name)
{
msg (SE, _("Multiple IN subcommands for a single FILE or "
"TABLE."));
goto error;
}
file->in_name = xstrdup (lex_tokcstr (lexer));
lex_get (lexer);
}
else if (lex_match_id (lexer, "SORT"))
{
file->is_sorted = false;
saw_sort = true;
}
if (!merge_dictionary (proc.dict, file))
goto error;
}
while (lex_token (lexer) != T_ENDCMD)
{
if (lex_match (lexer, T_BY))
{
const struct variable **by_vars;
size_t i;
bool ok;
if (saw_by)
{
lex_sbc_only_once ("BY");
goto error;
}
saw_by = true;
lex_match (lexer, T_EQUALS);
if (!parse_sort_criteria (lexer, proc.dict, &proc.by_vars,
&by_vars, NULL))
goto error;
ok = true;
for (i = 0; i < proc.n_files; i++)
{
struct comb_file *file = &proc.files[i];
size_t j;
for (j = 0; j < subcase_get_n_fields (&proc.by_vars); j++)
{
const char *name = var_get_name (by_vars[j]);
struct variable *var = dict_lookup_var (file->dict, name);
if (var != NULL)
subcase_add_var (&file->by_vars, var,
subcase_get_direction (&proc.by_vars, j));
else
{
if (file->handle != NULL)
msg (SE, _("File %s lacks BY variable %s."),
fh_get_name (file->handle), name);
else
msg (SE, _("Active dataset lacks BY variable %s."),
name);
ok = false;
}
}
assert (!ok || subcase_conformable (&file->by_vars,
&proc.files[0].by_vars));
}
free (by_vars);
if (!ok)
goto error;
}
else if (command != COMB_UPDATE && lex_match_id (lexer, "FIRST"))
{
if (first_name != NULL)
{
lex_sbc_only_once ("FIRST");
goto error;
}
lex_match (lexer, T_EQUALS);
if (!lex_force_id (lexer))
goto error;
first_name = xstrdup (lex_tokcstr (lexer));
lex_get (lexer);
}
else if (command != COMB_UPDATE && lex_match_id (lexer, "LAST"))
{
if (last_name != NULL)
{
lex_sbc_only_once ("LAST");
goto error;
}
lex_match (lexer, T_EQUALS);
if (!lex_force_id (lexer))
goto error;
last_name = xstrdup (lex_tokcstr (lexer));
lex_get (lexer);
}
else if (lex_match_id (lexer, "MAP"))
{
/* FIXME. */
}
else if (lex_match_id (lexer, "DROP"))
{
if (!parse_dict_drop (lexer, proc.dict))
goto error;
}
else if (lex_match_id (lexer, "KEEP"))
{
if (!parse_dict_keep (lexer, proc.dict))
goto error;
}
else
{
lex_error (lexer, NULL);
goto error;
}
if (!lex_match (lexer, T_SLASH) && lex_token (lexer) != T_ENDCMD)
{
lex_end_of_command (lexer);
goto error;
}
}
if (!saw_by)
{
if (command == COMB_UPDATE)
{
lex_sbc_missing ("BY");
goto error;
}
if (n_tables)
{
msg (SE, _("BY is required when %s is specified."), "TABLE");
goto error;
}
if (saw_sort)
{
msg (SE, _("BY is required when %s is specified."), "SORT");
goto error;
}
}
/* Add IN, FIRST, and LAST variables to master dictionary. */
for (i = 0; i < proc.n_files; i++)
{
struct comb_file *file = &proc.files[i];
if (!create_flag_var ("IN", file->in_name, proc.dict, &file->in_var))
goto error;
}
if (!create_flag_var ("FIRST", first_name, proc.dict, &proc.first)
|| !create_flag_var ("LAST", last_name, proc.dict, &proc.last))
goto error;
dict_delete_scratch_vars (proc.dict);
dict_compact_values (proc.dict);
/* Set up mapping from each file's variables to master
variables. */
for (i = 0; i < proc.n_files; i++)
{
struct comb_file *file = &proc.files[i];
size_t src_var_cnt = dict_get_var_cnt (file->dict);
size_t j;
file->mv = xnmalloc (src_var_cnt, sizeof *file->mv);
for (j = 0; j < src_var_cnt; j++)
{
struct variable *src_var = dict_get_var (file->dict, j);
struct variable *dst_var = dict_lookup_var (proc.dict,
var_get_name (src_var));
if (dst_var != NULL)
{
size_t n = subcase_get_n_fields (&file->src);
file->mv[n] = var_get_missing_values (src_var);
subcase_add_var (&file->src, src_var, SC_ASCEND);
subcase_add_var (&file->dst, dst_var, SC_ASCEND);
}
}
}
proc.output = autopaging_writer_create (dict_get_proto (proc.dict));
taint = taint_clone (casewriter_get_taint (proc.output));
/* Set up case matcher. */
proc.matcher = case_matcher_create ();
for (i = 0; i < proc.n_files; i++)
{
struct comb_file *file = &proc.files[i];
if (file->reader == NULL)
{
if (active_file == NULL)
{
proc_discard_output (ds);
file->reader = active_file = proc_open_filtering (ds, false);
}
else
file->reader = casereader_clone (active_file);
}
if (!file->is_sorted)
file->reader = sort_execute (file->reader, &file->by_vars);
taint_propagate (casereader_get_taint (file->reader), taint);
file->data = casereader_read (file->reader);
if (file->type == COMB_FILE)
case_matcher_add_input (proc.matcher, &file->by_vars,
&file->data, &file->is_minimal);
}
if (command == COMB_ADD)
execute_add_files (&proc);
else if (command == COMB_MATCH)
execute_match_files (&proc);
else if (command == COMB_UPDATE)
execute_update (&proc);
else
NOT_REACHED ();
case_matcher_destroy (proc.matcher);
proc.matcher = NULL;
close_all_comb_files (&proc);
if (active_file != NULL)
proc_commit (ds);
dataset_set_dict (ds, proc.dict);
dataset_set_source (ds, casewriter_make_reader (proc.output));
proc.dict = NULL;
proc.output = NULL;
free_comb_proc (&proc);
free (first_name);
free (last_name);
return taint_destroy (taint) ? CMD_SUCCESS : CMD_CASCADING_FAILURE;
error:
if (active_file != NULL)
proc_commit (ds);
free_comb_proc (&proc);
taint_destroy (taint);
free (first_name);
free (last_name);
return CMD_CASCADING_FAILURE;
}
/* Opens the CSV file designated by file handle FH for writing cases from
dictionary DICT according to the given OPTS.
No reference to D is retained, so it may be modified or
destroyed at will after this function returns. */
struct casewriter *
csv_writer_open (struct file_handle *fh, const struct dictionary *dict,
const struct csv_writer_options *opts)
{
struct csv_writer *w;
int i;
/* Create and initialize writer. */
w = xmalloc (sizeof *w);
w->fh = fh_ref (fh);
w->lock = NULL;
w->file = NULL;
w->rf = NULL;
w->opts = *opts;
w->encoding = xstrdup (dict_get_encoding (dict));
w->n_csv_vars = dict_get_var_cnt (dict);
w->csv_vars = xnmalloc (w->n_csv_vars, sizeof *w->csv_vars);
for (i = 0; i < w->n_csv_vars; i++)
{
const struct variable *var = dict_get_var (dict, i);
struct csv_var *cv = &w->csv_vars[i];
cv->width = var_get_width (var);
cv->case_index = var_get_case_index (var);
cv->format = *var_get_print_format (var);
if (opts->recode_user_missing)
mv_copy (&cv->missing, var_get_missing_values (var));
else
mv_init (&cv->missing, cv->width);
if (opts->use_value_labels)
cv->val_labs = val_labs_clone (var_get_value_labels (var));
else
cv->val_labs = NULL;
}
/* Open file handle as an exclusive writer. */
/* TRANSLATORS: this fragment will be interpolated into messages in fh_lock()
that identify types of files. */
w->lock = fh_lock (fh, FH_REF_FILE, N_("CSV file"), FH_ACC_WRITE, true);
if (w->lock == NULL)
goto error;
/* Create the file on disk. */
w->rf = replace_file_start (fh_get_file_name (fh), "w", 0666,
&w->file, NULL);
if (w->rf == NULL)
{
msg (ME, _("Error opening `%s' for writing as a system file: %s."),
fh_get_file_name (fh), strerror (errno));
goto error;
}
if (opts->include_var_names)
write_var_names (w, dict);
if (write_error (w))
goto error;
return casewriter_create (dict_get_proto (dict),
&csv_file_casewriter_class, w);
error:
close_writer (w);
return NULL;
}
