Esempio n. 1
0
/* Performs DELETE VARIABLES command. */
int
cmd_delete_variables (struct lexer *lexer, struct dataset *ds)
{
  struct variable **vars;
  size_t var_cnt;
  bool ok;

  if (proc_make_temporary_transformations_permanent (ds))
    msg (SE, _("DELETE VARIABLES may not be used after TEMPORARY.  "
               "Temporary transformations will be made permanent."));

  if (!parse_variables (lexer, dataset_dict (ds), &vars, &var_cnt, PV_NONE))
    goto error;
  if (var_cnt == dict_get_var_cnt (dataset_dict (ds)))
    {
      msg (SE, _("DELETE VARIABLES may not be used to delete all variables "
                 "from the active dataset dictionary.  "
                 "Use NEW FILE instead."));
      goto error;
    }

  ok = casereader_destroy (proc_open (ds));
  ok = proc_commit (ds) && ok;
  if (!ok)
    goto error;
  dict_delete_vars (dataset_dict (ds), vars, var_cnt);

  free (vars);

  return CMD_SUCCESS;

 error:
  free (vars);
  return CMD_CASCADING_FAILURE;
}
Esempio n. 2
0
/* Perform BEGIN DATA...END DATA as a procedure in itself. */
int
cmd_begin_data (struct lexer *lexer, struct dataset *ds)
{
  struct dfm_reader *r;
  bool ok;

  if (!fh_is_locked (fh_inline_file (), FH_ACC_READ))
    {
      msg (SE, _("This command is not valid here since the current "
                 "input program does not access the inline file."));
      return CMD_CASCADING_FAILURE;
    }
  lex_match (lexer, T_ENDCMD);

  /* Open inline file. */
  r = dfm_open_reader (fh_inline_file (), lexer);
  r->flags |= DFM_SAW_BEGIN_DATA;
  r->flags &= ~DFM_CONSUME;

  /* Input procedure reads from inline file. */
  casereader_destroy (proc_open (ds));
  ok = proc_commit (ds);
  dfm_close_reader (r);

  return ok ? CMD_SUCCESS : CMD_CASCADING_FAILURE;
}
Esempio n. 3
0
static bool
run_reliability (struct dataset *ds, const struct reliability *reliability)
{
  struct dictionary *dict = dataset_dict (ds);
  bool ok;
  struct casereader *group;

  struct casegrouper *grouper = casegrouper_create_splits (proc_open (ds), dict);


  while (casegrouper_get_next_group (grouper, &group))
    {
      do_reliability (group, ds, reliability);

      reliability_statistics (reliability);

      if (reliability->summary & SUMMARY_TOTAL )
	reliability_summary_total (reliability);
    }

  ok = casegrouper_destroy (grouper);
  ok = proc_commit (ds) && ok;

  return ok;
}
Esempio n. 4
0
/* Parses and performs the SAVE or EXPORT procedure. */
static int
parse_output_proc (struct lexer *lexer, struct dataset *ds,
                   enum writer_type writer_type)
{
  bool retain_unselected;
  struct casewriter *output;
  bool ok;

  output = parse_write_command (lexer, ds, writer_type, PROC_CMD,
                                &retain_unselected);
  if (output == NULL)
    return CMD_CASCADING_FAILURE;

  casereader_transfer (proc_open_filtering (ds, !retain_unselected), output);
  ok = casewriter_destroy (output);
  ok = proc_commit (ds) && ok;

  return ok ? CMD_SUCCESS : CMD_CASCADING_FAILURE;
}
Esempio n. 5
0
int
cmd_t_test (struct lexer *lexer, struct dataset *ds)
{
  bool ok;
  const struct dictionary *dict = dataset_dict (ds);
  struct tt tt;
  int mode_count = 0;

  /* Variables pertaining to the paired mode */
  const struct variable **v1 = NULL;
  size_t n_v1;
  const struct variable **v2 = NULL;
  size_t n_v2;
	  
  size_t n_pairs = 0;
  vp *pairs = NULL;


  /* One sample mode */
  double testval = SYSMIS;

  /* Independent samples mode */
  const struct variable *gvar;
  union value gval0;
  union value gval1;
  bool cut = false;

  tt.wv = dict_get_weight (dict);
  tt.dict = dict;
  tt.confidence = 0.95;
  tt.exclude = MV_ANY;
  tt.missing_type = MISS_ANALYSIS;
  tt.n_vars = 0;
  tt.vars = NULL;
  tt.mode = MODE_undef;

  lex_match (lexer, T_EQUALS);

  for (; lex_token (lexer) != T_ENDCMD; )
    {
      lex_match (lexer, T_SLASH);
      if (lex_match_id (lexer, "TESTVAL"))
	{
	  mode_count++;
	  tt.mode = MODE_SINGLE;
	  lex_match (lexer, T_EQUALS);
	  lex_force_num (lexer);
	  testval = lex_number (lexer);
	  lex_get (lexer);
	}
      else if (lex_match_id (lexer, "GROUPS"))
	{
	  mode_count++;
	  cut = false;
	  tt.mode = MODE_INDEP;
	  lex_match (lexer, T_EQUALS);

	  if (NULL == (gvar = parse_variable (lexer, dict)))
	    goto parse_failed;
      
	  if (lex_match (lexer, T_LPAREN))
	    {

	      value_init (&gval0, var_get_width (gvar));
	      parse_value (lexer, &gval0, gvar);
	      cut = true;
	      if (lex_match (lexer, T_COMMA))
		{
		  value_init (&gval1, var_get_width (gvar));
		  parse_value (lexer, &gval1, gvar);
		  cut = false;
		}

	      lex_force_match (lexer, T_RPAREN);
	    }
	  else
	    {
	      value_init (&gval0, 0);
	      value_init (&gval1, 0);
	      gval0.f = 1.0;
	      gval1.f = 2.0;
	      cut = false;
	    }

	  if ( cut == true && var_is_alpha (gvar))
	    {
	      msg (SE, _("When applying GROUPS to a string variable, two "
			 "values must be specified."));
	      goto parse_failed;
	    }
	}
      else if (lex_match_id (lexer, "PAIRS"))
	{
	  bool with = false;
	  bool paired = false;

	  if (tt.n_vars > 0)
	    {
	      msg (SE, _("VARIABLES subcommand may not be used with PAIRS."));
	      goto parse_failed;
	    }

	  mode_count++;
	  tt.mode = MODE_PAIRED;
	  lex_match (lexer, T_EQUALS);

	  if (!parse_variables_const (lexer, dict,
				      &v1, &n_v1,
				      PV_NO_DUPLICATE | PV_NUMERIC))
	    goto parse_failed;

	  if ( lex_match (lexer, T_WITH))
	    {
	      with = true;
	      if (!parse_variables_const (lexer, dict,
					  &v2, &n_v2,
					  PV_NO_DUPLICATE | PV_NUMERIC))
		goto parse_failed;

	      if (lex_match (lexer, T_LPAREN)
		  && lex_match_id (lexer, "PAIRED")
		  && lex_match (lexer, T_RPAREN))
		{
		  paired = true;
		  if (n_v1 != n_v2)
		    {
		      msg (SE, _("PAIRED was specified but the number of variables "
				 "preceding WITH (%zu) did not match the number "
				 "following (%zu)."),
			   n_v1, n_v2);
		      goto parse_failed;
		    }
		}
	    }
	  {
	    int i;

	    if ( !with )
	      n_pairs = (n_v1 * (n_v1 - 1)) / 2.0;
	    else if ( paired )
	      n_pairs = n_v1;
	    else
	      n_pairs = n_v1 * n_v2;
	  
	    pairs = xcalloc (n_pairs, sizeof *pairs);

	    if ( with)
	      {
		int x = 0;
		if (paired)
		  {
		    for (i = 0 ; i < n_v1; ++i)
		      {
			vp *pair = &pairs[i];
			(*pair)[0] = v1[i];
			(*pair)[1] = v2[i];
		      }	
		  }
		else
		  {
		    for (i = 0 ; i < n_v1; ++i)
		      {
			int j;
			for (j = 0 ; j < n_v2; ++j)
			  {
			    vp *pair = &pairs[x++];
			    (*pair)[0] = v1[i];
			    (*pair)[1] = v2[j];
			  }
		      }
		  }
	      }
	    else
	      {
		int x = 0;
		for (i = 0 ; i < n_v1; ++i)
		  {
		    int j;

		    for (j = i + 1 ; j < n_v1; ++j)
		      {
			vp *pair = &pairs[x++];
			(*pair)[0] = v1[i];
			(*pair)[1] = v1[j];
		      }
		  }
	      }

	  }
	}
      else if (lex_match_id (lexer, "VARIABLES"))
	{
	  if ( tt.mode == MODE_PAIRED)
	    {
	      msg (SE, _("VARIABLES subcommand may not be used with PAIRS."));
	      goto parse_failed;
	    }

	  lex_match (lexer, T_EQUALS);

	  if (!parse_variables_const (lexer, dict,
				      &tt.vars,
				      &tt.n_vars,
				      PV_NO_DUPLICATE | PV_NUMERIC))
	    goto parse_failed;
	}
      else if ( lex_match_id (lexer, "MISSING"))
	{
          lex_match (lexer, T_EQUALS);
          while (lex_token (lexer) != T_ENDCMD && lex_token (lexer) != T_SLASH)
            {
	      if (lex_match_id (lexer, "INCLUDE"))
		{
		  tt.exclude = MV_SYSTEM;
		}
	      else if (lex_match_id (lexer, "EXCLUDE"))
		{
		  tt.exclude = MV_ANY;
		}
	      else if (lex_match_id (lexer, "LISTWISE"))
		{
		  tt.missing_type = MISS_LISTWISE;
		}
	      else if (lex_match_id (lexer, "ANALYSIS"))
		{
		  tt.missing_type = MISS_ANALYSIS;
		}
	      else
		{
                  lex_error (lexer, NULL);
		  goto parse_failed;
		}
	      lex_match (lexer, T_COMMA);
	    }
	}
      else if (lex_match_id (lexer, "CRITERIA"))
	{
          lex_match (lexer, T_EQUALS);
	  if ( lex_force_match_id (lexer, "CIN"))
	    if ( lex_force_match (lexer, T_LPAREN))
	      {
		lex_force_num (lexer);
		tt.confidence = lex_number (lexer);
		lex_get (lexer);
		lex_force_match (lexer, T_RPAREN);
	      }
	}
      else 
	{
	  lex_error (lexer, NULL);
	  goto parse_failed;
	}
    }

  if ( mode_count != 1)
    {
      msg (SE, _("Exactly one of TESTVAL, GROUPS and PAIRS subcommands "
		 "must be specified."));
      goto parse_failed;
    }

  if (tt.n_vars == 0 && tt.mode != MODE_PAIRED)
    {
      lex_sbc_missing ("VARIABLES");
      goto parse_failed;
    }



  /* Deal with splits etc */
  {
    struct casereader *group;
    struct casegrouper *grouper = casegrouper_create_splits (proc_open (ds), dict);

    while (casegrouper_get_next_group (grouper, &group))
      {
	if ( tt.mode == MODE_SINGLE)
	  {
	    if ( tt.missing_type == MISS_LISTWISE )
	      group  = casereader_create_filter_missing (group,
							 tt.vars, tt.n_vars,
							 tt.exclude,
							 NULL,  NULL);
	    one_sample_run (&tt, testval, group);
	  }
	else if ( tt.mode == MODE_PAIRED)
	  {
	    if ( tt.missing_type == MISS_LISTWISE )
	      {
		group  = casereader_create_filter_missing (group,
							   v1, n_v1,
							   tt.exclude,
							   NULL,  NULL);
		group  = casereader_create_filter_missing (group,
							   v2, n_v2,
							   tt.exclude,
							   NULL,  NULL);
	      }

	    paired_run (&tt, n_pairs, pairs, group);
	  }
	else /* tt.mode == MODE_INDEP */
	  {
	    if ( tt.missing_type == MISS_LISTWISE )
	      {
		group  = casereader_create_filter_missing (group,
							   tt.vars, tt.n_vars,
							   tt.exclude,
							   NULL,  NULL);

		group  = casereader_create_filter_missing (group,
							   &gvar, 1,
							   tt.exclude,
							   NULL,  NULL);

	      }

	    indep_run (&tt, gvar, cut, &gval0, &gval1, group);
	  }
      }

    ok = casegrouper_destroy (grouper);
    ok = proc_commit (ds) && ok;
  }

  free (pairs);
  free (v1);
  free (v2);

  free (tt.vars);

  return ok ? CMD_SUCCESS : CMD_FAILURE;

 parse_failed:
  return CMD_FAILURE;
}
Esempio n. 6
0
int
cmd_save_translate (struct lexer *lexer, struct dataset *ds)
{
  enum { CSV_FILE = 1, TAB_FILE } type;

  struct dictionary *dict;
  struct case_map *map;
  struct casewriter *writer;
  struct file_handle *handle;

  struct csv_writer_options csv_opts;

  bool replace;

  bool retain_unselected;
  bool recode_user_missing;
  bool include_var_names;
  bool use_value_labels;
  bool use_print_formats;
  char decimal;
  char delimiter;
  char qualifier;

  bool ok;

  type = 0;

  dict = dict_clone (dataset_dict (ds));
  map = NULL;

  handle = NULL;
  replace = false;

  retain_unselected = true;
  recode_user_missing = false;
  include_var_names = false;
  use_value_labels = false;
  use_print_formats = false;
  decimal = settings_get_decimal_char (FMT_F);
  delimiter = 0;
  qualifier = '"';

  case_map_prepare_dict (dict);
  dict_delete_scratch_vars (dict);

  while (lex_match (lexer, T_SLASH))
    {
      if (lex_match_id (lexer, "OUTFILE"))
	{
          if (handle != NULL)
            {
              lex_sbc_only_once ("OUTFILE");
              goto error;
            }

	  lex_match (lexer, T_EQUALS);

	  handle = fh_parse (lexer, FH_REF_FILE, NULL);
	  if (handle == NULL)
	    goto error;
	}
      else if (lex_match_id (lexer, "TYPE"))
        {
          if (type != 0)
            {
              lex_sbc_only_once ("TYPE");
              goto error;
            }

          lex_match (lexer, T_EQUALS);
          if (lex_match_id (lexer, "CSV"))
            type = CSV_FILE;
          else if (lex_match_id (lexer, "TAB"))
            type = TAB_FILE;
          else
            {
              lex_error_expecting (lexer, "CSV", "TAB", NULL_SENTINEL);
              goto error;
            }
        }
      else if (lex_match_id (lexer, "REPLACE"))
        replace = true;
      else if (lex_match_id (lexer, "FIELDNAMES"))
        include_var_names = true;
      else if (lex_match_id (lexer, "MISSING"))
        {
          lex_match (lexer, T_EQUALS);
          if (lex_match_id (lexer, "IGNORE"))
            recode_user_missing = false;
          else if (lex_match_id (lexer, "RECODE"))
            recode_user_missing = true;
          else
            {
              lex_error_expecting (lexer, "IGNORE", "RECODE", NULL_SENTINEL);
              goto error;
            }
        }
      else if (lex_match_id (lexer, "CELLS"))
        {
          lex_match (lexer, T_EQUALS);
          if (lex_match_id (lexer, "VALUES"))
            use_value_labels = false;
          else if (lex_match_id (lexer, "LABELS"))
            use_value_labels = true;
          else
            {
              lex_error_expecting (lexer, "VALUES", "LABELS", NULL_SENTINEL);
              goto error;
            }
        }
      else if (lex_match_id (lexer, "TEXTOPTIONS"))
        {
          lex_match (lexer, T_EQUALS);
          for (;;)
            {
              if (lex_match_id (lexer, "DELIMITER"))
                {
                  lex_match (lexer, T_EQUALS);
                  if (!lex_force_string (lexer))
                    goto error;
                  /* XXX should support multibyte UTF-8 delimiters */
                  if (ss_length (lex_tokss (lexer)) != 1)
                    {
                      msg (SE, _("The %s string must contain exactly one "
                                 "character."), "DELIMITER");
                      goto error;
                    }
                  delimiter = ss_first (lex_tokss (lexer));
                  lex_get (lexer);
                }
              else if (lex_match_id (lexer, "QUALIFIER"))
                {
                  lex_match (lexer, T_EQUALS);
                  if (!lex_force_string (lexer))
                    goto error;
                  /* XXX should support multibyte UTF-8 qualifiers */
                  if (ss_length (lex_tokss (lexer)) != 1)
                    {
                      msg (SE, _("The %s string must contain exactly one "
                                 "character."), "QUALIFIER");
                      goto error;
                    }
                  qualifier = ss_first (lex_tokss (lexer));
                  lex_get (lexer);
                }
              else if (lex_match_id (lexer, "DECIMAL"))
                {
                  lex_match (lexer, T_EQUALS);
                  if (lex_match_id (lexer, "DOT"))
                    decimal = '.';
                  else if (lex_match_id (lexer, "COMMA"))
                    decimal = ',';
                  else
                    {
                      lex_error_expecting (lexer, "DOT", "COMMA",
                                           NULL_SENTINEL);
                      goto error;
                    }
                }
              else if (lex_match_id (lexer, "FORMAT"))
                {
                  lex_match (lexer, T_EQUALS);
                  if (lex_match_id (lexer, "PLAIN"))
                    use_print_formats = false;
                  else if (lex_match_id (lexer, "VARIABLE"))
                    use_print_formats = true;
                  else
                    {
                      lex_error_expecting (lexer, "PLAIN", "VARIABLE",
                                           NULL_SENTINEL);
                      goto error;
                    }
                }
              else
                break;
            }
        }
      else if (lex_match_id (lexer, "UNSELECTED"))
        {
          lex_match (lexer, T_EQUALS);
          if (lex_match_id (lexer, "RETAIN"))
            retain_unselected = true;
          else if (lex_match_id (lexer, "DELETE"))
            retain_unselected = false;
          else
            {
              lex_error_expecting (lexer, "RETAIN", "DELETE", NULL_SENTINEL);
              goto error;
            }
        }
      else if (!parse_dict_trim (lexer, dict))
        goto error;
    }
  if (lex_end_of_command (lexer) != CMD_SUCCESS)
    goto error;

  if (type == 0)
    {
      lex_sbc_missing ("TYPE");
      goto error;
    }
  else if (handle == NULL)
    {
      lex_sbc_missing ("OUTFILE");
      goto error;
    }
  else if (!replace && fn_exists (fh_get_file_name (handle)))
    {
      msg (SE, _("Output file `%s' exists but REPLACE was not specified."),
           fh_get_file_name (handle));
      goto error;
    }

  dict_delete_scratch_vars (dict);
  dict_compact_values (dict);

  csv_opts.recode_user_missing = recode_user_missing;
  csv_opts.include_var_names = include_var_names;
  csv_opts.use_value_labels = use_value_labels;
  csv_opts.use_print_formats = use_print_formats;
  csv_opts.decimal = decimal;
  csv_opts.delimiter = (delimiter ? delimiter
                        : type == TAB_FILE ? '\t'
                        : decimal == '.' ? ','
                        : ';');
  csv_opts.qualifier = qualifier;

  writer = csv_writer_open (handle, dict, &csv_opts);
  if (writer == NULL)
    goto error;
  fh_unref (handle);

  map = case_map_from_dict (dict);
  if (map != NULL)
    writer = case_map_create_output_translator (map, writer);
  dict_destroy (dict);

  casereader_transfer (proc_open_filtering (ds, !retain_unselected), writer);
  ok = casewriter_destroy (writer);
  ok = proc_commit (ds) && ok;

  return ok ? CMD_SUCCESS : CMD_CASCADING_FAILURE;

error:
  fh_unref (handle);
  dict_destroy (dict);
  case_map_destroy (map);
  return CMD_FAILURE;
}
Esempio n. 7
0
static int
combine_files (enum comb_command_type command,
               struct lexer *lexer, struct dataset *ds)
{
  struct comb_proc proc;

  bool saw_by = false;
  bool saw_sort = false;
  struct casereader *active_file = NULL;

  char *first_name = NULL;
  char *last_name = NULL;

  struct taint *taint = NULL;

  size_t n_tables = 0;
  size_t allocated_files = 0;

  size_t i;

  proc.files = NULL;
  proc.n_files = 0;
  proc.dict = dict_create (get_default_encoding ());
  proc.output = NULL;
  proc.matcher = NULL;
  subcase_init_empty (&proc.by_vars);
  proc.first = NULL;
  proc.last = NULL;
  proc.buffered_case = NULL;
  proc.prev_BY = NULL;

  dict_set_case_limit (proc.dict, dict_get_case_limit (dataset_dict (ds)));

  lex_match (lexer, T_SLASH);
  for (;;)
    {
      struct comb_file *file;
      enum comb_file_type type;

      if (lex_match_id (lexer, "FILE"))
        type = COMB_FILE;
      else if (command == COMB_MATCH && lex_match_id (lexer, "TABLE"))
        {
          type = COMB_TABLE;
          n_tables++;
        }
      else
        break;
      lex_match (lexer, T_EQUALS);

      if (proc.n_files >= allocated_files)
        proc.files = x2nrealloc (proc.files, &allocated_files,
                                sizeof *proc.files);
      file = &proc.files[proc.n_files++];
      file->type = type;
      subcase_init_empty (&file->by_vars);
      subcase_init_empty (&file->src);
      subcase_init_empty (&file->dst);
      file->mv = NULL;
      file->handle = NULL;
      file->dict = NULL;
      file->reader = NULL;
      file->data = NULL;
      file->is_sorted = true;
      file->in_name = NULL;
      file->in_var = NULL;

      if (lex_match (lexer, T_ASTERISK))
        {
          if (!dataset_has_source (ds))
            {
              msg (SE, _("Cannot specify the active dataset since none "
                         "has been defined."));
              goto error;
            }

          if (proc_make_temporary_transformations_permanent (ds))
            msg (SE, _("This command may not be used after TEMPORARY when "
                       "the active dataset is an input source.  "
                       "Temporary transformations will be made permanent."));

          file->dict = dict_clone (dataset_dict (ds));
        }
      else
        {
          file->handle = fh_parse (lexer, FH_REF_FILE, dataset_session (ds));
          if (file->handle == NULL)
            goto error;

          file->reader = any_reader_open (file->handle, NULL, &file->dict);
          if (file->reader == NULL)
            goto error;
        }

      while (lex_match (lexer, T_SLASH))
        if (lex_match_id (lexer, "RENAME"))
          {
            if (!parse_dict_rename (lexer, file->dict))
              goto error;
          }
        else if (lex_match_id (lexer, "IN"))
          {
            lex_match (lexer, T_EQUALS);
            if (lex_token (lexer) != T_ID)
              {
                lex_error (lexer, NULL);
                goto error;
              }

            if (file->in_name)
              {
                msg (SE, _("Multiple IN subcommands for a single FILE or "
                           "TABLE."));
                goto error;
              }
            file->in_name = xstrdup (lex_tokcstr (lexer));
            lex_get (lexer);
          }
        else if (lex_match_id (lexer, "SORT"))
          {
            file->is_sorted = false;
            saw_sort = true;
          }

      if (!merge_dictionary (proc.dict, file))
        goto error;
    }

  while (lex_token (lexer) != T_ENDCMD)
    {
      if (lex_match (lexer, T_BY))
	{
          const struct variable **by_vars;
          size_t i;
          bool ok;

	  if (saw_by)
	    {
              lex_sbc_only_once ("BY");
	      goto error;
	    }
          saw_by = true;

	  lex_match (lexer, T_EQUALS);
          if (!parse_sort_criteria (lexer, proc.dict, &proc.by_vars,
                                    &by_vars, NULL))
	    goto error;

          ok = true;
          for (i = 0; i < proc.n_files; i++)
            {
              struct comb_file *file = &proc.files[i];
              size_t j;

              for (j = 0; j < subcase_get_n_fields (&proc.by_vars); j++)
                {
                  const char *name = var_get_name (by_vars[j]);
                  struct variable *var = dict_lookup_var (file->dict, name);
                  if (var != NULL)
                    subcase_add_var (&file->by_vars, var,
                                     subcase_get_direction (&proc.by_vars, j));
                  else
                    {
                      if (file->handle != NULL)
                        msg (SE, _("File %s lacks BY variable %s."),
                             fh_get_name (file->handle), name);
                      else
                        msg (SE, _("Active dataset lacks BY variable %s."),
                             name);
                      ok = false;
                    }
                }
              assert (!ok || subcase_conformable (&file->by_vars,
                                                  &proc.files[0].by_vars));
            }
          free (by_vars);

          if (!ok)
            goto error;
	}
      else if (command != COMB_UPDATE && lex_match_id (lexer, "FIRST"))
        {
          if (first_name != NULL)
            {
              lex_sbc_only_once ("FIRST");
              goto error;
            }

	  lex_match (lexer, T_EQUALS);
          if (!lex_force_id (lexer))
            goto error;
          first_name = xstrdup (lex_tokcstr (lexer));
          lex_get (lexer);
        }
      else if (command != COMB_UPDATE && lex_match_id (lexer, "LAST"))
        {
          if (last_name != NULL)
            {
              lex_sbc_only_once ("LAST");
              goto error;
            }

	  lex_match (lexer, T_EQUALS);
          if (!lex_force_id (lexer))
            goto error;
          last_name = xstrdup (lex_tokcstr (lexer));
          lex_get (lexer);
        }
      else if (lex_match_id (lexer, "MAP"))
	{
	  /* FIXME. */
	}
      else if (lex_match_id (lexer, "DROP"))
        {
          if (!parse_dict_drop (lexer, proc.dict))
            goto error;
        }
      else if (lex_match_id (lexer, "KEEP"))
        {
          if (!parse_dict_keep (lexer, proc.dict))
            goto error;
        }
      else
	{
	  lex_error (lexer, NULL);
	  goto error;
	}

      if (!lex_match (lexer, T_SLASH) && lex_token (lexer) != T_ENDCMD)
        {
          lex_end_of_command (lexer);
          goto error;
        }
    }

  if (!saw_by)
    {
      if (command == COMB_UPDATE)
        {
          lex_sbc_missing ("BY");
          goto error;
        }
      if (n_tables)
        {
          msg (SE, _("BY is required when %s is specified."), "TABLE");
          goto error;
        }
      if (saw_sort)
        {
          msg (SE, _("BY is required when %s is specified."), "SORT");
          goto error;
        }
    }

  /* Add IN, FIRST, and LAST variables to master dictionary. */
  for (i = 0; i < proc.n_files; i++)
    {
      struct comb_file *file = &proc.files[i];
      if (!create_flag_var ("IN", file->in_name, proc.dict, &file->in_var))
        goto error;
    }
  if (!create_flag_var ("FIRST", first_name, proc.dict, &proc.first)
      || !create_flag_var ("LAST", last_name, proc.dict, &proc.last))
    goto error;

  dict_delete_scratch_vars (proc.dict);
  dict_compact_values (proc.dict);

  /* Set up mapping from each file's variables to master
     variables. */
  for (i = 0; i < proc.n_files; i++)
    {
      struct comb_file *file = &proc.files[i];
      size_t src_var_cnt = dict_get_var_cnt (file->dict);
      size_t j;

      file->mv = xnmalloc (src_var_cnt, sizeof *file->mv);
      for (j = 0; j < src_var_cnt; j++)
        {
          struct variable *src_var = dict_get_var (file->dict, j);
          struct variable *dst_var = dict_lookup_var (proc.dict,
                                                      var_get_name (src_var));
          if (dst_var != NULL)
            {
              size_t n = subcase_get_n_fields (&file->src);
              file->mv[n] = var_get_missing_values (src_var);
              subcase_add_var (&file->src, src_var, SC_ASCEND);
              subcase_add_var (&file->dst, dst_var, SC_ASCEND);
            }
        }
    }

  proc.output = autopaging_writer_create (dict_get_proto (proc.dict));
  taint = taint_clone (casewriter_get_taint (proc.output));

  /* Set up case matcher. */
  proc.matcher = case_matcher_create ();
  for (i = 0; i < proc.n_files; i++)
    {
      struct comb_file *file = &proc.files[i];
      if (file->reader == NULL)
        {
          if (active_file == NULL)
            {
              proc_discard_output (ds);
              file->reader = active_file = proc_open_filtering (ds, false);
            }
          else
            file->reader = casereader_clone (active_file);
        }
      if (!file->is_sorted)
        file->reader = sort_execute (file->reader, &file->by_vars);
      taint_propagate (casereader_get_taint (file->reader), taint);
      file->data = casereader_read (file->reader);
      if (file->type == COMB_FILE)
        case_matcher_add_input (proc.matcher, &file->by_vars,
                                &file->data, &file->is_minimal);
    }

  if (command == COMB_ADD)
    execute_add_files (&proc);
  else if (command == COMB_MATCH)
    execute_match_files (&proc);
  else if (command == COMB_UPDATE)
    execute_update (&proc);
  else
    NOT_REACHED ();

  case_matcher_destroy (proc.matcher);
  proc.matcher = NULL;
  close_all_comb_files (&proc);
  if (active_file != NULL)
    proc_commit (ds);

  dataset_set_dict (ds, proc.dict);
  dataset_set_source (ds, casewriter_make_reader (proc.output));
  proc.dict = NULL;
  proc.output = NULL;

  free_comb_proc (&proc);

  free (first_name);
  free (last_name);

  return taint_destroy (taint) ? CMD_SUCCESS : CMD_CASCADING_FAILURE;

 error:
  if (active_file != NULL)
    proc_commit (ds);
  free_comb_proc (&proc);
  taint_destroy (taint);
  free (first_name);
  free (last_name);
  return CMD_CASCADING_FAILURE;
}
Esempio n. 8
0
/* Parses and executes the AGGREGATE procedure. */
int
cmd_aggregate (struct lexer *lexer, struct dataset *ds)
{
  struct dictionary *dict = dataset_dict (ds);
  struct agr_proc agr;
  struct file_handle *out_file = NULL;
  struct casereader *input = NULL, *group;
  struct casegrouper *grouper;
  struct casewriter *output = NULL;

  bool copy_documents = false;
  bool presorted = false;
  bool saw_direction;
  bool ok;

  memset(&agr, 0 , sizeof (agr));
  agr.missing = ITEMWISE;
  agr.src_dict = dict;
  subcase_init_empty (&agr.sort);

  /* OUTFILE subcommand must be first. */
  lex_match (lexer, T_SLASH);
  if (!lex_force_match_id (lexer, "OUTFILE"))
    goto error;
  lex_match (lexer, T_EQUALS);
  if (!lex_match (lexer, T_ASTERISK))
    {
      out_file = fh_parse (lexer, FH_REF_FILE, dataset_session (ds));
      if (out_file == NULL)
        goto error;
    }

  if (out_file == NULL && lex_match_id (lexer, "MODE"))
    {
      lex_match (lexer, T_EQUALS);
      if (lex_match_id (lexer, "ADDVARIABLES"))
	{
	  agr.add_variables = true;

	  /* presorted is assumed in ADDVARIABLES mode */
	  presorted = true;
	}
      else if (lex_match_id (lexer, "REPLACE"))
	{
	  agr.add_variables = false;
	}
      else
	goto error;
    }

  if ( agr.add_variables )
    agr.dict = dict_clone (dict);
  else
    agr.dict = dict_create (dict_get_encoding (dict));

  dict_set_label (agr.dict, dict_get_label (dict));
  dict_set_documents (agr.dict, dict_get_documents (dict));

  /* Read most of the subcommands. */
  for (;;)
    {
      lex_match (lexer, T_SLASH);

      if (lex_match_id (lexer, "MISSING"))
	{
	  lex_match (lexer, T_EQUALS);
	  if (!lex_match_id (lexer, "COLUMNWISE"))
	    {
	      lex_error_expecting (lexer, "COLUMNWISE", NULL);
              goto error;
	    }
	  agr.missing = COLUMNWISE;
	}
      else if (lex_match_id (lexer, "DOCUMENT"))
        copy_documents = true;
      else if (lex_match_id (lexer, "PRESORTED"))
        presorted = true;
      else if (lex_force_match_id (lexer, "BREAK"))
	{
          int i;

	  lex_match (lexer, T_EQUALS);
          if (!parse_sort_criteria (lexer, dict, &agr.sort, &agr.break_vars,
                                    &saw_direction))
            goto error;
          agr.break_var_cnt = subcase_get_n_fields (&agr.sort);

	  if  (! agr.add_variables)
	    for (i = 0; i < agr.break_var_cnt; i++)
	      dict_clone_var_assert (agr.dict, agr.break_vars[i]);

          /* BREAK must follow the options. */
          break;
	}
      else
        goto error;

    }
  if (presorted && saw_direction)
    msg (SW, _("When PRESORTED is specified, specifying sorting directions "
               "with (A) or (D) has no effect.  Output data will be sorted "
               "the same way as the input data."));

  /* Read in the aggregate functions. */
  lex_match (lexer, T_SLASH);
  if (!parse_aggregate_functions (lexer, dict, &agr))
    goto error;

  /* Delete documents. */
  if (!copy_documents)
    dict_clear_documents (agr.dict);

  /* Cancel SPLIT FILE. */
  dict_set_split_vars (agr.dict, NULL, 0);

  /* Initialize. */
  agr.case_cnt = 0;

  if (out_file == NULL)
    {
      /* The active dataset will be replaced by the aggregated data,
         so TEMPORARY is moot. */
      proc_cancel_temporary_transformations (ds);
      proc_discard_output (ds);
      output = autopaging_writer_create (dict_get_proto (agr.dict));
    }
  else
    {
      output = any_writer_open (out_file, agr.dict);
      if (output == NULL)
        goto error;
    }

  input = proc_open (ds);
  if (!subcase_is_empty (&agr.sort) && !presorted)
    {
      input = sort_execute (input, &agr.sort);
      subcase_clear (&agr.sort);
    }

  for (grouper = casegrouper_create_vars (input, agr.break_vars,
                                          agr.break_var_cnt);
       casegrouper_get_next_group (grouper, &group);
       casereader_destroy (group))
    {
      struct casereader *placeholder = NULL;
      struct ccase *c = casereader_peek (group, 0);

      if (c == NULL)
        {
          casereader_destroy (group);
          continue;
        }

      initialize_aggregate_info (&agr);

      if ( agr.add_variables )
	placeholder = casereader_clone (group);

      {
	struct ccase *cg;
	for (; (cg = casereader_read (group)) != NULL; case_unref (cg))
	  accumulate_aggregate_info (&agr, cg);
      }


      if  (agr.add_variables)
	{
	  struct ccase *cg;
	  for (; (cg = casereader_read (placeholder)) != NULL; case_unref (cg))
	    dump_aggregate_info (&agr, output, cg);

	  casereader_destroy (placeholder);
	}
      else
	{
	  dump_aggregate_info (&agr, output, c);
	}
      case_unref (c);
    }
  if (!casegrouper_destroy (grouper))
    goto error;

  if (!proc_commit (ds))
    {
      input = NULL;
      goto error;
    }
  input = NULL;

  if (out_file == NULL)
    {
      struct casereader *next_input = casewriter_make_reader (output);
      if (next_input == NULL)
        goto error;

      dataset_set_dict (ds, agr.dict);
      dataset_set_source (ds, next_input);
      agr.dict = NULL;
    }
  else
    {
      ok = casewriter_destroy (output);
      output = NULL;
      if (!ok)
        goto error;
    }

  agr_destroy (&agr);
  fh_unref (out_file);
  return CMD_SUCCESS;

error:
  if (input != NULL)
    proc_commit (ds);
  casewriter_destroy (output);
  agr_destroy (&agr);
  fh_unref (out_file);
  return CMD_CASCADING_FAILURE;
}