void
merge_append (struct merge *m, struct casereader *r)
{
r = casereader_rename (r);
m->inputs[m->input_cnt++].reader = r;
if (m->input_cnt >= MAX_MERGE_ORDER)
do_merge (m);
}
static int multi_merge(int argc, char *argv[], int obj,
int reverse, int ignore, int show_wiggles,
int replace, int strip,
int quiet)
{
FILE *f;
char *filename;
struct plist *pl;
int num_patches;
int rv = 0;
int i;
if (!replace) {
fprintf(stderr,
"%s: -p in merge mode requires -r\n",
Cmd);
return 2;
}
if (argc != 1) {
fprintf(stderr,
"%s: -p in merge mode requires exactly one file\n",
Cmd);
return 2;
}
filename = argv[0];
f = fopen(filename, "r");
if (!f) {
fprintf(stderr, "%s: cannot open %s\n",
Cmd, filename);
return 2;
}
pl = parse_patch(f, NULL, &num_patches);
fclose(f);
if (set_prefix(pl, num_patches, strip) == 0) {
fprintf(stderr, "%s: aborting\n", Cmd);
return 2;
}
for (i = 0; i < num_patches; i++) {
char *name;
char *av[2];
asprintf(&name, "_wiggle_:%d:%d:%s",
pl[i].start, pl[i].end, filename);
av[0] = pl[i].file;
av[1] = name;
rv |= do_merge(2, av, obj, reverse, 1, ignore,
show_wiggles, quiet);
}
return rv;
}
static uint32 do_partial_filtering(msieve_obj *obj, filter_t *filter,
merge_t *merge, uint32 entries_r,
uint32 entries_a, double target_density) {
uint32 relations_needed;
uint32 max_weight = 20;
uint32 num_relations = filter->num_relations;
uint32 num_ideals = filter->num_ideals;
if (filter->num_relations < 20000000)
max_weight = 25;
for (; max_weight < MAX_KEEP_WEIGHT; max_weight += 5) {
filter->target_excess = entries_r + entries_a;
filter->num_relations = num_relations;
filter->num_ideals = num_ideals;
filter_read_lp_file(obj, filter, max_weight);
if ((relations_needed = do_merge(obj, filter,
merge, target_density)) > 0)
return relations_needed;
/* accept the collection of generated cycles
if the matrix they form is dense enough or
max_weight has been incremented enough */
if (merge->avg_cycle_weight > 63.0 ||
max_weight >= MAX_KEEP_WEIGHT - 5)
break;
logprintf(obj, "matrix not dense enough, retrying\n");
filter_free_relsets(merge);
}
return 0;
}
struct casereader *
merge_make_reader (struct merge *m)
{
struct casereader *r;
if (m->input_cnt > 1)
do_merge (m);
if (m->input_cnt == 1)
{
r = m->inputs[0].reader;
m->input_cnt = 0;
}
else if (m->input_cnt == 0)
{
struct casewriter *writer = mem_writer_create (m->proto);
r = casewriter_make_reader (writer);
}
else
NOT_REACHED ();
return r;
}
int main(int argc, const char * argv[])
{
if (argc != 3) {
printf("\nUsage: %s <combined.disco (input)> <merged.disco (output)>\n\n", argv[0]);
printf("I require files as arguments, but commonly they are called combined.disco and merged.disco.\n");
return 1;
}
// argv[1] is the input filename
// argv[2] is the output filename
long nlines = file_numrows(argv[1]);
printf("Number of rows: %ld\n", nlines);
row *rows = (row *)malloc(nlines * sizeof(row));
if(file_to_rows(argv[1], rows, nlines) == nlines)
{
do_merge(rows, nlines);
}
print_rows(rows, nlines, argv[2]);
free(rows);
return 0;
}
int main(int argc, char *argv[])
{
int opt;
int option_index;
int mode = 0;
int obj = 0;
int replace = 0;
int which = 0;
int ispatch = 0;
int reverse = 0;
int verbose = 0, quiet = 0;
int strip = -1;
int exit_status = 0;
int ignore = 1;
int show_wiggles = 0;
char *helpmsg;
char *trace;
int selftest = 0;
trace = getenv("WIGGLE_TRACE");
if (trace && *trace)
do_trace = 1;
while ((opt = getopt_long(argc, argv,
short_options, long_options,
&option_index)) != -1)
switch (opt) {
case 'h':
helpmsg = Help;
switch (mode) {
case 'x':
helpmsg = HelpExtract;
break;
case 'd':
helpmsg = HelpDiff;
break;
case 'm':
helpmsg = HelpMerge;
break;
case 'B':
helpmsg = HelpBrowse;
break;
}
fputs(helpmsg, stderr);
exit(0);
case 'V':
fputs(Version, stderr);
exit(0);
case ':':
case '?':
default:
fputs(Usage, stderr);
exit(2);
case 'B':
case 'x':
case 'd':
case 'm':
if (mode == 0) {
mode = opt;
continue;
}
fprintf(stderr,
"%s: mode is '%c' - cannot set to '%c'\n",
Cmd, mode, opt);
exit(2);
case 'w':
case 'l':
if (obj == 0 || obj == opt) {
obj = opt;
continue;
}
fprintf(stderr,
"%s: cannot select both words and lines.\n", Cmd);
exit(2);
case 'r':
replace = 1;
continue;
case 'R':
reverse = 1;
continue;
case 'i':
ignore = 0;
continue;
case 'W':
show_wiggles = 1;
ignore = 0;
continue;
case '1':
case '2':
case '3':
if (which == 0 || which == opt) {
which = opt;
continue;
}
fprintf(stderr,
"%s: can only select one of -1, -2, -3\n", Cmd);
exit(2);
case 'p': /* 'patch' or 'strip' */
if (optarg)
strip = atol(optarg);
ispatch = 1;
continue;
case 'v':
verbose++;
continue;
case 'q':
quiet = 1;
continue;
case SELF_TEST:
selftest = 1;
continue;
}
if (!mode)
mode = 'm';
if (mode == 'B') {
vpatch(argc-optind, argv+optind, ispatch,
strip, reverse, replace, selftest);
/* should not return */
exit(1);
}
if (obj && mode == 'x') {
fprintf(stderr,
"%s: cannot specify --line or --word with --extract\n",
Cmd);
exit(2);
}
if (mode != 'm' && !obj)
obj = 'w';
if (replace && mode != 'm') {
fprintf(stderr,
"%s: --replace only allowed with --merge\n", Cmd);
exit(2);
}
if (mode == 'x' && !which) {
fprintf(stderr,
"%s: must specify -1, -2 or -3 with --extract\n", Cmd);
exit(2);
}
if (mode != 'x' && mode != 'd' && which) {
fprintf(stderr,
"%s: -1, -2 or -3 only allowed with --extract or --diff\n",
Cmd);
exit(2);
}
if (ispatch && which == '3') {
fprintf(stderr,
"%s: cannot extract -3 from a patch.\n", Cmd);
exit(2);
}
switch (mode) {
case 'x':
exit_status = extract(argc-optind, argv+optind, ispatch, which);
break;
case 'd':
exit_status = do_diff(argc-optind, argv+optind, obj, ispatch, which, reverse);
break;
case 'm':
if (ispatch)
exit_status = multi_merge(argc-optind,
argv+optind, obj,
reverse, ignore,
show_wiggles,
replace, strip,
quiet);
else
exit_status = do_merge(argc-optind, argv+optind, obj, reverse, replace,
ignore, show_wiggles, quiet);
break;
}
exit(exit_status);
}
/*--------------------------------------------------------------------*/
uint32 nfs_filter_relations(msieve_obj *obj, mpz_t n) {
filter_t filter;
merge_t merge;
uint32 filtmin_r, filtmin_a;
uint32 entries_r, entries_a;
uint32 num_relations;
uint32 relations_needed = 0;
factor_base_t fb;
time_t wall_time = time(NULL);
uint64 savefile_size = get_file_size(obj->savefile.name);
uint64 ram_size = 0;
uint32 max_relations = 0;
uint32 filter_bound = 0;
double target_density = 0;
char lp_filename[256];
logprintf(obj, "\n");
logprintf(obj, "commencing relation filtering\n");
/* parse arguments */
if (obj->nfs_args != NULL) {
const char *tmp;
tmp = strstr(obj->nfs_args, "filter_mem_mb=");
if (tmp != NULL) {
ram_size = strtoull(tmp + 14, NULL, 10) << 20;
logprintf(obj, "setting memory use to %.1f MB\n",
(double)ram_size / 1048576);
}
tmp = strstr(obj->nfs_args, "filter_maxrels=");
if (tmp != NULL) {
max_relations = strtoul(tmp + 15, NULL, 10);
logprintf(obj, "setting max relations to %u\n",
max_relations);
}
tmp = strstr(obj->nfs_args, "filter_lpbound=");
if (tmp != NULL) {
filter_bound = strtoul(tmp + 15, NULL, 10);
logprintf(obj, "setting large prime bound to %u\n",
filter_bound);
}
tmp = strstr(obj->nfs_args, "target_density=");
if (tmp != NULL) {
target_density = strtod(tmp + 15, NULL);
logprintf(obj, "setting target matrix density to %.1f\n",
target_density);
}
/* old-style 'X,Y' format */
tmp = strchr(obj->nfs_args, ',');
if (tmp != NULL) {
const char *tmp0 = tmp - 1;
while (tmp0 > obj->nfs_args && isdigit(tmp0[-1]))
tmp0--;
max_relations = strtoul(tmp + 1, NULL, 10);
filter_bound = strtoul(tmp0, NULL, 10);
logprintf(obj, "setting max relations to %u\n",
max_relations);
logprintf(obj, "setting large prime bound to %u\n",
filter_bound);
}
}
if (ram_size == 0)
ram_size = get_ram_size();
memset(&filter, 0, sizeof(filter));
memset(&merge, 0, sizeof(merge));
memset(&fb, 0, sizeof(fb));
mpz_poly_init(&fb.rfb.poly);
mpz_poly_init(&fb.afb.poly);
if (read_poly(obj, n, &fb.rfb.poly, &fb.afb.poly, NULL)) {
printf("filtering failed to read polynomials\n");
exit(-1);
}
logprintf(obj, "estimated available RAM is %.1lf MB\n",
(double)ram_size / 1048576);
/* delete duplicate relations */
filtmin_r = filtmin_a = nfs_purge_duplicates(obj, &fb,
max_relations, &num_relations);
if (filter_bound > 0)
filtmin_r = filtmin_a = filter_bound;
/* set up the first disk-based pass; if the dataset is
"small", this will be the only such pass */
set_filtering_bounds(obj, &fb, filtmin_r, filtmin_a,
&entries_r, &entries_a, num_relations,
(uint32)(savefile_size < ram_size / 2),
&filter);
/* separate out the large ideals and delete singletons
once they are all in memory. If the dataset is large,
first delete most of the singletons from the disk file */
nfs_write_lp_file(obj, &fb, &filter, max_relations, 0);
if (filter.lp_file_size > ram_size / 2) {
filter_purge_lp_singletons(obj, &filter, ram_size);
#if 0
/* also delete most of the cliques from the disk
file if it is still large, and there is a
great deal of excess relations */
if (filter.lp_file_size > ram_size / 2) {
filter_purge_lp_cliques(obj, &filter);
}
#endif
}
filter_read_lp_file(obj, &filter, 0);
if (savefile_size < ram_size / 2) {
/* dataset is "small"; build the matrix immediately.
Depending on how much memory the machine has, really
big datasets may get to do this */
if ((relations_needed = do_merge(obj, &filter,
&merge, target_density)) > 0)
goto finished;
}
else {
/* dataset is "large", perform multiple singleton passes.
The first pass used a large bound; the second
filtering bound is much smaller. To allow reuse of
previous results, the second bound is used
during the rest of the filtering */
dump_relation_numbers(obj, &filter);
set_filtering_bounds(obj, &fb, filtmin_r, filtmin_a,
&entries_r, &entries_a,
filter.num_relations, 1, &filter);
free(filter.relation_array);
filter.relation_array = NULL;
nfs_write_lp_file(obj, &fb, &filter, max_relations, 1);
if (filter.lp_file_size < ram_size / 2) {
/* dataset is small enough for filtering to
complete in one pass */
filter_read_lp_file(obj, &filter, 0);
if ((relations_needed = do_merge(obj, &filter,
&merge, target_density)) > 0) {
goto finished;
}
}
else {
/* dataset is so large that even the pruned version
cannot fit comfortably in memory. We have to put
up with reading only the ideals that occur in the
fewest relations, forming the matrix, and then
determining whether the matrix incorporates enough
of the dataset so that the matrix will work */
if ((relations_needed = do_partial_filtering(obj,
&filter, &merge, entries_r,
entries_a, target_density)) > 0) {
goto finished;
}
}
}
/* filtering succeeded; delete the LP file */
sprintf(lp_filename, "%s.lp", obj->savefile.name);
remove(lp_filename);
/* optimize and then save the collection of relation-sets */
filter_postproc_relsets(obj, &merge);
filter_dump_relsets(obj, &merge);
filter_free_relsets(&merge);
wall_time = time(NULL) - wall_time;
logprintf(obj, "RelProcTime: %u\n", (uint32)wall_time);
finished:
mpz_poly_free(&fb.rfb.poly);
mpz_poly_free(&fb.afb.poly);
return relations_needed;
}
