/* scan_irr_file_main
* Parse file looking for objects.
* update_flag tell's us if we are parsing a mirror file (2)
* or !us...!ue update file (1), in contrast to a reload on bootstrap
* This tells us to look for mirror file format errors
* and save's us 'check for mirror header' cycles on reloads.
*/
void *scan_irr_file_main (FILE *fp, irr_database_t *database,
int update_flag, enum SCAN_T scan_scope) {
char buffer[4096], *cp, *p = NULL;
char first_attr[128];
u_long save_offset, offset, position, mode, len = 0;
irr_object_t *irr_object;
enum IRR_OBJECTS curr_f;
enum STATES save_state, state;
long lineno = 0;
/* init everything */
if (update_flag)
position = save_offset = offset = (u_long) ftell (fp);
else
position = save_offset = offset = 0;
mode = IRR_NOMODE;
state = BLANK_LINE;
curr_f = NO_FIELD;
irr_object = NULL;
if (scan_scope == SCAN_FILE)
database->hash_spec_tmp = g_hash_table_new_full(g_str_hash, g_str_equal, NULL, (GDestroyNotify)Delete_hash_spec);
/* okay, here we go scanning the file */
while (state != DB_EOF) { /* scan to end of file */
if ((cp = fgets (buffer, sizeof (buffer), fp)) != NULL) {
lineno++;
position = offset;
len = strlen(buffer);
offset += len;
}
state = get_state (cp, len, state, &save_state);
/* skip comment lines and lines that exceed the buffer size */
if (state & (OVRFLW | OVRFLW_END | COMMENT ))
continue;
if (update_flag &&
state == START_F &&
irr_object == NULL) {
if (!strncmp ("%END", buffer, 4))
break; /* normal exit from successful mirror */
else {
state = pick_off_mirror_hdr (fp, buffer, sizeof(buffer), state, &save_state,
&mode, &position, &offset, database);
/* something wrong with update, abort scan */
if (state == DB_EOF) {
p = "IRRd error: scan_irr_file_main (): Missing 'ADD' or 'DEL' "
"and/or malformed update!";
break;
}
}
}
if (state & (DB_EOF | OVRFLW | OVRFLW_END | BLANK_LINE))
curr_f = NO_FIELD;
else if (state != LINE_CONT)
curr_f = get_curr_f (buffer);
/* we have entered into a new object -- create the initial structure */
if (irr_object == NULL && state == START_F) {
irr_object = New_IRR_Object (buffer, position, mode);
/* save a copy of the first attribute for logging purposes */
strncpy(first_attr, buffer, sizeof(first_attr));
first_attr[sizeof(first_attr) - 1] = '\0';
}
/* Ignore these fields if they come at the end of the object.
* The trick is to treat the fields like a comment at the end of the
* object.
* dump_object_check() will set state = DB_EOF if other error's found
*/
if (curr_f == SYNTAX_ERR || curr_f == WARNING) {
/* this junk should not be in a well-formed transaction */
if (atomic_trans) {
p = "IRRd error: scan_irr_file_main (): 'WARNING' or 'ERROR' "
"attribute found in update. Abort transaction!";
if (irr_object != NULL)
Delete_IRR_Object (irr_object);
break;
}
trace (ERROR, default_trace,"In (db,serial)=(%s,%lu) found 'WARNING' "
"or '*ERROR*' line, attempting to remove extraneous lines starting with:\n%s", database->name, database->serial_number + 1, buffer);
/* mark end of object, read past err's and warn's */
save_offset = position;
state = find_blank_line (fp, buffer, sizeof(buffer), state, &save_state,
&position, &offset);
state = dump_object_check (irr_object, state, mode, update_flag,
database, fp);
if (state == DB_EOF) { /* something went wrong, dump object and abort */
trace (ERROR, default_trace,"Attempt to remove RIPE server extraneous line failed! Abort!\n");
Delete_IRR_Object (irr_object);
irr_object = NULL;
mode = IRR_NOMODE;
continue;
} else
trace (NORM, default_trace, "Attempt to remove extraneous line succeeded!\n");
}
if (curr_f != NO_FIELD && (state & (START_F | LINE_CONT)) ) {
/* if continuation line, attribute value starts at beginning + 1 */
if (state == LINE_CONT)
cp = buffer + 1; /* Ignore initial whitespace or '+' */
else /* skip over attribute name label */
cp = buffer + strlen(key_info[curr_f].name);
/* NAME_F indicates object class name attribute */
if (key_info[curr_f].f_type & NAME_F) {
whitespace_remove(cp);
if (*cp != '\0') { /* class name value may be on a continuation line */
if (irr_object->name != NULL) {
/* Shouldn't have more than one class name attribute */
trace (NORM, default_trace, "Warning! Multiple class name attributes: Previous - %s %s, New - %s %s\n", key_info[irr_object->type].name, irr_object->name, key_info[curr_f].name, cp );
} else {
irr_object->name = strdup (cp);
irr_object->type = curr_f;
irr_object->filter_val = key_info[curr_f].filter_val;
}
}
} else if (key_info[curr_f].f_type & SECONDARY_F)
/* add secondary keys, and store things like origin, nic-hdl, etc. */
pick_off_secondary_fields (cp, curr_f, irr_object);
continue;
}
/* Process OBJECT (we just read a '\n' on a line by itself or eof) */
if ( (state & (BLANK_LINE | DB_EOF)) && irr_object != NULL) {
if (scan_scope == SCAN_OBJECT)
return (void *) irr_object;
if (curr_f == SYNTAX_ERR || curr_f == WARNING)
position = save_offset;
else if (state == DB_EOF)
position = offset;
if ((p = build_indexes (fp, database, irr_object, position, update_flag, first_attr)) != NULL) {
if (!update_flag || (update_flag == 1 && atomic_trans))
state = DB_EOF; /* abort scan, something wrong found in input file */
else
p = NULL; /* ignore errors if mirroring or non-atomic update */
}
/* Need to skip over RIPE ERROR/WARNING messages in mirror file */
if (curr_f == SYNTAX_ERR || curr_f == WARNING) {
if (fseek (fp, offset, SEEK_SET) < 0) {
trace (ERROR, default_trace,"Attempt to seek past RIPE server extraneous line failed!\n");
}
}
Delete_IRR_Object (irr_object);
irr_object = NULL;
mode = IRR_NOMODE;
}
} /* while (state != DB_EOF) */
/* only do on reload's and mirror updates */
if (scan_scope == SCAN_FILE) {
if (p == NULL)
commit_spec_hash (database); /* commit if no critical errors */
g_hash_table_destroy(database->hash_spec_tmp);
}
return (void *) p; /* return error string (if any) */
}
int main(int argc, char *argv[])
{
if (argc!=5)
{
fprintf(stderr, "Usage: in_file search_field id_field out_basename\n");
return 1;
}
char *in, *outbase;
int in_len = strlen(argv[1]), out_len = strlen(argv[4]);
in = malloc((in_len + 1) * sizeof(char));
if (!in)
{
fprintf(stderr, "failed to alloc 'in'\n");
return 1;
}
out_len += 1;
outbase = malloc(out_len * sizeof(char));
if (!outbase)
{
fprintf(stderr, "failed to alloc 'outbase'\n");
return 1;
}
strcpy(in, argv[1]);
strcpy(outbase, argv[4]);
char *search_field, *id_field;
int sf_len = strlen(argv[2]), idf_len = strlen(argv[3]);
search_field = (char *) malloc((sf_len + 1) * sizeof(char));
if (!search_field)
{
fprintf(stderr, "failed to alloc 'search_field'\n");
return 1;
}
id_field = (char *) malloc((idf_len + 1) * sizeof(char));
if (!id_field)
{
fprintf(stderr, "failed to alloc 'id_field'\n");
return 1;
}
strcpy(search_field, argv[2]);
strcpy(id_field, argv[3]);
fprintf(stderr, "building indexes for %s...", in);
struct chunks *chunks = NULL;
struct indexes *indexes = NULL;
if (!build_indexes(in, &indexes, &chunks, -1))
{
fprintf(stderr, "failed to build indexes\n");
return 1;
}
fprintf(stderr, "done.\n");
char **out_files = malloc(sizeof(char *) * NUMCORES);
if (!out_files)
{
fprintf(stderr, "failed to alloc out files\n");
return 1;
}
pthread_t *threads = malloc(sizeof(pthread_t) * NUMCORES);
if (!threads)
{
fprintf(stderr, "failed to alloc threads\n");
return 1;
}
int *pt_ret = malloc(sizeof(int) * NUMCORES);
if (!pt_ret)
{
fprintf(stderr, "failed to alloc pt_ret\n");
return 1;
}
struct find_field_args **args = malloc(sizeof(struct find_field_args *) * NUMCORES);
if (!args)
{
fprintf(stderr, "failed to allocate args\n");
return 1;
}
char corestr[3];
int i, j;
for (i=0; i<NUMCORES; i++)
{
sprintf(corestr, "%d", i);
out_files[i] = malloc(sizeof(char) * (out_len + strlen(corestr) + 1));
if (!out_files[i])
{
fprintf(stderr, "failed to alloc out file");
return 1;
}
strcpy(out_files[i], outbase);
strcat(out_files[i], corestr);
args[i] = malloc(sizeof(struct find_field_args));
args[i]->ioargs = malloc(sizeof(struct ioargs));
args[i]->ioargs->in_file = in;
args[i]->ioargs->out_file = out_files[i];
args[i]->ioargs->chunk = &chunks[i];
args[i]->search_field = search_field;
args[i]->id_field = id_field;
int mb = args[i]->ioargs->chunk->size / (1024*1024);
fprintf(stderr, "creating new thread[%d] to process %dMB of data\n", i, mb);
pt_ret[i] = pthread_create(&threads[i], NULL, find_field, (void *) args[i]);
}
for (i=0; i<NUMCORES; i++)
{
pthread_join(threads[i], NULL);
fprintf(stderr, "thread[%d] returned with status %d\n", i, pt_ret[i]);
free(out_files[i]);
free(args[i]->ioargs);
free(args[i]);
free_line_positions(chunks[i].lp);
}
if (indexes)
{
free_index(indexes->index);
free_line_positions(indexes->lp);
free(indexes);
}
free(chunks);
free(out_files);
free(args);
free(in);
free(outbase);
free(search_field);
free(id_field);
free(pt_ret);
free(threads);
return 0;
}
