int main(int argc,char** argv) { NodesX *Nodes; SegmentsX *Segments,*SuperSegments=NULL,*MergedSegments=NULL; WaysX *Ways; int iteration=0,quit=0; int max_iterations=10; char *dirname=NULL,*prefix=NULL,*tagging=NULL; int option_parse_only=0,option_process_only=0; int option_filenames=0; int arg; /* Parse the command line arguments */ for(arg=1;arg<argc;arg++) { if(!strcmp(argv[arg],"--help")) print_usage(1); else if(!strcmp(argv[arg],"--slim")) option_slim=1; else if(!strncmp(argv[arg],"--sort-ram-size=",16)) option_filesort_ramsize=atoi(&argv[arg][16]); else if(!strncmp(argv[arg],"--dir=",6)) dirname=&argv[arg][6]; else if(!strncmp(argv[arg],"--tmpdir=",9)) option_tmpdirname=&argv[arg][9]; else if(!strncmp(argv[arg],"--prefix=",9)) prefix=&argv[arg][9]; else if(!strcmp(argv[arg],"--parse-only")) option_parse_only=1; else if(!strcmp(argv[arg],"--process-only")) option_process_only=1; else if(!strncmp(argv[arg],"--max-iterations=",17)) max_iterations=atoi(&argv[arg][17]); else if(!strncmp(argv[arg],"--tagging=",10)) tagging=&argv[arg][10]; else if(argv[arg][0]=='-' && argv[arg][1]=='-') print_usage(0); else option_filenames++; } /* Check the specified command line options */ if(option_parse_only && option_process_only) print_usage(0); if(option_filenames && option_process_only) print_usage(0); if(!option_filesort_ramsize) { if(option_slim) option_filesort_ramsize=64*1024*1024; else option_filesort_ramsize=256*1024*1024; } else option_filesort_ramsize*=1024*1024; if(!option_tmpdirname) { if(!dirname) option_tmpdirname="."; else option_tmpdirname=dirname; } if(tagging && ExistsFile(tagging)) ; else if(!tagging && ExistsFile(FileName(dirname,prefix,"tagging.xml"))) tagging=FileName(dirname,prefix,"tagging.xml"); if(tagging && ParseXMLTaggingRules(tagging)) { fprintf(stderr,"Error: Cannot read the tagging rules in the file '%s'.\n",tagging); return(1); } if(!tagging) { fprintf(stderr,"Error: Cannot run without reading some tagging rules.\n"); return(1); } /* Create new node, segment and way variables */ Nodes=NewNodeList(option_parse_only||option_process_only); Segments=NewSegmentList(option_parse_only||option_process_only); Ways=NewWayList(option_parse_only||option_process_only); /* Parse the file */ if(option_filenames) { for(arg=1;arg<argc;arg++) { FILE *file; if(argv[arg][0]=='-' && argv[arg][1]=='-') continue; file=fopen(argv[arg],"rb"); if(!file) { fprintf(stderr,"Cannot open file '%s' for reading [%s].\n",argv[arg],strerror(errno)); exit(EXIT_FAILURE); } printf("\nParse OSM Data [%s]\n==============\n\n",argv[arg]); fflush(stdout); if(ParseOSM(file,Nodes,Segments,Ways)) exit(EXIT_FAILURE); fclose(file); } } else if(!option_process_only) { printf("\nParse OSM Data\n==============\n\n"); fflush(stdout); if(ParseOSM(stdin,Nodes,Segments,Ways)) exit(EXIT_FAILURE); } if(option_parse_only) { FreeNodeList(Nodes,1); FreeSegmentList(Segments,1); FreeWayList(Ways,1); return(0); } /* Process the data */ printf("\nProcess OSM Data\n================\n\n"); fflush(stdout); /* Sort the nodes, segments and ways */ SortNodeList(Nodes); SortSegmentList(Segments); SortWayList(Ways); /* Remove bad segments (must be after sorting the nodes and segments) */ RemoveBadSegments(Nodes,Segments); /* Remove non-highway nodes (must be after removing the bad segments) */ RemoveNonHighwayNodes(Nodes,Segments); /* Measure the segments and replace node/way id with index (must be after removing non-highway nodes) */ UpdateSegments(Segments,Nodes,Ways); /* Repeated iteration on Super-Nodes and Super-Segments */ do { printf("\nProcess Super-Data (iteration %d)\n================================%s\n\n",iteration,iteration>9?"=":""); fflush(stdout); if(iteration==0) { /* Select the super-nodes */ ChooseSuperNodes(Nodes,Segments,Ways); /* Select the super-segments */ SuperSegments=CreateSuperSegments(Nodes,Segments,Ways,iteration); } else { SegmentsX *SuperSegments2; /* Select the super-nodes */ ChooseSuperNodes(Nodes,SuperSegments,Ways); /* Select the super-segments */ SuperSegments2=CreateSuperSegments(Nodes,SuperSegments,Ways,iteration); if(SuperSegments->xnumber==SuperSegments2->xnumber) quit=1; FreeSegmentList(SuperSegments,0); SuperSegments=SuperSegments2; } /* Sort the super-segments */ SortSegmentList(SuperSegments); /* Remove duplicated super-segments */ DeduplicateSegments(SuperSegments,Nodes,Ways); iteration++; if(iteration>max_iterations) quit=1; } while(!quit); /* Combine the super-segments */ printf("\nCombine Segments and Super-Segments\n===================================\n\n"); fflush(stdout); /* Merge the super-segments */ MergedSegments=MergeSuperSegments(Segments,SuperSegments); FreeSegmentList(Segments,0); FreeSegmentList(SuperSegments,0); Segments=MergedSegments; /* Rotate segments so that node1<node2 */ RotateSegments(Segments); /* Sort the segments */ SortSegmentList(Segments); /* Remove duplicated segments */ DeduplicateSegments(Segments,Nodes,Ways); /* Cross reference the nodes and segments */ printf("\nCross-Reference Nodes and Segments\n==================================\n\n"); fflush(stdout); /* Sort the node list geographically */ SortNodeListGeographically(Nodes); /* Create the real segments and nodes */ CreateRealNodes(Nodes,iteration); CreateRealSegments(Segments,Ways); /* Fix the segment and node indexes */ IndexNodes(Nodes,Segments); IndexSegments(Segments,Nodes); /* Output the results */ printf("\nWrite Out Database Files\n========================\n\n"); fflush(stdout); /* Write out the nodes */ SaveNodeList(Nodes,FileName(dirname,prefix,"nodes.mem")); FreeNodeList(Nodes,0); /* Write out the segments */ SaveSegmentList(Segments,FileName(dirname,prefix,"segments.mem")); FreeSegmentList(Segments,0); /* Write out the ways */ SaveWayList(Ways,FileName(dirname,prefix,"ways.mem")); FreeWayList(Ways,0); return(0); }
int main(int argc,char** argv) { NodesX *OSMNodes; WaysX *OSMWays; RelationsX *OSMRelations; ErrorLogsX *OSMErrorLogs; char *dirname=NULL,*prefix=NULL,*tagging=NULL; int option_keep=1; int option_filenames=0; int arg; printf_program_start(); /* Parse the command line arguments */ for(arg=1;arg<argc;arg++) { if(!strcmp(argv[arg],"--version")) print_usage(-1,NULL,NULL); else if(!strcmp(argv[arg],"--help")) print_usage(1,NULL,NULL); else if(!strncmp(argv[arg],"--dir=",6)) dirname=&argv[arg][6]; else if(!strncmp(argv[arg],"--sort-ram-size=",16)) option_filesort_ramsize=atoi(&argv[arg][16]); #if defined(USE_PTHREADS) && USE_PTHREADS else if(!strncmp(argv[arg],"--sort-threads=",15)) option_filesort_threads=atoi(&argv[arg][15]); #endif else if(!strncmp(argv[arg],"--tmpdir=",9)) option_tmpdirname=&argv[arg][9]; else if(!strncmp(argv[arg],"--tagging=",10)) tagging=&argv[arg][10]; else if(!strcmp(argv[arg],"--loggable")) option_loggable=1; else if(!strcmp(argv[arg],"--logtime")) option_logtime=1; else if(!strcmp(argv[arg],"--logmemory")) option_logmemory=1; else if(argv[arg][0]=='-' && argv[arg][1]=='-') print_usage(0,argv[arg],NULL); else option_filenames++; } /* Check the specified command line options */ if(!option_filesort_ramsize) { #if SLIM option_filesort_ramsize=64*1024*1024; #else option_filesort_ramsize=256*1024*1024; #endif } else option_filesort_ramsize*=1024*1024; if(!option_tmpdirname) { if(!dirname) option_tmpdirname="."; else option_tmpdirname=dirname; } if(tagging) { if(!ExistsFile(tagging)) { fprintf(stderr,"Error: The '--tagging' option specifies a file that does not exist.\n"); exit(EXIT_FAILURE); } } else { tagging=FileName(dirname,prefix,"fixme.xml"); if(!ExistsFile(tagging)) { fprintf(stderr,"Error: The '--tagging' option was not used and the default 'fixme.xml' does not exist.\n"); exit(EXIT_FAILURE); } } if(ParseXMLTaggingRules(tagging)) { fprintf(stderr,"Error: Cannot read the tagging rules in the file '%s'.\n",tagging); exit(EXIT_FAILURE); } /* Create new node, segment, way and relation variables */ OSMNodes=NewNodeList(0,0); OSMWays=NewWayList(0,0); OSMRelations=NewRelationList(0,0); /* Create the error log file */ open_errorlog(FileName(dirname,prefix,"fixme.log"),0,option_keep); /* Parse the file */ for(arg=1;arg<argc;arg++) { int fd; char *filename,*p; if(argv[arg][0]=='-' && argv[arg][1]=='-') continue; filename=strcpy(malloc(strlen(argv[arg])+1),argv[arg]); fd=OpenFile(filename); if((p=strstr(filename,".bz2")) && !strcmp(p,".bz2")) { fd=Uncompress_Bzip2(fd); *p=0; } if((p=strstr(filename,".gz")) && !strcmp(p,".gz")) { fd=Uncompress_Gzip(fd); *p=0; } if((p=strstr(filename,".xz")) && !strcmp(p,".xz")) { fd=Uncompress_Xz(fd); *p=0; } printf("\nParse OSM Data [%s]\n==============\n\n",filename); fflush(stdout); if((p=strstr(filename,".pbf")) && !strcmp(p,".pbf")) { if(ParsePBFFile(fd,OSMNodes,OSMWays,OSMRelations)) exit(EXIT_FAILURE); } else if((p=strstr(filename,".o5m")) && !strcmp(p,".o5m")) { if(ParseO5MFile(fd,OSMNodes,OSMWays,OSMRelations)) exit(EXIT_FAILURE); } else { if(ParseOSMFile(fd,OSMNodes,OSMWays,OSMRelations)) exit(EXIT_FAILURE); } CloseFile(fd); free(filename); } DeleteXMLTaggingRules(); FinishNodeList(OSMNodes); FinishWayList(OSMWays); FinishRelationList(OSMRelations); /* Sort the data */ printf("\nSort OSM Data\n=============\n\n"); fflush(stdout); /* Sort the nodes, ways and relations */ SortNodeList(OSMNodes); SortWayList(OSMWays); SortRelationList(OSMRelations); /* Process the data */ RenameFile(OSMNodes->filename_tmp,OSMNodes->filename); RenameFile(OSMWays->filename_tmp,OSMWays->filename); RenameFile(OSMRelations->rrfilename_tmp,OSMRelations->rrfilename); RenameFile(OSMRelations->trfilename_tmp,OSMRelations->trfilename); close_errorlog(); printf("\nCreate Error Log\n================\n\n"); fflush(stdout); OSMErrorLogs=NewErrorLogList(); ProcessErrorLogs(OSMErrorLogs,OSMNodes,OSMWays,OSMRelations); SortErrorLogsGeographically(OSMErrorLogs); SaveErrorLogs(OSMErrorLogs,FileName(dirname,prefix,"fixme.mem")); FreeErrorLogList(OSMErrorLogs); /* Free the memory (delete the temporary files) */ FreeNodeList(OSMNodes,0); FreeWayList(OSMWays,0); FreeRelationList(OSMRelations,0); printf("\n"); fflush(stdout); printf_program_end(); exit(EXIT_SUCCESS); }
int main(int argc, char *argv[]) { sigset_t signal_set; struct sigaction sa; int c, snaplen, err, do_daemonize; int subdir_index, compress, expire, cache_size; FlowSource_t *fs; dirstat_t *dirstat; time_t t_win; char *device, *pcapfile, *filter, *datadir, *pcap_datadir, *extension_tags, pidfile[MAXPATHLEN], pidstr[32]; char *Ident, *userid, *groupid; pcap_dev_t *pcap_dev; p_packet_thread_args_t *p_packet_thread_args; p_flow_thread_args_t *p_flow_thread_args; snaplen = 100; do_daemonize = 0; launcher_pid = 0; device = NULL; pcapfile = NULL; filter = NULL; pidfile[0] = '\0'; t_win = TIME_WINDOW; datadir = DEFAULT_DIR; pcap_datadir = NULL; userid = groupid = NULL; Ident = "none"; fs = NULL; extension_tags = DefaultExtensions; subdir_index = 0; compress = NOT_COMPRESSED; verbose = 0; expire = 0; cache_size = 0; while ((c = getopt(argc, argv, "B:DEI:g:hi:j:r:s:l:p:P:t:u:S:T:e:Vz")) != EOF) { switch (c) { struct stat fstat; case 'h': usage(argv[0]); exit(0); break; case 'u': userid = optarg; break; case 'g': groupid = optarg; break; case 'D': do_daemonize = 1; break; case 'B': cache_size = atoi(optarg); if (cache_size <= 0) { LogError("ERROR: Cache size must not be < 0"); exit(EXIT_FAILURE); } break; case 'I': Ident = strdup(optarg); break; case 'i': device = optarg; break; case 'l': datadir = optarg; err = stat(datadir, &fstat); if (!(fstat.st_mode & S_IFDIR)) { LogError("No such directory: " "'%s'", datadir); break; } break; case 'p': pcap_datadir = optarg; err = stat(pcap_datadir, &fstat); if (!(fstat.st_mode & S_IFDIR)) { LogError("No such directory: " "'%s'", pcap_datadir); break; } break; case 'r': { struct stat stat_buf; pcapfile = optarg; if ( stat(pcapfile, &stat_buf) ) { LogError("Can't stat '%s': %s", pcapfile, strerror(errno)); exit(EXIT_FAILURE); } if (!S_ISREG(stat_buf.st_mode) ) { LogError("'%s' is not a file", pcapfile); exit(EXIT_FAILURE); } } break; case 's': snaplen = atoi(optarg); if (snaplen < 14 + 20 + 20) { // ethernet, IP , TCP, no payload LogError("ERROR:, snaplen < sizeof IPv4 - Need 54 bytes for TCP/IPv4"); exit(EXIT_FAILURE); } break; case 't': t_win = atoi(optarg); if (t_win < 60) { LogError("WARNING, very small time frame (< 60)!"); } if (t_win <= 0) { LogError("ERROR: time frame too small (<= 0)"); exit(EXIT_FAILURE); } break; case 'j': if ( compress ) { LogError("Use either -z for LZO or -j for BZ2 compression, but not both\n"); exit(255); } compress = BZ2_COMPRESSED; break; case 'z': if ( compress ) { LogError("Use either -z for LZO or -j for BZ2 compression, but not both\n"); exit(255); } compress = LZO_COMPRESSED; break; case 'P': if ( optarg[0] == '/' ) { // absolute path given strncpy(pidfile, optarg, MAXPATHLEN-1); } else { // path relative to current working directory char tmp[MAXPATHLEN]; if ( !getcwd(tmp, MAXPATHLEN-1) ) { fprintf(stderr, "Failed to get current working directory: %s\n", strerror(errno)); exit(255); } tmp[MAXPATHLEN-1] = 0; snprintf(pidfile, MAXPATHLEN - 1 - strlen(tmp), "%s/%s", tmp, optarg); } // pidfile now absolute path pidfile[MAXPATHLEN-1] = 0; break; case 'S': subdir_index = atoi(optarg); break; case 'T': { size_t len = strlen(optarg); extension_tags = optarg; if ( len == 0 || len > 128 ) { fprintf(stderr, "Extension length error. Unexpected option '%s'\n", extension_tags); exit(255); } break; } case 'E': verbose = 1; Setv6Mode(1); break; case 'V': printf("%s: Version: %s\n",argv[0], nfdump_version); exit(0); break; default: usage(argv[0]); exit(EXIT_FAILURE); } } if (argc - optind > 1) { usage(argv[0]); exit(EXIT_FAILURE); } else { /* user specified a pcap filter */ filter = argv[optind]; } if ( fs == NULL && datadir != NULL && !AddDefaultFlowSource(&fs, Ident, datadir) ) { fprintf(stderr, "Failed to add default data collector directory\n"); exit(255); } if ( device && pcapfile ) { LogError("Specify either a device or a pcapfile, but not both"); exit(EXIT_FAILURE); } if ( !device && !pcapfile ) { LogError("Specify either a device or a pcapfile to read packets from"); exit(EXIT_FAILURE); } if ( !Init_FlowTree(cache_size)) { LogError("Init_FlowTree() failed."); exit(EXIT_FAILURE); } InitExtensionMaps(NO_EXTENSION_LIST); SetupExtensionDescriptors(strdup(extension_tags)); if ( pcapfile ) { pcap_dev = setup_pcap_file(pcapfile, filter, snaplen); } else { pcap_dev = setup_pcap_live(device, filter, snaplen); } if (!pcap_dev) { exit(EXIT_FAILURE); } SetPriv(userid, groupid); if ( subdir_index && !InitHierPath(subdir_index) ) { pcap_close(pcap_dev->handle); exit(255); } if ( do_daemonize && !InitLog(argv[0], SYSLOG_FACILITY)) { pcap_close(pcap_dev->handle); exit(255); } // check if pid file exists and if so, if a process with registered pid is running if ( strlen(pidfile) ) { int pidf; pidf = open(pidfile, O_RDONLY, 0); if ( pidf > 0 ) { // pid file exists char s[32]; ssize_t len; len = read(pidf, (void *)s, 31); close(pidf); s[31] = '\0'; if ( len < 0 ) { fprintf(stderr, "read() error existing pid file: %s\n", strerror(errno)); pcap_close(pcap_dev->handle); exit(255); } else { unsigned long pid = atol(s); if ( pid == 0 ) { // garbage - use this file unlink(pidfile); } else { if ( kill(pid, 0) == 0 ) { // process exists fprintf(stderr, "A process with pid %lu registered in pidfile %s is already running!\n", pid, strerror(errno)); pcap_close(pcap_dev->handle); exit(255); } else { // no such process - use this file unlink(pidfile); } } } } else { if ( errno != ENOENT ) { fprintf(stderr, "open() error existing pid file: %s\n", strerror(errno)); pcap_close(pcap_dev->handle); exit(255); } // else errno == ENOENT - no file - this is fine } } if ( do_daemonize ) { verbose = 0; daemonize(); } if (strlen(pidfile)) { pid_t pid = getpid(); int pidf = open(pidfile, O_RDWR|O_TRUNC|O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); if ( pidf == -1 ) { LogError("Error opening pid file: '%s' %s", pidfile, strerror(errno)); pcap_close(pcap_dev->handle); exit(255); } snprintf(pidstr,31,"%lu\n", (unsigned long)pid); if ( write(pidf, pidstr, strlen(pidstr)) <= 0 ) { LogError("Error write pid file: '%s' %s", pidfile, strerror(errno)); } close(pidf); } if ( InitBookkeeper(&fs->bookkeeper, fs->datadir, getpid(), launcher_pid) != BOOKKEEPER_OK ) { LogError("initialize bookkeeper failed."); pcap_close(pcap_dev->handle); exit(255); } // Init the extension map list if ( !InitExtensionMapList(fs) ) { // error message goes to syslog pcap_close(pcap_dev->handle); exit(255); } IPFragTree_init(); LogInfo("Startup."); // prepare signal mask for all threads // block signals, as they are handled by the main thread // mask is inherited by all threads sigemptyset(&signal_set); sigaddset(&signal_set, SIGINT); sigaddset(&signal_set, SIGHUP); sigaddset(&signal_set, SIGTERM); sigaddset(&signal_set, SIGUSR1); sigaddset(&signal_set, SIGPIPE); pthread_sigmask(SIG_BLOCK, &signal_set, NULL); // for USR2 set a signal handler, which interrupts blocking // system calls - and signals done event // handler applies for all threads in a process sa.sa_handler = Interrupt_handler; sigemptyset(&sa.sa_mask); sa.sa_flags = 0; sigaction(SIGPIPE, &sa, NULL); sigaction(SIGUSR2, &sa, NULL); // key for each thread err = pthread_key_create(&buffer_key, NULL); if ( err ) { LogError("pthread_key() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); exit(255); } // prepare flow thread args p_flow_thread_args = (p_flow_thread_args_t *)malloc(sizeof(p_flow_thread_args_t)); if ( !p_flow_thread_args ) { LogError("malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); exit(255); } p_flow_thread_args->fs = fs; p_flow_thread_args->t_win = t_win; p_flow_thread_args->compress = compress; p_flow_thread_args->subdir_index = subdir_index; p_flow_thread_args->parent = pthread_self(); p_flow_thread_args->NodeList = NewNodeList(); err = 0; err = pthread_create(&p_flow_thread_args->tid, NULL, p_flow_thread, (void *)p_flow_thread_args); if ( err ) { LogError("pthread_create() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); exit(255); } dbg_printf("Started flow thread[%lu]\n", (long unsigned)p_flow_thread_args->tid); // prepare packet thread args p_packet_thread_args = (p_packet_thread_args_t *)malloc(sizeof(p_packet_thread_args_t)); if ( !p_packet_thread_args ) { LogError("malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); exit(255); } p_packet_thread_args->pcap_dev = pcap_dev; p_packet_thread_args->t_win = t_win; p_packet_thread_args->subdir_index = subdir_index; p_packet_thread_args->pcap_datadir = pcap_datadir; p_packet_thread_args->live = device != NULL; p_packet_thread_args->parent = pthread_self(); p_packet_thread_args->NodeList = p_flow_thread_args->NodeList; err = pthread_create(&p_packet_thread_args->tid, NULL, p_packet_thread, (void *)p_packet_thread_args); if ( err ) { LogError("pthread_create() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); exit(255); } dbg_printf("Started packet thread[%lu]\n", (long unsigned)p_packet_thread_args->tid); // Wait till done WaitDone(); dbg_printf("Signal packet thread to terminate\n"); SignalThreadTerminate((thread_info_t *)p_packet_thread_args, NULL); dbg_printf("Signal flow thread to terminate\n"); SignalThreadTerminate((thread_info_t *)p_flow_thread_args, &p_packet_thread_args->NodeList->c_list); // free arg list free((void *)p_packet_thread_args); free((void *)p_flow_thread_args); LogInfo("Terminating nfpcapd."); if ( expire == 0 && ReadStatInfo(fs->datadir, &dirstat, LOCK_IF_EXISTS) == STATFILE_OK ) { UpdateBookStat(dirstat, fs->bookkeeper); WriteStatInfo(dirstat); LogInfo("Updating statinfo in directory '%s'", datadir); } ReleaseBookkeeper(fs->bookkeeper, DESTROY_BOOKKEEPER); pcap_close(pcap_dev->handle); if ( strlen(pidfile) ) unlink(pidfile); EndLog(); exit(EXIT_SUCCESS); } /* End of main */