int parseTweets(std::string &xmlstr, struct t_tweet ** dest, int usertop)
{
mxml_node_t *xml;
mxml_node_t *unit;
struct t_tweet * tweets = (struct t_tweet *)mem2_calloc(MAXUNITS, sizeof(struct t_tweet), MEM2_OTHER);
if(!tweets){
return 0;
}
size_t pos = 0;
while(1){
pos = xmlstr.find("&", pos);
if(pos == std::string::npos) break;
xmlstr.replace(pos, 9, "&");
}
xml = mxmlLoadString(NULL, xmlstr.c_str(), MXML_OPAQUE_CALLBACK);
if(!xml){
mem2_free(tweets, MEM2_OTHER);
return 0;
}
int t=0;
if(usertop){
for(unit = mxmlFindElement(xml, xml, "user", NULL, NULL, MXML_DESCEND); unit && t < MAXUNITS; unit = mxmlFindElement(unit, xml, "user", NULL, NULL, MXML_DESCEND)){
parseUser(unit, &(tweets[t].user), &(tweets[t]));
t++;
}
}else{
for(unit = mxmlFindElement(xml, xml, "status", NULL, NULL, MXML_DESCEND); unit && t < MAXUNITS; unit = mxmlFindElement(unit, xml, "status", NULL, NULL, MXML_DESCEND)){
parseTweet(unit, &tweets[t++]);
}
}
mxmlDelete(xml);
if(t < MAXUNITS){ //Free unused memory
tweets = (struct t_tweet *)mem2_realloc(tweets, t*sizeof(struct t_tweet), MEM2_OTHER);
}
*dest = tweets;
return t;
}
void parseUser(mxml_node_t *user, struct t_user *dest, struct t_tweet * dest_tweet){
mxml_node_t * value = NULL;
if(dest_tweet){
value = mxmlFindElement(user, user, "status", NULL, NULL, MXML_DESCEND);
if(value){ parseTweet(value, dest_tweet, 0); }
}
value = mxmlFindElement(user, user, "id", NULL, NULL, MXML_DESCEND);
if(value && value->child) sscanf(value->child->value.opaque, "%llu", &(dest->id));
value = mxmlFindElement(value, user, "name", NULL, NULL, MXML_DESCEND);
if(value && value->child) wt_strncpy(dest->name, value->child->value.opaque, 81);
value = mxmlFindElement(value, user, "screen_name", NULL, NULL, MXML_DESCEND);
if(value && value->child) snprintf(dest->screenname, 20, "@%s", value->child->value.opaque);
value = mxmlFindElement(value, user, "location", NULL, NULL, MXML_DESCEND);
if(value && value->child) wt_strncpy(dest->location, value->child->value.opaque, 160);
value = mxmlFindElement(value, user, "description", NULL, NULL, MXML_DESCEND);
if(value && value->child) wt_strncpy(dest->description, value->child->value.opaque, 641);
value = mxmlFindElement(value, user, "profile_image_url", NULL, NULL, MXML_DESCEND);
if(value && value->child) strncpy(dest->imageurl, value->child->value.opaque, 255);
value = mxmlFindElement(value, user, "url", NULL, NULL, MXML_DESCEND);
if(value && value->child) strncpy(dest->url, value->child->value.opaque, 160);
value = mxmlFindElement(value, user, "followers_count", NULL, NULL, MXML_DESCEND);
if(value && value->child) sscanf(value->child->value.opaque, "%llu", &(dest->followers));
value = mxmlFindElement(value, user, "friends_count", NULL, NULL, MXML_DESCEND);
if(value && value->child) sscanf(value->child->value.opaque, "%llu", &(dest->following));
value = mxmlFindElement(value, user, "favourites_count", NULL, NULL, MXML_DESCEND);
if(value && value->child) sscanf(value->child->value.opaque, "%llu", &(dest->favourites));
value = mxmlFindElement(value, user, "statuses_count", NULL, NULL, MXML_DESCEND);
if(value && value->child) sscanf(value->child->value.opaque, "%llu", &(dest->tweets));
value = mxmlFindElement(value, user, "following", NULL, NULL, MXML_DESCEND);
if(value && value->child){
if(value->child->value.opaque[0] == 't'){
dest->flags |= FOLLOWING;
}else{
dest->flags |= NFOLLOWING;
}
}
mxmlDelete(value);
}
int main(int argc, char* argv[]) {
// int my_rank; /* rank of process */
// int p; /* number of processes */
//int source; /* rank of sender */
//int dest; /* rank of receiver */
//int tag = 0; /* tag for messages */
//char message[100]; /* storage for message */
//MPI_Status status; /* return status for receive */
// MPI Initializations
//MPI_Init(&argc, &argv);
//MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
//MPI_Comm_size(MPI_COMM_WORLD, &p);
// Show keyword
printf("Tweets will be sorted by keyword '%s'.\n",KEYWORD);
struct timeval time1, time2;
unsigned long long timeToReadAndInit = 0l, timeToSort = 0l, timeToWrite = 0l, timeForAll = 0l,
microsec1, microsec2;
// File Initializations
FILE *fp;
fp = fopen(FILE_PATH, "rb+, ccs=UTF-8");
if (fp == NULL) {
fprintf(stderr,"file not accessible!\n");
exit(EXIT_FAILURE);
}
// if (my_rank == 0) {
/* create message */
//sprintf(message, "Hello MPI World from process %d!", my_rank);
// dest = 0;
//byte *test = readTweets("./");
//////////
tweetData_t *td;
printf("try to allocate %lu kbytes\n",
(sizeof(tweetData_t*) * NUMBEROFTWEETS) / 1000);
tweetData_t *test = (tweetData_t*) malloc(sizeof(tweetData_t) * (NUMBEROFTWEETS+1));
fflush (stdout);
long numberOfTweets = 0;
gettimeofday(&time1, NULL);
microsec1 = ((unsigned long long) time1.tv_sec * 1000000) + time1.tv_usec;
while (!feof(fp)) {
unsigned char *line = readLine(fp);
td = parseTweet(line, KEYWORD);
test[numberOfTweets] = *td;
numberOfTweets++;
}
fclose(fp);
// Calc time
gettimeofday(&time2, NULL);
microsec2 = ((unsigned long long) time2.tv_sec * 1000000) + time2.tv_usec;
timeToReadAndInit = (microsec2 - microsec1) / 1000000;
gettimeofday(&time1, NULL);
microsec1 = ((unsigned long long) time1.tv_sec * 1000000) + time1.tv_usec;
quickSort(test, numberOfTweets);
gettimeofday(&time2, NULL);
microsec2 = ((unsigned long long) time2.tv_sec * 1000000) + time2.tv_usec;
timeToSort = (microsec2 - microsec1) / 1000000;
gettimeofday(&time1, NULL);
microsec1 = ((unsigned long long) time1.tv_sec * 1000000) + time1.tv_usec;
FILE *f = fopen(OUTPUT_FILE_PATH, "w");
if (f == NULL)
{
printf("Error opening file!\n");
exit(1);
}
/* print some text */
for (int j = 0; j <= numberOfTweets; j++) {
if (test[j].line != NULL) {
fprintf(f, "%s\n", test[j].line);
}
}
fclose(f);
gettimeofday(&time2, NULL);
microsec2 = ((unsigned long long) time2.tv_sec * 1000000) + time2.tv_usec;
timeToWrite = (microsec2 - microsec1) / 1000000;
timeForAll = timeToReadAndInit + timeToSort + timeToWrite;
printf("Read & Init:\t%llu Sec.\n", timeToReadAndInit);
printf("Sort:\t\t%llu Sec.\n", timeToSort);
printf("Write:\t\t%llu Sec.\n", timeToWrite);
printf("All:\t\t%llu Sec.\n", timeForAll);
// MPI_Finalize();
return 0;
// use strlen+1 so that '\0' get transmitted
// MPI_Send(message, strlen(message)+1, MPI_CHAR,
// dest, tag, MPI_COMM_WORLD);
/* } else {
printf("Hello MPI World From process 0: Num processes: %d\n", p);
for (source = 1; source < p; source++) {
MPI_Recv(message, 100, MPI_CHAR, source, tag, MPI_COMM_WORLD,
&status);
printf("%s\n", message);
}
}
/* shut down MPI */
// MPI_Finalize();
return 0;
}
