static void process_newlink(unsigned short type, int index, unsigned flags,
unsigned change, struct ifinfomsg *msg, int bytes)
{
struct ether_addr address = {{ 0, 0, 0, 0, 0, 0 }};
struct ether_addr compare = {{ 0, 0, 0, 0, 0, 0 }};
struct rtnl_link_stats64 stats;
unsigned char operstate = 0xff;
struct interface_data *interface;
const char *ifname = NULL;
unsigned int mtu = 0;
char ident[13], str[18];
GSList *list;
memset(&stats, 0, sizeof(stats));
extract_link(msg, bytes, &address, &ifname, &mtu, &operstate, &stats);
snprintf(ident, 13, "%02x%02x%02x%02x%02x%02x",
address.ether_addr_octet[0],
address.ether_addr_octet[1],
address.ether_addr_octet[2],
address.ether_addr_octet[3],
address.ether_addr_octet[4],
address.ether_addr_octet[5]);
snprintf(str, 18, "%02X:%02X:%02X:%02X:%02X:%02X",
address.ether_addr_octet[0],
address.ether_addr_octet[1],
address.ether_addr_octet[2],
address.ether_addr_octet[3],
address.ether_addr_octet[4],
address.ether_addr_octet[5]);
switch (type) {
case ARPHRD_ETHER:
case ARPHRD_LOOPBACK:
case ARPHDR_PHONET_PIPE:
case ARPHRD_NONE:
__vpn_ipconfig_newlink(index, type, flags,
str, mtu, &stats);
break;
}
if (memcmp(&address, &compare, ETH_ALEN) != 0)
connman_info("%s {newlink} index %d address %s mtu %u",
ifname, index, str, mtu);
if (operstate != 0xff)
connman_info("%s {newlink} index %d operstate %u <%s>",
ifname, index, operstate,
operstate2str(operstate));
interface = g_hash_table_lookup(interface_list, GINT_TO_POINTER(index));
if (!interface) {
interface = g_new0(struct interface_data, 1);
interface->index = index;
interface->ident = g_strdup(ident);
g_hash_table_insert(interface_list,
GINT_TO_POINTER(index), interface);
}
/** ===========================================================================
*/
static void extract_host_port(osm_port_t *p_port, uint64_t *p_pkey_base_offset,
uint64_t *p_pkey_offset, uint64_t *p_port_offset,
uint64_t *p_link_offset,
struct ssa_db_extract *p_ssa_db)
{
const osm_pkey_tbl_t *p_pkey_tbl;
const ib_pkey_table_t *block;
osm_physp_t *p_physp = p_port->p_physp;
ib_net16_t pkey;
uint16_t block_index, pkey_idx;
p_pkey_tbl = osm_physp_get_pkey_tbl(p_physp);
for (block_index = 0; block_index < p_pkey_tbl->used_blocks;
block_index++) {
block = osm_pkey_tbl_block_get(p_pkey_tbl, block_index);
if (!block)
continue;
for (pkey_idx = 0; pkey_idx < IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
pkey_idx++) {
pkey = block->pkey_entry[pkey_idx];
if (ib_pkey_is_invalid(pkey))
continue;
p_ssa_db->p_pkey_tbl[*p_pkey_base_offset + *p_pkey_offset] = pkey;
*p_pkey_offset = *p_pkey_offset + 1;
}
}
if (*p_pkey_offset >= SSA_EXTRACT_PKEYS_MAX) {
ssa_log_err(SSA_LOG_DEFAULT,
"ERROR - truncating number of pkeys "
"from %d to %d (maximum) for LID %u\n",
*p_pkey_offset, SSA_EXTRACT_PKEYS_MAX - 1,
ntohs(osm_physp_get_base_lid(p_physp)));
*p_pkey_offset = SSA_EXTRACT_PKEYS_MAX - 1;
}
extract_port(p_physp, NULL, htonll(*p_pkey_base_offset * sizeof(pkey)),
htons(*p_pkey_offset * sizeof(pkey)),
p_port_offset, p_ssa_db);
if (!osm_physp_get_remote(p_physp))
return;
extract_link(p_physp, NULL, p_link_offset, p_ssa_db);
}
/* Attempt to perform Yadis discovery as specified in Yadis 1.0 section 6.2.5.
*
* If Yadis fails, we then attempt HTML discovery using the same document.
*
* If successful, returns nonzero and calls parse_xrds_document() to act upon the received data.
* If fails, returns 0 and does nothing else.
*/
int perform_openid2_discovery(StrBuf *SuppliedURL) {
ctdl_openid *oiddata = (ctdl_openid *) CC->openid_data;
int docbytes = (-1);
StrBuf *ReplyBuf = NULL;
int return_value = 0;
CURL *curl;
CURLcode result;
char errmsg[1024] = "";
struct curl_slist *my_headers = NULL;
StrBuf *x_xrds_location = NULL;
if (!SuppliedURL) return(0);
syslog(LOG_DEBUG, "perform_openid2_discovery(%s)", ChrPtr(SuppliedURL));
if (StrLength(SuppliedURL) == 0) return(0);
ReplyBuf = NewStrBuf();
if (!ReplyBuf) return(0);
curl = ctdl_openid_curl_easy_init(errmsg);
if (!curl) return(0);
curl_easy_setopt(curl, CURLOPT_URL, ChrPtr(SuppliedURL));
curl_easy_setopt(curl, CURLOPT_WRITEDATA, ReplyBuf);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlFillStrBuf_callback);
my_headers = curl_slist_append(my_headers, "Accept:"); /* disable the default Accept: header */
my_headers = curl_slist_append(my_headers, "Accept: application/xrds+xml");
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, my_headers);
curl_easy_setopt(curl, CURLOPT_WRITEHEADER, &x_xrds_location);
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, yadis_headerfunction);
result = curl_easy_perform(curl);
if (result) {
syslog(LOG_DEBUG, "libcurl error %d: %s", result, errmsg);
}
curl_slist_free_all(my_headers);
curl_easy_cleanup(curl);
docbytes = StrLength(ReplyBuf);
/*
* The response from the server will be one of:
*
* Option 1: An HTML document with a <head> element that includes a <meta> element with http-equiv
* attribute, X-XRDS-Location,
*
* Does any provider actually do this? If so then we will implement it in the future.
*/
/*
* Option 2: HTTP response-headers that include an X-XRDS-Location response-header,
* together with a document.
* Option 3: HTTP response-headers only, which MAY include an X-XRDS-Location response-header,
* a contenttype response-header specifying MIME media type,
* application/xrds+xml, or both.
*
* If the X-XRDS-Location header was delivered, we know about it at this point...
*/
if ( (x_xrds_location)
&& (strcmp(ChrPtr(x_xrds_location), ChrPtr(SuppliedURL)))
) {
syslog(LOG_DEBUG, "X-XRDS-Location: %s ... recursing!", ChrPtr(x_xrds_location));
return_value = perform_openid2_discovery(x_xrds_location);
FreeStrBuf(&x_xrds_location);
}
/*
* Option 4: the returned web page may *be* an XRDS document. Try to parse it.
*/
if ( (return_value == 0) && (docbytes >= 0)) {
return_value = parse_xrds_document(ReplyBuf);
}
/*
* Option 5: if all else fails, attempt HTML based discovery.
*/
if ( (return_value == 0) && (docbytes >= 0)) {
if (oiddata->op_url == NULL) {
oiddata->op_url = NewStrBuf();
}
extract_link(oiddata->op_url, HKEY("openid2.provider"), ReplyBuf);
if (StrLength(oiddata->op_url) > 0) {
return_value = openid_disco_html;
}
}
if (ReplyBuf != NULL) {
FreeStrBuf(&ReplyBuf);
}
return(return_value);
}
/** ===========================================================================
*/
static void extract_switch_port(osm_port_t *p_port, uint64_t *p_pkey_base_offset,
uint64_t *p_pkey_offset, uint64_t *p_port_offset,
uint64_t *p_link_offset,
struct ssa_db_extract *p_ssa_db)
{
osm_node_t *p_node = p_port->p_physp->p_node;
const osm_pkey_tbl_t *p_pkey_tbl;
const ib_pkey_table_t *block;
osm_physp_t *p_physp;
uint32_t i;
ib_net16_t pkey;
uint16_t lid_ho, block_index, pkey_idx;
for (i = 0; i < p_node->physp_tbl_size; i++) {
p_physp = osm_node_get_physp_ptr(p_node, i);
if (!p_physp)
continue;
/* TODO: add filtering for down ports */
if (i == 0) {
lid_ho = ntohs(osm_physp_get_base_lid(p_physp));
p_pkey_tbl = osm_physp_get_pkey_tbl(p_physp);
for (block_index = 0; block_index < p_pkey_tbl->used_blocks;
block_index++) {
block = osm_pkey_tbl_block_get(p_pkey_tbl, block_index);
if (!block)
continue;
for (pkey_idx = 0; pkey_idx < IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
pkey_idx++) {
pkey = block->pkey_entry[pkey_idx];
if (ib_pkey_is_invalid(pkey))
continue;
p_ssa_db->p_pkey_tbl[*p_pkey_base_offset + *p_pkey_offset] = pkey;
*p_pkey_offset = *p_pkey_offset + 1;
}
}
if (*p_pkey_offset >= SSA_EXTRACT_PKEYS_MAX) {
ssa_log_err(SSA_LOG_DEFAULT,
"ERROR - truncating number of pkeys "
"from %d to %d (maximum) for LID %u\n",
*p_pkey_offset, SSA_EXTRACT_PKEYS_MAX - 1, lid_ho);
*p_pkey_offset = SSA_EXTRACT_PKEYS_MAX - 1;
}
extract_port(p_physp, &lid_ho,
htonll(*p_pkey_base_offset * sizeof(pkey)),
htons(*p_pkey_offset * sizeof(pkey)),
p_port_offset, p_ssa_db);
} else {
extract_port(p_physp, &lid_ho, 0, 0, p_port_offset,
p_ssa_db);
}
if (!osm_physp_get_remote(p_physp))
continue;
extract_link(p_physp, &lid_ho, p_link_offset, p_ssa_db);
}
}
void * do_crawler(void *item)
{
char *url_ptr;
int clientfd ;
static int pages = 0;
static int error = 0;
char *buf = NULL;
urlq_t *url_list_head = NULL, *p, *p_pre;
char cur_dir[256];
hash_table *hash_in, *hash_out;
/* int tid = pthread_self();*/
int j = 0;
int i= 0;
int pos_found = 0;
char temp[256];
int status = 0;
static int a = 0;
static int b = 0;
static int c = 0;
static int d = 0;
static int e = 0;
static int f = 0;
static int g = 0;
while(1){
pthread_mutex_lock(&mutex);
while (urlqueue.head_ptr->next == NULL){
pthread_cond_wait(&ready, &mutex);
}
pthread_mutex_lock(&((threadpool_item *)item)->waitlock);
((threadpool_item *)item)->idle = 0;
pthread_mutex_unlock(&((threadpool_item *)item)->waitlock);
url_ptr = queue_pop(&urlqueue);
g++;
pthread_mutex_unlock(&mutex);
pthread_mutex_lock(&mutex);
/*if not visited, set flag = 1*/
if(has_visited(hash, url_ptr) == 1){
pthread_mutex_unlock(&mutex);
free(url_ptr);
pthread_mutex_lock(&((threadpool_item *)item)->waitlock);
((threadpool_item *)item)->idle =1;
pthread_mutex_unlock(&((threadpool_item *)item)->waitlock);
d++;
continue;
}
hash_out = has_url(hash, url_ptr);
e++;
pthread_mutex_unlock(&mutex);
if (hash_out == NULL){
printf("error\n");
getchar();
}
*temp = '\0';
cur_dir[0] = '\0';
strcpy(cur_dir, url_ptr);
j = strlen(cur_dir);
for (;cur_dir[j] != '/' && j != 0; j--) ;
if(j == 0)
cur_dir[j] = '\0';
else
cur_dir[j+1] = '\0';
for (i = 0; i < 3; i++){
if((clientfd = open_tcp("127.0.0.1", 80)) < 0){
close_fd(clientfd);
continue;
}
if( http_do_get(clientfd, rootdir, "127.0.0.1", url_ptr) < 0){
close_fd(clientfd);
continue;
}
if(recv_line(clientfd, temp) <= 0){
close_fd(clientfd);
continue;
}
if((status = http_response_status(temp)) == 4){
printf("%s error %d\n",url_ptr, error++);
pthread_mutex_lock(&mutex);
set_status(hash, url_ptr, 4);
set_webg_status(webg, hash_out, 4);
pthread_mutex_unlock(&mutex);
pthread_mutex_lock(&((threadpool_item *)item)->waitlock);
((threadpool_item *)item)->idle =1;
pthread_mutex_unlock(&((threadpool_item *)item)->waitlock);
close_fd(clientfd);
break;
}
buf = http_response_body(clientfd);
close_fd(clientfd);
break;
}
if (status == 4)
continue;
if(i == 3){
pthread_mutex_lock(&((threadpool_item *)item)->waitlock);
((threadpool_item *)item)->idle =1;
pthread_mutex_unlock(&((threadpool_item *)item)->waitlock);
close_fd(clientfd);
continue;
}
if (buf == NULL){
pthread_mutex_lock(&((threadpool_item *)item)->waitlock);
((threadpool_item *)item)->idle =1;
pthread_mutex_unlock(&((threadpool_item *)item)->waitlock);
continue;
}
printf("%s pages %d\n", url_ptr,pages++);
extract_link(buf, cur_dir, &url_list_head);
free(buf);
buf = NULL;
p = url_list_head->next;
p_pre = url_list_head;
while (p != NULL){
if(strcmp(url_ptr, p->url_ptr) == 0){
p_pre->next = p->next;
free(p->url_ptr);
free(p);
a++;
p = p_pre->next;
printf("a= %d, b= %d, c= %d, d= %d, e= %d, f= %d, g= %d\n", a,b,c,d,e,f,g);
continue;
}
pthread_mutex_lock(&mutex);
hash_in = has_url(hash, p->url_ptr);
if (hash_in != NULL ){
insert_edge(webg, hash_in, hash_out);
pthread_mutex_unlock(&mutex);
p_pre->next = p->next;
free(p->url_ptr);
free(p);
p = p_pre->next;
b++;
printf("a= %d, b= %d, c= %d, d= %d, e= %d, f= %d, g= %d\n", a,b,c,d,e,f,g);
continue;
}
else{
pos_found = insert_vertex(webg, hash_out, p->url_ptr);
insert_hash_item(hash, p->url_ptr, pos_found, 0);
pthread_mutex_unlock(&mutex);
c++;
p_pre = p;
p = p->next;
printf("a= %d, b= %d, c= %d, d= %d, e= %d, f= %d, g= %d\n", a,b,c,d,e,f,g);
}
}
if(p_pre != url_list_head){
pthread_mutex_lock(&mutex);
queue_push(&urlqueue, url_list_head->next, p_pre);
f++;
pthread_mutex_unlock(&mutex);
}
free(url_list_head);
p = p_pre = url_list_head = NULL;
pthread_mutex_lock(&((threadpool_item *)item)->waitlock);
((threadpool_item *)item)->idle = 1;
pthread_mutex_unlock(&((threadpool_item *)item)->waitlock);
/*printf("next time!\n");*/
}
/*printf("over!\n");*/
return NULL;
}
