int
input_queue_get(il_octet_string_t **buffer, long *offset, int timeout)
{
fd_set fds;
struct timeval tv;
int msg_len;
static il_http_message_t msg;
assert(buffer != NULL);
*buffer = (il_octet_string_t *)&msg;
FD_ZERO(&fds);
FD_SET(sock, &fds);
tv.tv_sec = timeout;
tv.tv_usec = 0;
msg_len = select(sock + 1, &fds, NULL, NULL, timeout >= 0 ? &tv : NULL);
switch(msg_len) {
case 0: /* timeout */
return(0);
case -1: /* error */
switch(errno) {
case EINTR:
il_log(LOG_DEBUG, " interrupted while waiting for event!\n");
return(0);
default:
set_error(IL_SYS, errno, "input_queue_get: error waiting for event");
return(-1);
}
default:
break;
}
if((accepted=accept(sock, NULL, NULL)) < 0) {
set_error(IL_SYS, errno, "input_queue_get: error accepting connection");
return(-1);
}
msg_len = receive_http(&accepted, plain_reader, &msg);
if(msg_len < 0) {
close(accepted);
if(error_get_maj() != IL_OK)
return -1;
else
return 0;
}
close(accepted);
*offset = -1;
return(msg.len);
}
int download_images() {
int thumb_request_fd = 0;
int image_request_fd = 0;
char *raw_thumb_resp = NULL;
char *raw_image_resp = NULL;
FILE *thumb_file = NULL;
FILE *image_file = NULL;
post_match *p_match = NULL;
struct stat st = {0};
if (stat(webm_location(), &st) == -1) {
log_msg(LOG_WARN, "Creating webms directory %s.", webm_location());
mkdir(webm_location(), 0755);
}
ol_stack *images_to_download = NULL;
images_to_download = build_thread_index();
if (!images_to_download) {
log_msg(LOG_WARN, "No images to download.");
goto error;
}
thumb_request_fd = connect_to_host(FOURCHAN_THUMBNAIL_HOST);
if (thumb_request_fd < 0) {
log_msg(LOG_ERR, "Could not connect to thumbnail host.");
goto error;
}
image_request_fd = connect_to_host(FOURCHAN_IMAGE_HOST);
if (image_request_fd < 0) {
log_msg(LOG_ERR, "Could not connect to image host.");
goto error;
}
/* Now actually download the images. */
while (images_to_download->next != NULL) {
p_match = (post_match *)spop(&images_to_download);
char image_filename[MAX_IMAGE_FILENAME_SIZE] = {0};
get_image_filename(image_filename, p_match);
/* We already have this file, don't need to download it again. */
struct stat ifname = {0};
if (stat(image_filename, &ifname) != -1 && ifname.st_size == p_match->size) {
log_msg(LOG_INFO, "Skipping %s.", image_filename);
free(p_match);
continue;
}
char thumb_filename[MAX_IMAGE_FILENAME_SIZE] = {0};
snprintf(thumb_filename, MAX_IMAGE_FILENAME_SIZE, "%s/%s/thumb_%s.jpg",
webm_location(), p_match->board, p_match->filename);
log_msg(LOG_INFO, "Downloading %s%.*s...", p_match->filename, 5, p_match->file_ext);
/* Build and send the thumbnail request. */
char thumb_request[256] = {0};
snprintf(thumb_request, sizeof(thumb_request), THUMB_REQUEST,
p_match->board, p_match->post_number);
int rc = send(thumb_request_fd, thumb_request, strlen(thumb_request), 0);
if (rc != strlen(thumb_request)) {
log_msg(LOG_ERR, "Could not send all bytes to host while requesting thumbnail.");
goto error;
}
/* Build and send the image request. */
char image_request[256] = {0};
snprintf(image_request, sizeof(image_request), IMAGE_REQUEST,
p_match->board, p_match->post_number, (int)sizeof(p_match->file_ext), p_match->file_ext);
rc = send(image_request_fd, image_request, strlen(image_request), 0);
if (rc != strlen(image_request)) {
log_msg(LOG_ERR, "Could not send all bytes to host while requesting image.");
goto error;
}
size_t thumb_size = 0, image_size = 0;
raw_thumb_resp = receive_http(thumb_request_fd, &thumb_size);
raw_image_resp = receive_http(image_request_fd, &image_size);
if (thumb_size <= 0 || image_size <= 0) {
/* 4chan cut us off. This happens sometimes. Just sleep for a bit. */
log_msg(LOG_WARN, "Hit API cutoff or whatever. Sleeping.");
sleep(30);
close(thumb_request_fd);
close(image_request_fd);
thumb_request_fd = connect_to_host(FOURCHAN_THUMBNAIL_HOST);
image_request_fd = connect_to_host(FOURCHAN_IMAGE_HOST);
free(p_match);
continue;
}
if (raw_thumb_resp == NULL) {
log_msg(LOG_ERR, "No thumbnail received.");
goto error;
}
if (raw_image_resp == NULL) {
log_msg(LOG_ERR, "No image received.");
goto error;
}
/* Write thumbnail to disk. */
thumb_file = fopen(thumb_filename, "wb");
if (!thumb_file || ferror(thumb_file)) {
log_msg(LOG_ERR, "Could not open thumbnail file: %s", thumb_filename);
perror(NULL);
goto error;
}
const size_t rt_written = fwrite(raw_thumb_resp, 1, thumb_size, thumb_file);
log_msg(LOG_INFO, "Wrote %i bytes of thumbnail to disk.", rt_written);
if (rt_written <= 0 || ferror(thumb_file)) {
log_msg(LOG_WARN, "Could not write thumbnail to disk: %s", thumb_filename);
perror(NULL);
goto error;
}
fclose(thumb_file);
thumb_file = 0;
image_file = fopen(image_filename, "wb");
if (!image_file || ferror(image_file)) {
log_msg(LOG_ERR, "Could not open image file: %s", image_filename);
perror(NULL);
goto error;
}
const size_t iwritten = fwrite(raw_image_resp, 1, image_size, image_file);
if (iwritten <= 0 || ferror(image_file)) {
log_msg(LOG_WARN, "Could not write image to disk: %s", image_filename);
perror(NULL);
goto error;
}
log_msg(LOG_INFO, "Wrote %i bytes of image to disk.", iwritten);
fclose(image_file);
image_file = 0;
/* Don't need the post match anymore: */
free(p_match);
p_match = NULL;
free(raw_image_resp);
raw_image_resp = NULL;
free(raw_thumb_resp);
raw_thumb_resp = NULL;
}
free(images_to_download);
close(thumb_request_fd);
close(image_request_fd);
log_msg(LOG_INFO, "Downloaded all images.");
return 0;
error:
if (thumb_request_fd)
close(thumb_request_fd);
if (image_request_fd)
close(image_request_fd);
if (p_match)
free(p_match);
if (images_to_download != NULL) {
while (images_to_download->next != NULL) {
post_match *_match = (post_match *)spop(&images_to_download);
free(_match);
}
free(images_to_download);
}
if (raw_thumb_resp)
free(raw_thumb_resp);
if (raw_image_resp)
free(raw_image_resp);
if (thumb_file != NULL)
fclose(thumb_file);
if (image_file != NULL)
fclose(image_file);
return -1;
}
