コード例 #1
0
ファイル: worker.c プロジェクト: tohojo/http-getter
static int run_worker(struct worker_data *data)
{
	char buf[PIPE_BUF+1] = {};
	char outbuf[PIPE_BUF+1] = {};
	char *urls[MAX_URLS];
	size_t urls_c;
	char *p;
	int res, i;
	ssize_t len;
	double bytes;
	long header_bytes;
	if(init_worker(data)) return -1;

	while(1)
	{
		if((len = msg_read(data->pipe_r, buf, sizeof(buf))) == -1) {
			return EXIT_FAILURE;
		}
		buf[len] = '\0';
		if(strncmp(buf, "STOP", 4) == 0 || len == 0) return destroy_worker(data);
		if(strncmp(buf, "RESET", 5) == 0) {
			if((res = reset_worker(data)) != 0) {
				len = sprintf(outbuf, "ERR %d", res);
				msg_write(data->pipe_w, outbuf, len);
			} else {
				msg_write(data->pipe_w, "OK", sizeof("OK"));
			}
			continue;
		}
		if(strncmp(buf, "URLLIST ", 8) == 0) {
			p = buf + 8;
			data->chunk.enabled = 1;
		} else if(strncmp(buf, "URL ", 4) == 0) {
			p = buf + 4;
		} else {
			fprintf(stderr, "Unrecognised command '%s'!\n", buf);
			break;
		}

		if(data->debug) {
			fprintf(stderr, "Getting URL '%s'.\n", p);
		}

		curl_easy_setopt(data->curl, CURLOPT_URL, p);
		data->chunk.size = 0;
		if((res = curl_easy_perform(data->curl)) != CURLE_OK) {
			len = sprintf(outbuf, "ERR %d", res);
			msg_write(data->pipe_w, outbuf, len);
		} else {
			if((res = curl_easy_getinfo(data->curl, CURLINFO_SIZE_DOWNLOAD, &bytes)) != CURLE_OK ||
				(res = curl_easy_getinfo(data->curl, CURLINFO_HEADER_SIZE, &header_bytes)) != CURLE_OK ) {
				fprintf(stderr, "cURL error: %s\n", curl_easy_strerror(res));
			}
			if(data->chunk.enabled == 0) {
				len = sprintf(outbuf, "OK %lu bytes", (long)bytes + header_bytes);
				msg_write(data->pipe_w, outbuf, len);
			} else {
				urls_c = parse_urls(data->chunk.memory, data->chunk.size, urls, MAX_URLS);
				len = sprintf(outbuf, "OK %lu bytes %lu urls", (long)bytes + header_bytes, (long)urls_c);
				msg_write(data->pipe_w, outbuf, len);
				for(i = 0; i < urls_c; i++) {
					len = sprintf(outbuf, "%s", urls[i]);
					msg_write(data->pipe_w, outbuf, len);
					free(urls[i]);
				}
				data->chunk.enabled = 0;
			}
		}
	}
	return 0;
}
コード例 #2
0
ファイル: buffer_array.c プロジェクト: henrycg/mulk
static mulk_type_return_t filter_buffer(int i, int valid_res, const char *base_url, long err_code, long resp_code)
{
	char *newfilename = NULL;
	char *newmimefilename = NULL;
	char *subtype = NULL;
	char *type = NULL;
	mulk_type_return_t ret = MULK_RET_OK;
	buffer_t *buffer = buffer_array + i;

#ifdef ENABLE_METALINK
	metalink_file_list_t *metalink = buffer->url->metalink_uri;

	if (metalink)
		buffer->url->http_code = 0;
	else
#endif
	{
		buffer->url->err_code = err_code;
		buffer->url->http_code = resp_code;
	}

	if (!valid_res) {
#ifdef ENABLE_METALINK
		if (metalink)
			buffer->url->err_code = METALINK_RES_NO_NORE_RESOURCES;
#endif
		remove(buffer->filename);
		return MULK_RET_ERR;
	}

#ifdef ENABLE_RECURSION
	if (is_html_file(buffer->url->mimetype)) 
		parse_urls(buffer->filename, base_url, buffer->url->level);
#endif

#ifdef ENABLE_METALINK
	if (option_values.follow_metalink && is_metalink_file(buffer->url->mimetype)) 
		add_new_metalink(buffer->filename, buffer->url->level);

	if (metalink) {
#ifdef ENABLE_CHECKSUM
		if (verify_metalink_file(metalink->file, buffer->filename) == CS_VERIFY_ERR) {
			MULK_NOTE((_("The file will be deleted.\n")));
			buffer->url->err_code = METALINK_RES_WRONG_CHECKSUM;

			ret = MULK_RET_ERR;
		}
		else
#endif /* ENABLE_CHECKSUM */
		{
			string_printf(&newfilename, "%s%s", option_values.file_output_directory,
				metalink->file->name);
			buffer->url->err_code = METALINK_RES_OK;
		}
	}
	else 
#endif /* ENABLE_METALINK */
	if (!option_values.disable_save_tree) {
		UriUriA *uri = create_absolute_uri(NULL, base_url);
		char *furi_str = uri2filename(uri);

		if (furi_str) {
			string_printf(&newfilename, "%s%s", option_values.file_output_directory, furi_str);

			/* add index.<mime-type> if the filename represents a directory name */
			if (newfilename[strlen(newfilename)-1] == *DIR_SEPAR_STR) {
				string_cat(&newfilename, "index.");
				if (extract_mime_type(buffer->url->mimetype, NULL, &subtype) == MULK_RET_OK) {
					string_cat(&newfilename, subtype);
					string_free(&subtype);
				} else
					string_cat(&newfilename, "bin");
			}
		}

		string_free(&furi_str);
		uri_free(uri);
	}

	if ((is_gif_image(buffer->url->mimetype) && is_valid_gif_image(buffer->filename))
		|| (is_png_image(buffer->url->mimetype) && is_valid_png_image(buffer->filename)) 
		|| (is_jpeg_image(buffer->url->mimetype) && is_valid_jpeg_image(buffer->filename))
		|| (is_saved_mime_type(buffer->url->mimetype))) {
		if (extract_mime_type(buffer->url->mimetype, &type, &subtype) == MULK_RET_OK) {
			string_printf(&newmimefilename, "%s%s%s%s_%05d.%s", option_values.mime_output_directory,
				type, DIR_SEPAR_STR, subtype, buffer->url->id, subtype);
			string_free(&type);
			string_free(&subtype);
		}
	}

	/* save file */
	buffer->url->filename = string_new(newfilename);
	buffer->url->mimefilename = string_new(newmimefilename);

	if (newfilename && newmimefilename) {
		if ((ret = save_file_to_outputdir(buffer->filename, newfilename, 1)) == MULK_RET_OK)
			ret = save_file_to_outputdir(buffer->filename, newmimefilename, 0);
	}
	else if (newfilename) 
		ret = save_file_to_outputdir(buffer->filename, newfilename, 0);
	else if (newmimefilename) 
		ret = save_file_to_outputdir(buffer->filename, newmimefilename, 0);
	else
		remove(buffer->filename);

	string_free(&newmimefilename);
	string_free(&newfilename);

	return ret;
}