GByteArray *
fb_util_zlib_deflate(const GByteArray *bytes, GError **error)
{
GByteArray *ret;
GZlibCompressor *conv;
conv = g_zlib_compressor_new(G_ZLIB_COMPRESSOR_FORMAT_ZLIB, -1);
ret = fb_util_zlib_conv(G_CONVERTER(conv), bytes, error);
g_object_unref(conv);
return ret;
}
static int
compress_blob_to_tmp (GInputStream *file_stream)
{
char tmpl[] = "/tmp/shardXXXXXX";
int fd = mkstemp (tmpl);
unlink (tmpl);
/* Compress the given GInputStream to the tmpfile. */
g_autoptr(GZlibCompressor) compressor = g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_ZLIB, -1);
g_autoptr(GInputStream) stream = g_converter_input_stream_new (G_INPUT_STREAM (file_stream), G_CONVERTER (compressor));
uint8_t buf[4096*4];
int size;
while ((size = g_input_stream_read (stream, buf, sizeof (buf), NULL, NULL)) != 0) {
g_assert (write (fd, buf, size) >= 0);
}
return fd;
}
static void
write_blob (int fd, struct eos_shard_writer_v1_blob_entry *blob)
{
g_autoptr(GError) error = NULL;
if (!blob->file)
return;
GFileInputStream *file_stream = g_file_read (blob->file, NULL, &error);
if (!file_stream) {
g_error ("Could not read from %s: %s", g_file_get_path (blob->file), error->message);
return;
}
g_autoptr(GInputStream) stream;
if (blob->flags & EOS_SHARD_BLOB_FLAG_COMPRESSED_ZLIB) {
g_autoptr(GZlibCompressor) compressor = g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_ZLIB, -1);
stream = g_converter_input_stream_new (G_INPUT_STREAM (file_stream), G_CONVERTER (compressor));
g_object_unref (file_stream);
} else {
stream = G_INPUT_STREAM (file_stream);
}
blob->offs = lalign (fd);
uint8_t buf[4096*4];
int size, total_size = 0;
g_autoptr(GChecksum) checksum = g_checksum_new (G_CHECKSUM_SHA256);
while ((size = g_input_stream_read (stream, buf, sizeof (buf), NULL, NULL)) != 0) {
g_assert (write (fd, buf, size) >= 0);
g_checksum_update (checksum, buf, size);
total_size += size;
}
size_t checksum_buf_len = sizeof (blob->checksum);
g_checksum_get_digest (checksum, blob->checksum, &checksum_buf_len);
g_assert (checksum_buf_len == sizeof (blob->checksum));
blob->size = total_size;
}
/*
* emer_gzip_compress:
* @input_data: the data to compress.
* @input_length: the length of the data to compress in bytes.
* @compressed_length: (out): the length of the compressed data.
* @error: (out) (optional): if compression failed, error will be set to a GError
* describing the failure; otherwise it won't be modified. Pass NULL to ignore
* this value.
*
* Compresses input_data with the gzip algorithm at compression level 9. Returns
* NULL and sets error if compression fails. Sets compressed_length to the
* length of the compressed data in bytes.
*
* Returns: the compressed data or NULL if compression fails. Free with g_free.
*/
gpointer
emer_gzip_compress (gconstpointer input_data,
gsize input_length,
gsize *compressed_length,
GError **error)
{
GZlibCompressor *zlib_compressor =
g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP, COMPRESSION_LEVEL);
GConverter *converter = G_CONVERTER (zlib_compressor);
gsize allocated_space = input_length + 1;
GByteArray *byte_array = g_byte_array_sized_new (allocated_space);
gsize total_bytes_read = 0;
gsize total_bytes_written = 0;
while (TRUE)
{
gsize bytes_left_in_buffer = allocated_space - total_bytes_written;
if (bytes_left_in_buffer == 0)
{
allocated_space *= 2;
g_byte_array_set_size (byte_array, allocated_space);
continue;
}
gsize bytes_left_in_input = input_length - total_bytes_read;
GConverterFlags conversion_flags = bytes_left_in_input > 0 ?
G_CONVERTER_NO_FLAGS : G_CONVERTER_INPUT_AT_END;
guint8 *curr_output = byte_array->data + total_bytes_written;
const guint8 *curr_input =
((const guint8 *) input_data) + total_bytes_read;
gsize curr_bytes_written, curr_bytes_read;
GError *local_error = NULL;
GConverterResult conversion_result =
g_converter_convert (converter,
curr_input, bytes_left_in_input,
curr_output, bytes_left_in_buffer,
conversion_flags,
&curr_bytes_read, &curr_bytes_written,
&local_error);
if (conversion_result == G_CONVERTER_ERROR)
{
if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NO_SPACE))
{
g_error_free (local_error);
allocated_space *= 2;
g_byte_array_set_size (byte_array, allocated_space);
continue;
}
g_object_unref (zlib_compressor);
g_byte_array_free (byte_array, TRUE);
g_propagate_error (error, local_error);
return NULL;
}
total_bytes_read += curr_bytes_read;
total_bytes_written += curr_bytes_written;
if (conversion_result == G_CONVERTER_FINISHED)
break;
/* Expand the byte array. */
allocated_space *= 2;
g_byte_array_set_size (byte_array, allocated_space);
}
g_object_unref (zlib_compressor);
gpointer compressed_data = g_memdup (byte_array->data, total_bytes_written);
g_byte_array_free (byte_array, TRUE);
*compressed_length = total_bytes_written;
return compressed_data;
}
static void
end_element (GMarkupParseContext *context,
const gchar *element_name,
gpointer user_data,
GError **error)
{
ParseState *state = user_data;
GError *my_error = NULL;
if (strcmp (element_name, "gresource") == 0)
{
g_free (state->prefix);
state->prefix = NULL;
}
else if (strcmp (element_name, "file") == 0)
{
gchar *file;
gchar *real_file = NULL;
gchar *key;
FileData *data = NULL;
char *tmp_file = NULL;
char *tmp_file2 = NULL;
file = state->string->str;
key = file;
if (state->alias)
key = state->alias;
if (state->prefix)
key = g_build_path ("/", "/", state->prefix, key, NULL);
else
key = g_build_path ("/", "/", key, NULL);
if (g_hash_table_lookup (state->table, key) != NULL)
{
g_set_error (error, G_MARKUP_ERROR, G_MARKUP_ERROR_INVALID_CONTENT,
_("File %s appears multiple times in the resource"),
key);
return;
}
if (sourcedirs != NULL)
{
real_file = find_file (file);
if (real_file == NULL && state->collect_data)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
_("Failed to locate “%s” in any source directory"), file);
return;
}
}
else
{
gboolean exists;
exists = g_file_test (file, G_FILE_TEST_EXISTS);
if (!exists && state->collect_data)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
_("Failed to locate “%s” in current directory"), file);
return;
}
}
if (real_file == NULL)
real_file = g_strdup (file);
data = g_new0 (FileData, 1);
data->filename = g_strdup (real_file);
if (!state->collect_data)
goto done;
if (state->preproc_options)
{
gchar **options;
guint i;
gboolean xml_stripblanks = FALSE;
gboolean to_pixdata = FALSE;
options = g_strsplit (state->preproc_options, ",", -1);
for (i = 0; options[i]; i++)
{
if (!strcmp (options[i], "xml-stripblanks"))
xml_stripblanks = TRUE;
else if (!strcmp (options[i], "to-pixdata"))
to_pixdata = TRUE;
else
{
g_set_error (error, G_MARKUP_ERROR, G_MARKUP_ERROR_INVALID_CONTENT,
_("Unknown processing option “%s”"), options[i]);
g_strfreev (options);
goto cleanup;
}
}
g_strfreev (options);
if (xml_stripblanks && xmllint != NULL)
{
int fd;
GSubprocess *proc;
tmp_file = g_strdup ("resource-XXXXXXXX");
if ((fd = g_mkstemp (tmp_file)) == -1)
{
int errsv = errno;
g_set_error (error, G_IO_ERROR, g_io_error_from_errno (errsv),
_("Failed to create temp file: %s"),
g_strerror (errsv));
g_free (tmp_file);
tmp_file = NULL;
goto cleanup;
}
close (fd);
proc = g_subprocess_new (G_SUBPROCESS_FLAGS_STDOUT_SILENCE, error,
xmllint, "--nonet", "--noblanks", "--output", tmp_file, real_file, NULL);
g_free (real_file);
real_file = NULL;
if (!proc)
goto cleanup;
if (!g_subprocess_wait_check (proc, NULL, error))
{
g_object_unref (proc);
goto cleanup;
}
g_object_unref (proc);
real_file = g_strdup (tmp_file);
}
if (to_pixdata)
{
int fd;
GSubprocess *proc;
if (gdk_pixbuf_pixdata == NULL)
{
g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"to-pixbuf preprocessing requested but GDK_PIXBUF_PIXDATA "
"not set and gdk-pixbuf-pixdata not found in path");
goto cleanup;
}
tmp_file2 = g_strdup ("resource-XXXXXXXX");
if ((fd = g_mkstemp (tmp_file2)) == -1)
{
int errsv = errno;
g_set_error (error, G_IO_ERROR, g_io_error_from_errno (errsv),
_("Failed to create temp file: %s"),
g_strerror (errsv));
g_free (tmp_file2);
tmp_file2 = NULL;
goto cleanup;
}
close (fd);
proc = g_subprocess_new (G_SUBPROCESS_FLAGS_STDOUT_SILENCE, error,
gdk_pixbuf_pixdata, real_file, tmp_file2, NULL);
g_free (real_file);
real_file = NULL;
if (!g_subprocess_wait_check (proc, NULL, error))
{
g_object_unref (proc);
goto cleanup;
}
g_object_unref (proc);
real_file = g_strdup (tmp_file2);
}
}
if (!g_file_get_contents (real_file, &data->content, &data->size, &my_error))
{
g_set_error (error, G_MARKUP_ERROR, G_MARKUP_ERROR_INVALID_CONTENT,
_("Error reading file %s: %s"),
real_file, my_error->message);
g_clear_error (&my_error);
goto cleanup;
}
/* Include zero termination in content_size for uncompressed files (but not in size) */
data->content_size = data->size + 1;
if (state->compressed)
{
GOutputStream *out = g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
GZlibCompressor *compressor =
g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_ZLIB, 9);
GOutputStream *out2 = g_converter_output_stream_new (out, G_CONVERTER (compressor));
if (!g_output_stream_write_all (out2, data->content, data->size,
NULL, NULL, NULL) ||
!g_output_stream_close (out2, NULL, NULL))
{
g_set_error (error, G_MARKUP_ERROR, G_MARKUP_ERROR_INVALID_CONTENT,
_("Error compressing file %s"),
real_file);
goto cleanup;
}
g_free (data->content);
data->content_size = g_memory_output_stream_get_size (G_MEMORY_OUTPUT_STREAM (out));
data->content = g_memory_output_stream_steal_data (G_MEMORY_OUTPUT_STREAM (out));
g_object_unref (compressor);
g_object_unref (out);
g_object_unref (out2);
data->flags |= G_RESOURCE_FLAGS_COMPRESSED;
}
done:
g_hash_table_insert (state->table, key, data);
data = NULL;
cleanup:
/* Cleanup */
g_free (state->alias);
state->alias = NULL;
g_string_free (state->string, TRUE);
state->string = NULL;
g_free (state->preproc_options);
state->preproc_options = NULL;
g_free (real_file);
if (tmp_file)
{
unlink (tmp_file);
g_free (tmp_file);
}
if (tmp_file2)
{
unlink (tmp_file2);
g_free (tmp_file2);
}
if (data != NULL)
file_data_free (data);
}
}
static void
end_element (GMarkupParseContext *context,
const gchar *element_name,
gpointer user_data,
GError **error)
{
ParseState *state = user_data;
GError *my_error = NULL;
if (strcmp (element_name, "gresource") == 0)
{
g_free (state->prefix);
state->prefix = NULL;
}
else if (strcmp (element_name, "file") == 0)
{
gchar *file, *real_file;
gchar *key;
FileData *data;
char *tmp_file = NULL;
char *tmp_file2 = NULL;
file = state->string->str;
key = file;
if (state->alias)
key = state->alias;
if (state->prefix)
key = g_build_path ("/", "/", state->prefix, key, NULL);
else
key = g_build_path ("/", "/", key, NULL);
if (g_hash_table_lookup (state->table, key) != NULL)
{
g_set_error (error, G_MARKUP_ERROR, G_MARKUP_ERROR_INVALID_CONTENT,
_("File %s appears multiple times in the resource"),
key);
return;
}
data = g_new0 (FileData, 1);
if (sourcedirs != NULL)
{
real_file = find_file (file);
if (real_file == NULL)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
_("Failed to locate '%s' in any source directory"), file);
return;
}
}
else
{
gboolean exists;
exists = g_file_test (file, G_FILE_TEST_EXISTS);
if (!exists)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
_("Failed to locate '%s' in current directory"), file);
return;
}
real_file = g_strdup (file);
}
data->filename = g_strdup (real_file);
if (!state->collect_data)
goto done;
if (state->preproc_options)
{
gchar **options;
guint i;
gboolean xml_stripblanks = FALSE;
gboolean to_pixdata = FALSE;
options = g_strsplit (state->preproc_options, ",", -1);
for (i = 0; options[i]; i++)
{
if (!strcmp (options[i], "xml-stripblanks"))
xml_stripblanks = TRUE;
else if (!strcmp (options[i], "to-pixdata"))
to_pixdata = TRUE;
else
{
g_set_error (error, G_MARKUP_ERROR, G_MARKUP_ERROR_INVALID_CONTENT,
_("Unknown processing option \"%s\""), options[i]);
g_strfreev (options);
goto cleanup;
}
}
g_strfreev (options);
if (xml_stripblanks && xmllint != NULL)
{
gchar *argv[8];
int status, fd, argc;
tmp_file = g_strdup ("resource-XXXXXXXX");
if ((fd = g_mkstemp (tmp_file)) == -1)
{
int errsv = errno;
g_set_error (error, G_IO_ERROR, g_io_error_from_errno (errsv),
_("Failed to create temp file: %s"),
g_strerror (errsv));
g_free (tmp_file);
tmp_file = NULL;
goto cleanup;
}
close (fd);
argc = 0;
argv[argc++] = (gchar *) xmllint;
argv[argc++] = "--nonet";
argv[argc++] = "--noblanks";
argv[argc++] = "--output";
argv[argc++] = tmp_file;
argv[argc++] = real_file;
argv[argc++] = NULL;
g_assert (argc <= G_N_ELEMENTS (argv));
if (!g_spawn_sync (NULL /* cwd */, argv, NULL /* envv */,
G_SPAWN_STDOUT_TO_DEV_NULL |
G_SPAWN_STDERR_TO_DEV_NULL,
NULL, NULL, NULL, NULL, &status, &my_error))
{
g_propagate_error (error, my_error);
goto cleanup;
}
#ifdef HAVE_SYS_WAIT_H
if (!WIFEXITED (status) || WEXITSTATUS (status) != 0)
{
g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
_("Error processing input file with xmllint"));
goto cleanup;
}
#endif
g_free (real_file);
real_file = g_strdup (tmp_file);
}
if (to_pixdata)
{
gchar *argv[4];
int status, fd, argc;
if (gdk_pixbuf_pixdata == NULL)
{
g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"to-pixbuf preprocessing requested but GDK_PIXBUF_PIXDATA "
"not set and gdk-pixbuf-pixdata not found in path");
goto cleanup;
}
tmp_file2 = g_strdup ("resource-XXXXXXXX");
if ((fd = g_mkstemp (tmp_file2)) == -1)
{
int errsv = errno;
g_set_error (error, G_IO_ERROR, g_io_error_from_errno (errsv),
_("Failed to create temp file: %s"),
g_strerror (errsv));
g_free (tmp_file2);
tmp_file2 = NULL;
goto cleanup;
}
close (fd);
argc = 0;
argv[argc++] = (gchar *) gdk_pixbuf_pixdata;
argv[argc++] = real_file;
argv[argc++] = tmp_file2;
argv[argc++] = NULL;
g_assert (argc <= G_N_ELEMENTS (argv));
if (!g_spawn_sync (NULL /* cwd */, argv, NULL /* envv */,
G_SPAWN_STDOUT_TO_DEV_NULL |
G_SPAWN_STDERR_TO_DEV_NULL,
NULL, NULL, NULL, NULL, &status, &my_error))
{
g_propagate_error (error, my_error);
goto cleanup;
}
#ifdef HAVE_SYS_WAIT_H
if (!WIFEXITED (status) || WEXITSTATUS (status) != 0)
{
g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
_("Error processing input file with to-pixdata"));
goto cleanup;
}
#endif
g_free (real_file);
real_file = g_strdup (tmp_file2);
}
}
if (!g_file_get_contents (real_file, &data->content, &data->size, &my_error))
{
g_set_error (error, G_MARKUP_ERROR, G_MARKUP_ERROR_INVALID_CONTENT,
_("Error reading file %s: %s"),
real_file, my_error->message);
g_clear_error (&my_error);
goto cleanup;
}
/* Include zero termination in content_size for uncompressed files (but not in size) */
data->content_size = data->size + 1;
if (state->compressed)
{
GOutputStream *out = g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
GZlibCompressor *compressor =
g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_ZLIB, 9);
GOutputStream *out2 = g_converter_output_stream_new (out, G_CONVERTER (compressor));
if (!g_output_stream_write_all (out2, data->content, data->size,
NULL, NULL, NULL) ||
!g_output_stream_close (out2, NULL, NULL))
{
g_set_error (error, G_MARKUP_ERROR, G_MARKUP_ERROR_INVALID_CONTENT,
_("Error compressing file %s"),
real_file);
goto cleanup;
}
g_free (data->content);
data->content_size = g_memory_output_stream_get_size (G_MEMORY_OUTPUT_STREAM (out));
data->content = g_memory_output_stream_steal_data (G_MEMORY_OUTPUT_STREAM (out));
g_object_unref (compressor);
g_object_unref (out);
g_object_unref (out2);
data->flags |= G_RESOURCE_FLAGS_COMPRESSED;
}
done:
g_hash_table_insert (state->table, key, data);
cleanup:
/* Cleanup */
g_free (state->alias);
state->alias = NULL;
g_string_free (state->string, TRUE);
state->string = NULL;
g_free (state->preproc_options);
state->preproc_options = NULL;
g_free (real_file);
if (tmp_file)
{
unlink (tmp_file);
g_free (tmp_file);
}
if (tmp_file2)
{
unlink (tmp_file2);
g_free (tmp_file2);
}
}
}
