static gboolean
cmd_options_to_task(GSList **modifyrepotasks,
RawCmdOptions *options,
gchar *metadatapath,
GError **err)
{
assert(modifyrepotasks);
assert(!err || *err == NULL);
if (!options)
return TRUE;
//assert(metadatapath || options->remove);
if (options->remove)
g_debug("Preparing remove-task for: %s", options->remove);
else
g_debug("Preparing task for: %s", metadatapath);
if (metadatapath && !g_file_test(metadatapath, G_FILE_TEST_IS_REGULAR)) {
g_set_error(err, ERR_DOMAIN, CRE_ERROR,
"File \"%s\" is not regular file or doesn't exists",
metadatapath);
return FALSE;
}
if (options->remove)
metadatapath = options->remove;
cr_ModifyRepoTask *task = cr_modifyrepotask_new();
task->path = cr_safe_string_chunk_insert_null(task->chunk, metadatapath);
task->type = cr_safe_string_chunk_insert_null(task->chunk, options->mdtype);
task->remove = (options->remove) ? TRUE : FALSE;
task->compress = options->compress;
task->compress_type = cr_compression_type(options->compress_type);
task->unique_md_filenames = options->unique_md_filenames;
task->checksum_type = cr_checksum_type(options->checksum);
task->new_name = cr_safe_string_chunk_insert_null(task->chunk,
options->new_name);
task->zck = options->zck;
task->zck_dict_dir = options->zck_dict_dir;
*modifyrepotasks = g_slist_append(*modifyrepotasks, task);
g_debug("Task: [path: %s, type: %s, remove: %d, compress: %d, "
"compress_type: %d (%s), unique_md_filenames: %d, "
"checksum_type: %d (%s), new_name: %s]",
task->path, task->type, task->remove, task->compress,
task->compress_type, cr_compression_suffix(task->compress_type),
task->unique_md_filenames, task->checksum_type,
cr_checksum_name_str(task->checksum_type), task->new_name);
return TRUE;
}
static void
test_cr_compression_suffix(void)
{
const char *suffix;
suffix = cr_compression_suffix(CR_CW_AUTO_DETECT_COMPRESSION);
g_assert(!suffix);
suffix = cr_compression_suffix(CR_CW_UNKNOWN_COMPRESSION);
g_assert(!suffix);
suffix = cr_compression_suffix(CR_CW_NO_COMPRESSION);
g_assert(!suffix);
suffix = cr_compression_suffix(CR_CW_GZ_COMPRESSION);
g_assert_cmpstr(suffix, ==, ".gz");
suffix = cr_compression_suffix(CR_CW_BZ2_COMPRESSION);
g_assert_cmpstr(suffix, ==, ".bz2");
suffix = cr_compression_suffix(CR_CW_XZ_COMPRESSION);
g_assert_cmpstr(suffix, ==, ".xz");
}
static gboolean
compress_sqlite_dbs(const gchar *tmp_out_repo,
const gchar *pri_db_filename,
cr_RepomdRecord **in_pri_db_rec,
const gchar *fil_db_filename,
cr_RepomdRecord **in_fil_db_rec,
const gchar *oth_db_filename,
cr_RepomdRecord **in_oth_db_rec,
cr_CompressionType compression_type,
cr_ChecksumType checksum_type,
GError **err)
{
cr_CompressionTask *pri_db_task;
cr_CompressionTask *fil_db_task;
cr_CompressionTask *oth_db_task;
const char *sqlite_compression_suffix;
cr_RepomdRecord *pri_db_rec = NULL;
cr_RepomdRecord *fil_db_rec = NULL;
cr_RepomdRecord *oth_db_rec = NULL;
// Prepare thread pool for compression tasks
GThreadPool *compress_pool = g_thread_pool_new(cr_compressing_thread,
NULL, 3, FALSE, NULL);
// Prepare output filenames
sqlite_compression_suffix = cr_compression_suffix(compression_type);
gchar *pri_db_name = g_strconcat(tmp_out_repo, "/primary.sqlite",
sqlite_compression_suffix, NULL);
gchar *fil_db_name = g_strconcat(tmp_out_repo, "/filelists.sqlite",
sqlite_compression_suffix, NULL);
gchar *oth_db_name = g_strconcat(tmp_out_repo, "/other.sqlite",
sqlite_compression_suffix, NULL);
// Prepare compression tasks
pri_db_task = cr_compressiontask_new(pri_db_filename,
pri_db_name,
compression_type,
checksum_type,
1, NULL);
g_thread_pool_push(compress_pool, pri_db_task, NULL);
fil_db_task = cr_compressiontask_new(fil_db_filename,
fil_db_name,
compression_type,
checksum_type,
1, NULL);
g_thread_pool_push(compress_pool, fil_db_task, NULL);
oth_db_task = cr_compressiontask_new(oth_db_filename,
oth_db_name,
compression_type,
checksum_type,
1, NULL);
g_thread_pool_push(compress_pool, oth_db_task, NULL);
// Wait till all tasks are complete and free the thread pool
g_thread_pool_free(compress_pool, FALSE, TRUE);
// Remove uncompressed DBs
cr_rm(pri_db_filename, CR_RM_FORCE, NULL, NULL);
cr_rm(fil_db_filename, CR_RM_FORCE, NULL, NULL);
cr_rm(oth_db_filename, CR_RM_FORCE, NULL, NULL);
// Prepare repomd records
pri_db_rec = cr_repomd_record_new("primary_db", pri_db_name);
fil_db_rec = cr_repomd_record_new("filelists_db", fil_db_name);
oth_db_rec = cr_repomd_record_new("other_db", oth_db_name);
*in_pri_db_rec = pri_db_rec;
*in_fil_db_rec = fil_db_rec;
*in_oth_db_rec = oth_db_rec;
// Free paths to compressed files
g_free(pri_db_name);
g_free(fil_db_name);
g_free(oth_db_name);
// Fill repomd records from stats gathered during compression
cr_repomd_record_load_contentstat(pri_db_rec, pri_db_task->stat);
cr_repomd_record_load_contentstat(fil_db_rec, fil_db_task->stat);
cr_repomd_record_load_contentstat(oth_db_rec, oth_db_task->stat);
// Free the compression tasks
cr_compressiontask_free(pri_db_task, NULL);
cr_compressiontask_free(fil_db_task, NULL);
cr_compressiontask_free(oth_db_task, NULL);
// Prepare thread pool for repomd record filling tasks
GThreadPool *fill_pool = g_thread_pool_new(cr_repomd_record_fill_thread,
NULL, 3, FALSE, NULL);
// Prepare the tasks themselves
cr_RepomdRecordFillTask *pri_db_fill_task;
cr_RepomdRecordFillTask *fil_db_fill_task;
cr_RepomdRecordFillTask *oth_db_fill_task;
pri_db_fill_task = cr_repomdrecordfilltask_new(pri_db_rec,
checksum_type,
NULL);
g_thread_pool_push(fill_pool, pri_db_fill_task, NULL);
fil_db_fill_task = cr_repomdrecordfilltask_new(fil_db_rec,
checksum_type,
NULL);
g_thread_pool_push(fill_pool, fil_db_fill_task, NULL);
oth_db_fill_task = cr_repomdrecordfilltask_new(oth_db_rec,
checksum_type,
NULL);
g_thread_pool_push(fill_pool, oth_db_fill_task, NULL);
// Wait till the all tasks are finished and free the pool
g_thread_pool_free(fill_pool, FALSE, TRUE);
// Clear the tasks
cr_repomdrecordfilltask_free(pri_db_fill_task, NULL);
cr_repomdrecordfilltask_free(fil_db_fill_task, NULL);
cr_repomdrecordfilltask_free(oth_db_fill_task, NULL);
return TRUE;
}
int
cr_repomd_record_compress_and_fill(cr_RepomdRecord *record,
cr_RepomdRecord *crecord,
cr_ChecksumType checksum_type,
cr_CompressionType record_compression,
GError **err)
{
const char *suffix;
gchar *path, *cpath;
gchar *clocation_real, *clocation_href;
gchar *checksum, *cchecksum;
int readed;
char buf[BUFFER_SIZE];
CR_FILE *cw_plain;
CR_FILE *cw_compressed;
gint64 gf_size = -1, cgf_size = -1;
gint64 gf_time = -1, cgf_time = -1;
struct stat gf_stat, cgf_stat;
const char *checksum_str = cr_checksum_name_str(checksum_type);
GError *tmp_err = NULL;
assert(record);
assert(crecord);
assert(!err || *err == NULL);
if (!(record->location_real) || !strlen(record->location_real)) {
g_set_error(err, CR_REPOMD_RECORD_ERROR, CRE_BADARG,
"Empty locations in repomd record object");
return CRE_BADARG;
}
if (!g_file_test(record->location_real, G_FILE_TEST_IS_REGULAR)) {
// File doesn't exists
g_warning("%s: File %s doesn't exists", __func__, record->location_real);
g_set_error(err, CR_REPOMD_RECORD_ERROR, CRE_NOFILE,
"File %s doesn't exists or not a regular file",
record->location_real);
return CRE_NOFILE;;
}
// Paths
suffix = cr_compression_suffix(record_compression);
clocation_real = g_strconcat(record->location_real, suffix, NULL);
clocation_href = g_strconcat(record->location_href, suffix, NULL);
crecord->location_real = g_string_chunk_insert(crecord->chunk,
clocation_real);
crecord->location_href = g_string_chunk_insert(crecord->chunk,
clocation_href);
g_free(clocation_real);
g_free(clocation_href);
path = record->location_real;
cpath = crecord->location_real;
// Compress file + get size of non compressed file
cw_plain = cr_open(path,
CR_CW_MODE_READ,
CR_CW_NO_COMPRESSION,
&tmp_err);
if (!cw_plain) {
int code = tmp_err->code;
g_propagate_prefixed_error(err, tmp_err, "Cannot open %s: ", path);
return code;
}
cw_compressed = cr_open(cpath,
CR_CW_MODE_WRITE,
record_compression,
&tmp_err);
if (!cw_compressed) {
int code = tmp_err->code;
g_propagate_prefixed_error(err, tmp_err, "Cannot open %s: ", cpath);
return code;
}
while ((readed = cr_read(cw_plain, buf, BUFFER_SIZE, &tmp_err)) > 0) {
cr_write(cw_compressed, buf, (unsigned int) readed, &tmp_err);
if (tmp_err)
break;
}
cr_close(cw_plain, NULL);
if (tmp_err) {
int code = tmp_err->code;
cr_close(cw_compressed, NULL);
g_debug("%s: Error while repomd record compression: %s", __func__,
tmp_err->message);
g_propagate_prefixed_error(err, tmp_err,
"Error while compression %s -> %s:", path, cpath);
return code;
}
cr_close(cw_compressed, &tmp_err);
if (tmp_err) {
int code = tmp_err->code;
g_propagate_prefixed_error(err, tmp_err,
"Error while closing %s: ", path);
return code;
}
// Compute checksums
checksum = cr_checksum_file(path, checksum_type, &tmp_err);
if (tmp_err) {
int code = tmp_err->code;
g_propagate_prefixed_error(err, tmp_err,
"Error while checksum calculation:");
return code;
}
cchecksum = cr_checksum_file(cpath, checksum_type, &tmp_err);
if (tmp_err) {
int code = tmp_err->code;
g_propagate_prefixed_error(err, tmp_err,
"Error while checksum calculation:");
return code;
}
// Get stats
if (stat(path, &gf_stat)) {
g_debug("%s: Error while stat() on %s", __func__, path);
g_set_error(err, CR_REPOMD_RECORD_ERROR, CRE_IO,
"Cannot stat %s", path);
return CRE_IO;
} else {
gf_size = gf_stat.st_size;
gf_time = gf_stat.st_mtime;
}
if (stat(cpath, &cgf_stat)) {
g_debug("%s: Error while stat() on %s", __func__, cpath);
g_set_error(err, CR_REPOMD_RECORD_ERROR, CRE_IO,
"Cannot stat %s", cpath);
return CRE_IO;
} else {
cgf_size = cgf_stat.st_size;
cgf_time = cgf_stat.st_mtime;
}
// Results
record->checksum = g_string_chunk_insert(record->chunk, checksum);
record->checksum_type = g_string_chunk_insert(record->chunk, checksum_str);
record->checksum_open = NULL;
record->checksum_open_type = NULL;
record->timestamp = gf_time;
record->size = gf_size;
record->size_open = -1;
crecord->checksum = g_string_chunk_insert(crecord->chunk, cchecksum);
crecord->checksum_type = g_string_chunk_insert(crecord->chunk, checksum_str);
crecord->checksum_open = g_string_chunk_insert(record->chunk, checksum);
crecord->checksum_open_type = g_string_chunk_insert(record->chunk, checksum_str);
crecord->timestamp = cgf_time;
crecord->size = cgf_size;
crecord->size_open = gf_size;
g_free(checksum);
g_free(cchecksum);
return CRE_OK;
}
