static gboolean cmd_options_to_task(GSList **modifyrepotasks, RawCmdOptions *options, gchar *metadatapath, GError **err) { assert(modifyrepotasks); assert(!err || *err == NULL); if (!options) return TRUE; //assert(metadatapath || options->remove); if (options->remove) g_debug("Preparing remove-task for: %s", options->remove); else g_debug("Preparing task for: %s", metadatapath); if (metadatapath && !g_file_test(metadatapath, G_FILE_TEST_IS_REGULAR)) { g_set_error(err, ERR_DOMAIN, CRE_ERROR, "File \"%s\" is not regular file or doesn't exists", metadatapath); return FALSE; } if (options->remove) metadatapath = options->remove; cr_ModifyRepoTask *task = cr_modifyrepotask_new(); task->path = cr_safe_string_chunk_insert_null(task->chunk, metadatapath); task->type = cr_safe_string_chunk_insert_null(task->chunk, options->mdtype); task->remove = (options->remove) ? TRUE : FALSE; task->compress = options->compress; task->compress_type = cr_compression_type(options->compress_type); task->unique_md_filenames = options->unique_md_filenames; task->checksum_type = cr_checksum_type(options->checksum); task->new_name = cr_safe_string_chunk_insert_null(task->chunk, options->new_name); task->zck = options->zck; task->zck_dict_dir = options->zck_dict_dir; *modifyrepotasks = g_slist_append(*modifyrepotasks, task); g_debug("Task: [path: %s, type: %s, remove: %d, compress: %d, " "compress_type: %d (%s), unique_md_filenames: %d, " "checksum_type: %d (%s), new_name: %s]", task->path, task->type, task->remove, task->compress, task->compress_type, cr_compression_suffix(task->compress_type), task->unique_md_filenames, task->checksum_type, cr_checksum_name_str(task->checksum_type), task->new_name); return TRUE; }
static void test_cr_compression_suffix(void) { const char *suffix; suffix = cr_compression_suffix(CR_CW_AUTO_DETECT_COMPRESSION); g_assert(!suffix); suffix = cr_compression_suffix(CR_CW_UNKNOWN_COMPRESSION); g_assert(!suffix); suffix = cr_compression_suffix(CR_CW_NO_COMPRESSION); g_assert(!suffix); suffix = cr_compression_suffix(CR_CW_GZ_COMPRESSION); g_assert_cmpstr(suffix, ==, ".gz"); suffix = cr_compression_suffix(CR_CW_BZ2_COMPRESSION); g_assert_cmpstr(suffix, ==, ".bz2"); suffix = cr_compression_suffix(CR_CW_XZ_COMPRESSION); g_assert_cmpstr(suffix, ==, ".xz"); }
static gboolean compress_sqlite_dbs(const gchar *tmp_out_repo, const gchar *pri_db_filename, cr_RepomdRecord **in_pri_db_rec, const gchar *fil_db_filename, cr_RepomdRecord **in_fil_db_rec, const gchar *oth_db_filename, cr_RepomdRecord **in_oth_db_rec, cr_CompressionType compression_type, cr_ChecksumType checksum_type, GError **err) { cr_CompressionTask *pri_db_task; cr_CompressionTask *fil_db_task; cr_CompressionTask *oth_db_task; const char *sqlite_compression_suffix; cr_RepomdRecord *pri_db_rec = NULL; cr_RepomdRecord *fil_db_rec = NULL; cr_RepomdRecord *oth_db_rec = NULL; // Prepare thread pool for compression tasks GThreadPool *compress_pool = g_thread_pool_new(cr_compressing_thread, NULL, 3, FALSE, NULL); // Prepare output filenames sqlite_compression_suffix = cr_compression_suffix(compression_type); gchar *pri_db_name = g_strconcat(tmp_out_repo, "/primary.sqlite", sqlite_compression_suffix, NULL); gchar *fil_db_name = g_strconcat(tmp_out_repo, "/filelists.sqlite", sqlite_compression_suffix, NULL); gchar *oth_db_name = g_strconcat(tmp_out_repo, "/other.sqlite", sqlite_compression_suffix, NULL); // Prepare compression tasks pri_db_task = cr_compressiontask_new(pri_db_filename, pri_db_name, compression_type, checksum_type, 1, NULL); g_thread_pool_push(compress_pool, pri_db_task, NULL); fil_db_task = cr_compressiontask_new(fil_db_filename, fil_db_name, compression_type, checksum_type, 1, NULL); g_thread_pool_push(compress_pool, fil_db_task, NULL); oth_db_task = cr_compressiontask_new(oth_db_filename, oth_db_name, compression_type, checksum_type, 1, NULL); g_thread_pool_push(compress_pool, oth_db_task, NULL); // Wait till all tasks are complete and free the thread pool g_thread_pool_free(compress_pool, FALSE, TRUE); // Remove uncompressed DBs cr_rm(pri_db_filename, CR_RM_FORCE, NULL, NULL); cr_rm(fil_db_filename, CR_RM_FORCE, NULL, NULL); cr_rm(oth_db_filename, CR_RM_FORCE, NULL, NULL); // Prepare repomd records pri_db_rec = cr_repomd_record_new("primary_db", pri_db_name); fil_db_rec = cr_repomd_record_new("filelists_db", fil_db_name); oth_db_rec = cr_repomd_record_new("other_db", oth_db_name); *in_pri_db_rec = pri_db_rec; *in_fil_db_rec = fil_db_rec; *in_oth_db_rec = oth_db_rec; // Free paths to compressed files g_free(pri_db_name); g_free(fil_db_name); g_free(oth_db_name); // Fill repomd records from stats gathered during compression cr_repomd_record_load_contentstat(pri_db_rec, pri_db_task->stat); cr_repomd_record_load_contentstat(fil_db_rec, fil_db_task->stat); cr_repomd_record_load_contentstat(oth_db_rec, oth_db_task->stat); // Free the compression tasks cr_compressiontask_free(pri_db_task, NULL); cr_compressiontask_free(fil_db_task, NULL); cr_compressiontask_free(oth_db_task, NULL); // Prepare thread pool for repomd record filling tasks GThreadPool *fill_pool = g_thread_pool_new(cr_repomd_record_fill_thread, NULL, 3, FALSE, NULL); // Prepare the tasks themselves cr_RepomdRecordFillTask *pri_db_fill_task; cr_RepomdRecordFillTask *fil_db_fill_task; cr_RepomdRecordFillTask *oth_db_fill_task; pri_db_fill_task = cr_repomdrecordfilltask_new(pri_db_rec, checksum_type, NULL); g_thread_pool_push(fill_pool, pri_db_fill_task, NULL); fil_db_fill_task = cr_repomdrecordfilltask_new(fil_db_rec, checksum_type, NULL); g_thread_pool_push(fill_pool, fil_db_fill_task, NULL); oth_db_fill_task = cr_repomdrecordfilltask_new(oth_db_rec, checksum_type, NULL); g_thread_pool_push(fill_pool, oth_db_fill_task, NULL); // Wait till the all tasks are finished and free the pool g_thread_pool_free(fill_pool, FALSE, TRUE); // Clear the tasks cr_repomdrecordfilltask_free(pri_db_fill_task, NULL); cr_repomdrecordfilltask_free(fil_db_fill_task, NULL); cr_repomdrecordfilltask_free(oth_db_fill_task, NULL); return TRUE; }
int cr_repomd_record_compress_and_fill(cr_RepomdRecord *record, cr_RepomdRecord *crecord, cr_ChecksumType checksum_type, cr_CompressionType record_compression, GError **err) { const char *suffix; gchar *path, *cpath; gchar *clocation_real, *clocation_href; gchar *checksum, *cchecksum; int readed; char buf[BUFFER_SIZE]; CR_FILE *cw_plain; CR_FILE *cw_compressed; gint64 gf_size = -1, cgf_size = -1; gint64 gf_time = -1, cgf_time = -1; struct stat gf_stat, cgf_stat; const char *checksum_str = cr_checksum_name_str(checksum_type); GError *tmp_err = NULL; assert(record); assert(crecord); assert(!err || *err == NULL); if (!(record->location_real) || !strlen(record->location_real)) { g_set_error(err, CR_REPOMD_RECORD_ERROR, CRE_BADARG, "Empty locations in repomd record object"); return CRE_BADARG; } if (!g_file_test(record->location_real, G_FILE_TEST_IS_REGULAR)) { // File doesn't exists g_warning("%s: File %s doesn't exists", __func__, record->location_real); g_set_error(err, CR_REPOMD_RECORD_ERROR, CRE_NOFILE, "File %s doesn't exists or not a regular file", record->location_real); return CRE_NOFILE;; } // Paths suffix = cr_compression_suffix(record_compression); clocation_real = g_strconcat(record->location_real, suffix, NULL); clocation_href = g_strconcat(record->location_href, suffix, NULL); crecord->location_real = g_string_chunk_insert(crecord->chunk, clocation_real); crecord->location_href = g_string_chunk_insert(crecord->chunk, clocation_href); g_free(clocation_real); g_free(clocation_href); path = record->location_real; cpath = crecord->location_real; // Compress file + get size of non compressed file cw_plain = cr_open(path, CR_CW_MODE_READ, CR_CW_NO_COMPRESSION, &tmp_err); if (!cw_plain) { int code = tmp_err->code; g_propagate_prefixed_error(err, tmp_err, "Cannot open %s: ", path); return code; } cw_compressed = cr_open(cpath, CR_CW_MODE_WRITE, record_compression, &tmp_err); if (!cw_compressed) { int code = tmp_err->code; g_propagate_prefixed_error(err, tmp_err, "Cannot open %s: ", cpath); return code; } while ((readed = cr_read(cw_plain, buf, BUFFER_SIZE, &tmp_err)) > 0) { cr_write(cw_compressed, buf, (unsigned int) readed, &tmp_err); if (tmp_err) break; } cr_close(cw_plain, NULL); if (tmp_err) { int code = tmp_err->code; cr_close(cw_compressed, NULL); g_debug("%s: Error while repomd record compression: %s", __func__, tmp_err->message); g_propagate_prefixed_error(err, tmp_err, "Error while compression %s -> %s:", path, cpath); return code; } cr_close(cw_compressed, &tmp_err); if (tmp_err) { int code = tmp_err->code; g_propagate_prefixed_error(err, tmp_err, "Error while closing %s: ", path); return code; } // Compute checksums checksum = cr_checksum_file(path, checksum_type, &tmp_err); if (tmp_err) { int code = tmp_err->code; g_propagate_prefixed_error(err, tmp_err, "Error while checksum calculation:"); return code; } cchecksum = cr_checksum_file(cpath, checksum_type, &tmp_err); if (tmp_err) { int code = tmp_err->code; g_propagate_prefixed_error(err, tmp_err, "Error while checksum calculation:"); return code; } // Get stats if (stat(path, &gf_stat)) { g_debug("%s: Error while stat() on %s", __func__, path); g_set_error(err, CR_REPOMD_RECORD_ERROR, CRE_IO, "Cannot stat %s", path); return CRE_IO; } else { gf_size = gf_stat.st_size; gf_time = gf_stat.st_mtime; } if (stat(cpath, &cgf_stat)) { g_debug("%s: Error while stat() on %s", __func__, cpath); g_set_error(err, CR_REPOMD_RECORD_ERROR, CRE_IO, "Cannot stat %s", cpath); return CRE_IO; } else { cgf_size = cgf_stat.st_size; cgf_time = cgf_stat.st_mtime; } // Results record->checksum = g_string_chunk_insert(record->chunk, checksum); record->checksum_type = g_string_chunk_insert(record->chunk, checksum_str); record->checksum_open = NULL; record->checksum_open_type = NULL; record->timestamp = gf_time; record->size = gf_size; record->size_open = -1; crecord->checksum = g_string_chunk_insert(crecord->chunk, cchecksum); crecord->checksum_type = g_string_chunk_insert(crecord->chunk, checksum_str); crecord->checksum_open = g_string_chunk_insert(record->chunk, checksum); crecord->checksum_open_type = g_string_chunk_insert(record->chunk, checksum_str); crecord->timestamp = cgf_time; crecord->size = cgf_size; crecord->size_open = gf_size; g_free(checksum); g_free(cchecksum); return CRE_OK; }