char *load_utf_string_table(FILE *infile, const long offset)
{
const struct utf_query_result result = query_utf_nofail(infile, offset, NULL);
const size_t string_table_size = result.data_offset - result.string_table_offset;
const long string_table_offset = offset + 8 + result.string_table_offset;
char *string_table = malloc(string_table_size + 1);
CHECK_ERRNO (!string_table, "malloc");
memset(string_table, 0, string_table_size+1);
get_bytes_seek(string_table_offset, infile,
(unsigned char *)string_table, string_table_size);
return string_table;
}
uint8_t * get_whole_file(FILE *infile, long *file_size_p)
{
const long file_size = get_file_size(infile);
if (file_size_p)
{
*file_size_p = file_size;
}
uint8_t *indata = malloc(file_size);
CHECK_ERRNO(!indata, "malloc");
// dump whole file
get_bytes_seek(0, infile, indata, file_size);
return indata;
}
long uncompress(reader_t *infile, long offset, long input_size, FILE *outfile)
{
unsigned char *output_buffer = NULL;
CHECK_ERROR( !(
(get_32_le_seek(offset+0x00, infile) == 0 &&
get_32_le_seek(offset+0x04, infile) == 0) ||
(get_64_be_seek(offset+0x00, infile) == CRILAYLA_sig)
), "didn't find 0 or CRILAYLA signature for compressed data");
const long uncompressed_size =
get_32_le_seek(offset+0x08, infile);
const long uncompressed_header_offset =
offset + get_32_le_seek(offset+0x0C, infile)+0x10;
CHECK_ERROR( uncompressed_header_offset + 0x100 != offset + input_size, "size mismatch");
output_buffer = malloc(uncompressed_size + 0x100);
CHECK_ERROR(!output_buffer, "malloc");
get_bytes_seek(uncompressed_header_offset, infile, output_buffer, 0x100);
const long input_end = offset + input_size - 0x100 - 1;
long input_offset = input_end;
const long output_end = 0x100 + uncompressed_size - 1;
uint8_t bit_pool = 0;
int bits_left = 0;
long bytes_output = 0;
while ( bytes_output < uncompressed_size )
{
if (GET_NEXT_BITS(1))
{
long backreference_offset =
output_end-bytes_output+GET_NEXT_BITS(13)+3;
long backreference_length = 3;
// decode variable length coding for length
enum { vle_levels = 4 };
int vle_lens[vle_levels] = { 2, 3, 5, 8 };
int vle_level;
for (vle_level = 0; vle_level < vle_levels; vle_level++)
{
int this_level = GET_NEXT_BITS(vle_lens[vle_level]);
backreference_length += this_level;
if (this_level != ((1 << vle_lens[vle_level])-1)) break;
}
if (vle_level == vle_levels)
{
int this_level;
do
{
this_level = GET_NEXT_BITS(8);
backreference_length += this_level;
} while (this_level == 255);
}
//printf("0x%08lx backreference to 0x%lx, length 0x%lx\n", output_end-bytes_output, backreference_offset, backreference_length);
for (int i=0;i<backreference_length;i++)
{
output_buffer[output_end-bytes_output] = output_buffer[backreference_offset--];
bytes_output++;
}
}
else
{
// verbatim byte
output_buffer[output_end-bytes_output] = GET_NEXT_BITS(8);
//printf("0x%08lx verbatim byte\n", output_end-bytes_output);
bytes_output++;
}
}
put_bytes_seek(0, outfile, output_buffer, 0x100 + uncompressed_size);
free(output_buffer);
return 0x100 + bytes_output;
}
void analyze_Huf8(FILE *infile, FILE *outfile, long file_length)
{
unsigned char *decode_table = NULL;
int decode_table_size;
long decoded_length;
int symbol_count;
/* read header */
{
unsigned char buf[5];
get_bytes_seek(0, infile, buf, 5);
CHECK_ERROR (buf[0] != 0x28, "not 8-bit Huffman");
decoded_length = read_24_le(&buf[1]);
symbol_count = buf[4] + 1;
}
/* allocate decode table */
decode_table_size = symbol_count * 2 - 1;
decode_table = malloc(decode_table_size);
CHECK_ERRNO(decode_table == NULL, "malloc");
/* read decode table */
get_bytes(infile, decode_table, decode_table_size);
#if 0
printf("encoded size = %ld bytes (%d header + %ld body)\n",
file_length, 5 + decode_table_size,
file_length - (5 + decode_table_size));
printf("decoded size = %ld bytes\n", decoded_length);
#endif
/* decode */
{
uint32_t bits;
int bits_left = 0;
int table_offset = 0;
long bytes_decoded = 0;
while ( bytes_decoded < decoded_length )
{
if (bits_left == 0)
{
bits = get_32_le(infile);
bits_left = 32;
}
int current_bit = ((bits & 0x80000000) != 0);
int next_offset = ((table_offset + 1) / 2 * 2) + 1 +
(decode_table[table_offset] & 0x3f) * 2 +
(current_bit ? 1 : 0);
#if 0
printf("%d %02x %lx => %lx\n", current_bit,
decode_table[table_offset],
(unsigned long)table_offset,
(unsigned long)next_offset);
#endif
CHECK_ERROR (next_offset >= decode_table_size,
"reading past end of decode table");
if ((!current_bit && (decode_table[table_offset] & 0x80)) ||
( current_bit && (decode_table[table_offset] & 0x40)))
{
CHECK_FILE(
fwrite(&decode_table[next_offset], 1, 1, outfile) != 1,
outfile, "fwrite");
bytes_decoded++;
#if 0
printf("%02x\n", decode_table[next_offset]);
return;
#endif
next_offset = 0;
}
CHECK_ERROR (next_offset == table_offset,
"stuck in a loop somehow");
table_offset = next_offset;
bits_left--;
bits <<= 1;
}
}
#if 0
printf("done\n");
#endif
}
struct utf_query_result analyze_utf(FILE *infile, const long offset, int indent, int print, const struct utf_query *query)
{
unsigned char buf[4];
struct utf_table_info table_info;
char *string_table = NULL;
struct utf_column_info * schema = NULL;
struct utf_query_result result;
result.valid = 0;
if (print)
{
fprintf_indent(stdout, indent);
printf("{\n");
}
indent += INDENT_LEVEL;
table_info.table_offset = offset;
/* check header */
static const char UTF_signature[4] = "@UTF"; /* intentionally unterminated */
get_bytes_seek(offset, infile, buf, 4);
if (memcmp(buf, UTF_signature, sizeof(UTF_signature)))
{
if (print)
{
fprintf_indent(stdout, indent);
printf("not a @UTF table at %08" PRIx32 "\n", (uint32_t)offset);
}
goto cleanup;
}
/* get table size */
table_info.table_size = get_32_be(infile);
table_info.schema_offset = 0x20;
table_info.rows_offset = get_32_be(infile);
table_info.string_table_offset = get_32_be(infile);
table_info.data_offset = get_32_be(infile);
const uint32_t table_name_string = get_32_be(infile);
table_info.columns = get_16_be(infile);
table_info.row_width = get_16_be(infile);
table_info.rows = get_32_be(infile);
/* allocate for string table */
const int string_table_size =
table_info.data_offset-table_info.string_table_offset;
string_table = malloc(string_table_size+1);
CHECK_ERRNO(!string_table, "malloc");
table_info.string_table = string_table;
memset(string_table, 0, string_table_size+1);
/* load schema */
schema = malloc(sizeof(struct utf_column_info) * table_info.columns);
CHECK_ERRNO(!schema, "malloc");
{
int i;
for (i = 0; i < table_info.columns; i++)
{
schema[i].type = get_byte(infile);
schema[i].column_name = string_table + get_32_be(infile);
if ((schema[i].type & COLUMN_STORAGE_MASK) == COLUMN_STORAGE_CONSTANT)
{
schema[i].constant_offset = ftell(infile);
switch (schema[i].type & COLUMN_TYPE_MASK)
{
case COLUMN_TYPE_STRING:
get_32_be(infile);
break;
case COLUMN_TYPE_8BYTE:
case COLUMN_TYPE_DATA:
get_32_be(infile);
get_32_be(infile);
break;
case COLUMN_TYPE_FLOAT:
case COLUMN_TYPE_4BYTE2:
case COLUMN_TYPE_4BYTE:
get_32_be(infile);
break;
case COLUMN_TYPE_2BYTE2:
case COLUMN_TYPE_2BYTE:
get_16_be(infile);
break;
case COLUMN_TYPE_1BYTE2:
case COLUMN_TYPE_1BYTE:
get_byte(infile);
break;
default:
CHECK_ERROR(1, "unknown type for constant");
}
}
}
}
table_info.schema = schema;
/* read string table */
get_bytes_seek(table_info.string_table_offset+8+offset,
infile, (unsigned char *)string_table, string_table_size);
table_info.table_name = table_info.string_table+table_name_string;
#if 0
if (print)
{
fprintf_table_info(stdout, &table_info, indent);
}
#endif
/* fill in the default stuff */
result.valid = 1;
result.found = 0;
result.rows = table_info.rows;
result.name_offset = table_name_string;
result.string_table_offset = table_info.string_table_offset;
result.data_offset = table_info.data_offset;
/* explore the values */
if (query || print) {
int i, j;
for (i = 0; i < table_info.rows; i++)
{
if (!print && query && i != query->index) continue;
uint32_t row_offset =
table_info.table_offset + 8 + table_info.rows_offset +
i * table_info.row_width;
const uint32_t row_start_offset = row_offset;
if (print)
{
fprintf_indent(stdout, indent);
printf("%s[%d] = {\n", table_info.table_name, i);
}
indent += INDENT_LEVEL;
for (j = 0; j < table_info.columns; j++)
{
uint8_t type = table_info.schema[j].type;
long constant_offset = table_info.schema[j].constant_offset;
int constant = 0;
int qthis = (query && i == query->index &&
!strcmp(table_info.schema[j].column_name, query->name));
if (print)
{
fprintf_indent(stdout, indent);
#if 1
printf("%08x %02x %s = ", row_offset-row_start_offset, type, table_info.schema[j].column_name);
#else
printf("%s = ", table_info.schema[j].column_name);
#endif
}
if (qthis)
{
result.found = 1;
result.type = schema[j].type & COLUMN_TYPE_MASK;
}
switch (schema[j].type & COLUMN_STORAGE_MASK)
{
case COLUMN_STORAGE_PERROW:
break;
case COLUMN_STORAGE_CONSTANT:
constant = 1;
break;
case COLUMN_STORAGE_ZERO:
if (print)
{
printf("UNDEFINED\n");
}
if (qthis)
{
memset(&result.value, 0,
sizeof(result.value));
}
continue;
default:
CHECK_ERROR(1, "unknown storage class");
}
if (1)
{
long data_offset;
int bytes_read;
if (constant)
{
data_offset = constant_offset;
if (print)
{
printf("constant ");
}
}
else
{
data_offset = row_offset;
}
switch (type & COLUMN_TYPE_MASK)
{
case COLUMN_TYPE_STRING:
{
uint32_t string_offset;
string_offset = get_32_be_seek(data_offset, infile);
bytes_read = 4;
if (print)
{
printf("\"%s\"\n", table_info.string_table + string_offset);
}
if (qthis)
{
result.value.value_string = string_offset;
}
}
break;
case COLUMN_TYPE_DATA:
{
uint32_t vardata_offset, vardata_size;
vardata_offset = get_32_be_seek(data_offset, infile);
vardata_size = get_32_be(infile);
bytes_read = 8;
if (print)
{
printf("[0x%08" PRIx32 "]", vardata_offset);
printf(" (size 0x%08" PRIx32 ")\n", vardata_size);
}
if (qthis)
{
result.value.value_data.offset = vardata_offset;
result.value.value_data.size = vardata_size;
}
if (vardata_size != 0 && print)
{
/* assume that the data is another table */
analyze_utf(infile,
table_info.table_offset + 8 +
table_info.data_offset +
vardata_offset,
indent,
print,
NULL
);
}
}
break;
case COLUMN_TYPE_8BYTE:
{
uint64_t value =
get_64_be_seek(data_offset, infile);
if (print)
{
printf("0x%" PRIx64 "\n", value);
}
if (qthis)
{
result.value.value_u64 = value;
}
bytes_read = 8;
break;
}
case COLUMN_TYPE_4BYTE2:
if (print)
{
printf("type 2 ");
}
case COLUMN_TYPE_4BYTE:
{
uint32_t value =
get_32_be_seek(data_offset, infile);
if (print)
{
printf("%" PRId32 "\n", value);
}
if (qthis)
{
result.value.value_u32 = value;
}
bytes_read = 4;
}
break;
case COLUMN_TYPE_2BYTE2:
if (print)
{
printf("type 2 ");
}
case COLUMN_TYPE_2BYTE:
{
uint16_t value =
get_16_be_seek(data_offset, infile);
if (print)
{
printf("%" PRId16 "\n", value);
}
if (qthis)
{
result.value.value_u16 = value;
}
bytes_read = 2;
}
break;
case COLUMN_TYPE_FLOAT:
if (sizeof(float) == 4)
{
union {
float float_value;
uint32_t int_value;
} int_float;
int_float.int_value = get_32_be_seek(data_offset, infile);
if (print)
{
printf("%f\n", int_float.float_value);
}
if (qthis)
{
result.value.value_float = int_float.float_value;
}
}
else
{
get_32_be_seek(data_offset, infile);
if (print)
{
printf("float\n");
}
if (qthis)
{
CHECK_ERROR(1, "float is wrong size, can't return");
}
}
bytes_read = 4;
break;
case COLUMN_TYPE_1BYTE2:
if (print)
{
printf("type 2 ");
}
case COLUMN_TYPE_1BYTE:
{
uint8_t value =
get_byte_seek(data_offset, infile);
if (print)
{
printf("%" PRId8 "\n", value);
}
if (qthis)
{
result.value.value_u8 = value;
}
bytes_read = 1;
}
break;
default:
CHECK_ERROR(1, "unknown normal type");
}
if (!constant)
{
row_offset += bytes_read;
}
} /* useless if end */
} /* column for loop end */
indent -= INDENT_LEVEL;
if (print)
{
fprintf_indent(stdout,indent);
printf("}\n");
}
CHECK_ERROR(row_offset - row_start_offset != table_info.row_width,
"column widths do now add up to row width");
if (query && !print && i >= query->index) break;
} /* row for loop end */
} /* explore values block end */
cleanup:
indent -= INDENT_LEVEL;
if (print)
{
fprintf_indent(stdout, indent);
printf("}\n");
}
if (string_table)
{
free(string_table);
string_table = NULL;
}
if (schema)
{
free(schema);
schema = NULL;
}
return result;
}
void process_dir_list(long dir_offset, long table_size, long data_size, FILE *infile)
{
long data_offset = dir_offset + table_size;
const uint32_t dir_count = get_32_le_seek(dir_offset, infile);
dir_offset += 4;
// offset to info on files
long file_offset = dir_offset;
// advance file offset to where file info starts
for (uint32_t i = 0; i < dir_count; i++)
{
CHECK_ERROR(file_offset >= data_offset, "read beyond table");
const uint32_t name_size = get_32_le_seek(file_offset, infile);
file_offset += 4 + name_size;
}
// dirs
for (uint32_t i = 0; i < dir_count; i++)
{
CHECK_ERROR(dir_offset >= data_offset, "read beyond table");
const uint32_t dir_name_size = get_32_le_seek(dir_offset, infile);
dir_offset +=4 ;
dump(infile, stdout, dir_offset, dir_name_size);
printf("\n");
dir_offset += dir_name_size;
if (file_offset == data_offset) break;
const uint32_t file_count = get_32_le_seek(file_offset, infile);
file_offset += 4;
// contents of dir
for (uint32_t j = 0; j < file_count; j++)
{
CHECK_ERROR(file_offset >= data_offset, "read beyond table");
const uint32_t file_name_size = get_32_le_seek(file_offset, infile);
file_offset += 4;
printf(" ");
dump(infile, stdout, file_offset, file_name_size);
file_offset += file_name_size;
CHECK_ERROR(get_32_le_seek(file_offset, infile) != 0, "zernooo");
const uint32_t offset = get_32_le_seek(file_offset+4, infile)+data_offset;
const uint32_t size = get_32_le_seek(file_offset+8, infile);
printf(": offset 0x%"PRIx32" size 0x%"PRIx32"\n",offset,size);
{
unsigned char namebuf[file_name_size+1];
get_bytes_seek(file_offset-file_name_size,infile,namebuf,file_name_size);
namebuf[file_name_size-1]='\0';
FILE *outfile = fopen((char*)namebuf,"wb");
CHECK_ERRNO(outfile == NULL,"fopen");
dump(infile, outfile, offset+0x38, size-0x38);
CHECK_ERRNO(fclose(outfile) == EOF,"fclose");
}
file_offset += 0xc;
}
}
}
void analyze_CPK(FILE *infile, long file_length, struct cpk_file *file, int new_file_count)
{
const long CpkHeader_offset = 0x0;
char *toc_string_table = NULL;
/* check header */
{
unsigned char buf[4];
static const char CPK_signature[4] = "CPK "; /* intentionally unterminated */
get_bytes_seek(CpkHeader_offset, infile, buf, 4);
CHECK_ERROR (memcmp(buf, CPK_signature, sizeof(CPK_signature)), "CPK signature not found");
}
/* check CpkHeader */
{
struct utf_query_result result = query_utf_nofail(infile, CpkHeader_offset+0x10, NULL);
CHECK_ERROR (result.rows != 1, "wrong number of rows in CpkHeader");
}
/* get TOC offset */
long toc_offset = query_utf_8byte(infile, CpkHeader_offset+0x10, 0, "TocOffset");
/* get content offset */
long content_offset = query_utf_8byte(infile, CpkHeader_offset+0x10, 0, "ContentOffset");
/* get file count from CpkHeader */
long CpkHeader_count = query_utf_4byte(infile, CpkHeader_offset+0x10, 0, "Files");
#ifdef DEBUG
printf("(Debug)TocOffset: %d\n(Debug)ContentOffset: %d\n(Debug)Files(Count): %d\n",toc_offset,content_offset,CpkHeader_count);
#endif
/* check TOC header */
{
unsigned char buf[4];
static const char TOC_signature[4] = "TOC "; /* intentionally unterminated */
get_bytes_seek(toc_offset, infile, buf, 4);
CHECK_ERROR (memcmp(buf, TOC_signature, sizeof(TOC_signature)), "TOC signature not found");
}
/* get TOC entry count, string table offset */
long toc_entries;
long toc_string_table_offset;
long toc_string_table_size;
{
struct utf_query_result result = query_utf_nofail(infile, toc_offset+0x10, NULL);
toc_entries = result.rows;
toc_string_table_offset = toc_offset + 0x10 + 8 + result.string_table_offset;
toc_string_table_size = result.data_offset - result.string_table_offset;
}
#ifdef DEBUG
printf("(Debug)TocEntries: %d\n(Debug)TocStringTableOffset: %d\n(Debug)TocStringTableSize:%d\n",toc_entries,toc_string_table_offset,toc_string_table_size);
#endif
/* check that counts match */
CHECK_ERROR( toc_entries != CpkHeader_count, "CpkHeader file count and TOC entry count do not match" );
/* load string table */
toc_string_table = malloc(toc_string_table_size + 1);
{
CHECK_ERRNO(!toc_string_table, "malloc");
memset(toc_string_table, 0, toc_string_table_size+1);
get_bytes_seek(toc_string_table_offset, infile,
(unsigned char *)toc_string_table, toc_string_table_size);
}
/* find files in cpk */
for (int i = 0; i < toc_entries; i++)
{
/* get file name */
const char *file_name = query_utf_string(infile, toc_offset+0x10, i, "FileName", toc_string_table);
const char *dir_name=query_utf_string(infile,toc_offset+0x10,i,"DirName",toc_string_table);
/* to query a file with relative path, remember to free it */
char* query_file=malloc(strlen(dir_name)+strlen(file_name)+2);
strcpy(query_file,dir_name);
char sep[]="/";
strcat(query_file,sep);
strcat(query_file,file_name);
int entry_num = contains_file_name(query_file, file, new_file_count);
if (entry_num >= 0)
{
#ifdef DEBUG
printf("(Debug)FileName length:%d\n(Debug)DirName length:%d\n",strlen(file_name),strlen(dir_name));
printf("(Debug)query file location:%s\n",query_file);
#endif
printf("Found %s/%s in cpk\n",dir_name, file_name);
/* get file offset */
file[entry_num].orig_offset = content_offset + query_utf_8byte(infile, toc_offset+0x10, i, "FileOffset");
/* get file size */
file[entry_num].orig_size = query_utf_4byte(infile, toc_offset+0x10, i, "FileSize");
/* save index */
file[entry_num].index = i;
file[entry_num].found = 1;
#ifdef DEBUG
printf("(Debug)EntryNum:%d\n",entry_num);
#endif
}
/* free the query_file to avoid leak */
if(query_file)
free(query_file);
}
sort_by_offset(file, new_file_count);
FILE *outfile = fopen("out.cpk", "w+b");
printf("Starting file copy\n");
long current = 0;
for (int i = 0; i < new_file_count; i++)
{
if (file[i].found)
{
/* copy from infile to start of current new file */
dump(infile, outfile, current, file[i].orig_offset-current);
current = file[i].orig_offset;
/* open new file and copy data */
FILE *newfile = fopen(file[i].location, "rb");
CHECK_ERRNO(!newfile, "fopen");
CHECK_ERRNO(fseek(newfile, 0 , SEEK_END) != 0, "fseek");
file[i].new_size = ftell(newfile);
CHECK_ERRNO(file_length == -1, "ftell");
rewind(infile);
printf("Copying %s(file name:%s)\n", file[i].location,file[i].filename);
dump(newfile, outfile, 0, file[i].new_size);
printf("Finished copying %s(file name:%s)\n", file[i].location,file[i].filename);
fclose(newfile);
/* update information */
current += file[i].orig_size;
file[i].copied = 1;
file[i].offset_diff = file[i].new_size -file[i].orig_size;
}
}
/* copy remaining data */
dump(infile, outfile, current, file_length-current);
printf("Finished file copy\n");
/* fix toc_offset */
toc_offset = fix_offset(outfile, CpkHeader_offset+0x10, 0, 0, "TocOffset", file, new_file_count);
printf("Fixed toc offset\n");
/* fix file offsets */
for (int i = 0; i < toc_entries; i++)
fix_offset(outfile, toc_offset+0x10, content_offset, i, "FileOffset", file, new_file_count);
printf("Fixed file offsets\n");
/* fix file sizes */
for (int i = 0; i < new_file_count; i++)
fix_file_sizes(outfile, toc_offset+0x10, file, i);
printf("Finished replacing files and fixing data\n");
CHECK_ERRNO(fclose(outfile) != 0, "fclose");
if (toc_string_table)
{
free(toc_string_table);
toc_string_table = NULL;
}
}
