MorphologicalDictionary::MorphologicalDictionary(const string & binaryFileName,
bool load_data,
shared_ptr<DictionaryTools> _tools)
{
if (_tools == NULL)
{
tools = make_shared<DictionaryTools>();
}
else
{
tools = _tools;
}
root = NULL;
suffix_root = NULL;
if (load_data)
{
string path = BIN_DIC_DATA_PATH;
string binary_file_path = path + binaryFileName;
readBinaryFile(binary_file_path);
string rules_path = path + MORPHOLOGY_RULES_PATH;
readRulesFromTextFile(rules_path);
init();
}
useRules = false;
usePrediction = false;
min_suffix_length = 2;
useE = true;
}
int main(int argc, char **argv) {
if (argc != 4) {
printf("Incorrect number of arguments: %d\n", argc);
return -1;
}
char* binFile;
float histLen;
long buffSize; long buffBytes;
int* in; int* out; int* window;
int gridLen; int gridSize; int gridBytes; int filtLen; int filtBytes;
sscanf(argv[1], "%d", &gridLen);
sscanf(argv[2], "%d", &filtLen);
binFile = argv[3];
// window size must be odd.
if (filtLen % 2 == 0) {
filtLen++;
printf("filter size must be odd, increasing to: %d", filtLen);
}
// initialise the grid
histLen = 1.0 / (gridLen+1);
gridSize = gridLen * gridLen;
gridBytes = gridSize * sizeof(int);
in = (int*) malloc(gridBytes); memset(in, 0, gridBytes);
out = (int*) malloc(gridBytes); memset(out, 0, gridBytes);
filtBytes = filtLen * filtLen * sizeof(int);
window = (int*) malloc(filtBytes);
// read input file
clock_t t_rea_s = clock();
float* buff = readBinaryFile(binFile, &buffBytes, &buffSize);
clock_t t_rea_e = clock();
printf("Elapsed reading time: %12.3f\n", (double)(t_rea_e - t_rea_s)/CLOCKS_PER_SEC);
// perform binning
double t_bin_s = omp_get_wtime();
populateHistogram(in, buff, buffSize, histLen, gridLen, gridBytes);
double t_bin_e = omp_get_wtime();
printf("Elapsed binning time: %12.3f\n", (double)(t_bin_e - t_bin_s));
// perform filtering
/*
clock_t t_fil_s = omp_get_wtime();
median_filter(in, out, window, gridLen, filtLen);
clock_t t_fil_e = omp_get_wtime();
printf("Elapsed filtering time: %12.3f\n", (double)(t_fil_e - t_fil_s));
writeArrayToCSV("output/output-omp.csv", out, gridLen, histLen);
*/
free(in);
free(out);
free(window);
}
/*
* Implementation notes: readBinaryFile
* ------------------------------------
* The binary lexicon file format must follow this pattern:
* DAWG:<startnode index>:<num bytes>:<num bytes block of edge data>
*/
void DawgLexicon::readBinaryFile(const std::string& filename) {
#ifdef _foreachpatch_h
std::ifstream input(filename.c_str(), __IOS_IN__ | __IOS_BINARY__);
#else
std::ifstream input(filename.c_str(), std::ios::in | std::ios::binary);
#endif // _foreachpatch_h
if (input.fail()) {
error("DawgLexicon::addWordsFromFile: Couldn't open lexicon file " + filename);
}
readBinaryFile(input);
input.close();
}
void Lexicon::addWordsFromFile(std::istream& input) {
bool isDAWG = isDAWGFile(input);
rewindStream(input);
if (isDAWG) {
readBinaryFile(input);
} else {
if (input.fail()) {
error("Lexicon::addWordsFromFile: Couldn't read from input");
}
std::string line;
while (getline(input, line)) {
add(trim(line));
}
}
}
VkShaderModule load(const char *fileName, VkDevice device)
{
size_t size = 0;
const char *shaderCode = readBinaryFile(fileName, &size);
assert(size > 0);
VkShaderModule shaderModule;
VkShaderModuleCreateInfo moduleCreateInfo;
moduleCreateInfo.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO;
moduleCreateInfo.pNext = NULL;
moduleCreateInfo.codeSize = size;
moduleCreateInfo.pCode = (uint32_t*)shaderCode;
moduleCreateInfo.flags = 0;
VkResult err = vkCreateShaderModule(device, &moduleCreateInfo, NULL, &shaderModule);
assert(!err);
return shaderModule;
}
/*
* Check for DAWG in first 4 to identify as special binary format,
* otherwise assume ASCII, one word per line
*/
void DawgLexicon::addWordsFromFile(std::istream& input) {
char firstFour[4], expected[] = "DAWG";
if (input.fail()) {
error("DawgLexicon::addWordsFromFile: Couldn't read input");
}
input.read(firstFour, 4);
if (strncmp(firstFour, expected, 4) == 0) {
if (otherWords.size() != 0) {
error("DawgLexicon::addWordsFromFile: Binary files require an empty lexicon");
}
readBinaryFile(input);
} else {
// plain text file
input.seekg(0);
std::string line;
while (getline(input, line)) {
add(line);
}
}
}
/*
* Check for DAWG in first 4 to identify as special binary format,
* otherwise assume ASCII, one word per line
*/
void DawgLexicon::addWordsFromFile(const std::string& filename) {
char firstFour[4], expected[] = "DAWG";
std::ifstream istr(filename.c_str());
if (istr.fail()) {
error("DawgLexicon::addWordsFromFile: Couldn't open lexicon file " + filename);
}
istr.read(firstFour, 4);
if (strncmp(firstFour, expected, 4) == 0) {
if (otherWords.size() != 0) {
error("DawgLexicon::addWordsFromFile: Binary files require an empty lexicon");
}
readBinaryFile(filename);
} else {
// plain text file
istr.seekg(0);
std::string line;
while (getline(istr, line)) {
add(line);
}
istr.close();
}
}
int main(int argc, char *argv[ ])
{
int i;
FILE *infile, *outfile;
if (argc < 3)
{
printf("XGMTool %s - Stephane Dallongeville - copyright 2016\n", version);
printf("\n");
printf("Usage: xgmtool inputFile outputFile <options>\n");
printf("XGMTool can do the following operations:\n");
printf(" - Optimize and reduce size of Sega Megadrive VGM file\n");
printf(" Note that it won't work correctly on VGM file which require sub frame accurate timing.\n");
printf(" - Convert a Sega Megadrive VGM file to XGM file\n");
printf(" - Convert a XGM file to Sega Megadrive VGM file\n");
printf(" - Compile a XGM file into a binary file (XGC) ready to played by the Z80 XGM driver\n");
printf(" - Convert a XGC binary file to XGM file (experimental)\n");
printf(" - Convert a XGC binary file to Sega Megadrive VGM file (experimental)\n");
printf("\n");
printf("Optimize VGM:\n");
printf(" xgmtool input.vgm output.vgm\n");
printf("\n");
printf("Convert VGM to XGM:\n");
printf(" xgmtool input.vgm output.xgm\n");
printf("\n");
printf("Convert and compile VGM to binary/XGC:\n");
printf(" xgmtool input.vgm output.bin\n");
printf(" xgmtool input.vgm output.xgc\n");
printf("\n");
printf("Convert XGM to VGM:\n");
printf(" xgmtool input.xgm output.vgm\n");
printf("\n");
printf("Compile XGM to binary/XGC:\n");
printf(" xgmtool input.xgm output.bin\n");
printf(" xgmtool input.xgm output.xgc\n");
printf("\n");
printf("Convert XGC to XGM (experimental):\n");
printf(" xgmtool input.xgc output.xgm\n");
printf("\n");
printf("Compile XGC to VGM (experimental):\n");
printf(" xgmtool input.xgc output.vgm\n");
printf("\n");
printf("The action xmgtool performs is dependant from the input and output file extension.\n");
printf("Supported options:\n");
printf("-s\tenable silent mode (no message except error and warning).\n");
printf("-v\tenable verbose mode.\n");
printf("-n\tforce NTSC timing (only meaningful for VGM to XGM conversion).\n");
printf("-p\tforce PAL timing (only meaningful for VGM to XGM conversion).\n");
printf("-di\tdisable PCM sample auto ignore (it can help when PCM are not properly extracted).\n");
printf("-dr\tdisable PCM sample rate auto fix (it can help when PCM are not properly extracted).\n");
printf("-kf\tenable delayed KEY OFF event when we have KEY ON/OFF in a single frame (it can fix incorrect instrument sound).\n");
exit(1);
}
sys = SYSTEM_AUTO;
silent = false;
verbose = false;
sampleIgnore = true;
sampleRateFix = true;
delayKeyOff = false;
// Open source for binary read (will fail if file does not exist)
if ((infile = fopen(argv[1], "rb")) == NULL)
{
printf("Error: the source file %s could not be opened\n", argv[1]);
exit(2);
}
// test open output for write
if ((outfile = fopen(argv[2], "wb")) == NULL)
{
printf("Error: the output file %s could not be opened\n", argv[2]);
exit(3);
}
// can close
fclose(outfile);
// options
for(i = 3; i < argc; i++)
{
if (!strcasecmp(argv[i], "-s"))
{
silent = true;
verbose = false;
}
else if (!strcasecmp(argv[i], "-v"))
{
verbose = true;
silent = false;
}
else if (!strcasecmp(argv[i], "-di"))
sampleIgnore = false;
else if (!strcasecmp(argv[i], "-dr"))
sampleRateFix = false;
else if (!strcasecmp(argv[i], "-kf"))
delayKeyOff = true;
else if (!strcasecmp(argv[i], "-n"))
sys = SYSTEM_NTSC;
else if (!strcasecmp(argv[i], "-p"))
sys = SYSTEM_PAL;
else
printf("Warning: option %s not recognized (ignored)\n", argv[i]);
}
// silent mode has priority
if (silent)
verbose = false;
char* inExt = getFileExtension(argv[1]);
char* outExt = getFileExtension(argv[2]);
int errCode = 0;
if (!strcasecmp(inExt, "VGM"))
{
if ((!strcasecmp(outExt, "VGM")) || (!strcasecmp(outExt, "XGM")) || (!strcasecmp(outExt, "BIN")) || (!strcasecmp(outExt, "XGC")))
{
// VGM optimization
int inDataSize;
unsigned char* inData;
int outDataSize;
unsigned char* outData;
VGM* vgm;
VGM* optVgm;
// load file
inData = readBinaryFile(argv[1], &inDataSize);
if (inData == NULL) exit(1);
// load VGM
if (sys == SYSTEM_NTSC)
inData[0x24] = 60;
else if (sys == SYSTEM_PAL)
inData[0x24] = 50;
vgm = VGM_create1(inData, inDataSize, 0);
if (vgm == NULL) exit(1);
// optimize
optVgm = VGM_createFromVGM(vgm, true);
if (optVgm == NULL) exit(1);
VGM_convertWaits(optVgm);
VGM_cleanCommands(optVgm);
VGM_cleanSamples(optVgm);
VGM_fixKeyCommands(optVgm);
// VGM output
if (!strcasecmp(outExt, "VGM"))
{
// get byte array
outData = VGM_asByteArray(optVgm, &outDataSize);
if (outData == NULL) exit(1);
// write to file
writeBinaryFile(outData, outDataSize, argv[2]);
}
else
{
XGM* xgm;
// convert to XGM
xgm = XGM_createFromVGM(optVgm);
if (xgm == NULL) exit(1);
// XGM output
if (!strcasecmp(outExt, "XGM"))
{
// get byte array
outData = XGM_asByteArray(xgm, &outDataSize);
}
else
{
XGM* xgc;
// convert to XGC (compiled XGM)
xgc = XGC_create(xgm);
if (xgc == NULL) exit(1);
// get byte array
outData = XGC_asByteArray(xgc, &outDataSize);
}
if (outData == NULL) exit(1);
// write to file
writeBinaryFile(outData, outDataSize, argv[2]);
}
}
else
{
printf("Error: the output file %s is incorrect (should be a VGM, XGM or BIN/XGC file)\n", argv[2]);
errCode = 4;
}
}
else if (!strcasecmp(inExt, "XGM"))
{
if ((!strcasecmp(outExt, "VGM")) || (!strcasecmp(outExt, "BIN")) || (!strcasecmp(outExt, "XGC")))
{
// XGM to VGM
int inDataSize;
unsigned char* inData;
int outDataSize;
unsigned char* outData;
XGM* xgm;
// load file
inData = readBinaryFile(argv[1], &inDataSize);
if (inData == NULL) exit(1);
// load XGM
xgm = XGM_createFromData(inData, inDataSize);
if (xgm == NULL) exit(1);
// VGM conversion
if (!strcasecmp(outExt, "VGM"))
{
VGM* vgm;
// convert to VGM
vgm = VGM_createFromXGM(xgm);
if (vgm == NULL) exit(1);
// get byte array
outData = VGM_asByteArray(vgm, &outDataSize);
}
else
{
XGM* xgc;
// convert to XGC (compiled XGM)
xgc = XGC_create(xgm);
if (xgc == NULL) exit(1);
// get byte array
outData = XGC_asByteArray(xgc, &outDataSize);
}
if (outData == NULL) exit(1);
// write to file
writeBinaryFile(outData, outDataSize, argv[2]);
}
else
{
printf("Error: the output file %s is incorrect (should be a VGM or BIN/XGC file)\n", argv[2]);
errCode = 4;
}
}
else if (!strcasecmp(inExt, "XGC"))
{
if ((!strcasecmp(outExt, "VGM")) || (!strcasecmp(outExt, "XGM")))
{
// XGC to XGM
int inDataSize;
unsigned char* inData;
int outDataSize;
unsigned char* outData;
XGM* xgm;
// load file
inData = readBinaryFile(argv[1], &inDataSize);
if (inData == NULL) exit(1);
// load XGM
xgm = XGM_createFromXGCData(inData, inDataSize);
if (xgm == NULL) exit(1);
// VGM conversion
if (!strcasecmp(outExt, "VGM"))
{
VGM* vgm;
// convert to VGM
vgm = VGM_createFromXGM(xgm);
if (vgm == NULL) exit(1);
// get byte array
outData = VGM_asByteArray(vgm, &outDataSize);
}
else
{
// get byte array
outData = XGM_asByteArray(xgm, &outDataSize);
}
if (outData == NULL) exit(1);
// write to file
writeBinaryFile(outData, outDataSize, argv[2]);
}
else
{
printf("Error: the output file %s is incorrect (should be a XGM or VGM file)\n", argv[2]);
errCode = 4;
}
}
else
{
printf("Error: the input file %s is incorrect (should be a VGM, XGM or XGC file)\n", argv[1]);
errCode = 4;
}
fclose(infile);
remove("tmp.bin");
return errCode;
}