AST_Module* parse_file(const char* fn) {
STAT_TIMER(t0, "us_timer_cpyton_parsing");
Timer _t("parsing");
if (ENABLE_PYPA_PARSER) {
AST_Module* rtn = pypa_parse(fn);
assert(rtn);
return rtn;
}
FILE* fp = popen(getParserCommandLine(fn).c_str(), "r");
BufferedReader* reader = new BufferedReader(fp);
AST* rtn = readASTMisc(reader);
reader->fill();
ASSERT(reader->bytesBuffered() == 0, "%d", reader->bytesBuffered());
delete reader;
int code = pclose(fp);
assert(code == 0);
assert(rtn->type == AST_TYPE::Module);
long us = _t.end();
static StatCounter us_parsing("us_parsing");
us_parsing.log(us);
return ast_cast<AST_Module>(rtn);
}
// Parsing the file is somewhat expensive since we have to shell out to cpython;
// it's not a huge deal right now, but this caching version can significantly cut down
// on the startup time (40ms -> 10ms).
AST_Module* caching_parse(const char* fn) {
Timer _t("parsing");
int code;
std::string cache_fn = std::string(fn) + "c";
struct stat source_stat, cache_stat;
code = stat(fn, &source_stat);
assert(code == 0);
code = stat(cache_fn.c_str(), &cache_stat);
if (code != 0 || cache_stat.st_mtime < source_stat.st_mtime ||
(cache_stat.st_mtime == source_stat.st_mtime && cache_stat.st_mtim.tv_nsec < source_stat.st_mtim.tv_nsec)) {
_reparse(fn, cache_fn);
}
FILE *fp = fopen(cache_fn.c_str(), "r");
assert(fp);
while (true) {
char buf[MAGIC_STRING_LENGTH];
int read = fread(buf, 1, MAGIC_STRING_LENGTH, fp);
if (read != 4 || strncmp(buf, MAGIC_STRING, MAGIC_STRING_LENGTH) != 0) {
fclose(fp);
_reparse(fn, cache_fn);
fp = fopen(cache_fn.c_str(), "r");
assert(fp);
} else {
break;
}
}
BufferedReader *reader = new BufferedReader(fp);
AST* rtn = readASTMisc(reader);
reader->fill();
assert(reader->bytesBuffered() == 0);
delete reader;
assert(rtn->type == AST_TYPE::Module);
long us = _t.end();
static StatCounter us_parsing("us_parsing");
us_parsing.log(us);
return static_cast<AST_Module*>(rtn);
}
AST_Module* parse(const char* fn) {
Timer _t("parsing");
std::string cmdline = "python codegen/parse_ast.py ";
cmdline += fn;
FILE *fp = popen(cmdline.c_str(), "r");
BufferedReader *reader = new BufferedReader(fp);
AST* rtn = readASTMisc(reader);
reader->fill();
ASSERT(reader->bytesBuffered() == 0, "%d", reader->bytesBuffered());
delete reader;
int code = pclose(fp);
assert(code == 0);
assert(rtn->type == AST_TYPE::Module);
long us = _t.end();
static StatCounter us_parsing("us_parsing");
us_parsing.log(us);
return static_cast<AST_Module*>(rtn);
}
/**
* Rewind the file to the beginning.
*/
void Rewind() {
file.Rewind();
buffered.Reset();
}
// Parsing the file is somewhat expensive since we have to shell out to cpython;
// it's not a huge deal right now, but this caching version can significantly cut down
// on the startup time (40ms -> 10ms).
AST_Module* caching_parse(const char* fn) {
Timer _t("parsing");
int code;
std::string cache_fn = std::string(fn) + "c";
struct stat source_stat, cache_stat;
code = stat(fn, &source_stat);
assert(code == 0);
code = stat(cache_fn.c_str(), &cache_stat);
if (code != 0 || cache_stat.st_mtime < source_stat.st_mtime
|| (cache_stat.st_mtime == source_stat.st_mtime && cache_stat.st_mtim.tv_nsec < source_stat.st_mtim.tv_nsec)) {
_reparse(fn, cache_fn);
code = stat(cache_fn.c_str(), &cache_stat);
assert(code == 0);
}
FILE* fp = fopen(cache_fn.c_str(), "r");
assert(fp);
while (true) {
bool good = true;
if (good) {
char buf[MAGIC_STRING_LENGTH];
int read = fread(buf, 1, MAGIC_STRING_LENGTH, fp);
if (read != MAGIC_STRING_LENGTH || strncmp(buf, MAGIC_STRING, MAGIC_STRING_LENGTH) != 0) {
if (VERBOSITY()) {
printf("Warning: corrupt or non-Pyston .pyc file found; ignoring\n");
}
good = false;
}
}
if (good) {
int length = 0;
fseek(fp, MAGIC_STRING_LENGTH, SEEK_SET);
static_assert(sizeof(length) >= CHECKSUM_LENGTH, "");
int read = fread(&length, 1, CHECKSUM_LENGTH, fp);
int expected_total_length = MAGIC_STRING_LENGTH + CHECKSUM_LENGTH + length;
if (read != CHECKSUM_LENGTH || expected_total_length != cache_stat.st_size) {
if (VERBOSITY()) {
printf("Warning: truncated .pyc file found; ignoring\n");
}
good = false;
}
}
if (!good) {
fclose(fp);
_reparse(fn, cache_fn);
code = stat(cache_fn.c_str(), &cache_stat);
assert(code == 0);
fp = fopen(cache_fn.c_str(), "r");
assert(fp);
} else {
break;
}
}
BufferedReader* reader = new BufferedReader(fp);
AST* rtn = readASTMisc(reader);
reader->fill();
assert(reader->bytesBuffered() == 0);
delete reader;
assert(rtn->type == AST_TYPE::Module);
long us = _t.end();
static StatCounter us_parsing("us_parsing");
us_parsing.log(us);
return ast_cast<AST_Module>(rtn);
}
// Parsing the file is somewhat expensive since we have to shell out to cpython;
// it's not a huge deal right now, but this caching version can significantly cut down
// on the startup time (40ms -> 10ms).
AST_Module* caching_parse_file(const char* fn) {
STAT_TIMER(t0, "us_timer_caching_parse_file");
static StatCounter us_parsing("us_parsing");
Timer _t("parsing");
_t.setExitCallback([](uint64_t t) { us_parsing.log(t); });
int code;
std::string cache_fn = std::string(fn) + "c";
struct stat source_stat, cache_stat;
code = stat(fn, &source_stat);
assert(code == 0);
code = stat(cache_fn.c_str(), &cache_stat);
if (code != 0 || cache_stat.st_mtime < source_stat.st_mtime
|| (cache_stat.st_mtime == source_stat.st_mtime && cache_stat.st_mtim.tv_nsec < source_stat.st_mtim.tv_nsec)) {
AST_Module* mod = 0;
auto result = _reparse(fn, cache_fn, mod);
if (mod)
return mod;
if (result == ParseResult::FAILURE)
return NULL;
if (result == ParseResult::PYC_UNWRITABLE)
return parse_file(fn);
code = stat(cache_fn.c_str(), &cache_stat);
assert(code == 0);
}
FILE* fp = fopen(cache_fn.c_str(), "r");
assert(fp);
while (true) {
bool good = true;
if (good) {
char buf[MAGIC_STRING_LENGTH];
int read = fread(buf, 1, MAGIC_STRING_LENGTH, fp);
if (read != MAGIC_STRING_LENGTH || strncmp(buf, getMagic(), MAGIC_STRING_LENGTH) != 0) {
if (VERBOSITY()) {
printf("Warning: corrupt or non-Pyston .pyc file found; ignoring\n");
}
good = false;
}
}
if (good) {
int length = 0;
fseek(fp, MAGIC_STRING_LENGTH, SEEK_SET);
static_assert(sizeof(length) >= CHECKSUM_LENGTH, "");
int read = fread(&length, 1, CHECKSUM_LENGTH, fp);
int expected_total_length = MAGIC_STRING_LENGTH + CHECKSUM_LENGTH + length;
if (read != CHECKSUM_LENGTH || expected_total_length != cache_stat.st_size) {
if (VERBOSITY()) {
printf("Warning: truncated .pyc file found; ignoring\n");
}
good = false;
}
}
if (!good) {
fclose(fp);
AST_Module* mod = 0;
auto result = _reparse(fn, cache_fn, mod);
if (mod)
return mod;
if (result == ParseResult::FAILURE)
return NULL;
if (result == ParseResult::PYC_UNWRITABLE)
return parse_file(fn);
code = stat(cache_fn.c_str(), &cache_stat);
assert(code == 0);
fp = fopen(cache_fn.c_str(), "r");
assert(fp);
} else {
break;
}
}
BufferedReader* reader = new BufferedReader(fp);
AST* rtn = readASTMisc(reader);
reader->fill();
assert(reader->bytesBuffered() == 0);
delete reader;
fclose(fp);
assert(rtn->type == AST_TYPE::Module);
return ast_cast<AST_Module>(rtn);
}