AST_Module* parse_file(const char* fn) {
STAT_TIMER(t0, "us_timer_cpyton_parsing");
Timer _t("parsing");
if (ENABLE_PYPA_PARSER) {
AST_Module* rtn = pypa_parse(fn);
assert(rtn);
return rtn;
}
FILE* fp = popen(getParserCommandLine(fn).c_str(), "r");
BufferedReader* reader = new BufferedReader(fp);
AST* rtn = readASTMisc(reader);
reader->fill();
ASSERT(reader->bytesBuffered() == 0, "%d", reader->bytesBuffered());
delete reader;
int code = pclose(fp);
assert(code == 0);
assert(rtn->type == AST_TYPE::Module);
long us = _t.end();
static StatCounter us_parsing("us_parsing");
us_parsing.log(us);
return ast_cast<AST_Module>(rtn);
}
AST_Lambda* read_lambda(BufferedReader* reader) {
AST_Lambda* rtn = new AST_Lambda();
rtn->args = ast_cast<AST_arguments>(readASTMisc(reader));
rtn->body = readASTExpr(reader);
rtn->col_offset = readColOffset(reader);
rtn->lineno = reader->readULL();
return rtn;
}
static void readMiscVector(std::vector<T*> &vec, BufferedReader *reader) {
int num_elts = reader->readShort();
if (VERBOSITY("parsing") >= 2)
printf("%d elts to read\n", num_elts);
for (int i = 0; i < num_elts; i++) {
AST* read = readASTMisc(reader);
assert(read->type == T::TYPE);
vec.push_back(static_cast<T*>(read));
}
}
AST_FunctionDef* read_functiondef(BufferedReader *reader) {
if (VERBOSITY("parsing") >= 2)
printf("reading functiondef\n");
AST_FunctionDef *rtn = new AST_FunctionDef();
rtn->args = static_cast<AST_arguments*>(readASTMisc(reader));
readStmtVector(rtn->body, reader);
rtn->col_offset = readColOffset(reader);
readExprVector(rtn->decorator_list, reader);
rtn->lineno = reader->readULL();
rtn->name = readString(reader);
return rtn;
}
AST_Import* read_import(BufferedReader *reader) {
AST_Import *rtn = new AST_Import();
rtn->col_offset = readColOffset(reader);
rtn->lineno = reader->readULL();
int num_elts = reader->readShort();
for (int i = 0; i < num_elts; i++) {
AST* elt = readASTMisc(reader);
assert(elt->type == AST_TYPE::alias);
rtn->names.push_back(static_cast<AST_alias*>(elt));
}
return rtn;
}
// Parsing the file is somewhat expensive since we have to shell out to cpython;
// it's not a huge deal right now, but this caching version can significantly cut down
// on the startup time (40ms -> 10ms).
AST_Module* caching_parse(const char* fn) {
Timer _t("parsing");
int code;
std::string cache_fn = std::string(fn) + "c";
struct stat source_stat, cache_stat;
code = stat(fn, &source_stat);
assert(code == 0);
code = stat(cache_fn.c_str(), &cache_stat);
if (code != 0 || cache_stat.st_mtime < source_stat.st_mtime ||
(cache_stat.st_mtime == source_stat.st_mtime && cache_stat.st_mtim.tv_nsec < source_stat.st_mtim.tv_nsec)) {
_reparse(fn, cache_fn);
}
FILE *fp = fopen(cache_fn.c_str(), "r");
assert(fp);
while (true) {
char buf[MAGIC_STRING_LENGTH];
int read = fread(buf, 1, MAGIC_STRING_LENGTH, fp);
if (read != 4 || strncmp(buf, MAGIC_STRING, MAGIC_STRING_LENGTH) != 0) {
fclose(fp);
_reparse(fn, cache_fn);
fp = fopen(cache_fn.c_str(), "r");
assert(fp);
} else {
break;
}
}
BufferedReader *reader = new BufferedReader(fp);
AST* rtn = readASTMisc(reader);
reader->fill();
assert(reader->bytesBuffered() == 0);
delete reader;
assert(rtn->type == AST_TYPE::Module);
long us = _t.end();
static StatCounter us_parsing("us_parsing");
us_parsing.log(us);
return static_cast<AST_Module*>(rtn);
}
AST_Module* parse(const char* fn) {
Timer _t("parsing");
std::string cmdline = "python codegen/parse_ast.py ";
cmdline += fn;
FILE *fp = popen(cmdline.c_str(), "r");
BufferedReader *reader = new BufferedReader(fp);
AST* rtn = readASTMisc(reader);
reader->fill();
ASSERT(reader->bytesBuffered() == 0, "%d", reader->bytesBuffered());
delete reader;
int code = pclose(fp);
assert(code == 0);
assert(rtn->type == AST_TYPE::Module);
long us = _t.end();
static StatCounter us_parsing("us_parsing");
us_parsing.log(us);
return static_cast<AST_Module*>(rtn);
}
// Parsing the file is somewhat expensive since we have to shell out to cpython;
// it's not a huge deal right now, but this caching version can significantly cut down
// on the startup time (40ms -> 10ms).
AST_Module* caching_parse(const char* fn) {
Timer _t("parsing");
int code;
std::string cache_fn = std::string(fn) + "c";
struct stat source_stat, cache_stat;
code = stat(fn, &source_stat);
assert(code == 0);
code = stat(cache_fn.c_str(), &cache_stat);
if (code != 0 || cache_stat.st_mtime < source_stat.st_mtime
|| (cache_stat.st_mtime == source_stat.st_mtime && cache_stat.st_mtim.tv_nsec < source_stat.st_mtim.tv_nsec)) {
_reparse(fn, cache_fn);
code = stat(cache_fn.c_str(), &cache_stat);
assert(code == 0);
}
FILE* fp = fopen(cache_fn.c_str(), "r");
assert(fp);
while (true) {
bool good = true;
if (good) {
char buf[MAGIC_STRING_LENGTH];
int read = fread(buf, 1, MAGIC_STRING_LENGTH, fp);
if (read != MAGIC_STRING_LENGTH || strncmp(buf, MAGIC_STRING, MAGIC_STRING_LENGTH) != 0) {
if (VERBOSITY()) {
printf("Warning: corrupt or non-Pyston .pyc file found; ignoring\n");
}
good = false;
}
}
if (good) {
int length = 0;
fseek(fp, MAGIC_STRING_LENGTH, SEEK_SET);
static_assert(sizeof(length) >= CHECKSUM_LENGTH, "");
int read = fread(&length, 1, CHECKSUM_LENGTH, fp);
int expected_total_length = MAGIC_STRING_LENGTH + CHECKSUM_LENGTH + length;
if (read != CHECKSUM_LENGTH || expected_total_length != cache_stat.st_size) {
if (VERBOSITY()) {
printf("Warning: truncated .pyc file found; ignoring\n");
}
good = false;
}
}
if (!good) {
fclose(fp);
_reparse(fn, cache_fn);
code = stat(cache_fn.c_str(), &cache_stat);
assert(code == 0);
fp = fopen(cache_fn.c_str(), "r");
assert(fp);
} else {
break;
}
}
BufferedReader* reader = new BufferedReader(fp);
AST* rtn = readASTMisc(reader);
reader->fill();
assert(reader->bytesBuffered() == 0);
delete reader;
assert(rtn->type == AST_TYPE::Module);
long us = _t.end();
static StatCounter us_parsing("us_parsing");
us_parsing.log(us);
return ast_cast<AST_Module>(rtn);
}
// Parsing the file is somewhat expensive since we have to shell out to cpython;
// it's not a huge deal right now, but this caching version can significantly cut down
// on the startup time (40ms -> 10ms).
AST_Module* caching_parse_file(const char* fn) {
STAT_TIMER(t0, "us_timer_caching_parse_file");
static StatCounter us_parsing("us_parsing");
Timer _t("parsing");
_t.setExitCallback([](uint64_t t) { us_parsing.log(t); });
int code;
std::string cache_fn = std::string(fn) + "c";
struct stat source_stat, cache_stat;
code = stat(fn, &source_stat);
assert(code == 0);
code = stat(cache_fn.c_str(), &cache_stat);
if (code != 0 || cache_stat.st_mtime < source_stat.st_mtime
|| (cache_stat.st_mtime == source_stat.st_mtime && cache_stat.st_mtim.tv_nsec < source_stat.st_mtim.tv_nsec)) {
AST_Module* mod = 0;
auto result = _reparse(fn, cache_fn, mod);
if (mod)
return mod;
if (result == ParseResult::FAILURE)
return NULL;
if (result == ParseResult::PYC_UNWRITABLE)
return parse_file(fn);
code = stat(cache_fn.c_str(), &cache_stat);
assert(code == 0);
}
FILE* fp = fopen(cache_fn.c_str(), "r");
assert(fp);
while (true) {
bool good = true;
if (good) {
char buf[MAGIC_STRING_LENGTH];
int read = fread(buf, 1, MAGIC_STRING_LENGTH, fp);
if (read != MAGIC_STRING_LENGTH || strncmp(buf, getMagic(), MAGIC_STRING_LENGTH) != 0) {
if (VERBOSITY()) {
printf("Warning: corrupt or non-Pyston .pyc file found; ignoring\n");
}
good = false;
}
}
if (good) {
int length = 0;
fseek(fp, MAGIC_STRING_LENGTH, SEEK_SET);
static_assert(sizeof(length) >= CHECKSUM_LENGTH, "");
int read = fread(&length, 1, CHECKSUM_LENGTH, fp);
int expected_total_length = MAGIC_STRING_LENGTH + CHECKSUM_LENGTH + length;
if (read != CHECKSUM_LENGTH || expected_total_length != cache_stat.st_size) {
if (VERBOSITY()) {
printf("Warning: truncated .pyc file found; ignoring\n");
}
good = false;
}
}
if (!good) {
fclose(fp);
AST_Module* mod = 0;
auto result = _reparse(fn, cache_fn, mod);
if (mod)
return mod;
if (result == ParseResult::FAILURE)
return NULL;
if (result == ParseResult::PYC_UNWRITABLE)
return parse_file(fn);
code = stat(cache_fn.c_str(), &cache_stat);
assert(code == 0);
fp = fopen(cache_fn.c_str(), "r");
assert(fp);
} else {
break;
}
}
BufferedReader* reader = new BufferedReader(fp);
AST* rtn = readASTMisc(reader);
reader->fill();
assert(reader->bytesBuffered() == 0);
delete reader;
fclose(fp);
assert(rtn->type == AST_TYPE::Module);
return ast_cast<AST_Module>(rtn);
}