PyObject* plPythonPack::OpenPacked(const char* fileName)
{
if (!Open())
return nil;
std::string pythonName = fileName;
pythonName += ".py";
FileOffset::iterator it = fFileOffsets.find(pythonName);
if (it != fFileOffsets.end())
{
plPackOffsetInfo offsetInfo = (*it).second;
hsStream* fPackStream = fPackStreams[offsetInfo.fStreamIndex];
fPackStream->SetPosition(offsetInfo.fOffset);
int32_t size = fPackStream->ReadLE32();
if (size > 0)
{
char *buf = new char[size];
uint32_t readSize = fPackStream->Read(size, buf);
hsAssert(readSize <= size, xtl::format("Python PackFile %s: Incorrect amount of data, read %d instead of %d",
fileName, readSize, size).c_str());
// let the python marshal make it back into a code object
PyObject *pythonCode = PyMarshal_ReadObjectFromString(buf, size);
delete [] buf;
return pythonCode;
}
}
return nil;
}
bool plPythonPack::Open()
{
if (fPackStreams.size() > 0)
return true;
// We already tried and it wasn't there
if (fPackNotFound)
return false;
fPackNotFound = true;
// Get the names of all the pak files
std::vector<plFileName> files = plStreamSource::GetInstance()->GetListOfNames("python", "pak");
std::vector<time_t> modTimes; // the modification time for each of the streams (to resolve duplicate file issues)
// grab all the .pak files in the folder
for (int curName = 0; curName < files.size(); curName++)
{
// obtain the stream
hsStream *fPackStream = plStreamSource::GetInstance()->GetFile(files[curName]);
if (fPackStream)
{
fPackStream->Rewind(); // make sure we're at the beginning of the file
fPackNotFound = false;
time_t curModTime = 0;
plFileInfo info(files[curName]);
if (info.Exists())
curModTime = info.ModifyTime();
modTimes.push_back(curModTime);
// read the index data
int numFiles = fPackStream->ReadLE32();
uint32_t streamIndex = (uint32_t)(fPackStreams.size());
for (int i = 0; i < numFiles; i++)
{
// and pack the index into our own data structure
plString pythonName = fPackStream->ReadSafeString();
uint32_t offset = fPackStream->ReadLE32();
plPackOffsetInfo offsetInfo;
offsetInfo.fOffset = offset;
offsetInfo.fStreamIndex = streamIndex;
if (fFileOffsets.find(pythonName) != fFileOffsets.end())
{
uint32_t index = fFileOffsets[pythonName].fStreamIndex;
if (modTimes[index] < curModTime) // is the existing file older then the new one?
fFileOffsets[pythonName] = offsetInfo; // yup, so replace it with the new info
}
else
fFileOffsets[pythonName] = offsetInfo; // no conflicts, add the info
}
fPackStreams.push_back(fPackStream);
}
}
return !fPackNotFound;
}
static void applyRewrite(EditsReceiver &receiver,
StringRef text, FileOffset offs, unsigned len,
const SourceManager &SM, const LangOptions &LangOpts) {
assert(!offs.getFID().isInvalid());
SourceLocation Loc = SM.getLocForStartOfFile(offs.getFID());
Loc = Loc.getLocWithOffset(offs.getOffset());
assert(Loc.isFileID());
if (text.empty())
adjustRemoval(SM, LangOpts, Loc, offs, len, text);
CharSourceRange range = CharSourceRange::getCharRange(Loc,
Loc.getLocWithOffset(len));
if (text.empty()) {
assert(len);
receiver.remove(range);
return;
}
if (len)
receiver.replace(range, text);
else
receiver.insert(Loc, text);
}
bool plPythonPack::IsPackedFile(const plString& fileName)
{
if (!Open())
return nil;
plString pythonName = fileName + ".py";
FileOffset:: iterator it = fFileOffsets.find(pythonName);
if (it != fFileOffsets.end())
return true;
return false;
}
EditedSource::FileEditsTy::iterator
EditedSource::getActionForOffset(FileOffset Offs) {
FileEditsTy::iterator I = FileEdits.upper_bound(Offs);
if (I == FileEdits.begin())
return FileEdits.end();
--I;
FileEdit &FA = I->second;
FileOffset B = I->first;
FileOffset E = B.getWithOffset(FA.RemoveLen);
if (Offs >= B && Offs < E)
return I;
return FileEdits.end();
}
bool Commit::canReplaceText(SourceLocation loc, StringRef text,
FileOffset &Offs, unsigned &Len) {
assert(!text.empty());
if (!canInsert(loc, Offs))
return false;
// Try to load the file buffer.
bool invalidTemp = false;
StringRef file = SourceMgr.getBufferData(Offs.getFID(), &invalidTemp);
if (invalidTemp)
return false;
Len = text.size();
return file.substr(Offs.getOffset()).startswith(text);
}
/// \brief Check the range that we are going to remove and:
/// -Remove any trailing whitespace if possible.
/// -Insert a space if removing the range is going to mess up the source tokens.
static void adjustRemoval(const SourceManager &SM, const LangOptions &LangOpts,
SourceLocation Loc, FileOffset offs,
unsigned &len, StringRef &text) {
assert(len && text.empty());
SourceLocation BeginTokLoc = Lexer::GetBeginningOfToken(Loc, SM, LangOpts);
if (BeginTokLoc != Loc)
return; // the range is not at the beginning of a token, keep the range.
bool Invalid = false;
StringRef buffer = SM.getBufferData(offs.getFID(), &Invalid);
if (Invalid)
return;
unsigned begin = offs.getOffset();
unsigned end = begin + len;
// Do not try to extend the removal if we're at the end of the buffer already.
if (end == buffer.size())
return;
assert(begin < buffer.size() && end < buffer.size() && "Invalid range!");
// FIXME: Remove newline.
if (begin == 0) {
if (buffer[end] == ' ')
++len;
return;
}
if (buffer[end] == ' ') {
assert((end + 1 != buffer.size() || buffer.data()[end + 1] == 0) &&
"buffer not zero-terminated!");
if (canRemoveWhitespace(/*left=*/buffer[begin-1],
/*beforeWSpace=*/buffer[end-1],
/*right=*/buffer.data()[end + 1], // zero-terminated
LangOpts))
++len;
return;
}
if (!canBeJoined(buffer[begin-1], buffer[end], LangOpts))
text = " ";
}
bool Commit::canInsertInOffset(SourceLocation OrigLoc, FileOffset Offs) {
for (const auto &act : CachedEdits)
if (act.Kind == Act_Remove) {
if (act.Offset.getFID() == Offs.getFID() &&
Offs > act.Offset && Offs < act.Offset.getWithOffset(act.Length))
return false; // position has been removed.
}
if (!Editor)
return true;
return Editor->canInsertInOffset(OrigLoc, Offs);
}
bool Commit::canInsertInOffset(SourceLocation OrigLoc, FileOffset Offs) {
for (unsigned i = 0, e = CachedEdits.size(); i != e; ++i) {
Edit &act = CachedEdits[i];
if (act.Kind == Act_Remove) {
if (act.Offset.getFID() == Offs.getFID() &&
Offs > act.Offset && Offs < act.Offset.getWithOffset(act.Length))
return false; // position has been removed.
}
}
if (!Editor)
return true;
return Editor->canInsertInOffset(OrigLoc, Offs);
}
bool EditedSource::commitInsertFromRange(SourceLocation OrigLoc,
FileOffset Offs,
FileOffset InsertFromRangeOffs, unsigned Len,
bool beforePreviousInsertions) {
if (Len == 0)
return true;
SmallString<128> StrVec;
FileOffset BeginOffs = InsertFromRangeOffs;
FileOffset EndOffs = BeginOffs.getWithOffset(Len);
FileEditsTy::iterator I = FileEdits.upper_bound(BeginOffs);
if (I != FileEdits.begin())
--I;
for (; I != FileEdits.end(); ++I) {
FileEdit &FA = I->second;
FileOffset B = I->first;
FileOffset E = B.getWithOffset(FA.RemoveLen);
if (BeginOffs == B)
break;
if (BeginOffs < E) {
if (BeginOffs > B) {
BeginOffs = E;
++I;
}
break;
}
}
for (; I != FileEdits.end() && EndOffs > I->first; ++I) {
FileEdit &FA = I->second;
FileOffset B = I->first;
FileOffset E = B.getWithOffset(FA.RemoveLen);
if (BeginOffs < B) {
bool Invalid = false;
StringRef text = getSourceText(BeginOffs, B, Invalid);
if (Invalid)
return false;
StrVec += text;
}
StrVec += FA.Text;
BeginOffs = E;
}
if (BeginOffs < EndOffs) {
bool Invalid = false;
StringRef text = getSourceText(BeginOffs, EndOffs, Invalid);
if (Invalid)
return false;
StrVec += text;
}
return commitInsert(OrigLoc, Offs, StrVec.str(), beforePreviousInsertions);
}
/// \brief Check the range that we are going to remove and:
/// -Remove any trailing whitespace if possible.
/// -Insert a space if removing the range is going to mess up the source tokens.
static void adjustRemoval(const SourceManager &SM, const LangOptions &LangOpts,
SourceLocation Loc, FileOffset offs,
unsigned &len, StringRef &text) {
assert(len && text.empty());
SourceLocation BeginTokLoc = Lexer::GetBeginningOfToken(Loc, SM, LangOpts);
if (BeginTokLoc != Loc)
return; // the range is not at the beginning of a token, keep the range.
bool Invalid = false;
StringRef buffer = SM.getBufferData(offs.getFID(), &Invalid);
if (Invalid)
return;
unsigned begin = offs.getOffset();
unsigned end = begin + len;
// FIXME: Remove newline.
if (begin == 0) {
if (buffer[end] == ' ')
++len;
return;
}
if (buffer[end] == ' ') {
if (canRemoveWhitespace(/*left=*/buffer[begin-1],
/*beforeWSpace=*/buffer[end-1],
/*right=*/buffer[end+1],
LangOpts))
++len;
return;
}
if (!canBeJoined(buffer[begin-1], buffer[end], LangOpts))
text = " ";
}
void plPythonPack::Close()
{
if (fPackStreams.size() == 0)
return;
int i;
for (i=0; i<fPackStreams.size(); i++)
{
hsStream* fPackStream = fPackStreams[i];
// do NOT close or delete the streams, the preloader will do that for us
fPackStreams[i] = nil;
}
fPackStreams.clear();
fFileOffsets.clear();
}
StringRef EditedSource::getSourceText(FileOffset BeginOffs, FileOffset EndOffs,
bool &Invalid) {
assert(BeginOffs.getFID() == EndOffs.getFID());
assert(BeginOffs <= EndOffs);
SourceLocation BLoc = SourceMgr.getLocForStartOfFile(BeginOffs.getFID());
BLoc = BLoc.getLocWithOffset(BeginOffs.getOffset());
assert(BLoc.isFileID());
SourceLocation
ELoc = BLoc.getLocWithOffset(EndOffs.getOffset() - BeginOffs.getOffset());
return Lexer::getSourceText(CharSourceRange::getCharRange(BLoc, ELoc),
SourceMgr, LangOpts, &Invalid);
}
bool plPythonPack::Open()
{
if (fPackStreams.size() > 0)
return true;
// We already tried and it wasn't there
if (fPackNotFound)
return false;
fPackNotFound = true;
// Get the names of all the pak files
std::vector<std::wstring> files = plStreamSource::GetInstance()->GetListOfNames(L"python", L".pak");
std::vector<time_t> modTimes; // the modification time for each of the streams (to resolve duplicate file issues)
// grab all the .pak files in the folder
for (int curName = 0; curName < files.size(); curName++)
{
// obtain the stream
hsStream *fPackStream = plStreamSource::GetInstance()->GetFile(files[curName]);
if (fPackStream)
{
fPackStream->Rewind(); // make sure we're at the beginning of the file
fPackNotFound = false;
char* tempFilename = hsWStringToString(files[curName].c_str());
struct stat buf;
time_t curModTime = 0;
if (stat(tempFilename,&buf)==0)
curModTime = buf.st_mtime;
modTimes.push_back(curModTime);
delete [] tempFilename;
// read the index data
int numFiles = fPackStream->ReadLE32();
uint32_t streamIndex = (uint32_t)(fPackStreams.size());
for (int i = 0; i < numFiles; i++)
{
// and pack the index into our own data structure
char* buf = fPackStream->ReadSafeString();
std::string pythonName = buf; // reading a "string" from a hsStream directly into a stl string causes memory loss
delete [] buf;
uint32_t offset = fPackStream->ReadLE32();
plPackOffsetInfo offsetInfo;
offsetInfo.fOffset = offset;
offsetInfo.fStreamIndex = streamIndex;
if (fFileOffsets.find(pythonName) != fFileOffsets.end())
{
uint32_t index = fFileOffsets[pythonName].fStreamIndex;
if (modTimes[index] < curModTime) // is the existing file older then the new one?
fFileOffsets[pythonName] = offsetInfo; // yup, so replace it with the new info
}
else
fFileOffsets[pythonName] = offsetInfo; // no conflicts, add the info
}
fPackStreams.push_back(fPackStream);
}
}
return !fPackNotFound;
}
void EditedSource::commitRemove(SourceLocation OrigLoc,
FileOffset BeginOffs, unsigned Len) {
if (Len == 0)
return;
FileOffset EndOffs = BeginOffs.getWithOffset(Len);
FileEditsTy::iterator I = FileEdits.upper_bound(BeginOffs);
if (I != FileEdits.begin())
--I;
for (; I != FileEdits.end(); ++I) {
FileEdit &FA = I->second;
FileOffset B = I->first;
FileOffset E = B.getWithOffset(FA.RemoveLen);
if (BeginOffs < E)
break;
}
FileOffset TopBegin, TopEnd;
FileEdit *TopFA = nullptr;
if (I == FileEdits.end()) {
FileEditsTy::iterator
NewI = FileEdits.insert(I, std::make_pair(BeginOffs, FileEdit()));
NewI->second.RemoveLen = Len;
return;
}
FileEdit &FA = I->second;
FileOffset B = I->first;
FileOffset E = B.getWithOffset(FA.RemoveLen);
if (BeginOffs < B) {
FileEditsTy::iterator
NewI = FileEdits.insert(I, std::make_pair(BeginOffs, FileEdit()));
TopBegin = BeginOffs;
TopEnd = EndOffs;
TopFA = &NewI->second;
TopFA->RemoveLen = Len;
} else {
TopBegin = B;
TopEnd = E;
TopFA = &I->second;
if (TopEnd >= EndOffs)
return;
unsigned diff = EndOffs.getOffset() - TopEnd.getOffset();
TopEnd = EndOffs;
TopFA->RemoveLen += diff;
if (B == BeginOffs)
TopFA->Text = StringRef();
++I;
}
while (I != FileEdits.end()) {
FileEdit &FA = I->second;
FileOffset B = I->first;
FileOffset E = B.getWithOffset(FA.RemoveLen);
if (B >= TopEnd)
break;
if (E <= TopEnd) {
FileEdits.erase(I++);
continue;
}
if (B < TopEnd) {
unsigned diff = E.getOffset() - TopEnd.getOffset();
TopEnd = E;
TopFA->RemoveLen += diff;
FileEdits.erase(I);
}
break;
}
}
bool Commit::replaceWithInner(CharSourceRange range,
CharSourceRange replacementRange) {
FileOffset OuterBegin;
unsigned OuterLen;
if (!canRemoveRange(range, OuterBegin, OuterLen)) {
IsCommitable = false;
return false;
}
FileOffset InnerBegin;
unsigned InnerLen;
if (!canRemoveRange(replacementRange, InnerBegin, InnerLen)) {
IsCommitable = false;
return false;
}
FileOffset OuterEnd = OuterBegin.getWithOffset(OuterLen);
FileOffset InnerEnd = InnerBegin.getWithOffset(InnerLen);
if (OuterBegin.getFID() != InnerBegin.getFID() ||
InnerBegin < OuterBegin ||
InnerBegin > OuterEnd ||
InnerEnd > OuterEnd) {
IsCommitable = false;
return false;
}
addRemove(range.getBegin(),
OuterBegin, InnerBegin.getOffset() - OuterBegin.getOffset());
addRemove(replacementRange.getEnd(),
InnerEnd, OuterEnd.getOffset() - InnerEnd.getOffset());
return true;
}
