/* copy the tokens from StartParser to EndParser into new memory, removing TokenEOFs and terminate with a TokenEndOfFunction */
void *LexCopyTokens(struct ParseState *StartParser, struct ParseState *EndParser)
{
int MemSize = 0;
int CopySize;
unsigned char *Pos = (unsigned char *)StartParser->Pos;
unsigned char *NewTokens;
unsigned char *NewTokenPos;
struct TokenLine *ILine;
Picoc *pc = StartParser->pc;
if (pc->InteractiveHead == NULL)
{
/* non-interactive mode - copy the tokens */
MemSize = EndParser->Pos - StartParser->Pos;
NewTokens = VariableAlloc(pc, StartParser, MemSize + TOKEN_DATA_OFFSET, TRUE);
memcpy(NewTokens, (void *)StartParser->Pos, MemSize);
}
else
{
/* we're in interactive mode - add up line by line */
for (pc->InteractiveCurrentLine = pc->InteractiveHead; pc->InteractiveCurrentLine != NULL && (Pos < &pc->InteractiveCurrentLine->Tokens[0] || Pos >= &pc->InteractiveCurrentLine->Tokens[pc->InteractiveCurrentLine->NumBytes]); pc->InteractiveCurrentLine = pc->InteractiveCurrentLine->Next)
{} /* find the line we just counted */
if (EndParser->Pos >= StartParser->Pos && EndParser->Pos < &pc->InteractiveCurrentLine->Tokens[pc->InteractiveCurrentLine->NumBytes])
{
/* all on a single line */
MemSize = EndParser->Pos - StartParser->Pos;
NewTokens = VariableAlloc(pc, StartParser, MemSize + TOKEN_DATA_OFFSET, TRUE);
memcpy(NewTokens, (void *)StartParser->Pos, MemSize);
}
else
{
/* it's spread across multiple lines */
MemSize = &pc->InteractiveCurrentLine->Tokens[pc->InteractiveCurrentLine->NumBytes-TOKEN_DATA_OFFSET] - Pos;
for (ILine = pc->InteractiveCurrentLine->Next; ILine != NULL && (EndParser->Pos < &ILine->Tokens[0] || EndParser->Pos >= &ILine->Tokens[ILine->NumBytes]); ILine = ILine->Next)
MemSize += ILine->NumBytes - TOKEN_DATA_OFFSET;
assert(ILine != NULL);
MemSize += EndParser->Pos - &ILine->Tokens[0];
NewTokens = VariableAlloc(pc, StartParser, MemSize + TOKEN_DATA_OFFSET, TRUE);
CopySize = &pc->InteractiveCurrentLine->Tokens[pc->InteractiveCurrentLine->NumBytes-TOKEN_DATA_OFFSET] - Pos;
memcpy(NewTokens, Pos, CopySize);
NewTokenPos = NewTokens + CopySize;
for (ILine = pc->InteractiveCurrentLine->Next; ILine != NULL && (EndParser->Pos < &ILine->Tokens[0] || EndParser->Pos >= &ILine->Tokens[ILine->NumBytes]); ILine = ILine->Next)
{
memcpy(NewTokenPos, &ILine->Tokens[0], ILine->NumBytes - TOKEN_DATA_OFFSET);
NewTokenPos += ILine->NumBytes-TOKEN_DATA_OFFSET;
}
assert(ILine != NULL);
memcpy(NewTokenPos, &ILine->Tokens[0], EndParser->Pos - &ILine->Tokens[0]);
}
}
NewTokens[MemSize] = (unsigned char)TokenEndOfFunction;
return NewTokens;
}
/* allocate a value either on the heap or the stack using space dependent on what type we want */
struct Value *VariableAllocValueAndData(struct ParseState *Parser, int DataSize, int IsLValue, struct Value *LValueFrom, int OnHeap)
{
struct Value *NewValue = VariableAlloc(Parser, MEM_ALIGN(sizeof(struct Value)) + DataSize, OnHeap);
NewValue->Val = (union AnyValue *)((char *)NewValue + MEM_ALIGN(sizeof(struct Value)));
NewValue->ValOnHeap = OnHeap;
NewValue->ValOnStack = !OnHeap;
NewValue->IsLValue = IsLValue;
NewValue->LValueFrom = LValueFrom;
return NewValue;
}
/* create a system struct which has no user-visible members */
struct ValueType *TypeCreateOpaqueStruct(struct ParseState *Parser, const char *StructName, int Size)
{
struct ValueType *Typ = TypeGetMatching(Parser, &UberType, TypeStruct, 0, StructName, FALSE);
/* create the (empty) table */
Typ->Members = VariableAlloc(Parser, sizeof(struct Table) + STRUCT_TABLE_SIZE * sizeof(struct TableEntry), TRUE);
Typ->Members->HashTable = (struct TableEntry **)((char *)Typ->Members + sizeof(struct Table));
TableInitTable(Typ->Members, (struct TableEntry **)((char *)Typ->Members + sizeof(struct Table)), STRUCT_TABLE_SIZE, TRUE);
Typ->Sizeof = Size;
return Typ;
}
/* allocate a value either on the heap or the stack from an existing AnyValue and type */
struct Value *VariableAllocValueFromExistingData(struct ParseState *Parser, struct ValueType *Typ, union AnyValue *FromValue, int IsLValue, struct Value *LValueFrom)
{
struct Value *NewValue = VariableAlloc(Parser, sizeof(struct Value), FALSE);
NewValue->Typ = Typ;
NewValue->Val = FromValue;
NewValue->ValOnHeap = FALSE;
NewValue->ValOnStack = FALSE;
NewValue->IsLValue = IsLValue;
NewValue->LValueFrom = LValueFrom;
return NewValue;
}
/* add a new type to the set of types we know about */
struct ValueType *TypeAdd(struct ParseState *Parser, struct ValueType *ParentType, enum BaseType Base, int ArraySize, const char *Identifier, int Sizeof, int AlignBytes)
{
struct ValueType *NewType = VariableAlloc(Parser, sizeof(struct ValueType), TRUE);
NewType->Base = Base;
NewType->ArraySize = ArraySize;
NewType->Sizeof = Sizeof;
NewType->AlignBytes = AlignBytes;
NewType->Identifier = Identifier;
NewType->Members = NULL;
NewType->FromType = ParentType;
NewType->DerivedTypeList = NULL;
NewType->OnHeap = TRUE;
NewType->Next = ParentType->DerivedTypeList;
ParentType->DerivedTypeList = NewType;
return NewType;
}
/* set an identifier to a value. returns FALSE if it already exists.
* Key must be a shared string from TableStrRegister() */
int TableSet(Picoc *pc, struct Table *Tbl, char *Key, struct Value *Val, const char *DeclFileName, int DeclLine, int DeclColumn){
int AddAt;
struct TableEntry *FoundEntry = TableSearch(Tbl, Key, &AddAt);
if (FoundEntry == NULL)
{ /* add it to the table */
struct TableEntry *NewEntry = VariableAlloc(pc, NULL, sizeof(struct TableEntry), Tbl->OnHeap);
NewEntry->DeclFileName = DeclFileName;
NewEntry->DeclLine = DeclLine;
NewEntry->DeclColumn = DeclColumn;
NewEntry->p.v.Key = Key;
NewEntry->p.v.Val = Val;
NewEntry->Next = Tbl->HashTable[AddAt];
Tbl->HashTable[AddAt] = NewEntry;
return TRUE;
}
return FALSE;
}
/* parse a struct or union declaration */
void TypeParseStruct(struct ParseState *Parser, struct ValueType **Typ, int IsStruct)
{
struct Value *LexValue;
struct ValueType *MemberType;
char *MemberIdentifier;
char *StructIdentifier;
struct Value *MemberValue;
enum LexToken Token;
int AlignBoundary;
Token = LexGetToken(Parser, &LexValue, FALSE);
if (Token == TokenIdentifier)
{
LexGetToken(Parser, &LexValue, TRUE);
StructIdentifier = LexValue->Val->Identifier;
Token = LexGetToken(Parser, NULL, FALSE);
}
else
{
static char TempNameBuf[7] = "^s0000";
StructIdentifier = PlatformMakeTempName(TempNameBuf);
}
*Typ = TypeGetMatching(Parser, &UberType, IsStruct ? TypeStruct : TypeUnion, 0, StructIdentifier, Token != TokenLeftBrace);
Token = LexGetToken(Parser, NULL, FALSE);
if (Token != TokenLeftBrace)
{
/* use the already defined structure */
if ((*Typ)->Members == NULL)
ProgramFail(Parser, "la estructura '%s' no esta definida en este ambito", LexValue->Val->Identifier);
return;
}
if (TopStackFrame != NULL)
ProgramFail(Parser, "las estructuras/uniones solo pueden definirse en un ambito global");
LexGetToken(Parser, NULL, TRUE);
(*Typ)->Members = VariableAlloc(Parser, sizeof(struct Table) + STRUCT_TABLE_SIZE * sizeof(struct TableEntry), TRUE);
(*Typ)->Members->HashTable = (struct TableEntry **)((char *)(*Typ)->Members + sizeof(struct Table));
TableInitTable((*Typ)->Members, (struct TableEntry **)((char *)(*Typ)->Members + sizeof(struct Table)), STRUCT_TABLE_SIZE, TRUE);
do {
TypeParse(Parser, &MemberType, &MemberIdentifier, NULL);
if (MemberType == NULL || MemberIdentifier == NULL)
ProgramFail(Parser, "tipo invalido en estructura");
MemberValue = VariableAllocValueAndData(Parser, sizeof(int), FALSE, NULL, TRUE);
MemberValue->Typ = MemberType;
if (IsStruct)
{
/* allocate this member's location in the struct */
AlignBoundary = MemberValue->Typ->AlignBytes;
if (((*Typ)->Sizeof & (AlignBoundary-1)) != 0)
(*Typ)->Sizeof += AlignBoundary - ((*Typ)->Sizeof & (AlignBoundary-1));
MemberValue->Val->Integer = (*Typ)->Sizeof;
(*Typ)->Sizeof += TypeSizeValue(MemberValue, TRUE);
}
else
{
/* union members always start at 0, make sure it's big enough to hold the largest member */
MemberValue->Val->Integer = 0;
if (MemberValue->Typ->Sizeof > (*Typ)->Sizeof)
(*Typ)->Sizeof = TypeSizeValue(MemberValue, TRUE);
}
/* make sure to align to the size of the largest member's alignment */
if ((*Typ)->AlignBytes < MemberValue->Typ->AlignBytes)
(*Typ)->AlignBytes = MemberValue->Typ->AlignBytes;
/* define it */
if (!TableSet((*Typ)->Members, MemberIdentifier, MemberValue, Parser->FileName, Parser->Line, Parser->CharacterPos))
ProgramFail(Parser, "el miembro '%s' ya esta definido", &MemberIdentifier);
if (LexGetToken(Parser, NULL, TRUE) != TokenSemicolon)
ProgramFail(Parser, "se esperaba un punto y coma");
} while (LexGetToken(Parser, NULL, FALSE) != TokenRightBrace);
/* now align the structure to the size of its largest member's alignment */
AlignBoundary = (*Typ)->AlignBytes;
if (((*Typ)->Sizeof & (AlignBoundary-1)) != 0)
(*Typ)->Sizeof += AlignBoundary - ((*Typ)->Sizeof & (AlignBoundary-1));
LexGetToken(Parser, NULL, TRUE);
}
/* get the next token, without pre-processing */
enum LexToken LexGetRawToken(struct ParseState *Parser, struct Value **Value, int IncPos)
{
enum LexToken Token = TokenNone;
int ValueSize;
char *Prompt = NULL;
do
{
/* get the next token */
if (Parser->Pos == NULL && InteractiveHead != NULL)
Parser->Pos = InteractiveHead->Tokens;
if (Parser->FileName != StrEmpty || InteractiveHead != NULL)
{
/* skip leading newlines */
while ((Token = (enum LexToken)*(unsigned char *)Parser->Pos) == TokenEndOfLine)
{
Parser->Line++;
Parser->Pos += TOKEN_DATA_OFFSET;
}
}
if (Parser->FileName == StrEmpty && (InteractiveHead == NULL || Token == TokenEOF))
{
/* we're at the end of an interactive input token list */
char LineBuffer[LINEBUFFER_MAX];
void *LineTokens;
int LineBytes;
struct TokenLine *LineNode;
if (InteractiveHead == NULL || (unsigned char *)Parser->Pos == &InteractiveTail->Tokens[InteractiveTail->NumBytes-TOKEN_DATA_OFFSET])
{
/* get interactive input */
if (LexUseStatementPrompt)
{
Prompt = INTERACTIVE_PROMPT_STATEMENT;
LexUseStatementPrompt = FALSE;
}
else
Prompt = INTERACTIVE_PROMPT_LINE;
if (PlatformGetLine(&LineBuffer[0], LINEBUFFER_MAX, Prompt) == NULL)
return TokenEOF;
/* put the new line at the end of the linked list of interactive lines */
LineTokens = LexAnalyse(StrEmpty, &LineBuffer[0], strlen(LineBuffer), &LineBytes);
LineNode = VariableAlloc(Parser, sizeof(struct TokenLine), TRUE);
LineNode->Tokens = LineTokens;
LineNode->NumBytes = LineBytes;
if (InteractiveHead == NULL)
{
/* start a new list */
InteractiveHead = LineNode;
Parser->Line = 1;
Parser->CharacterPos = 0;
}
else
InteractiveTail->Next = LineNode;
InteractiveTail = LineNode;
InteractiveCurrentLine = LineNode;
Parser->Pos = LineTokens;
}
else
{
/* go to the next token line */
if (Parser->Pos != &InteractiveCurrentLine->Tokens[InteractiveCurrentLine->NumBytes-TOKEN_DATA_OFFSET])
{
/* scan for the line */
for (InteractiveCurrentLine = InteractiveHead; Parser->Pos != &InteractiveCurrentLine->Tokens[InteractiveCurrentLine->NumBytes-TOKEN_DATA_OFFSET]; InteractiveCurrentLine = InteractiveCurrentLine->Next)
{ assert(InteractiveCurrentLine->Next != NULL); }
}
assert(InteractiveCurrentLine != NULL);
InteractiveCurrentLine = InteractiveCurrentLine->Next;
assert(InteractiveCurrentLine != NULL);
Parser->Pos = InteractiveCurrentLine->Tokens;
}
Token = (enum LexToken)*(unsigned char *)Parser->Pos;
}
} while ((Parser->FileName == StrEmpty && Token == TokenEOF) || Token == TokenEndOfLine);
Parser->CharacterPos = *((unsigned char *)Parser->Pos + 1);
ValueSize = LexTokenSize(Token);
if (ValueSize > 0)
{
/* this token requires a value - unpack it */
if (Value != NULL)
{
switch (Token)
{
case TokenStringConstant: LexValue.Typ = CharPtrType; break;
case TokenIdentifier: LexValue.Typ = NULL; break;
case TokenIntegerConstant: LexValue.Typ = &IntType; break;
case TokenCharacterConstant: LexValue.Typ = &CharType; break;
#ifndef NO_FP
case TokenFPConstant: LexValue.Typ = &FPType; break;
#endif
default: break;
}
memcpy((void *)LexValue.Val, (void *)((char *)Parser->Pos + TOKEN_DATA_OFFSET), ValueSize);
LexValue.ValOnHeap = FALSE;
LexValue.ValOnStack = FALSE;
LexValue.IsLValue = FALSE;
LexValue.LValueFrom = NULL;
*Value = &LexValue;
}
if (IncPos)
Parser->Pos += ValueSize + TOKEN_DATA_OFFSET;
}
else
{
if (IncPos && Token != TokenEOF)
Parser->Pos += TOKEN_DATA_OFFSET;
}
#ifdef DEBUG_LEXER
printf("Got token=%02x inc=%d pos=%d\n", Token, IncPos, Parser->CharacterPos);
#endif
assert(Token >= TokenNone && Token <= TokenEndOfFunction);
return Token;
}
/* parse a struct or union declaration */
void TypeParseStruct(struct ParseState *Parser, struct ValueType **Typ, int IsStruct){
struct Value *LexValue;
struct ValueType *MemberType;
char *MemberIdentifier;
char *StructIdentifier;
struct Value *MemberValue;
enum LexToken Token;
int AlignBoundary;
Picoc *pc = Parser->pc;
Token = LexGetToken(Parser, &LexValue, FALSE);
if (Token == TokenIdentifier){
LexGetToken(Parser, &LexValue, TRUE);
StructIdentifier = LexValue->Val->Identifier;
Token = LexGetToken(Parser, NULL, FALSE);
}else{
static char TempNameBuf[7] = "^s0000";
StructIdentifier = PlatformMakeTempName(pc, TempNameBuf);
}
*Typ = TypeGetMatching(pc, Parser, &Parser->pc->UberType, IsStruct ? TypeStruct : TypeUnion, 0, StructIdentifier, TRUE);
if (Token == TokenLeftBrace && (*Typ)->Members)
ProgramFail(Parser, "data type '%t' is already defined", *Typ);
Token = LexGetToken(Parser, NULL, FALSE);
if (Token != TokenLeftBrace){
/* use the already defined structure */
#if 0
if ((*Typ)->Members == NULL)
ProgramFail(Parser, "structure '%s' isn't defined", LexValue->Val->Identifier);
#endif
return;
}
if (pc->TopStackFrame != NULL) ProgramFail(Parser, "struct/union definitions can only be globals");
LexGetToken(Parser, NULL, TRUE);
(*Typ)->Members = VariableAlloc(pc, Parser, sizeof(struct Table) + STRUCT_TABLE_SIZE * sizeof(struct TableEntry), TRUE);
(*Typ)->Members->HashTable = (struct TableEntry **)((char *)(*Typ)->Members + sizeof(struct Table));
TableInitTable((*Typ)->Members, (struct TableEntry **)((char *)(*Typ)->Members + sizeof(struct Table)), STRUCT_TABLE_SIZE, TRUE);
do {
TypeParse(Parser, &MemberType, &MemberIdentifier, NULL);
if (!MemberType || !MemberIdentifier)
ProgramFail(Parser, "invalid type in struct");
MemberValue = VariableAllocValueAndData(pc, Parser, sizeof(int), FALSE, NULL, TRUE);
MemberValue->Typ = MemberType;
if (IsStruct){
/* allocate this member's location in the struct */
AlignBoundary = MemberValue->Typ->AlignBytes;
if (((*Typ)->Sizeof & (AlignBoundary-1)) != 0)
(*Typ)->Sizeof += AlignBoundary - ((*Typ)->Sizeof & (AlignBoundary-1));
MemberValue->Val->Integer = (*Typ)->Sizeof;
(*Typ)->Sizeof += TypeSizeValue(MemberValue, TRUE);
}else{
/* union members always start at 0, make sure it's big enough to hold the largest member */
MemberValue->Val->Integer = 0;
if ((*Typ)->Sizeof < MemberValue->Typ->Sizeof )
(*Typ)->Sizeof = TypeSizeValue(MemberValue, TRUE);
}
/* make sure to align to the size of the largest member's alignment */
if ((*Typ)->AlignBytes < MemberValue->Typ->AlignBytes)
(*Typ)->AlignBytes = MemberValue->Typ->AlignBytes;
/* define it */
if (!TableSet(pc, (*Typ)->Members, MemberIdentifier, MemberValue, Parser->FileName, Parser->Line, Parser->CharacterPos))
ProgramFail(Parser, "member '%s' already defined", &MemberIdentifier);
if (LexGetToken(Parser, NULL, TRUE) != TokenSemicolon)
ProgramFail(Parser, "semicolon expected");
} while (LexGetToken(Parser, NULL, FALSE) != TokenRightBrace);
/* now align the structure to the size of its largest member's alignment */
AlignBoundary = (*Typ)->AlignBytes;
if (((*Typ)->Sizeof & (AlignBoundary-1)) != 0)
(*Typ)->Sizeof += AlignBoundary-((*Typ)->Sizeof & (AlignBoundary-1));
LexGetToken(Parser, NULL, TRUE);
}
