static void parseFunctionOrMethod (tokenInfo *const token)
{
// FunctionDecl = "func" identifier Signature [ Body ] .
// Body = Block.
//
// MethodDecl = "func" Receiver MethodName Signature [ Body ] .
// Receiver = "(" [ identifier ] [ "*" ] BaseTypeName ")" .
// BaseTypeName = identifier .
// Skip over receiver.
readToken (token);
if (isType (token, TOKEN_OPEN_PAREN))
skipToMatched (token);
if (isType (token, TOKEN_IDENTIFIER))
{
makeTag (token, GOTAG_FUNCTION);
// Skip over parameters.
readToken (token);
skipToMatched (token);
// Skip over result.
skipType (token);
// Skip over function body.
if (isType (token, TOKEN_OPEN_CURLY))
skipToMatched (token);
}
}
static void parseFunctionOrMethod (tokenInfo *const token)
{
// FunctionDecl = "func" identifier Signature [ Body ] .
// Body = Block.
//
// MethodDecl = "func" Receiver MethodName Signature [ Body ] .
// Receiver = "(" [ identifier ] [ "*" ] BaseTypeName ")" .
// BaseTypeName = identifier .
// Skip over receiver.
readToken (token);
if (isType (token, TOKEN_OPEN_PAREN))
skipToMatched (token);
if (isType (token, TOKEN_IDENTIFIER))
{
vString *argList;
tokenInfo *functionToken = copyToken (token);
// Start recording signature
signature = vStringNew ();
// Skip over parameters.
readToken (token);
skipToMatchedNoRead (token);
vStringStripLeading (signature);
vStringStripTrailing (signature);
argList = signature;
signature = vStringNew ();
readToken (token);
// Skip over result.
skipType (token);
// Remove the extra { we have just read
vStringStripTrailing (signature);
vStringChop (signature);
vStringStripLeading (signature);
vStringStripTrailing (signature);
makeTag (functionToken, GOTAG_FUNCTION, NULL, GOTAG_UNDEFINED, argList->buffer, signature->buffer);
deleteToken (functionToken);
vStringDelete(signature);
vStringDelete(argList);
// Stop recording signature
signature = NULL;
// Skip over function body.
if (isType (token, TOKEN_OPEN_CURLY))
skipToMatched (token);
}
}
// this finds an element in a composite list.
// it works progressively, so it only finds elements it hasn't come
// across yet.
static int nbtFindElement(bfFile bf,char *name)
{
for( ;; )
{
unsigned char type=0;
bfread(bf,&type,1);
if (type==0) return 0;
if (compare(bf,name))
return type;
skipType(bf,type);
}
}
static void parseConstTypeVar (tokenInfo *const token, goKind kind)
{
// ConstDecl = "const" ( ConstSpec | "(" { ConstSpec ";" } ")" ) .
// ConstSpec = IdentifierList [ [ Type ] "=" ExpressionList ] .
// IdentifierList = identifier { "," identifier } .
// ExpressionList = Expression { "," Expression } .
// TypeDecl = "type" ( TypeSpec | "(" { TypeSpec ";" } ")" ) .
// TypeSpec = identifier Type .
// VarDecl = "var" ( VarSpec | "(" { VarSpec ";" } ")" ) .
// VarSpec = IdentifierList ( Type [ "=" ExpressionList ] | "=" ExpressionList ) .
boolean usesParens = FALSE;
readToken (token);
if (isType (token, TOKEN_OPEN_PAREN))
{
usesParens = TRUE;
readToken (token);
}
do
{
while (!isType (token, TOKEN_EOF))
{
if (isType (token, TOKEN_IDENTIFIER))
{
makeTag (token, kind);
readToken (token);
}
if (!isType (token, TOKEN_COMMA))
break;
readToken (token);
}
skipType (token);
while (!isType (token, TOKEN_SEMICOLON) && !isType (token, TOKEN_CLOSE_PAREN)
&& !isType (token, TOKEN_EOF))
{
readToken (token);
skipToMatched (token);
}
if (usesParens && !isType (token, TOKEN_CLOSE_PAREN))
{
// we are at TOKEN_SEMICOLON
readToken (token);
}
}
while (!isType (token, TOKEN_EOF) &&
usesParens && !isType (token, TOKEN_CLOSE_PAREN));
}
static void skipCompound(bfFile bf)
{
int len;
unsigned char type=0;
do {
bfread(bf,&type,1);
if (type)
{
len=readWord(bf);
bfseek(bf,len,SEEK_CUR); //skip name
skipType(bf,type);
}
} while (type);
}
/**
* read column chunk metadata information
*/
int
readColumnMetadata(
CompactProtocol *prot,
struct ColumnChunkMetadata_4C *colChunk)
{
uint32_t xfer = 0;
TType ftype;
int16_t fid;
readStructBegin(prot);
bool isset_type = false;
bool isset_encodings = false;
bool isset_path_in_schema = false;
bool isset_codec = false;
bool isset_num_values = false;
bool isset_total_uncompressed_size = false;
bool isset_total_compressed_size = false;
bool isset_data_page_offset = false;
while (true) {
xfer += readFieldBegin(prot, &ftype, &fid);
if (ftype == T_STOP) {
break;
}
switch (fid) {
case 1:
if (ftype == T_I32) {
int32_t type;
xfer += readI32(prot, &type);
colChunk->type = (PrimitiveTypeName) type;
isset_type = true;
}
break;
case 2:
if (ftype == T_LIST) {
uint32_t encodingCount;
TType etype;
xfer += readListBegin(prot, &etype, &encodingCount);
colChunk->EncodingCount = encodingCount;
colChunk->pEncodings =
(enum Encoding *) palloc0(sizeof(enum Encoding) * encodingCount);
for (int i = 0; i < encodingCount; i++) {
int32_t encoding;
xfer += readI32(prot, &encoding);
colChunk->pEncodings[i] = (enum Encoding) encoding;
}
isset_encodings = true;
}
break;
case 3:
if (ftype == T_LIST) {
{
/*process path in schema, setting colchunk->depth and colchunk->pathInSchema*/
TType etype;
uint32_t lsize;
StringInfoData colNameBuf;
xfer += readListBegin(prot, &etype, &lsize);
colChunk->depth = lsize;
initStringInfo(&colNameBuf);
char *path_in_schema;
for (int i = 0; i < lsize - 1; i++) {
xfer += readString(prot, &path_in_schema);
appendStringInfo(&colNameBuf, "%s:", path_in_schema);
pfree(path_in_schema);
}
xfer += readString(prot, &path_in_schema);
appendStringInfo(&colNameBuf, "%s", path_in_schema);
colChunk->pathInSchema = colNameBuf.data;
colChunk->colName = path_in_schema;
}
isset_path_in_schema = true;
}
break;
case 4:
if (ftype == T_I32) {
int32_t compresscode;
xfer += readI32(prot, &compresscode);
colChunk->codec = (enum CompressionCodecName) compresscode;
isset_codec = true;
}
break;
case 5:
if (ftype == T_I64) {
int64_t valCnt;
xfer += readI64(prot, &valCnt);
colChunk->valueCount = valCnt;
isset_num_values = true;
}
break;
case 6:
if (ftype == T_I64) {
xfer += readI64(prot, &(colChunk->totalUncompressedSize));
isset_total_uncompressed_size = true;
}
break;
case 7:
if (ftype == T_I64) {
xfer += readI64(prot, &(colChunk->totalSize));
isset_total_compressed_size = true;
}
break;
case 8:
if (ftype == T_LIST) {
xfer += skipType(prot, ftype);
}
break;
case 9:
if (ftype == T_I64) {
xfer += readI64(prot, &(colChunk->firstDataPage));
isset_data_page_offset = true;
}
break;
case 10:
if (ftype == T_I64) {
xfer += skipType(prot, ftype);
}
break;
case 11:
if (ftype == T_I64) {
xfer += skipType(prot, ftype);
}
break;
default:
break;
}
}
readStructEnd(prot);
if (!isset_type)
ereport(ERROR,
(errcode(ERRCODE_GP_INTERNAL_ERROR), errmsg("file metadata: row group column chunk type not set")));
if (!isset_encodings)
ereport(ERROR,
(errcode(ERRCODE_GP_INTERNAL_ERROR), errmsg("file metadata: row group column chunk encoding not set")));
if (!isset_path_in_schema)
ereport(ERROR,
(errcode(ERRCODE_GP_INTERNAL_ERROR), errmsg("file metadata: row group column chunk path_in_schema not set")));
if (!isset_codec)
ereport(ERROR,
(errcode(ERRCODE_GP_INTERNAL_ERROR), errmsg("file metadata: row group column chunk compression code not set")));
if (!isset_num_values)
ereport(ERROR,
(errcode(ERRCODE_GP_INTERNAL_ERROR), errmsg("file metadata: row group column chunk value number not set")));
if (!isset_total_uncompressed_size)
ereport(ERROR,
(errcode(ERRCODE_GP_INTERNAL_ERROR), errmsg("file metadata: row group column chunk total uncompressed size not set")));
if (!isset_total_compressed_size)
ereport(ERROR,
(errcode(ERRCODE_GP_INTERNAL_ERROR), errmsg("file metadata: row group column chunk total compressed size not set")));
if (!isset_data_page_offset)
ereport(ERROR,
(errcode(ERRCODE_GP_INTERNAL_ERROR), errmsg("file metadata: row group column chunk first data page not set")));
return xfer;
}
/**
* Read single field of schema element in parquet file
* @prot: The protocol for reading (input)
* @fieldType: The type of the field
* @type_length: The length of the field type
* @repetition_type:The repetition level of parent
* @fieldName The name of the field
* @num_children The children number of field
*/
int
readSchemaElement_Single(
CompactProtocol *prot,
PrimitiveTypeName *fieldType,
int32_t *type_length,
RepetitionType *repetition_type,
char **fieldName,
int32_t *num_children)
{
uint32_t xfer = 0;
TType ftype;
int16_t fid;
bool isset_name = false;
readStructBegin(prot);
while (true) {
xfer += readFieldBegin(prot, &ftype, &fid);
if (ftype == T_STOP) {
break;
}
switch (fid) {
case 1:
if (ftype == T_I32) {
int32_t val;
xfer += readI32(prot, &val);
*fieldType = (PrimitiveTypeName) val;
}
break;
case 2:
if (ftype == T_I32) {
int32_t bit_length = 0;
xfer += readI32(prot, &bit_length);
*type_length = bit_length / 8;
}
break;
case 3:
if (ftype == T_I32) {
int32_t ecast1;
xfer += readI32(prot, &ecast1);
*repetition_type = (RepetitionType) ecast1;
}
break;
case 4:
if (ftype == T_STRING) {
isset_name = true;
xfer += readString(prot, fieldName);
}
break;
case 5:
if (ftype == T_I32) {
xfer += readI32(prot, num_children);
}
break;
case 6:
if (ftype == T_I32) {
xfer += skipType(prot, ftype);
}
break;
default:
break;
}
}
readStructEnd(prot);
if (!isset_name)
ereport(ERROR,
(errcode(ERRCODE_GP_INTERNAL_ERROR), errmsg(
"file metadata schema element information not correct")));
return xfer;
}
static void parseConstTypeVar (tokenInfo *const token, goKind kind)
{
// ConstDecl = "const" ( ConstSpec | "(" { ConstSpec ";" } ")" ) .
// ConstSpec = IdentifierList [ [ Type ] "=" ExpressionList ] .
// IdentifierList = identifier { "," identifier } .
// ExpressionList = Expression { "," Expression } .
// TypeDecl = "type" ( TypeSpec | "(" { TypeSpec ";" } ")" ) .
// TypeSpec = identifier Type .
// VarDecl = "var" ( VarSpec | "(" { VarSpec ";" } ")" ) .
// VarSpec = IdentifierList ( Type [ "=" ExpressionList ] | "=" ExpressionList ) .
boolean usesParens = FALSE;
readToken (token);
if (isType (token, TOKEN_OPEN_PAREN))
{
usesParens = TRUE;
readToken (token);
}
do
{
tokenInfo *typeToken = NULL;
while (!isType (token, TOKEN_EOF))
{
if (isType (token, TOKEN_IDENTIFIER))
{
if (kind == GOTAG_TYPE)
{
typeToken = copyToken (token);
readToken (token);
if (isKeyword (token, KEYWORD_struct))
makeTag (typeToken, GOTAG_STRUCT, NULL, GOTAG_UNDEFINED, NULL);
else if (isKeyword (token, KEYWORD_interface))
makeTag (typeToken, GOTAG_INTERFACE, NULL, GOTAG_UNDEFINED, NULL);
else
makeTag (typeToken, kind, NULL, GOTAG_UNDEFINED, NULL);
break;
}
else
makeTag (token, kind, NULL, GOTAG_UNDEFINED, NULL);
readToken (token);
}
if (!isType (token, TOKEN_COMMA))
break;
readToken (token);
}
if (typeToken)
{
if (isKeyword (token, KEYWORD_struct))
parseStructMembers (token, typeToken);
else
skipType (token);
deleteToken (typeToken);
}
else
skipType (token);
while (!isType (token, TOKEN_SEMICOLON) && !isType (token, TOKEN_CLOSE_PAREN)
&& !isType (token, TOKEN_EOF))
{
readToken (token);
skipToMatched (token);
}
if (usesParens && !isType (token, TOKEN_CLOSE_PAREN))
{
// we are at TOKEN_SEMICOLON
readToken (token);
}
}
while (!isType (token, TOKEN_EOF) &&
usesParens && !isType (token, TOKEN_CLOSE_PAREN));
}
static void parseStructMembers (tokenInfo *const token, tokenInfo *const parent_token)
{
// StructType = "struct" "{" { FieldDecl ";" } "}" .
// FieldDecl = (IdentifierList Type | AnonymousField) [ Tag ] .
// AnonymousField = [ "*" ] TypeName .
// Tag = string_lit .
readToken (token);
if (!isType (token, TOKEN_OPEN_CURLY))
return;
readToken (token);
while (!isType (token, TOKEN_EOF) && !isType (token, TOKEN_CLOSE_CURLY))
{
tokenInfo *memberCandidate = NULL;
boolean first = TRUE;
while (!isType (token, TOKEN_EOF))
{
if (isType (token, TOKEN_IDENTIFIER))
{
if (first)
{
// could be anonymous field like in 'struct {int}' - we don't know yet
memberCandidate = copyToken (token);
first = FALSE;
}
else
{
if (memberCandidate)
{
// if we are here, there was a comma and memberCandidate isn't an anonymous field
makeTag (memberCandidate, GOTAG_MEMBER, parent_token, GOTAG_STRUCT, NULL);
deleteToken (memberCandidate);
memberCandidate = NULL;
}
makeTag (token, GOTAG_MEMBER, parent_token, GOTAG_STRUCT, NULL);
}
readToken (token);
}
if (!isType (token, TOKEN_COMMA))
break;
readToken (token);
}
// in the case of an anonymous field, we already read part of the
// type into memberCandidate and skipType() should return FALSE so no tag should
// be generated in this case.
if (skipType (token) && memberCandidate)
makeTag (memberCandidate, GOTAG_MEMBER, parent_token, GOTAG_STRUCT, NULL);
if (memberCandidate)
deleteToken (memberCandidate);
while (!isType (token, TOKEN_SEMICOLON) && !isType (token, TOKEN_CLOSE_CURLY)
&& !isType (token, TOKEN_EOF))
{
readToken (token);
skipToMatched (token);
}
if (!isType (token, TOKEN_CLOSE_CURLY))
{
// we are at TOKEN_SEMICOLON
readToken (token);
}
}
}
static boolean skipType (tokenInfo *const token)
{
// Type = TypeName | TypeLit | "(" Type ")" .
// Skips also function multiple return values "(" Type {"," Type} ")"
if (isType (token, TOKEN_OPEN_PAREN))
{
skipToMatched (token);
return TRUE;
}
// TypeName = QualifiedIdent.
// QualifiedIdent = [ PackageName "." ] identifier .
// PackageName = identifier .
if (isType (token, TOKEN_IDENTIFIER))
{
readToken (token);
if (isType (token, TOKEN_DOT))
{
readToken (token);
if (isType (token, TOKEN_IDENTIFIER))
readToken (token);
}
return TRUE;
}
// StructType = "struct" "{" { FieldDecl ";" } "}"
// InterfaceType = "interface" "{" { MethodSpec ";" } "}" .
if (isKeyword (token, KEYWORD_struct) || isKeyword (token, KEYWORD_interface))
{
readToken (token);
// skip over "{}"
skipToMatched (token);
return TRUE;
}
// ArrayType = "[" ArrayLength "]" ElementType .
// SliceType = "[" "]" ElementType .
// ElementType = Type .
if (isType (token, TOKEN_OPEN_SQUARE))
{
skipToMatched (token);
return skipType (token);
}
// PointerType = "*" BaseType .
// BaseType = Type .
// ChannelType = ( "chan" [ "<-" ] | "<-" "chan" ) ElementType .
if (isType (token, TOKEN_STAR) || isKeyword (token, KEYWORD_chan) || isType (token, TOKEN_LEFT_ARROW))
{
readToken (token);
return skipType (token);
}
// MapType = "map" "[" KeyType "]" ElementType .
// KeyType = Type .
if (isKeyword (token, KEYWORD_map))
{
readToken (token);
// skip over "[]"
skipToMatched (token);
return skipType (token);
}
// FunctionType = "func" Signature .
// Signature = Parameters [ Result ] .
// Result = Parameters | Type .
// Parameters = "(" [ ParameterList [ "," ] ] ")" .
if (isKeyword (token, KEYWORD_func))
{
readToken (token);
// Parameters, skip over "()"
skipToMatched (token);
// Result is parameters or type or nothing. skipType treats anything
// surrounded by parentheses as a type, and does nothing if what
// follows is not a type.
return skipType (token);
}
return FALSE;
}
int nbtGetBlocks(bfFile bf, unsigned char *buff, unsigned char *data, unsigned char *blockLight, unsigned char *biome)
{
int len,nsections;
int biome_save;
//int found;
#ifndef C99
char *thisName;
#endif
//Level/Blocks
bfseek(bf,1,SEEK_CUR); //skip type
len=readWord(bf); //name length
bfseek(bf,len,SEEK_CUR); //skip name ()
if (nbtFindElement(bf,"Level")!=10)
return 0;
// For some reason, on most maps the biome info is before the Sections;
// on others they're after. So, read biome data, then rewind to find Sections.
// Format info at http://wiki.vg/Map_Format, though don't trust order.
biome_save = *bf.offset;
memset(biome, 0, 16*16);
if (nbtFindElement(bf,"Biomes")!=7)
return 0;
{
len=readDword(bf); //array length
bfread(bf,biome,len);
}
bfseek(bf,biome_save,SEEK_SET); //rewind to start of section
if (nbtFindElement(bf,"Sections")!= 9)
return 0;
{
unsigned char type=0;
bfread(bf,&type,1);
if (type != 10)
return 0;
}
memset(buff, 0, 16*16*256);
memset(data, 0, 16*16*128);
memset(blockLight, 0, 16*16*128);
nsections=readDword(bf);
while (nsections--)
{
unsigned char y;
int save = *bf.offset;
if (nbtFindElement(bf,"Y")!=1) //which section is this?
return 0;
bfread(bf,&y,1);
bfseek(bf,save,SEEK_SET); //rewind to start of section
//found=0;
for (;;)
{
int ret=0;
unsigned char type=0;
bfread(bf,&type,1);
if (type==0)
break;
len=readWord(bf);
#ifdef C99
char thisName[len+1];
#else
thisName=(char *)malloc(len+1);
#endif
bfread(bf,thisName,len);
thisName[len]=0;
if (strcmp(thisName,"BlockLight")==0)
{
//found++;
ret=1;
len=readDword(bf); //array length
bfread(bf,blockLight+16*16*8*y,len);
}
if (strcmp(thisName,"Blocks")==0)
{
//found++;
ret=1;
len=readDword(bf); //array length
bfread(bf,buff+16*16*16*y,len);
}
else if (strcmp(thisName,"Data")==0)
{
//found++;
ret=1;
len=readDword(bf); //array length
bfread(bf,data+16*16*8*y,len);
}
#ifndef C99
free(thisName);
#endif
if (!ret)
skipType(bf,type);
}
}
return 1;
}
int nbtGetBlocks(bfFile bf, unsigned char *buff, unsigned char *data, unsigned char *blockLight)
{
int len,nsections;
//int found;
#ifndef C99
char *thisName;
#endif
//Level/Blocks
bfseek(bf,1,SEEK_CUR); //skip type
len=readWord(bf); //name length
bfseek(bf,len,SEEK_CUR); //skip name ()
if (nbtFindElement(bf,"Level")!=10)
return 0;
if (nbtFindElement(bf,"Sections")!= 9)
return 0;
{
unsigned char type=0;
bfread(bf,&type,1);
if (type != 10)
return 0;
}
memset(buff, 0, 16*16*256);
memset(data, 0, 16*16*128);
memset(blockLight, 0, 16*16*128);
nsections=readDword(bf);
while (nsections--)
{
unsigned char y;
int save = *bf.offset;
if (nbtFindElement(bf,"Y")!=1) //which section is this?
return 0;
bfread(bf,&y,1);
bfseek(bf,save,SEEK_SET); //rewind to start of section
//found=0;
for (;;)
{
int ret=0;
unsigned char type=0;
bfread(bf,&type,1);
if (type==0)
break;
len=readWord(bf);
#ifdef C99
char thisName[len+1];
#else
thisName=(char *)malloc(len+1);
#endif
bfread(bf,thisName,len);
thisName[len]=0;
if (strcmp(thisName,"BlockLight")==0)
{
//found++;
ret=1;
len=readDword(bf); //array length
bfread(bf,blockLight+16*16*8*y,len);
}
if (strcmp(thisName,"Blocks")==0)
{
//found++;
ret=1;
len=readDword(bf); //array length
bfread(bf,buff+16*16*16*y,len);
}
else if (strcmp(thisName,"Data")==0)
{
//found++;
ret=1;
len=readDword(bf); //array length
bfread(bf,data+16*16*8*y,len);
}
#ifndef C99
free(thisName);
#endif
if (!ret)
skipType(bf,type);
}
}
return 1;
}
