AMI_err _LSList::getSortOnFieldSize(s32 *fields) {
s32 counts[MAX_LIST_ENTRIES];
s32 c_idx;
AMI_err result = AMI_ERROR_NO_ERROR;
memset(counts,0,sizeof(s32)*MAX_LIST_ENTRIES);
for (u32 i=0; i<_m_count(); ++i) {
c_idx=fieldSize(i);
if (c_idx<MAX_LIST_ENTRIES) {
++counts[c_idx+1];
} else {
printf("getsortOnFieldSize: Severe: MAX_LIST_ENTRIES too small.\n");
return AMI_ERROR_GENERIC_ERROR;
}
}
for (u32 i=1; i<MAX_LIST_ENTRIES; ++i) {
counts[i]+=counts[i-1];
}
for (u32 i=0; i<_m_count(); ++i) {
c_idx=fieldSize(i);
if (c_idx<MAX_LIST_ENTRIES) {
fields[counts[c_idx]++]=i;
} else {
printf("getsortOnFieldSize: Severe: MAX_LIST_ENTRIES too small.\n");
return AMI_ERROR_GENERIC_ERROR;
}
}
return result;
}
AMI_err _LSList::deleteFields(s32 *fields, s32 numFields) {
u32 buffSize = 0;
s32* buff = NULL;
char *base_name = tpie_tempnam("LSList");
AMI_err result = AMI_ERROR_NO_ERROR;
AMI_stream<s32>* newBaseStream = new AMI_stream<s32>(base_name);
newBaseStream->persist(_m_per);
s32 cdfield = 0;
_m_baseStream->seek(0);
for (u32 i=0; i<_m_count();++i) {
TPIE_OS_OFFSET fs = fieldSize(i);
if (fs>buffSize) {
buffSize = fs;
delete[] buff;
buff = new s32[buffSize];
memset(buff,0,buffSize);
}
_m_baseStream->read_array(buff,&fs);
if (cdfield>=numFields || (s32)i != fields[cdfield]) {
newBaseStream->write_array(buff,fs);
_m_fields->at(i-cdfield+1)=_m_fields->at(i-cdfield)+fs;
} else {
++cdfield;
}
}
_m_fields->erase(_m_fields->begin()+(_m_count()-numFields+1),_m_fields->end());
delete[] buff; delete _m_baseStream;
_m_baseStream = newBaseStream;
return result;
}
void ResultMetadata::init(MYSQL_STMT* stmt)
{
ResultMetadataHandle h(stmt);
if (!h)
{
// all right, it is normal
// querys such an "INSERT INTO" just does not have result at all
reset();
return;
}
std::size_t count = mysql_num_fields(h);
MYSQL_FIELD* fields = mysql_fetch_fields(h);
std::size_t commonSize = 0;
_columns.reserve(count);
{for (std::size_t i = 0; i < count; i++)
{
std::size_t size = fieldSize(fields[i]);
std::size_t zero = 0;
if (size == ~zero) size = 0;
_columns.push_back(MetaColumn(
i, // position
fields[i].name, // name
fieldType(fields[i]), // type
size, // length
0, // TODO: precision
!IS_NOT_NULL(fields[i].flags) // nullable
));
commonSize += _columns[i].length();
}}
_buffer.resize(commonSize);
_row.resize(count);
_lengths.resize(count);
_isNull.resize(count);
std::size_t offset = 0;
for (std::size_t i = 0; i < count; i++)
{
std::memset(&_row[i], 0, sizeof(MYSQL_BIND));
unsigned int len = static_cast<unsigned int>(_columns[i].length());
_row[i].buffer_type = fields[i].type;
_row[i].buffer_length = len;
_row[i].buffer = (len > 0) ? (&_buffer[0] + offset) : 0;
_row[i].length = &_lengths[i];
_row[i].is_null = &_isNull[i];
_row[i].is_unsigned = (fields[i].flags & UNSIGNED_FLAG) > 0;
offset += _row[i].buffer_length;
}
}
void _LSList::sortEntries(u32 field) {
TPIE_OS_OFFSET cSize = fieldSize(field);
s32* entries = new s32[cSize];
_m_baseStream->seek(_m_fields->at(field));
_m_baseStream->read_array(entries,&cSize);
qsort(entries, cSize, sizeof(s32), cmpS32s);
_m_baseStream->seek(_m_fields->at(field));
_m_baseStream->write_array(entries,cSize);
delete[] entries;
}
int FSCurve::lengthInStream(FSOutputStream* aStream)
{
int numberOfBits = 6;
numberOfBits += fieldSize()*4;
aStream->setCursor(aStream->getCursor()+numberOfBits);
return numberOfBits;
}
_LSListField* _LSList::fetchField(u32 index) {
// FIXME: assert
TPIE_OS_OFFSET fs = fieldSize(index);
s32* entries = new s32[fs];
memset(entries,0,fs*sizeof(s32));
_m_baseStream->seek(_m_fields->at(index));
_m_baseStream->read_array(entries,&fs);
_LSListField* F = new _LSListField(entries,fs);
delete[] entries;
return F;
}
bool IndexDataSource::init()
{
if (!df)
return false;
numParts = df->numParts();
singlePart = (numParts == 1);
ignoreSkippedRows = true; // better if skipping to a particular point
StringBuffer partName;
Owned<IDistributedFilePart> kf = df->getPart(numParts-1);
tlk.setown(openKeyFile(kf));
if (!tlk)
return false;
IPropertyTree & properties = df->queryAttributes();
//Need to assign the transformed record to meta
if (!diskMeta)
diskMeta.setown(new DataSourceMetaData(diskRecord, 0, true, false, tlk->keyedSize()));
if (!returnedMeta)
{
returnedMeta.set(diskMeta);
returnedRecordSize.set(returnedMeta);
}
if (!transformedMeta)
transformedMeta.set(returnedMeta);
if (properties.hasProp("@recordCount"))
totalRows = properties.getPropInt64("@recordCount");
else
totalRows = UNKNOWN_NUM_ROWS; // more: could probably count them
isLocal = properties.hasProp("@local");
diskMeta->extractKeyedInfo(keyedOffsets, keyedTypes);
ForEachItemIn(i, keyedTypes)
{
IStringSet * set = createRtlStringSet(fieldSize(i));
set->addAll();
values.append(*set);
}
void FSCurve::encodeToStream(FSOutputStream* aStream)
{
int _fieldSize = fieldSize();
#ifdef _DEBUG
aStream->startEncoding(className());
#endif
/*
* The parent object decides whether to perform validation.
*/
aStream->write(1, FSStream::UnsignedBit, 1);
aStream->write(0, FSStream::UnsignedBit, 1);
aStream->write(_fieldSize-2, FSStream::UnsignedBit, 4);
aStream->write(controlX, FSStream::SignedBit, _fieldSize);
aStream->write(controlY, FSStream::SignedBit, _fieldSize);
aStream->write(anchorX, FSStream::SignedBit, _fieldSize);
aStream->write(anchorY, FSStream::SignedBit, _fieldSize);
#ifdef _DEBUG
aStream->endEncoding(className());
#endif
}
AMI_err _LSList::catFields(lpair_t *pairs, s32 numPairs, bool mod2) {
s32 newEntries[MAX_FIELD_ENTRIES], numNewEntries;
AMI_err result = AMI_ERROR_NO_ERROR;
char *base_name = tpie_tempnam("LSList");
u32 buffSize = 0;
s32* buff = NULL;
std::vector<off_t>* newOffsets = new std::vector<off_t>;
newOffsets->push_back(0);
AMI_stream<s32>* newBaseStream = new AMI_stream<s32>(base_name);
newBaseStream->persist(_m_per);
_LSListField** pairs_y = new _LSListField*[numPairs];
s32* pairs_x = new s32[numPairs];
for (s32 i=0; i<numPairs; ++i) {
pairs_y[i] = fetchField(pairs[i].y);
pairs_x[i] = pairs[i].x;
}
_m_baseStream->seek(0);
s32 cdfield = 0;
for (u32 i=0; i<_m_count(); ++i) {
TPIE_OS_OFFSET fs = fieldSize(i);
if (fs>buffSize) {
buffSize = fs;
delete[] buff;
buff = new s32[buffSize];
}
_m_baseStream->read_array(buff,&fs);
if (cdfield<numPairs && (s32)i==pairs[cdfield].x && checkPair(pairs_x,cdfield,pairs[cdfield].y)) {
numNewEntries=0;
for (u32 j=0; (j<fs) && (numNewEntries < MAX_FIELD_ENTRIES); j++) {
newEntries[numNewEntries++] = buff[j];
}
for (u32 j=0; (j<pairs_y[cdfield]->numEntries()) && (numNewEntries < MAX_FIELD_ENTRIES); j++) {
newEntries[numNewEntries++] = pairs_y[cdfield]->at(j);
}
if (numNewEntries >= MAX_FIELD_ENTRIES) {
printf("MAX_FIELD_ENTRIES exceeded (i=%" PRId32 ", c0=%" PRId32 ", c1=%" PRId32 ".). Ignoring...\n",i,pairs[i].x,pairs[i].y);
return AMI_ERROR_GENERIC_ERROR;
}
qsort(newEntries, numNewEntries, sizeof(s32), cmpS32s);
int newEndLoc=0;
int i=0;
int j=0;
while (i<numNewEntries) {
/* How many copies of newEntries[i] are there? */
j=i+1;
while ((j<numNewEntries) && (newEntries[j]==newEntries[i])) {
j++;
}
if (mod2) {
if ((j-i)%2 == 0) {
/* An even number of them, so omit them altogether. */
i=j;
} else {
/* Copy one of them over. */
newEntries[newEndLoc++] = newEntries[i];
i=j;
}
} else {
/* Copy one of them over. */
newEntries[newEndLoc++] = newEntries[i];
i=j;
}
}
newBaseStream->write_array(newEntries,newEndLoc);
releaseField(pairs_y[cdfield++]);
} else {
newBaseStream->write_array(buff,fs);
}
newOffsets->push_back(newBaseStream->tell());
}
delete[] pairs_y; delete[] buff; delete[] pairs_x;
delete _m_fields; delete _m_baseStream;
_m_fields=newOffsets; _m_baseStream = newBaseStream;
return result;
}
bool Format::parse( const String &fmt )
{
String tmp;
uint32 pos = 0;
uint32 len = fmt.length();
typedef enum {
e_sInitial,
e_sSize,
e_sDecimals,
e_sPadding,
e_sDecSep,
e_sGroupSep,
e_sGroupSep2,
e_sErrorEffect,
e_sErrorEffect2,
e_sNilMode,
e_sNegFmt,
e_sNegFmt2
}
t_state;
t_state state = e_sInitial;
while( pos < len )
{
uint32 chr = fmt.getCharAt( pos );
switch( state )
{
//=============================
// Basic state.
//
case e_sInitial:
if( chr >= '0' && chr <= '9' )
{
// size already given
if ( m_size != 0 )
return false;
state = e_sSize;
tmp.size(0);
tmp += chr;
break;
}
// else:
switch( chr )
{
case 'N':
// it should be an octal.
m_convType = e_tNum;
numFormat( e_decimal );
break;
case '.':
// it should be an octal.
m_convType = e_tNum;
state = e_sDecimals;
tmp.size(0);
break;
case 'b':
// it should be an octal.
m_convType = e_tNum;
numFormat( e_binary );
break;
case 'B':
// it should be an octal.
m_convType = e_tNum;
numFormat( e_binaryB );
break;
case 'd':
m_convType = e_tNum;
state = e_sDecSep;
break;
case 'p':
state = e_sPadding;
break;
case 'g':
m_convType = e_tNum;
state = e_sGroupSep;
break;
case 'G':
m_convType = e_tNum;
state = e_sGroupSep2;
break;
case '0':
// it should be an octal.
m_convType = e_tNum;
numFormat( e_octalZero );
break;
case 'o':
// it should be an octal.
m_convType = e_tNum;
numFormat( e_octal );
break;
case 'x':
// it should be an octal.
m_convType = e_tNum;
numFormat( e_hexLower );
break;
case 'X':
// it should be an octal.
m_convType = e_tNum;
numFormat( e_hexUpper );
break;
case 'c':
// it should be an octal.
m_convType = e_tNum;
numFormat( e_cHexLower );
break;
case 'C':
// it should be an octal.
m_convType = e_tNum;
numFormat( e_cHexUpper );
break;
case 'e':
// it should be in scientific format
m_convType = e_tNum;
numFormat( e_scientific );
break;
case '/':
// it should be an octal.
state = e_sErrorEffect;
break;
case 'n':
state = e_sNilMode;
break;
case '|':
m_posOfObjectFmt = pos;
m_convType = e_tStr;
// complete parsing
pos = len;
break;
case '+':
m_negFormat = e_plusMinusFront;
state = e_sNegFmt;
break;
case '-':
m_negFormat = e_minusFront;
state = e_sNegFmt2;
break;
case '[':
m_negFormat = e_parenthesis;
break;
case ']':
m_negFormat = e_parpad;
break;
case 'r':
m_rightAlign = true;
break;
default:
// unrecognized character
m_convType = e_tError;
return false;
}
break;
//=============================
// Parse padding
//
case e_sDecSep:
m_decimalSep = chr;
state = e_sInitial;
break;
case e_sPadding:
m_paddingChr = chr;
state = e_sInitial;
break;
case e_sGroupSep:
if( chr >= '0' && chr <='9' )
{
m_grouping = chr - '0';
state = e_sGroupSep2;
}
else {
m_thousandSep = chr;
state = e_sInitial;
}
break;
case e_sGroupSep2:
m_thousandSep = chr;
state = e_sInitial;
break;
//=============================
// Size parsing state
//
case e_sSize:
if( chr >= '0' && chr <= '9' )
{
tmp += chr;
// size too wide
if ( tmp.length() > 4 ) {
m_convType = e_tError;
return false;
}
}
else
{
int64 tgt;
tmp.parseInt( tgt );
fieldSize( (uint16) tgt );
if( chr == '*' )
{
fixedSize( true );
}
else {
// reparse current char
--pos;
}
state = e_sInitial;
}
break;
//=============================
// Decimals parsing state
//
case e_sDecimals:
if( chr >= '0' && chr <= '9' )
{
tmp += chr;
// size too wide
if ( tmp.length() > 2 ) {
m_convType = e_tError;
return false;
}
}
else
{
int64 tgt;
tmp.parseInt( tgt );
decimals( (uint8) tgt );
// reparse current char
--pos;
state = e_sInitial;
}
break;
//===============================================
// Parsing what should be done in case of error.
//
case e_sErrorEffect:
if ( chr == 'c' )
{
state = e_sErrorEffect2;
break;
}
// else
switch( chr )
{
case 'n': mismatchAction( e_actNil ); break;
case '0': mismatchAction( e_actZero ); break;
case 'r': mismatchAction( e_actRaise ); break;
default:
// invalid choiche
m_convType = e_tError;
return false;
}
state = e_sInitial;
break;
case e_sErrorEffect2:
switch( chr )
{
case 'n': mismatchAction( e_actConvertNil ); break;
case '0': mismatchAction( e_actConvertZero ); break;
case 'r': mismatchAction( e_actConvertRaise ); break;
default:
// invalid choiche
m_convType = e_tError;
return false;
}
state = e_sInitial;
break;
//=================================
// parsing what do to with a Nil
//
case e_sNilMode:
switch( chr )
{
case 'n': m_nilFormat = e_nilEmpty; break;
case 'N': m_nilFormat = e_nilN; break;
case 'l': m_nilFormat = e_nilnil; break;
case 'L': m_nilFormat = e_nilNil; break;
case 'u': m_nilFormat = e_nilNull; break;
case 'U': m_nilFormat = e_nilNULL; break;
case 'o': m_nilFormat = e_nilNone; break;
case 'A': m_nilFormat = e_nilNA; break;
default:
m_convType = e_tError;
return false;
}
state = e_sInitial;
break;
//=================================
// Parsing neg format
case e_sNegFmt:
switch( chr ) {
case '+': m_negFormat = e_plusMinusBack; break;
case '^': m_negFormat = e_plusMinusEnd; break;
default:
pos--;
}
state = e_sInitial;
break;
//=================================
// Parsing neg format 2
case e_sNegFmt2:
switch( chr ) {
case '-': m_negFormat = e_minusBack; break;
case '^': m_negFormat = e_minusEnd; break;
default:
pos--;
}
state = e_sInitial;
break;
}
++pos;
} // end main loop
// verify output status
switch( state )
{
case e_sInitial: // ok
case e_sNegFmt:
break;
case e_sSize:
{
int64 tgt;
tmp.parseInt( tgt );
fieldSize( (uint8) tgt );
}
break;
case e_sDecimals:
{
int64 tgt;
tmp.parseInt( tgt );
decimals( (uint8) tgt );
}
break;
// any other state means we're left in the middle of something
default:
m_convType = e_tError;
return false;
}
// if everything goes fine...
m_originalFormat = fmt;
return true;
}
void MvForward::init()
{
m_indexInit = 0;
m_indexLimit = fieldSize();
}
void MvBackward::init()
{
m_indexInit = fieldSize() - 1;
m_indexLimit = -1;
}
void Reader::readHeader(std::istream &in)
{
try
{
static const char * const propertyNames[numProperties] =
{
"x", "y", "z", "nx", "ny", "nz", "radius"
};
vertexSize = 0;
size_type elements = 0;
bool haveProperty[numProperties] = {};
std::string line = getHeaderLine(in);
if (line != "ply")
throw boost::enable_error_info(FormatError("PLY signature missing"));
// read the header
bool haveFormat = false;
while (true)
{
std::vector<std::string> tokens;
line = getHeaderLine(in);
tokens = splitLine(line);
if (tokens.empty())
continue; // ignore blank lines
if (tokens[0] == "end_header")
break;
else if (tokens[0] == "format")
{
if (tokens.size() != 3)
throw boost::enable_error_info(FormatError("Malformed format line"));
if (tokens[1] == "ascii")
throw boost::enable_error_info(FormatError("PLY ASCII format not supported"));
else if (tokens[1] == "binary_big_endian")
{
if (!cpuBigEndian())
throw boost::enable_error_info(FormatError("PLY big endian format not supported on this CPU"));
}
else if (tokens[1] == "binary_little_endian")
{
if (!cpuLittleEndian())
throw boost::enable_error_info(FormatError("PLY little endian format not supported on this CPU"));
}
else
{
throw boost::enable_error_info(FormatError("Unknown PLY format " + tokens[1]));
}
if (tokens[2] != "1.0")
throw boost::enable_error_info(FormatError("Unknown PLY version " + tokens[2]));
haveFormat = true;
}
else if (tokens[0] == "element")
{
if (tokens.size() != 3)
throw boost::enable_error_info(FormatError("Malformed element line"));
std::string elementName = tokens[1];
size_type elementCount;
try
{
elementCount = boost::lexical_cast<size_type>(tokens[2]);
}
catch (boost::bad_lexical_cast &e)
{
throw boost::enable_error_info(FormatError("Malformed element line or too many elements"));
}
if (elements == 0)
{
/* Expect the vertex element */
if (elementName != "vertex")
throw boost::enable_error_info(FormatError("First element is not vertex"));
vertexCount = elementCount;
}
elements++;
}
else if (tokens[0] == "property")
{
if (tokens.size() < 3)
throw boost::enable_error_info(FormatError("Malformed property line"));
bool isList;
FieldType lengthType, valueType;
std::string name;
if (tokens[1] == "list")
{
if (tokens.size() != 5)
throw boost::enable_error_info(FormatError("Malformed property line"));
isList = true;
lengthType = parseType(tokens[2]);
valueType = parseType(tokens[3]);
if (lengthType == FLOAT32 || lengthType == FLOAT64)
throw boost::enable_error_info(FormatError("List cannot have floating-point count"));
name = tokens[4];
}
else
{
if (tokens.size() != 3)
throw boost::enable_error_info(FormatError("Malformed property line"));
isList = false;
lengthType = INT32; // unused, just to avoid undefined values
valueType = parseType(tokens[1]);
name = tokens[2];
}
if (elements == 0)
throw boost::enable_error_info(FormatError("Property `" + name + "' appears before any element declaration"));
if (elements == 1)
{
/* Vertex element - match it up to the expected fields */
if (isList)
throw boost::enable_error_info(FormatError("Lists in a vertex are not supported"));
for (unsigned int i = 0; i < numProperties; i++)
{
if (name == propertyNames[i])
{
if (haveProperty[i])
throw boost::enable_error_info(FormatError("Duplicate property " + name));
if (valueType != FLOAT32)
throw boost::enable_error_info(FormatError("Property " + name + " must be FLOAT32"));
haveProperty[i] = true;
offsets[i] = vertexSize;
break;
}
}
vertexSize += fieldSize(valueType);
}
}
}
if (!haveFormat)
throw boost::enable_error_info(FormatError("No format line found"));
if (elements < 1)
throw boost::enable_error_info(FormatError("No elements found"));
for (unsigned int i = 0; i < numProperties; i++)
if (!haveProperty[i])
throw boost::enable_error_info(FormatError(std::string("Property ") + propertyNames[i] + " not found"));
headerSize = in.tellg();
}
catch (boost::exception &e)
{
e << boost::errinfo_file_name(path.string());
throw;
}
}
