void CompactIndex::initializeForQuerying() {
readOnly = true;
fileHandle = open(fileName, O_RDONLY | O_LARGEFILE);
if (fileHandle < 0) {
snprintf(errorMessage, sizeof(errorMessage), "Unable to open on-disk index: %s", fileName);
log(LOG_ERROR, LOG_ID, errorMessage);
perror(NULL);
exit(1);
}
// create File object to be used by all posting lists; initial usage count: 1
// setting the usage count to 1 makes sure the object is not destroyed by
// its children (see FileFile for details)
baseFile = new FileFile(fileName, (off_t)0, 1);
readRawData(getByteSize() - sizeof(header), &header, sizeof(header));
long long descSize =
header.descriptorCount * sizeof(CompactIndex_BlockDescriptor);
descriptorSlotCount = header.descriptorCount;
descriptors = typed_malloc(CompactIndex_BlockDescriptor, descriptorSlotCount + 1);
readRawData(getByteSize() - sizeof(header) - descSize, descriptors, descSize);
long long pc = header.postingCount;
sprintf(errorMessage, "On-disk index loaded: %s", fileName);
log(LOG_DEBUG, LOG_ID, errorMessage);
sprintf(errorMessage, " terms: %d, segments: %d, postings: %lld, descriptors: %d (%lld bytes)",
header.termCount, header.listCount, pc, header.descriptorCount, descSize);
log(LOG_DEBUG, LOG_ID, errorMessage);
} // end of initializeForQuerying()
float C1WireByOWFS::GetIlluminescence(const _t1WireDevice& device) const
{
// Both terms "illumination" and "illuminance" are in the OWFS documentation, so try both
std::string readValue=readRawData(std::string(device.filename+"/S3-R1-A/illuminance"));
if (readValue.empty())
readValue=readRawData(std::string(device.filename+"/S3-R1-A/illumination"));
if (readValue.empty())
return -1000.0;
return (float)(atof(readValue.c_str())*1000.0);
}
unsigned long C1WireByOWFS::GetCounter(const _t1WireDevice& device,int unit) const
{
// Depending on OWFS version, file can be "counter" or "counters". So try both.
std::string readValue=readRawData(std::string(device.filename+"/counter.").append(1,'A'+unit));
if (readValue.empty())
readValue=readRawData(std::string(device.filename+"/counters.").append(1,'A'+unit));
if (readValue.empty())
return 0;
return (unsigned long)atol(readValue.c_str());
}
ManParser::EChunkType ManParser::parse()
{
if (m_Zen.getSeek() >= m_Zen.getFileSize())
return CHUNK_EOF;
BinaryChunkInfo chunkInfo;
m_Zen.readStructure(chunkInfo);
// store position after header so that we can skip the chunk
size_t chunkEnd = m_Zen.getSeek() + chunkInfo.length;
switch (chunkInfo.id)
{
case CHUNK_HEADER:
readHeader();
return CHUNK_HEADER;
case CHUNK_RAWDATA:
readRawData();
return CHUNK_RAWDATA;
default:
m_Zen.setSeek(chunkEnd);
return parse(); // skip unknown chunk
}
return CHUNK_EOF;
}
DBusInputStream& DBusInputStream::readBasicTypeValue<float>(float& val) {
if (sizeof(val) > 1)
alignToBoundary(sizeof(double));
val = (float) (*(reinterpret_cast<double*>(readRawData(sizeof(double)))));
return *this;
}
float C1WireByOWFS::GetPressure(const _t1WireDevice& device) const
{
std::string readValue=readRawData(std::string(device.filename+"/B1-R1-A/pressure"));
if (readValue.empty())
return -1000.0;
return static_cast<float>(atof(readValue.c_str()));
}
int C1WireByOWFS::GetVoltage(const _t1WireDevice& device,int unit) const
{
std::string fileName(device.filename);
switch(device.family)
{
case smart_battery_monitor:
{
static const std::string unitNames[] = { "VAD", "VDD", "vis" };
if (unit >= (sizeof(unitNames)/sizeof(unitNames[0])))
return 0;
fileName.append("/").append(unitNames[unit]);
break;
}
default:
{
fileName.append("/volt.").append(1,'A'+unit);
break;
}
}
std::string readValue=readRawData(fileName);
if (readValue.empty())
return -1000;
return static_cast<int>((atof(readValue.c_str())*1000.0));
}
float C1WireByOWFS::GetHumidity(const _t1WireDevice& device) const
{
std::string readValue=readRawData(std::string(device.filename+"/humidity"));
if (readValue.empty())
return 0.0;
return (float)atof(readValue.c_str());
}
unsigned int C1WireByOWFS::GetNbChannels(const _t1WireDevice& device) const
{
std::string readValue=readRawData(std::string(device.filename+"/channels"));
if (readValue.empty())
return 0;
return atoi(readValue.c_str());
}
int C1WireByOWFS::GetVoltage(const _t1WireDevice& device,int unit) const
{
std::string readValue=readRawData(std::string(device.filename+"/volt.").append(1,'A'+unit));
if (readValue.empty())
return 0;
return (int)(atof(readValue.c_str())*1000.0);
}
float C1WireByOWFS::GetTemperature(const _t1WireDevice& device) const
{
std::string readValue=readRawData(std::string(device.filename+"/temperature"));
if (readValue.empty())
return -1000.0;
return (float)atof(readValue.c_str());
}
unsigned int serializedSize() const {
uint64 tmp=readRawData();
unsigned int retval=0;
do {
retval++;
tmp/=128;
}while (tmp);
return retval;
}
unsigned int serialize(uint8 *destination, unsigned int maxsize) const {
uint64 tmp=readRawData();
unsigned int retval=0;
do {
if (retval>=maxsize) return 0;
*destination = tmp&127;
retval++;
tmp/=128;
}while (tmp&&(*destination++|=128));
return retval;
}
InputStream& DBusInputStream::readValue(std::string& stringValue) {
uint32_t lengthOfString;
readValue(lengthOfString);
// length field does not include terminating 0-byte, therefore length of data to read is +1
char* dataPtr = readRawData(lengthOfString + 1);
// The string contained in a DBus-message is required to be 0-terminated, therefore the following line works
stringValue = dataPtr;
return *this;
}
void usbDialog::on_pushButton_import_clicked()
{
HEALTHDATA set;
((MainWindow*)parent())->cfg.hem7322u = radioButton_hem7322u->isChecked();
mem = radioButton_hem7322u->isChecked() ? 100 : 60;
if(!readRawData())
{
return;
}
user1 = 0;
user2 = 0;
for(int i = 0; i < 14*mem; i += 14)
{
if((payload[i] != 0xFF) & (payload[i + 1] != 0xFF) && (payload[i + 2] != 0xFF))
{
user1++;
}
if((payload[i + 14*mem] != 0xFF) & (payload[i + 1 + 14*mem] != 0xFF) && (payload[i + 2 + 14*mem] != 0xFF))
{
user2++;
}
}
for(int i = 0; i < 14*user1; i += 14)
{
set.time = QDateTime(QDate(2000 + payload[i + 2], (payload[i + 4]>>2) & 0x0F, ((payload[i + 4]<<8 | payload[i + 5])>>5) & 0x1F), QTime(payload[i + 5] & 0x1F, ((payload[i + 6]<<8 | payload[i + 7])>>6) & 0x3F)).toTime_t();
set.sys = payload[i + 1] + 25;
set.dia = payload[i];
set.bpm = payload[i + 3];
((MainWindow*)parent())->healthdata[0].append(set);
}
for(int i = 0; i < 14*user2; i += 14)
{
set.time = QDateTime(QDate(2000 + payload[i + 2 + 14*mem], (payload[i + 4 + 14*mem]>>2) & 0x0F, ((payload[i + 4 + 14*mem]<<8 | payload[i + 5 + 14*mem])>>5) & 0x1F), QTime(payload[i + 5 + 14*mem] & 0x1F, ((payload[i + 6 + 14*mem]<<8 | payload[i + 7 + 14*mem])>>6) & 0x3F)).toTime_t();
set.sys = payload[i + 1 + 14*mem] + 25;
set.dia = payload[i + 14*mem];
set.bpm = payload[i + 3 + 14*mem];
((MainWindow*)parent())->healthdata[1].append(set);
}
accept();
}
static int readSector(u32 sector, void *buf)
{
int ret;
u32 pos;
if (g_is_compressed) {
ret = readSectorCompressed(sector, buf);
} else {
pos = isoLBA2Pos(sector, 0);
ret = readRawData(buf, SECTOR_SIZE, pos);
}
return ret;
}
uint8_t HMC5883L::refresh()
{
// read the raw data
if(readRawData() != 0) return 0;
// test if the data has been updated
if(!hasRawDataChanged()) return 0;
// compute the magnetometer data
magnetX = rawData[rawDataIndex].magnetX / HMC5883L_SENSITIVITY;
magnetY = rawData[rawDataIndex].magnetY / HMC5883L_SENSITIVITY;
magnetZ = rawData[rawDataIndex].magnetZ / HMC5883L_SENSITIVITY;
// increment the raw data index
rawDataIndex ^= 1;
return 1;
}
int main()
{
// parameters
char *serial=NULL; // can be set to a specific serial, NULL uses the first found device
bool debug=true; // debug mode (health monitor writes to stderr)
char *message;
bool errorFlag = false;
// initialize hardware and health monitor
struct ftdi_context ftdic;
if (!initInfnoise(&ftdic, serial, &message, false, debug)) {
fputs(message, stderr);
return 1; // ERROR
}
// read and print in a loop (until 1M)
uint32_t totalBytesWritten = 0;
while (totalBytesWritten < 1000000) {
uint8_t result[64];// result is always 64 bytes in raw mode
// read data returns the number of bytes written to result array
uint64_t bytesWritten = readRawData(&ftdic, result, &message, &errorFlag);
totalBytesWritten += bytesWritten;
// check for errors
// note: bytesWritten is also 0 in this case, but an errorFlag is needed as
// bytesWritten can also be 0 when data hasn't passed the health monitor.
if (errorFlag) {
fprintf(stderr, "Error: %s\n", message);
return 1;
}
// print bytes to stdout
fwrite(result, 1, bytesWritten, stdout);
}
return 0;
}
int
getSCN(char * rawdatafile, /*input: root of the dir tree*/
int totalNum) /*input: total number of nodes*/
{
int error;
time_t time_begin,time_end;
struct tm * TM;
char dir_p[10];
char * filename;
int name_size;
FILE *outfile;
struct network * scn;
name_size=30;
error=0;
/* 1) intialize the static variables: net and list*/
scn=(struct network *)mem_alloc(sizeof(struct network));
initNetwork(scn, totalNum, UN_DIRECT_WITH_WEIGHT);
/* 2) get the SCN from rawdatafile*/
readRawData(scn, rawdatafile);
/* 3) get statistical properties of the network, and record all the information into output files */
// create ouput directories
TM=NULL;
time(&time_begin);
TM=localtime(&time_begin);
strftime(dir_p, name_size, "%y%m%d%H%M", TM);
strcat(dir_p,"/");
#if defined(_WIN32)
error=_mkdir(dir_p);
#else
error=mkdir(dir_p, 0777);
#endif
filename=(char *)mem_alloc(name_size*sizeof(char));
// <1> show the structure of the network
//showNetwork(scn);
// <2> write the network info to the files
memset(filename, 0, name_size);
strcat(filename, dir_p);
strcat(filename, "net.txt");
outfile=fopen(filename, "w");
outputNetwork(scn, outfile);
fclose (outfile);
// <4> analyze the structural properties
analyzeSCN(scn,dir_p);
time(&time_end);
printf("\n exhausted time: %ld \n",time_end-time_begin);
/* 4) free network*/
freeNetwork(scn);
mem_free(filename);
return error;
}
void CompactIndex2::initializeForQuerying() {
readOnly = true;
fileHandle = open(fileName, O_RDONLY | O_LARGEFILE);
if (fileHandle < 0) {
snprintf(errorMessage, sizeof(errorMessage), "Unable to open on-disk index: %s", fileName);
log(LOG_ERROR, LOG_ID, errorMessage);
perror(NULL);
exit(1);
}
else {
// create File object to be used by all posting lists; initial usage count: 1
// setting the usage count to 1 makes sure the object is not destroyed by
// its children (see FileFile for details)
baseFile = new FileFile(fileName, (off_t)0, 1);
}
// read header from end of file
readRawData(getByteSize() - sizeof(header), &header, sizeof(header));
// read compressed descriptor sequence
usedByDescriptors = allocatedForDescriptors = header.compressedDescriptorSize;
endOfPostingsData = getByteSize() - sizeof(header) - usedByDescriptors;
compressedDescriptors = (byte*)malloc(usedByDescriptors);
readRawData(endOfPostingsData, compressedDescriptors, usedByDescriptors);
sprintf(errorMessage, "On-disk index loaded: %s", fileName);
log(LOG_DEBUG, LOG_ID, errorMessage);
sprintf(errorMessage, " terms: %lld, segments: %lld, postings: %lld, descriptors: %lld (%d bytes)",
(long long)header.termCount,
(long long)header.listCount,
(long long)header.postingCount,
(long long)header.descriptorCount,
usedByDescriptors);
log(LOG_DEBUG, LOG_ID, errorMessage);
// build search array from compressed descriptor sequence
dictionaryGroupCount =
(header.descriptorCount + DICTIONARY_GROUP_SIZE - 1) / DICTIONARY_GROUP_SIZE;
groupDescriptors =
typed_malloc(CompactIndex2_DictionaryGroup, dictionaryGroupCount + 1);
int64_t filePos = 0;
char prevTerm[MAX_TOKEN_LENGTH * 2] = { 0 };
uint32_t inPos = 0;
for (int i = 0; i < dictionaryGroupCount; i++) {
assert(inPos < 2000000000);
offset delta;
groupDescriptors[i].groupStart = inPos;
inPos += decodeFrontCoding(&compressedDescriptors[inPos], prevTerm, groupDescriptors[i].groupLeader);
inPos += decodeVByteOffset(&delta, &compressedDescriptors[inPos]);
filePos += delta;
groupDescriptors[i].filePosition = filePos;
strcpy(prevTerm, groupDescriptors[i].groupLeader);
for (int k = 1; (k < DICTIONARY_GROUP_SIZE) && (inPos < usedByDescriptors); k++) {
char term[MAX_TOKEN_LENGTH * 2];
inPos += decodeFrontCoding(&compressedDescriptors[inPos], prevTerm, term);
inPos += decodeVByteOffset(&delta, &compressedDescriptors[inPos]);
strcpy(prevTerm, term);
filePos += delta;
}
} // end for (int i = 0; i < dictionaryGroupCount; i++)
temporaryPLSH = NULL;
} // end of initializeForQuerying()
ExtentList * CompactIndex2::getPostings2(const char *term) {
if ((header.descriptorCount <= 0) || (header.termCount <= 0))
return new ExtentList_Empty();
// obtain the file position of the block containing the term
char prevTerm[MAX_TOKEN_LENGTH * 2], t[MAX_TOKEN_LENGTH * 2];
int64_t filePosition = getBlockStart(term, prevTerm);
if (filePosition < 0)
return new ExtentList_Empty();
// we have identified the index block that potentially contains the
// term that we are looking for; load first BYTES_PER_INDEX_BLOCK bytes
// into memory and conduct another sequential scan on those data
PostingListSegmentHeader plsh;
byte buffer[BYTES_PER_INDEX_BLOCK + 256];
int status = readRawData(filePosition, buffer, sizeof(buffer));
int64_t postingsPosition = -1;
int pos = 0;
while (pos < MIN(status, BYTES_PER_INDEX_BLOCK)) {
// extract term and check whether this is the one that we are looking for
pos += decodeFrontCoding(&buffer[pos], prevTerm, t);
strcpy(prevTerm, t);
int comparison = strcmp(t, term);
if (comparison >= 0) {
if (comparison == 0)
postingsPosition = filePosition + pos;
break;
}
// skip over the postings for the current term
int segmentsSeen = 0;
do {
if (++segmentsSeen == 2)
pos += sizeof(int64_t);
pos += decompressPLSH(&buffer[pos], 0, &plsh);
pos += plsh.byteLength;
if (pos >= status)
break;
} while (buffer[pos++] == 255);
if ((segmentsSeen > 1) && (pos < status)) {
int32_t segmentCount, segmentSize;
pos += decodeVByte32(&segmentCount, &buffer[pos]);
pos += decodeVByte32(&segmentSize, &buffer[pos]);
assert(segmentCount == segmentsSeen);
pos += segmentSize;
}
} // end while (pos < status)
// if we were unable to find the term, return an empty list
if (postingsPosition < 0)
return new ExtentList_Empty();
LocalLock lock(this);
// load the first list segment for the term into memory ("tempBuf")
pos += decompressPLSH(&buffer[pos], 0, &plsh);
byte *tempBuf = &buffer[pos];
bool mustFreeTempBuf = false;
if (pos + plsh.byteLength + 32 > sizeof(buffer))
tempBuf = getRawData(filePosition + pos, plsh.byteLength + 32, &mustFreeTempBuf);
int32_t segmentCount;
PostingListSegmentHeader *segmentHeaders;
int64_t *segmentPositions;
if (tempBuf[plsh.byteLength] == 0) {
// this is the only list segment for the given term (continuation flag == 0);
// build SegmentedPostingList instance directly from data found in plsh
segmentCount = 1;
segmentHeaders = typed_malloc(PostingListSegmentHeader, segmentCount);
segmentPositions = typed_malloc(int64_t, segmentCount);
segmentHeaders[0] = plsh;
segmentPositions[0] = filePosition + pos;
}
else {
// more segments to follow (continuation flag == 255); seek to beginning of
// sync point list and build SegmentedPostingList instance from those data
int64_t markerValue;
memcpy(&markerValue, &tempBuf[plsh.byteLength + 1], sizeof(markerValue));
int64_t markerFilePos = filePosition + pos + plsh.byteLength + 1;
int64_t headerFilePos = markerFilePos + sizeof(markerValue) + markerValue;
headerFilePos += readRawData(headerFilePos, buffer, sizeof(buffer));
int32_t dummy32;
pos = decodeVByte32(&segmentCount, &buffer[0]);
pos += decodeVByte32(&dummy32, &buffer[pos]);
segmentHeaders = typed_malloc(PostingListSegmentHeader, segmentCount);
segmentPositions = typed_malloc(int64_t, segmentCount);
offset referencePosting = 0;
for (int i = 0; i < segmentCount; i++) {
if (pos > sizeof(buffer) - 256) {
memmove(buffer, &buffer[pos], sizeof(buffer) - pos);
headerFilePos += readRawData(headerFilePos, &buffer[sizeof(buffer) - pos], pos);
pos = 0;
}
int headerSize =
decompressPLSH(&buffer[pos], referencePosting, &segmentHeaders[i]);
pos += headerSize;
postingsPosition += headerSize;
if (i > 0) {
postingsPosition++;
if (i == 1)
postingsPosition += sizeof(markerValue);
}
segmentPositions[i] = postingsPosition;
referencePosting = segmentHeaders[i].lastElement;
postingsPosition += segmentHeaders[i].byteLength;
}
} // end else [tempBuf[plsh.byteLength] != 0]
if (mustFreeTempBuf)
free(tempBuf);
#if ALWAYS_LOAD_POSTINGS_INTO_MEMORY
SPL_InMemorySegment *splSegments = typed_malloc(SPL_InMemorySegment, segmentCount);
FileFile *file = NULL;
#else
SPL_OnDiskSegment *splSegments = typed_malloc(SPL_OnDiskSegment, segmentCount);
FileFile *file = getFile();
#endif
for (int i = 0; i < segmentCount; i++) {
splSegments[i].count = segmentHeaders[i].postingCount;
splSegments[i].byteLength = segmentHeaders[i].byteLength;
splSegments[i].firstPosting = segmentHeaders[i].firstElement;
splSegments[i].lastPosting = segmentHeaders[i].lastElement;
#if ALWAYS_LOAD_POSTINGS_INTO_MEMORY
splSegments[i].postings = (byte*)malloc(segmentHeaders[i].byteLength);
readRawData(segmentPositions[i], splSegments[i].postings, segmentHeaders[i].byteLength);
#else
splSegments[i].file = new FileFile(file, segmentPositions[i]);
#endif
}
if ((segmentCount == 0) && (file != NULL))
delete file;
free(segmentHeaders);
free(segmentPositions);
#if ALWAYS_LOAD_POSTINGS_INTO_MEMORY
return new SegmentedPostingList(splSegments, segmentCount, true);
#else
return new SegmentedPostingList(splSegments, segmentCount);
#endif
} // end of getPostings2(char*)
ExtentList * CompactIndex2::getPostingsForWildcardQuery(const char *pattern, const char *stem) {
if ((header.descriptorCount <= 0) || (header.termCount <= 0))
return new ExtentList_Empty();
char *prefix = duplicateString(pattern);
for (int i = 0; prefix[i] != 0; i++)
if (IS_WILDCARD_CHAR(prefix[i])) {
prefix[i] = 0;
break;
}
bool isDocumentLevel = startsWith(prefix, "<!>");
int prefixLen = strlen(prefix);
if (prefixLen < (isDocumentLevel ? 5 : 2)) {
free(prefix);
return new ExtentList_Empty();
}
char t[MAX_TOKEN_LENGTH * 2], prevTerm[MAX_TOKEN_LENGTH * 2];
int64_t filePosition = getBlockStart(prefix, prevTerm);
if (filePosition < 0)
filePosition = groupDescriptors[0].filePosition;
// we have identified the index block that potentially contains the
// term that we are looking for; load first BYTES_PER_INDEX_BLOCK bytes
// into memory and conduct another sequential scan on those data
PostingListSegmentHeader plsh;
byte buffer[BYTES_PER_INDEX_BLOCK + 256];
int status = readRawData(filePosition, buffer, sizeof(buffer));
int pos = 0;
LocalLock lock(this);
int termsFound = 0, termsAllocated = 256;
ExtentList **lists = typed_malloc(ExtentList*, termsAllocated);
#if ALWAYS_LOAD_POSTINGS_INTO_MEMORY
SPL_InMemorySegment *splSegments = NULL;
FileFile *file = NULL;
#else
SPL_OnDiskSegment *splSegments = NULL;
FileFile *file = getFile();
#endif
pos += decodeFrontCoding(&buffer[pos], prevTerm, t);
strcpy(prevTerm, t);
while (strncmp(prevTerm, prefix, prefixLen) <= 0) {
int64_t postingsPosition = filePosition + pos;
int comparison = strncmp(prevTerm, prefix, prefixLen);
PostingListSegmentHeader plsh;
// make sure the current term matches the prefix query and also satisfies the
// stemming criterion
if (comparison == 0)
if (fnmatch(pattern, prevTerm, 0) != 0)
comparison = -1;
if ((comparison == 0) && (stem != NULL)) {
char tempForStemming[MAX_TOKEN_LENGTH * 2];
strcpy(tempForStemming, prevTerm);
if (isDocumentLevel)
Stemmer::stemEnglish(&tempForStemming[3]);
else
Stemmer::stemEnglish(tempForStemming);
if (strcmp(tempForStemming, stem) != 0)
comparison = -1;
} // end if ((comparison == 0) && (stem != NULL))
int segmentsSeen = 0;
do {
if (++segmentsSeen == 2) {
int64_t markerValue;
memcpy(&markerValue, &buffer[pos], sizeof(markerValue));
postingsPosition += sizeof(markerValue);
filePosition += pos + sizeof(markerValue) + markerValue;
status = readRawData(filePosition, buffer, sizeof(buffer));
pos = 0;
int32_t segmentCount, segmentSize;
pos += decodeVByte32(&segmentCount, &buffer[pos]);
pos += decodeVByte32(&segmentSize, &buffer[pos]);
if (comparison == 0) {
byte *compressedHeaders = (byte*)malloc(segmentSize);
readRawData(filePosition + pos, compressedHeaders, segmentSize);
int inPos = decompressPLSH(&compressedHeaders[0], 0, &plsh);
#if ALWAYS_LOAD_POSTINGS_INTO_MEMORY
splSegments = typed_realloc(SPL_InMemorySegment, splSegments, segmentCount);
#else
splSegments = typed_realloc(SPL_OnDiskSegment, splSegments, segmentCount);
#endif
for (int i = 1; i < segmentCount; i++) {
int headerSize = decompressPLSH(&compressedHeaders[inPos], plsh.lastElement, &plsh);
inPos += headerSize;
postingsPosition += headerSize;
splSegments[i].count = plsh.postingCount;
splSegments[i].byteLength = plsh.byteLength;
splSegments[i].firstPosting = plsh.firstElement;
splSegments[i].lastPosting = plsh.lastElement;
#if ALWAYS_LOAD_POSTINGS_INTO_MEMORY
splSegments[i].postings = (byte*)malloc(plsh.byteLength);
readRawData(postingsPosition, splSegments[i].postings, plsh.byteLength);
#else
splSegments[i].file = new FileFile(file, postingsPosition);
#endif
postingsPosition += plsh.byteLength + 1;
}
free(compressedHeaders);
segmentsSeen = segmentCount;
} // end if (comparison == 0)
pos += segmentSize;
break;
} // end if (++segmentsSeen == 2)
int headerSize = decompressPLSH(&buffer[pos], 0, &plsh);
pos += headerSize;
postingsPosition += headerSize;
// if the current term matches the query, collect postings data
if (comparison == 0) {
#if ALWAYS_LOAD_POSTINGS_INTO_MEMORY
splSegments = typed_malloc(SPL_InMemorySegment, 1);
splSegments[0].postings = (byte*)malloc(plsh.byteLength);
readRawData(postingsPosition, splSegments[0].postings, plsh.byteLength);
#else
splSegments = typed_malloc(SPL_OnDiskSegment, 1);
splSegments[0].file = new FileFile(file, postingsPosition);
#endif
splSegments[0].count = plsh.postingCount;
splSegments[0].byteLength = plsh.byteLength;
splSegments[0].firstPosting = plsh.firstElement;
splSegments[0].lastPosting = plsh.lastElement;
} // end if (comparison == 0)
pos += plsh.byteLength;
postingsPosition += plsh.byteLength + 1;
if (pos + 256 > status) {
filePosition += pos;
status = readRawData(filePosition, buffer, sizeof(buffer));
pos = 0;
}
} while (buffer[pos++] == 255);
// add current list to set of lists to return to caller
if ((comparison == 0) && (segmentsSeen > 0)) {
if (termsFound >= termsAllocated)
lists = typed_realloc(ExtentList*, lists, termsAllocated = (termsAllocated * 2));
#if ALWAYS_LOAD_POSTINGS_INTO_MEMORY
lists[termsFound++] = new SegmentedPostingList(splSegments, segmentsSeen, true);
#else
lists[termsFound++] = new SegmentedPostingList(splSegments, segmentsSeen);
#endif
}
if (pos + 256 > status) {
// refill buffer if necessary
filePosition += pos;
status = readRawData(filePosition, buffer, sizeof(buffer));
pos = 0;
}
// fetch next term from buffer
pos += decodeFrontCoding(&buffer[pos], prevTerm, t);
strcpy(prevTerm, t);
} // end while (strncmp(prevTerm, prefix, prefixLen) <= 0)
free(prefix);
if (termsFound == 0) {
if (file != NULL)
delete file;
free(lists);
return new ExtentList_Empty();
}
else if (termsFound == 1) {
ExtentList *result = lists[0];
free(lists);
return result;
}
else if (isDocumentLevel)
return ExtentList::mergeDocumentLevelLists(lists, termsFound);
else
return new ExtentList_OR_Postings(lists, termsFound);
} // end of getPostingsForWildcardQuery(char*, char*)
bool C1WireByOWFS::GetLightState(const _t1WireDevice& device,int unit) const
{
std::string fileName=std::string(device.filename);
switch(device.family)
{
case Addresable_Switch:
{
fileName.append("/sensed");
break;
}
case dual_addressable_switch_plus_1k_memory:
case Temperature_IO:
case dual_channel_addressable_switch:
{
if (unit<0 || unit>1)
return false;
fileName.append("/sensed.").append(1,'A'+unit);
break;
}
case _8_channel_addressable_switch:
{
if (unit<0 || unit>7)
return false;
fileName.append("/sensed.").append(1,'0'+unit);
break;
}
case _4k_EEPROM_with_PIO:
{
if (unit<0 || unit>1)
return false;
fileName.append("/sensed.").append(1,'0'+unit);
break;
}
case microlan_coupler:
{
fileName.append("/control");
break;
}
}
std::string readValue=readRawData(fileName);
if (readValue.empty())
return false;
switch(device.family)
{
case Addresable_Switch:
case dual_addressable_switch_plus_1k_memory:
case Temperature_IO:
case dual_channel_addressable_switch:
case _8_channel_addressable_switch:
case _4k_EEPROM_with_PIO:
// Caution : read value correspond to voltage level, inverted from the transistor state
// We have to invert the read value
return readValue!="1";
case microlan_coupler:
{
// 1 = Unconditionally off (non-conducting)
// 2 = Unconditionally on (conducting)
int iValue=atoi(readValue.c_str())==2;
if (iValue!=1 && iValue!=2)
return false;
else
return iValue==2;
}
}
return false;
}
void WavStreamer::fillBuffer(float *stereoBuffer, uint32_t stereoSamples)
{
if (m_waveStream==NULL)
{
// clear the buffer if we have no file to read
// Note: this is compatible with the IEEE 756 floating-point format!
memset(stereoBuffer, 0, sizeof(float)*2*stereoSamples);
return;
}
else
{
int monoSamples = stereoSamples * 2;
if (m_waveFormat.wFormatTag == 1) // PCM
{
switch(m_waveFormat.wBitsPerSample)
{
case 16:
{
int16_t *ptr = static_cast<short*>(tempBuffer);
for(int i=0; i<monoSamples; i++)
{
if (sampleIndex > (TEMPBUFFERSIZE*2-1)) readRawData(TEMPBUFFERSIZE);
stereoBuffer[i] = static_cast<float>(ptr[sampleIndex]) / 32768.0f;
sampleIndex++;
}
}
break;
case 24:
{
uint8_t *ptr = static_cast<uint8_t*>(tempBuffer);
long v = 0;
int index = sampleIndex * 3;
for(int i=0; i<monoSamples; i++)
{
if (sampleIndex > (TEMPBUFFERSIZE*2-1))
{
readRawData(TEMPBUFFERSIZE);
index = 0;
}
v = static_cast<uint32_t>(ptr[index]) << 8;
v |= static_cast<uint32_t>(ptr[index+1]) << 16;
uint8_t top = static_cast<uint32_t>(ptr[index+2]);
v |= (top << 24);
if (top >= 0x80)
{
// signed!
v |= 0xFF000000;
}
index += 3;
stereoBuffer[i] = static_cast<float>(v) / 2147483648.0f;
sampleIndex++;
}
}
break;
case 32:
{
int32_t *ptr = static_cast<int*>(tempBuffer);
for(int i=0; i<monoSamples; i++)
{
if (sampleIndex > (TEMPBUFFERSIZE*2-1))
{
readRawData(TEMPBUFFERSIZE);
}
stereoBuffer[i] = static_cast<float>(ptr[sampleIndex]) / 2147483648.0f;
sampleIndex++;
}
}
break;
}
}
else if (m_waveFormat.wFormatTag == 3)
{
// uncompressed 32-bit float data
float *ptr = static_cast<float*>(tempBuffer);
for(int i=0; i<monoSamples; i++)
{
if (sampleIndex > (TEMPBUFFERSIZE*2-1))
{
readRawData(TEMPBUFFERSIZE);
}
stereoBuffer[i] = ptr[sampleIndex]; // FIXME: scaling??
sampleIndex++;
}
}
}
}
void BinaryReader::readRawData(uint8_t *destination, const size_t size)
{
readRawData(reinterpret_cast<char*>(destination), size);
}
int32_t WavStreamer::openFile(const QString &filename)
{
if (m_waveStream != 0)
{
delete m_waveStream;
m_waveStream = 0;
}
if (m_file.isOpen())
{
m_file.close();
}
m_isOK = false;
// try to open the file
m_file.setFileName(filename);
if (m_file.open(QIODevice::ReadOnly) == false)
{
return -2; // error opening file
}
m_waveStream = new QDataStream(&m_file);
// check for RIFF file & size
size_t bytes = m_waveStream->readRawData(m_chunkType, 4);
if ((bytes != 4) || (strncmp(m_chunkType, "RIFF", 4)!=0))
{
delete m_waveStream;
m_waveStream = NULL;
m_file.close();
return -1;
}
bytes = m_waveStream->readRawData((char*)&m_chunkSize, 4);
if (bytes != 4)
{
delete m_waveStream;
m_waveStream = NULL;
m_file.close();
return -1;
}
bytes = m_waveStream->readRawData((char*)&m_chunkType, 4);
if ((bytes != 4) && (strncmp(m_chunkType, "WAVE", 4)!=0))
{
delete m_waveStream;
m_waveStream = NULL;
m_file.close();
return -1;
}
m_riffSize = m_chunkSize + 8; // size of RIFF chunk + 4-byte ID + 4-byte chunksize.
if (!findChunk("fmt "))
{
// error, format chunk not found!
delete m_waveStream;
m_waveStream = NULL;
m_file.close();
return -1;
}
// now, read the format chunk
bytes = m_waveStream->readRawData((char*)&m_waveFormat, sizeof(WavFormatChunk));
if (m_waveFormat.wFormatTag == 65534) // WAVE_FORMAT_EXTENSIBLE case..
{
m_waveFormat.wFormatTag = 1; // treat as PCM data, which should be the same when we have only 2 channels.
}
if (((m_waveFormat.wFormatTag != 3) && (m_waveFormat.wFormatTag != 1)) || (m_waveFormat.wChannels != 2))
{
delete m_waveStream;
m_waveStream = NULL;
m_file.close();
return -1; // wrong format!
}
// formats smaller than 16 bits are not supported!
if (m_waveFormat.wBitsPerSample < 16)
{
delete m_waveStream;
m_waveStream = NULL;
m_file.close();
return -1;
}
// if there are additional bytes in the format chunk, skip them.
// important in the WAVE_FORMAT_EXTENSIBLE case!
if (sizeof(WavFormatChunk) != m_chunkSize)
{
m_waveStream->skipRawData(m_chunkSize - sizeof(WavFormatChunk));
}
// now, search for the audio data and set the file offset pointers
if (!findChunk("data"))
{
delete m_waveStream;
m_waveStream = NULL;
m_file.close();
return -1;
}
m_playStart = m_waveStream->device()->pos();
m_playOffset = m_playStart;
m_playEnd = m_playStart + m_chunkSize;
// allocate the correct temporary buffer.
if (tempBuffer!=NULL) delete[] static_cast<char*>(tempBuffer);
tempBuffer = static_cast<void*>(new char[m_waveFormat.wBitsPerSample*TEMPBUFFERSIZE*2/8]);
// don't forget to actually read the data
readRawData(TEMPBUFFERSIZE);
sampleIndex = 0;
m_filename = filename;
m_isOK = true;
return 0;
}
/**
* \brief Store data record
*/
void Storage::storeDataRecord(struct metadata *mdata)
{
offset = 0;
record.clear();
record += "{\"@type\": \"ipfix.entry\", \"ipfix\": {";
struct ipfix_template *templ = mdata->record.templ;
uint8_t *data_record = (uint8_t*) mdata->record.record;
/* get all fields */
for (uint16_t count = 0, index = 0; count < templ->field_count; ++count, ++index) {
/* Get Enterprise number and ID */
id = templ->fields[index].ie.id;
length = templ->fields[index].ie.length;
enterprise = 0;
if (id & 0x8000) {
id &= 0x7fff;
enterprise = templ->fields[++index].enterprise_number;
}
/* Get element informations */
struct ipfix_element& element = elements[enterprise][id];
if (element.type == UNKNOWN && element.name.empty()) {
element.name = rawName(enterprise, id);
elements[enterprise][id] = element;
MSG_DEBUG(msg_module, "Unknown element (%s)", element.name.c_str());
}
if (count > 0) {
record += ", ";
}
record += "\"";
record += element.name;
record += "\": \"";
switch (element.type) {
case PROTOCOL:
record += translator.formatProtocol(read8(data_record + offset));
break;
case FLAGS:
record += translator.formatFlags(read16(data_record + offset));
break;
case IPV4:
record += translator.formatIPv4(read32(data_record + offset));
break;
case IPV6:{
READ_BYTE_ARR(addr6, data_record + offset, IPV6_LEN);
record += translator.formatIPv6(addr6);
break;}
case MAC: {
READ_BYTE_ARR(addrMac, data_record + offset, MAC_LEN);
record += translator.formatMac(addrMac);
break;}
case TSTAMP_SEC:
record += translator.formatTimestamp(read64(data_record + offset), t_units::SEC);
break;
case TSTAMP_MILLI:
record += translator.formatTimestamp(read64(data_record + offset), t_units::MILLISEC);
break;
case TSTAMP_MICRO:
record += translator.formatTimestamp(read64(data_record + offset), t_units::MICROSEC);
break;
case TSTAMP_NANO:
record += translator.formatTimestamp(read64(data_record + offset), t_units::NANOSEC);
break;
case STRING:
readString(length, data_record, offset);
break;
case RAW:
readRawData(length, data_record, offset);
break;
default:
readRawData(length, data_record, offset);
break;
}
record += "\"";
offset += length;
}
/* Store metadata */
if (processMetadata) {
record += "}, \"metadata\": {";
storeMetadata(mdata);
}
record += "}}\n";
sendData();
}
static int readSectorCompressed(int sector, void *addr)
{
int ret;
int n_sector;
int offset, next_offset;
int size;
n_sector = sector - g_CISO_cur_idx;
// not within sector idx cache?
if (g_CISO_cur_idx == -1 || n_sector < 0 || n_sector >= NELEMS(g_CISO_idx_cache)) {
ret = readRawData(g_CISO_idx_cache, sizeof(g_CISO_idx_cache),
(sector << 2) + sizeof(CISOHeader));
if (ret < 0) {
return ret;
}
g_CISO_cur_idx = sector;
n_sector = 0;
}
offset = (g_CISO_idx_cache[n_sector] & 0x7FFFFFFF) << g_ciso_h.align;
// is uncompressed data?
if (g_CISO_idx_cache[n_sector] & 0x80000000) {
return readRawData(addr, SECTOR_SIZE, offset);
}
sector++;
n_sector = sector - g_CISO_cur_idx;
if (g_CISO_cur_idx == -1 || n_sector < 0 || n_sector >= NELEMS(g_CISO_idx_cache)) {
ret = readRawData(g_CISO_idx_cache, sizeof(g_CISO_idx_cache), (sector << 2) + sizeof(CISOHeader));
if (ret < 0) {
return ret;
}
g_CISO_cur_idx = sector;
n_sector = 0;
}
next_offset = (g_CISO_idx_cache[n_sector] & 0x7FFFFFFF) << g_ciso_h.align;
size = next_offset - offset;
if (size <= SECTOR_SIZE)
size = SECTOR_SIZE;
if (offset < g_ciso_dec_buf_offset || size + offset >= g_ciso_dec_buf_offset + CISO_DEC_BUFFER_SIZE) {
ret = readRawData(g_ciso_dec_buf, CISO_DEC_BUFFER_SIZE, offset);
if (ret < 0) {
g_ciso_dec_buf_offset = 0xFFF00000;
return ret;
}
g_ciso_dec_buf_offset = offset;
}
ret = gzip_decompress(addr, SECTOR_SIZE,
g_ciso_dec_buf + offset - g_ciso_dec_buf_offset, size
);
return ret;
}
/**
* \brief Store data record
*/
void Storage::storeDataRecord(struct metadata *mdata, struct json_conf * config)
{
const char *element_name = NULL;
ELEMENT_TYPE element_type;
offset = 0;
uint16_t trans_len = 0;
const char *trans_str = NULL;
record.clear();
STR_APPEND(record, "{\"@type\": \"ipfix.entry\", ");
struct ipfix_template *templ = mdata->record.templ;
uint8_t *data_record = (uint8_t*) mdata->record.record;
/* get all fields */
uint16_t added = 0;
for (uint16_t count = 0, index = 0; count < templ->field_count; ++count, ++index) {
/* Get Enterprise number and ID */
id = templ->fields[index].ie.id;
length = templ->fields[index].ie.length;
enterprise = 0;
if (id & 0x8000) {
id &= 0x7fff;
enterprise = templ->fields[++index].enterprise_number;
}
/* Get element informations */
const ipfix_element_t * element = get_element_by_id(id, enterprise);
if (element != NULL) {
element_name = element->name;
element_type = element->type;
} else {
// Element not found
if (config->ignoreUnknown) {
offset += realLength(length, data_record, offset);
continue;
}
element_name = rawName(enterprise, id);
element_type = ET_UNASSIGNED;
MSG_DEBUG(msg_module, "Unknown element (%s)", element_name);
}
if (added > 0) {
STR_APPEND(record, ", ");
}
STR_APPEND(record, "\"");
record += config->prefix;
record += element_name;
STR_APPEND(record, "\": ");
switch (element_type) {
case ET_UNSIGNED_8:
case ET_UNSIGNED_16:
case ET_UNSIGNED_32:
case ET_UNSIGNED_64:{
trans_str = translator.toUnsigned(length, &trans_len, data_record, offset,
element, config);
record.append(trans_str, trans_len);
}
break;
case ET_SIGNED_8:
case ET_SIGNED_16:
case ET_SIGNED_32:
case ET_SIGNED_64:
trans_str = translator.toSigned(length, &trans_len, data_record, offset);
record.append(trans_str, trans_len);
break;
case ET_FLOAT_32:
case ET_FLOAT_64:
trans_str = translator.toFloat(length, &trans_len, data_record, offset);
record.append(trans_str, trans_len);
break;
case ET_IPV4_ADDRESS:
record += '"';
trans_str = translator.formatIPv4(read32(data_record + offset), &trans_len);
record.append(trans_str, trans_len);
record += '"';
break;
case ET_IPV6_ADDRESS:
READ_BYTE_ARR(addr6, data_record + offset, IPV6_LEN);
record += '"';
record += translator.formatIPv6(addr6);
record += '"';
break;
case ET_MAC_ADDRESS:
READ_BYTE_ARR(addrMac, data_record + offset, MAC_LEN);
record += '"';
record += translator.formatMac(addrMac);
record += '"';
break;
case ET_DATE_TIME_SECONDS:
record += translator.formatTimestamp(read32(data_record + offset),
t_units::SEC, config);
break;
case ET_DATE_TIME_MILLISECONDS:
record += translator.formatTimestamp(read64(data_record + offset),
t_units::MILLISEC, config);
break;
case ET_DATE_TIME_MICROSECONDS:
record += translator.formatTimestamp(read64(data_record + offset),
t_units::MICROSEC, config);
break;
case ET_DATE_TIME_NANOSECONDS:
record += translator.formatTimestamp(read64(data_record + offset),
t_units::NANOSEC, config);
break;
case ET_STRING:
length = realLength(length, data_record, offset);
record += translator.escapeString(length, data_record + offset,
config);
break;
case ET_BOOLEAN:
case ET_UNASSIGNED:
default:
readRawData(length, data_record, offset);
break;
}
offset += length;
added++;
}
/* Store metadata */
if (processMetadata) {
STR_APPEND(record, ", \"ipfix.metadata\": {");
storeMetadata(mdata);
STR_APPEND(record, "}");
}
STR_APPEND(record, "}\n");
sendData();
}
