size_t readValue(const char* buf, sqlite_value* val)
{
u8 type = buf[0];
val->type = type;
buf++;
void* data = (void*)(buf);
size_t read = 1;
switch(type)
{
case SQLITE_INTEGER: {
val->data1.iVal = sessionGetI64((u8*)data);
read += 8;
break;
}
case SQLITE_FLOAT: {
int64_t iVal = sessionGetI64((u8*)data);
val->data1.dVal = *(double*)(&iVal);
read += 8;
break;
}
case SQLITE_TEXT: {
u32 textLen;
u8 varIntLen = getVarint32((u8*)data, textLen);
val->data1.iVal = textLen;
val->data2 = (char*)data + varIntLen;
read += textLen + varIntLen;
break;
}
case SQLITE_BLOB: {
u32 blobLen;
u8 varIntLen = getVarint32((u8*)data, blobLen);
val->data1.iVal = blobLen;
val->data2 = (char*)data + varIntLen;
read += blobLen + varIntLen;
break;
}
case SQLITE_NULL:
break;
case 0:
val->type = 0;
break;
default:
val->type = -1;
return 0;
}
return read;
}
/* Read the next position from a position list.
* Returns the position, or -1 at the end of the list. */
static int readPosition(DocListReader *pReader){
int i;
int iType = pReader->pDoclist->iType;
assert( iType>=DL_POSITIONS );
assert( !readerAtEnd(pReader) );
pReader->p += getVarint32(pReader->p, &i);
if( i==0 ){
pReader->iLastPos = -1;
return -1;
}
pReader->iLastPos += ((int) i)-1;
if( iType>=DL_POSITIONS_OFFSETS ){
/* Skip over offsets, ignoring them for now. */
int iStart, iEnd;
pReader->p += getVarint32(pReader->p, &iStart);
pReader->p += getVarint32(pReader->p, &iEnd);
}
return pReader->iLastPos;
}
/*
** Advance iterator pIter to the next key in its PMA. Return SQLITE_OK if
** no error occurs, or an SQLite error code if one does.
*/
static int vdbeSorterIterNext(
sqlite3 *db, /* Database handle (for sqlite3DbMalloc() ) */
VdbeSorterIter *pIter /* Iterator to advance */
){
int rc; /* Return Code */
int nRead; /* Number of bytes read */
int nRec = 0; /* Size of record in bytes */
int iOff = 0; /* Size of serialized size varint in bytes */
assert( pIter->iEof>=pIter->iReadOff );
if( pIter->iEof-pIter->iReadOff>5 ){
nRead = 5;
}else{
nRead = (int)(pIter->iEof - pIter->iReadOff);
}
if( nRead<=0 ){
/* This is an EOF condition */
vdbeSorterIterZero(db, pIter);
return SQLITE_OK;
}
rc = sqlite3OsRead(pIter->pFile, pIter->aAlloc, nRead, pIter->iReadOff);
if( rc==SQLITE_OK ){
iOff = getVarint32(pIter->aAlloc, nRec);
if( (iOff+nRec)>nRead ){
int nRead2; /* Number of extra bytes to read */
if( (iOff+nRec)>pIter->nAlloc ){
int nNew = pIter->nAlloc*2;
while( (iOff+nRec)>nNew ) nNew = nNew*2;
pIter->aAlloc = sqlite3DbReallocOrFree(db, pIter->aAlloc, nNew);
if( !pIter->aAlloc ) return SQLITE_NOMEM;
pIter->nAlloc = nNew;
}
nRead2 = iOff + nRec - nRead;
rc = sqlite3OsRead(
pIter->pFile, &pIter->aAlloc[nRead], nRead2, pIter->iReadOff+nRead
);
}
}
assert( rc!=SQLITE_OK || nRec>0 );
pIter->iReadOff += iOff+nRec;
pIter->nKey = nRec;
pIter->aKey = &pIter->aAlloc[iOff];
return rc;
}
/*
** usage: varint_test START MULTIPLIER COUNT INCREMENT
**
** This command tests the putVarint() and getVarint()
** routines, both for accuracy and for speed.
**
** An integer is written using putVarint() and read back with
** getVarint() and varified to be unchanged. This repeats COUNT
** times. The first integer is START*MULTIPLIER. Each iteration
** increases the integer by INCREMENT.
**
** This command returns nothing if it works. It returns an error message
** if something goes wrong.
*/
static int btree_varint_test(
void *NotUsed,
Tcl_Interp *interp, /* The TCL interpreter that invoked this command */
int argc, /* Number of arguments */
const char **argv /* Text of each argument */
){
u32 start, mult, count, incr;
u64 in, out;
int n1, n2, i, j;
unsigned char zBuf[100];
if( argc!=5 ){
Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
" START MULTIPLIER COUNT INCREMENT\"", 0);
return TCL_ERROR;
}
if( Tcl_GetInt(interp, argv[1], (int*)&start) ) return TCL_ERROR;
if( Tcl_GetInt(interp, argv[2], (int*)&mult) ) return TCL_ERROR;
if( Tcl_GetInt(interp, argv[3], (int*)&count) ) return TCL_ERROR;
if( Tcl_GetInt(interp, argv[4], (int*)&incr) ) return TCL_ERROR;
in = start;
in *= mult;
for(i=0; i<count; i++){
char zErr[200];
n1 = putVarint(zBuf, in);
if( n1>9 || n1<1 ){
sprintf(zErr, "putVarint returned %d - should be between 1 and 9", n1);
Tcl_AppendResult(interp, zErr, 0);
return TCL_ERROR;
}
n2 = getVarint(zBuf, &out);
if( n1!=n2 ){
sprintf(zErr, "putVarint returned %d and getVarint returned %d", n1, n2);
Tcl_AppendResult(interp, zErr, 0);
return TCL_ERROR;
}
if( in!=out ){
sprintf(zErr, "Wrote 0x%016llx and got back 0x%016llx", in, out);
Tcl_AppendResult(interp, zErr, 0);
return TCL_ERROR;
}
if( (in & 0xffffffff)==in ){
u32 out32;
n2 = getVarint32(zBuf, out32);
out = out32;
if( n1!=n2 ){
sprintf(zErr, "putVarint returned %d and GetVarint32 returned %d",
n1, n2);
Tcl_AppendResult(interp, zErr, 0);
return TCL_ERROR;
}
if( in!=out ){
sprintf(zErr, "Wrote 0x%016llx and got back 0x%016llx from GetVarint32",
in, out);
Tcl_AppendResult(interp, zErr, 0);
return TCL_ERROR;
}
}
/* In order to get realistic timings, run getVarint 19 more times.
** This is because getVarint is called about 20 times more often
** than putVarint.
*/
for(j=0; j<19; j++){
getVarint(zBuf, &out);
}
in += incr;
}
return TCL_OK;
}
static int statDecodePage(Btree *pBt, StatPage *p){
int nUnused;
int iOff;
int nHdr;
int isLeaf;
int szPage;
u8 *aData = sqlite3PagerGetData(p->pPg);
u8 *aHdr = &aData[p->iPgno==1 ? 100 : 0];
p->flags = aHdr[0];
p->nCell = get2byte(&aHdr[3]);
p->nMxPayload = 0;
isLeaf = (p->flags==0x0A || p->flags==0x0D);
nHdr = 12 - isLeaf*4 + (p->iPgno==1)*100;
nUnused = get2byte(&aHdr[5]) - nHdr - 2*p->nCell;
nUnused += (int)aHdr[7];
iOff = get2byte(&aHdr[1]);
while( iOff ){
nUnused += get2byte(&aData[iOff+2]);
iOff = get2byte(&aData[iOff]);
}
p->nUnused = nUnused;
p->iRightChildPg = isLeaf ? 0 : sqlite3Get4byte(&aHdr[8]);
szPage = sqlite3BtreeGetPageSize(pBt);
if( p->nCell ){
int i; /* Used to iterate through cells */
int nUsable; /* Usable bytes per page */
sqlite3BtreeEnter(pBt);
nUsable = szPage - sqlite3BtreeGetReserveNoMutex(pBt);
sqlite3BtreeLeave(pBt);
p->aCell = sqlite3_malloc64((p->nCell+1) * sizeof(StatCell));
if( p->aCell==0 ) return SQLITE_NOMEM_BKPT;
memset(p->aCell, 0, (p->nCell+1) * sizeof(StatCell));
for(i=0; i<p->nCell; i++){
StatCell *pCell = &p->aCell[i];
iOff = get2byte(&aData[nHdr+i*2]);
if( !isLeaf ){
pCell->iChildPg = sqlite3Get4byte(&aData[iOff]);
iOff += 4;
}
if( p->flags==0x05 ){
/* A table interior node. nPayload==0. */
}else{
u32 nPayload; /* Bytes of payload total (local+overflow) */
int nLocal; /* Bytes of payload stored locally */
iOff += getVarint32(&aData[iOff], nPayload);
if( p->flags==0x0D ){
u64 dummy;
iOff += sqlite3GetVarint(&aData[iOff], &dummy);
}
if( nPayload>(u32)p->nMxPayload ) p->nMxPayload = nPayload;
getLocalPayload(nUsable, p->flags, nPayload, &nLocal);
pCell->nLocal = nLocal;
assert( nLocal>=0 );
assert( nPayload>=(u32)nLocal );
assert( nLocal<=(nUsable-35) );
if( nPayload>(u32)nLocal ){
int j;
int nOvfl = ((nPayload - nLocal) + nUsable-4 - 1) / (nUsable - 4);
pCell->nLastOvfl = (nPayload-nLocal) - (nOvfl-1) * (nUsable-4);
pCell->nOvfl = nOvfl;
pCell->aOvfl = sqlite3_malloc64(sizeof(u32)*nOvfl);
if( pCell->aOvfl==0 ) return SQLITE_NOMEM_BKPT;
pCell->aOvfl[0] = sqlite3Get4byte(&aData[iOff+nLocal]);
for(j=1; j<nOvfl; j++){
int rc;
u32 iPrev = pCell->aOvfl[j-1];
DbPage *pPg = 0;
rc = sqlite3PagerGet(sqlite3BtreePager(pBt), iPrev, &pPg, 0);
if( rc!=SQLITE_OK ){
assert( pPg==0 );
return rc;
}
pCell->aOvfl[j] = sqlite3Get4byte(sqlite3PagerGetData(pPg));
sqlite3PagerUnref(pPg);
}
}
}
}
}
return SQLITE_OK;
}
int readChangeset(const char* buf, size_t size, InstrCallback instr_callback, void* context)
{
const char* const bufStart = buf;
const char* const bufEnd = buf + size;
double lastPos = .0;
auto t1 = std::chrono::high_resolution_clock::now();
while(buf < bufEnd) {
char op = buf[0];
buf++;
// Read OP
if (op != 'T') {
return CHANGESET_CORRUPT;
}
// Read number of columns
u32 nCol;
u8 varintLen = getVarint32((u8*)buf, nCol);
buf += varintLen;
// Read Primary Key flags
std::vector<int> PKs(nCol);
for (u32 i=0; i < nCol; i++) {
PKs[i] = (bool) buf[i];
}
buf += nCol;
// Read table name
const char* tableName = buf;
size_t tableNameLen = std::strlen(tableName);
buf += tableNameLen + 1;
size_t instrRead = 0;
TableInfo table;
table.PKs = PKs.data();
table.nCol = nCol;
table.tableName = tableName;
Instruction instr;
instr.table = &table;
instr.values = new sqlite_value[nCol*2];
while (buf < bufEnd && buf[0] != 'T') {
instrRead = readInstructionFromBuffer(buf, &instr);
if (instrRead == 0) {
std::cerr << "Error reading instruction from buffer." << std::endl;
delete[] instr.values;
return CHANGESET_INSTRUCTION_CORRUPT;
}
int rc;
if (instr_callback && (rc = instr_callback(&instr, context))) {
std::cerr << "Error applying instruction. Callback returned " << rc << std::endl;
delete[] instr.values;
return CHANGESET_CALLBACK_ERROR;
}
buf += instrRead;
double pos = (double)(buf - bufStart) / (bufEnd - bufStart) * 100;
if ((pos - lastPos) > 0.1) {
auto t2 = std::chrono::high_resolution_clock::now();
auto time_span = std::chrono::duration_cast<std::chrono::duration<double>>(t2 - t1);
std::cerr << pos << "%, " << (time_span.count() / pos) * 100 << std::endl;
lastPos = pos;
}
}
delete[] instr.values;
}
return 0;
}