static void
y_add_line(char *line)
{
y_line_t *yline = p_malloc(sizeof(y_line_t));
if (yline) {
yline->next = 0;
yline->line = p_strcpy(line);
if (y_ltail) {
y_ltail->next = yline;
y_ltail = yline;
} else {
y_lhead = y_ltail = yline;
}
}
}
char *p_strdup_vprintf(pool_t pool, const char *format, va_list args)
{
char *tmp, *buf;
unsigned int size;
tmp = t_noalloc_strdup_vprintf(format, args, &size);
if (pool->datastack_pool) {
t_buffer_alloc(size);
return tmp;
} else {
buf = p_malloc(pool, size);
memcpy(buf, tmp, size - 1);
return buf;
}
}
p_file *
p_popen(const char *command, const char *mode)
{
/* WARNING- for non-console programs, returned FILE* causes
* program to hang forever (according to msdn documentation) */
FILE *fp = _popen(command, mode);
p_file *f = fp? p_malloc(sizeof(p_file)) : 0;
if (f) {
f->fp = fp;
f->fd = fileno(fp);
for (; mode[0] ; mode++) if (mode[0]=='b') break;
f->binary = (mode[0]=='b') | 2;
}
return f;
}
void http_client_request_set_payload_data(struct http_client_request *req,
const unsigned char *data, size_t size)
{
struct istream *input;
unsigned char *payload_data;
if (size == 0)
return;
payload_data = p_malloc(req->pool, size);
memcpy(payload_data, data, size);
input = i_stream_create_from_data(payload_data, size);
http_client_request_set_payload(req, input, FALSE);
i_stream_unref(&input);
}
psckt_t *
psckt_listen(int *pport, void *ctx, psckt_cb_t *callback)
{
int lfd;
int port = *pport;
psckt_t *listener = p_malloc(sizeof(psckt_t));
struct addrinfo *ai, *ailist = listener? psckt_get_ailist(0, port, 1) : 0;
if (!ailist) {
if (listener) p_free(listener);
return 0;
}
for (ai=ailist ; ai ; ai=ai->ai_next) {
lfd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol);
if (lfd == -1) continue;
if (!bind(lfd, ai->ai_addr, ai->ai_addrlen)) break;
close(lfd);
}
if (ailist) freeaddrinfo(ailist);
if (lfd < 0) {
p_free(listener);
return 0;
}
if (port == 0) {
struct sockaddr_storage sad;
socklen_t lsad = sizeof(struct sockaddr_storage);
struct sockaddr *psad = (struct sockaddr *)&sad;
char sport[NI_MAXSERV];
if (getsockname(lfd, psad, &lsad) ||
getnameinfo(psad, lsad, 0, 0, sport, NI_MAXSERV, NI_NUMERICSERV)) {
close(lfd);
p_free(listener);
return 0;
}
pport[0] = port = strtod(sport, 0);
}
if (listen(lfd, 10)) {
close(lfd);
p_free(listener);
return 0;
}
listener->fd = lfd;
listener->peer = 0; /* listener has no peer, could usefully be port */
return psckt_setup(listener, ctx, callback);
}
static int *NewReg(long iMax, long ijMax)
{
if (tmpReg) FreeTmpReg();
tmpReg= (int *)p_malloc(sizeof(int)*(ijMax+iMax+1));
if (tmpReg) {
long i, j=0;
for (i=0 ; i<ijMax+iMax+1 ; i++) {
if (i<1 || i>=iMax || j<1) tmpReg[i]= 0;
else tmpReg[i]= 1;
j++;
if (j==iMax) j= 0;
}
} else {
MMError();
}
return tmpReg;
}
char *p_strndup(pool_t pool, const void *str, size_t max_chars)
{
char *mem;
size_t len;
i_assert(str != NULL);
i_assert(max_chars != (size_t)-1);
len = 0;
while (len < max_chars && ((const char *) str)[len] != '\0')
len++;
mem = p_malloc(pool, len+1);
memcpy(mem, str, len);
mem[len] = '\0';
return mem;
}
P_LIB_API pchar *
p_strdup (const pchar *str)
{
pchar *ret;
psize len;
if (P_UNLIKELY (str == NULL))
return NULL;
len = strlen (str) + 1;
if (P_UNLIKELY ((ret = p_malloc (len)) == NULL))
return NULL;
memcpy (ret, str, len);
return ret;
}
static struct imap_match_glob *
imap_match_init_multiple_real(pool_t pool, const char *const *patterns,
bool inboxcase, char separator)
{
struct imap_match_glob *glob;
struct imap_match_pattern *match_patterns;
unsigned int i, len, pos, patterns_count, patterns_data_len = 0;
patterns_count = str_array_length(patterns);
match_patterns = p_new(pool, struct imap_match_pattern,
patterns_count + 1);
/* compress the patterns */
for (i = 0; i < patterns_count; i++) {
match_patterns[i].pattern = pattern_compress(patterns[i]);
match_patterns[i].inboxcase = inboxcase &&
pattern_is_inboxcase(match_patterns[i].pattern,
separator);
patterns_data_len += strlen(match_patterns[i].pattern) + 1;
}
patterns_count = i;
/* now we know how much memory we need */
glob = p_malloc(pool, sizeof(struct imap_match_glob) +
patterns_data_len);
glob->pool = pool;
glob->sep = separator;
/* copy pattern strings to our allocated memory */
for (i = 0, pos = 0; i < patterns_count; i++) {
len = strlen(match_patterns[i].pattern) + 1;
i_assert(pos + len <= patterns_data_len);
/* @UNSAFE */
memcpy(glob->patterns_data + pos,
match_patterns[i].pattern, len);
match_patterns[i].pattern = glob->patterns_data + pos;
pos += len;
}
glob->patterns = match_patterns;
return glob;
}
static bool _file_memory_load(struct sieve_binary_file *file)
{
struct _file_memory *fmem = (struct _file_memory *) file;
int ret;
size_t size;
void *indata;
i_assert(file->fd > 0);
/* Allocate memory buffer
*/
indata = p_malloc(file->pool, file->st.st_size);
size = file->st.st_size;
file->offset = 0;
fmem->memory = indata;
fmem->memory_size = file->st.st_size;
/* Return to beginning of the file */
if ( lseek(file->fd, 0, SEEK_SET) == (off_t) -1 ) {
sieve_sys_error("failed to seek() in binary %s: %m", file->path);
return FALSE;
}
/* Read the whole file into memory */
while (size > 0) {
if ( (ret=read(file->fd, indata, size)) <= 0 ) {
sieve_sys_error("failed to read from binary %s: %m", file->path);
break;
}
indata = PTR_OFFSET(indata, ret);
size -= ret;
}
if ( size != 0 ) {
/* Failed to read the whole file */
return FALSE;
}
return TRUE;
}
p_file *
p_fopen(const char *unix_name, const char *mode)
{
FILE *fp = fopen(w_pathname(unix_name), mode);
p_file *f = fp? p_malloc(sizeof(p_file)) : 0;
if (f) {
f->fp = fp;
f->fd = _fileno(fp);
for (; mode[0] ; mode++) if (mode[0]=='b') break;
f->binary = (mode[0]=='b');
/* problem:
* in _O_TEXT mode, ftell only works properly if the file
* really is a DOS file with CRLF newlines; if it is a UNIX
* file (like all of the distribution .i files), ftell fails
* on the other hand, using _O_BINARY makes fgets fail to remove
* the CR and fputs not insert CR
* (may depend on Windows flavor, this is Win2k */
if (!f->binary) _setmode(f->fd, _O_BINARY);
}
return f;
}
static int SaveName(int device, char *name)
{
int len;
if (name == outNames[device]) return 0;
len= name? (int)strlen(name) : 0;
if (len>outLengths[device]) {
if (outLengths[device]>0) p_free(outNames[device]);
outNames[device]= (char *)p_malloc(len+1);
if (!outNames[device]) {
p_stderr("gist: (SEVERE) memory manager failed in SaveName\n");
outLengths[device]= 0;
return 1;
}
outLengths[device]= len;
}
if (name) strcpy(outNames[device], name);
else if (outLengths[device]>0) outNames[device][0]= '\0';
else outNames[device]= 0;
return 0;
}
char *i_strconcat(const char *str1, ...)
{
va_list args;
char *ret;
size_t len;
va_start(args, str1);
T_BEGIN {
const char *temp = vstrconcat(str1, args, &len);
if (temp == NULL)
ret = NULL;
else {
t_buffer_alloc(len);
ret = p_malloc(default_pool, len);
memcpy(ret, temp, len);
}
} T_END;
va_end(args);
return ret;
}
static int
hash_format_string_analyze(struct hash_format *format, const char *str,
const char **error_r)
{
struct hash_format_list *list;
unsigned int i;
for (i = 0; str[i] != '\0'; i++) {
if (str[i] != '%')
continue;
i++;
list = p_new(format->pool, struct hash_format_list, 1);
list->encoding = HASH_ENCODING_HEX;
*format->pos = list;
format->pos = &list->next;
if (str[i] == 'B') {
list->encoding = HASH_ENCODING_BASE64;
i++;
} else if (str[i] == 'X') {
list->encoding = HASH_ENCODING_HEX_SHORT;
i++;
}
if (str[i++] != '{') {
*error_r = "No '{' after '%'";
return -1;
}
if (hash_format_parse(str, &i, &list->method,
&list->bits, error_r) < 0)
return -1;
list->context = p_malloc(format->pool,
list->method->context_size);
list->method->init(list->context);
}
return 0;
}
psckt_t *
psckt_accept(psckt_t *listener, char **ppeer, void *ctx, psckt_cb_t *callback)
{
struct sockaddr_storage sad;
socklen_t lsad = sizeof(struct sockaddr_storage);
struct sockaddr *psad = (struct sockaddr *)&sad;
char host[NI_MAXHOST];
psckt_t *sock = p_malloc(sizeof(psckt_t));
SOCKET sfd, lfd = sock->fd;
*ppeer = 0;
if (listener->peer || !sock) { /* listeners have no peer */
if (sock) p_free(sock);
return 0;
}
if (listener->callback && listener->waiting) {
if (WaitForSingleObject(listener->ready, INFINITE) != WAIT_OBJECT_0)
psckt_stop_thread(listener);
listener->waiting = 0;
if (listener->ready != INVALID_HANDLE_VALUE)
ResetEvent(listener->ready);
}
sfd = accept(listener->fd, psad, &lsad);
if (sfd==INVALID_SOCKET ||
getnameinfo(psad, lsad, host, NI_MAXHOST, 0, 0, 0)) {
closesocket(sfd);
p_free(sock);
return 0;
}
sock->fd = sfd;
sock->peer = *ppeer = p_strcpy(host);
return psckt_setup(sock, ctx, callback);
}
void hash_format_deinit(struct hash_format **_format, string_t *dest)
{
struct hash_format *format = *_format;
struct hash_format_list *list;
const char *p;
unsigned char *digest;
unsigned int i, max_digest_size = 0;
*_format = NULL;
for (list = format->list; list != NULL; list = list->next) {
if (max_digest_size < list->method->digest_size)
max_digest_size = list->method->digest_size;
}
digest = p_malloc(format->pool, max_digest_size);
list = format->list;
for (i = 0; format->str[i] != '\0'; i++) {
if (format->str[i] != '%') {
str_append_c(dest, format->str[i]);
continue;
}
/* we already verified that the string is ok */
i_assert(list != NULL);
list->method->result(list->context, digest);
hash_format_digest(dest, list, digest);
list = list->next;
p = strchr(format->str+i, '}');
i_assert(p != NULL);
i = p - format->str;
}
pool_unref(&format->pool);
}
void Y_ml4read(int nArgs)
{
char *filename="";
char *varname="";
int leave_open = 0;
if (nArgs==2) {
filename=YGetString(sp-nArgs+1);
varname=YGetString(sp-nArgs+2);
leave_open = 0;
} else if (nArgs==3) {
filename=YGetString(sp-nArgs+1);
varname=YGetString(sp-nArgs+2);
leave_open=YGetInteger(sp-nArgs+3);
}
unsigned long bytes_read;
int type,namelen;
unsigned long nElements,nBytesToRead;
int mrows,mcols,imagf;
FILE *fs;
int fileptr;
int endian = 'L';
int size=0,i;
fs = openmat(filename);
if (fs == NULL) YError(p_strncat("Can't open file ",filename,0));
if (!matfind(fs,varname,50000)) YError(p_strncat("No Such variable ",varname,0));
fileptr = ftell(fs);
if (DEBUG) printf("@ position %d\n",fileptr);
bytes_read = fread(&type,sizeof(int),1,fs);
if (bytes_read==0) {
matclose(filename);
YError("Premature end of file");; // end of file
}
fread(&mrows,sizeof(int),1,fs);
fread(&mcols,sizeof(int),1,fs);
fread(&imagf,sizeof(int),1,fs);
fread(&namelen,sizeof(int),1,fs);
if (namelen & 0xffff0000) {
if (DEBUG) printf("Big endian file\n");
endian = 'B';
SWAP_INT(type);
SWAP_INT(mrows);
SWAP_INT(mcols);
SWAP_INT(imagf);
SWAP_INT(namelen);
}
type = type%1000;
if (DEBUG) printf("rows,cols,namelen= %d %d %d\n",mrows,mcols,namelen);
if (namelen>255) {
fseek(fs,fileptr,SEEK_SET); // leave file ptr at begginning of this variable
matclose(filename);
YError("Variable name too long!");
}
fread(tempvarname,(unsigned int)namelen,1,fs);
// if ((*varname!='*') && strcmp(varname,tempvarname)) { // error if not same varname
if (!matchvarname(tempvarname,varname)) { // error if not same varname
fseek(fs,fileptr,SEEK_SET); // leave file ptr at begginning of this variable
matclose(filename);
YError(p_strncat("Can't find variable",varname,0));
}
nElements = (unsigned)mrows*(unsigned)mcols;
Dimension *tmp=tmpDims;
tmpDims=0;
FreeDimension(tmp);
if (mrows<=1) {
tmpDims= NewDimension(mcols, 1L, (Dimension *)0);
} else if (mcols<=1) {
tmpDims= NewDimension(mrows, 1L, (Dimension *)0);
} else {
tmpDims= NewDimension(mrows, 1L, (Dimension *)0);
tmpDims= NewDimension(mcols, 1L, tmpDims);
}
if (type==0) {
// 8-byte doubles
size = 8;
Array *a= PushDataBlock(NewArray(&doubleStruct, tmpDims));
double *data = a->value.d;
bytes_read = fread((void *)data,size,nElements,fs);
if (endian=='B') { for (i=0;i<nElements;i++) SWAP_DOUBLE(data[i]); }
} else if (type==10) {
// 4-byte reals
size = 4;
Array *a= PushDataBlock(NewArray(&floatStruct, tmpDims));
float *data = a->value.f;
bytes_read = fread((void *)data,size,nElements,fs);
if (endian=='B') { for (i=0;i<nElements;i++) SWAP_FLOAT(data[i]); }
} else if ((type==120) || (type==20)) {
// 4-byte int
size = 4;
Array *a= PushDataBlock(NewArray(&intStruct, tmpDims));
int *data = a->value.l;
bytes_read = fread((void *)data,size,nElements,fs);
if (endian=='B') { for (i=0;i<nElements;i++) SWAP_INT(data[i]); }
} else if (type==30) {
// 2-byte signed (30) shorts
size = 2;
Array *a= PushDataBlock(NewArray(&shortStruct, tmpDims));
short *data = a->value.s;
bytes_read = fread((void *)data,size,nElements,fs);
if (endian=='B') { for (i=0;i<nElements;i++) SWAP_SHORT(data[i]); }
} else if (type==40) {
// 2-byte unsigned (40) shorts
size = 2;
Array *a= PushDataBlock(NewArray(&shortStruct, tmpDims));
short *data = a->value.s;
Array *b= PushDataBlock(NewArray(&longStruct, tmpDims));
long *data2 = b->value.l;
bytes_read = fread((void *)data,size,nElements,fs);
if (endian=='B') { for (i=0;i<nElements;i++) SWAP_SHORT(data[i]); }
for (i=1;i<=nElements;i++) *(data2++) = (((long)*(data++))|0xFFFF0000)+65535;
} else if (type==50) {
// 1-byte signed or unsigned chars (50)
size = 1;
Array *a= PushDataBlock(NewArray(&charStruct, tmpDims));
char *data = a->value.c;
bytes_read = fread((void *)data,size,nElements,fs);
} else if (type==51) {
// text (51)
size = 1;
Array *a= PushDataBlock(NewArray(&stringStruct, (Dimension *)0));
char *buf;
a->value.q[0] = buf = p_malloc(nElements+1);
if (DEBUG) printf("strlen: %d\n",(int)strlen((void *)a->value.q[0]));
// bytes_read = fread(a->value.q[0],1,nElements,fs);
bytes_read = fread(buf,1,nElements,fs);
*((char *)buf + nElements) = 0; // append a NULL to text string
} else {
matclose(filename);
sprintf(message,"Unknown type %d",type);
YError(message);
}
if (bytes_read!=nElements) {
fseek(fs,nElements*size,SEEK_CUR);
matclose(filename);
if (DEBUG) printf("read:%ld expected:%ld\n",bytes_read,nBytesToRead);
YError("Premature end of file");
}
if (!leave_open) matclose(filename);
}
void
Y_spawn(int nargs)
{
Dimension *dims = 0;
char **argv = yarg_q(nargs-1, &dims);
long argc = 1;
Operand op;
Operand *pop = yarg_op(nargs-2, &op);
long callout=-1, callerr=-1;
spawn_proc *proc;
if (nargs<2 || nargs>3)
YError("spawn: accepts precisely two or three arguments");
if (dims) {
if (dims->next)
YError("spawn: first argument must be string or 1D array of strings");
argc = dims->number;
}
if (!argv || !argv[0] || !argv[0][0])
YError("spawn: first element of first argument must be process name");
if (!pop || (op.ops!=&functionOps && op.ops!=&builtinOps))
YError("spawn: second argument must be callback function");
if (op.ops==&builtinOps) callout = ((BIFunction *)op.value)->index;
else callout = ((Function *)op.value)->code[0].index;
if (nargs == 3) {
pop = yarg_op(0, &op);
if (!pop || (op.ops!=&functionOps && op.ops!=&builtinOps))
YError("spawn: third argument must be callback function");
if (op.ops==&builtinOps) callerr = ((BIFunction *)op.value)->index;
else callerr = ((Function *)op.value)->code[0].index;
}
if (argv[argc-1]) {
/* must construct a 0-terminated argv list */
typedef struct tmpobj {
int references;
Operations *ops;
void (*zapper)(void *to);
char *argv[1];
} tmpobj;
tmpobj *tmp;
long i;
/* CheckStack(2); fnctn.c guarantees at least 2 free stack slots */
tmp = PushDataBlock(y_new_tmpobj(sizeof(tmpobj)+argc*sizeof(char*),
p_free));
for (i=0 ; i<argc ; i++) tmp->argv[i] = argv[i];
tmp->argv[i] = 0;
argv = tmp->argv;
/* tmp will be cleaned up when stack cleared */
}
/* push result object onto stack */
proc = p_malloc(sizeof(spawn_proc));
proc->references = 0;
proc->ops = &spawn_ops;
proc->proc = p_spawn(argv[0], argv, spawn_callback, proc, callerr>=0);
proc->argv0 = p_strcpy(argv[0]);
proc->callout = callout;
proc->callerr = callerr;
proc->next = spawn_list;
spawn_list = proc;
PushDataBlock(proc);
if (!proc->proc) {
Drop(1);
PushDataBlock(RefNC(&nilDB));
}
if (!spawn_setclean) {
spawn_setclean = 1;
spawn_prevclean = CleanUpForExit;
CleanUpForExit = spawn_cleanup;
}
}
/* do interpreted callback to process message from process */
static void
spawn_callback(void *vproc, int err)
{
spawn_proc *proc = vproc;
char *msg = 0;
Array *msga;
Instruction code[12];
Symbol *ctable;
long callback = -1;
if (err != 2) {
long nbytes = 0;
if (proc && proc->proc) callback = err? proc->callerr : proc->callout;
if (callback<0) {
/* probably a bug, but unclear that calling YError
* would prevent the possible fault loop
*/
return;
}
/* read the message from process
* - this just reads all available bytes, up to 2048
*/
msg = p_malloc(2048);
nbytes = p_recv(proc->proc, msg, 2048);
if (nbytes <= 0) {
p_free(msg);
/* can get this when an unrelated process finishes, just ignore */
/* YError("spawn process read error in callback"); */
/* also could get this if the fd gave POLLERR, possibly should
* find some way to figure out what's going on
*/
return;
}
msg = p_realloc(msg, nbytes+1);
msg[nbytes] ='\0';
} else {
if (!proc || proc->callout<0) return;
callback = proc->callout;
proc->callout = -1;
p_spawf(proc->proc, 1);
proc->proc = 0;
}
if (callback < 0) return;
/* task constant table contains only message string */
msga = NewArray(&stringStruct, (Dimension *)0);
msga->value.q[0] = msg;
ctable = p_malloc(sizeof(Symbol));
ctable->ops = &dataBlockSym;
ctable->value.db = (DataBlock *)msga;
/* fill in function code */
code[0].Action = &PushVariable;
code[1].index = callback;
code[2].Action = &PushString;
code[3].constant = ctable;
code[4].Action = &Eval;
code[5].count = 1;
code[6].Action = &DropTop;
code[7].Action = &PushNil;
code[8].Action = &Return;
code[9].Action = 0;
code[10].index = 11;
/* NewFunction moves this to beginning */
code[11].index = Globalize("*callback*", 0L);
PushTask(NewFunction(ctable, 1, 0, 0, 0, 0, 2, code, 11));
}
//void doExec(char* filename, char* argv, char *en[])
void doExecvp(char *fn, char **argv)
{
PCB *pro;
pro = running;
char filename[20];
char param[10][20];
for(int i=0;i<10; i++)
{
//param[i] = (char*)(k_malloc(sizeof(char)*10));
strcpy(param[i], argv[i]);
//printf("Parameters - %s %s\n", param[i], argv[i]);
}
//filename = (char*)(k_malloc((sizeof(char)*32)));
strcpy(filename, fn);
//printf("We will execute - %s %s\n", filename, fn);
// delete all page table entries
deletePageTables();
_ptcr3(pro->cr3);
char **params;
params = (char**)(p_malloc((sizeof(char*)*10)));
for(int i=0;i<10; i++)
{
params[i] = (char*)(p_malloc(sizeof(char)*20));
strcpy(params[i], param[i]);
//printf("Parameters - %s %s\n", params[i], param[i]);
}
printf("In Exec :: check PID=%p, cr3=%p ppid=%p\n", pro->pid, pro->cr3, pro->ppid);
//char *filename2;
//filename2 = filename;
//read_tarfs(pro, "bin/world");
strcpy(pro->name, filename);
pro->start_time = sec;
read_tarfs(pro, filename);
printf("We will execute - %s\n", filename);
if ((pro->u_stack = process_stack()) == NULL)
{
printf("\n Cant allocate memory for process User stack");
//exit();
}
/*
__asm volatile( "movq %0,%%rdi\n\t"
"movq %1,%%rsi\n\t"
::"r"((uint64_t)(filename)),"r"((uint64_t)(param)):"rdi","rsi","memory"
);*/
//pushf;
printf("We will execute - %s %p\n", filename, param);
pro->u_stack[0] = pro->rip;
pro->rsp = (uint64_t)(pro->u_stack);
printf("user_stack_rsp%p",pro->rsp);
tss.rsp0 = (uint64_t) &(pro->kernel_stack[255]);
printf("In Execvp() GDT SET\n");
__asm volatile("\
push $0x23;\
push %0;\
push $0x200;\
push $0x1B;\
push %1"::"g"(pro->u_stack),"g"(pro->rip):"memory");
__asm volatile( "movq %0,%%rdi\n\t"
::"r"((uint64_t)(params)):"rdi","memory");
__asm volatile("\
iretq;\
");
}
void *YAsyncBytes(long n)
{
void *ptr= p_malloc(n);
return keeplist? ListAppend(ptr) : ptr;
}
static void simple_set(struct ybc *const cache, const size_t requests_count,
const size_t items_count, const size_t max_item_size)
{
struct m_rand_state rand_state;
uint64_t tmp;
char *const buf = p_malloc(max_item_size);
const struct ybc_key key = {
.ptr = &tmp,
.size = sizeof(tmp),
};
struct ybc_value value = {
.ptr = buf,
.size = 0,
.ttl = YBC_MAX_TTL,
};
m_rand_init(&rand_state);
for (size_t i = 0; i < requests_count; ++i) {
tmp = m_rand_next(&rand_state) % items_count;
value.size = m_rand_next(&rand_state) % (max_item_size + 1);
m_memset(buf, (char)value.size, value.size);
if (!ybc_item_set(cache, &key, &value)) {
M_ERROR("Cannot store item in the cache");
}
}
free(buf);
}
static void simple_get_miss(struct ybc *const cache,
const size_t requests_count, const size_t items_count)
{
char item_buf[ybc_item_get_size()];
struct ybc_item *const item = (struct ybc_item *)item_buf;
struct m_rand_state rand_state;
uint64_t tmp;
const struct ybc_key key = {
.ptr = &tmp,
.size = sizeof(tmp),
};
m_rand_init(&rand_state);
for (size_t i = 0; i < requests_count; ++i) {
tmp = m_rand_next(&rand_state) % items_count;
if (ybc_item_get(cache, item, &key)) {
M_ERROR("Unexpected item found");
}
}
}
static void simple_get_hit(struct ybc *const cache,
const size_t requests_count, const size_t items_count,
const size_t max_item_size)
{
char item_buf[ybc_item_get_size()];
struct ybc_item *const item = (struct ybc_item *)item_buf;
struct m_rand_state rand_state;
uint64_t tmp;
const struct ybc_key key = {
.ptr = &tmp,
.size = sizeof(tmp),
};
struct ybc_value value;
m_rand_init(&rand_state);
for (size_t i = 0; i < requests_count; ++i) {
tmp = m_rand_next(&rand_state) % items_count;
if (ybc_item_get(cache, item, &key)) {
/* Emulate access to the item */
ybc_item_get_value(item, &value);
if (value.size > max_item_size) {
M_ERROR("Unexpected value size");
}
if (!m_memset_check(value.ptr, (char)value.size, value.size)) {
fprintf(stderr, "i=%zu, requests_count=%zu, value.size=%zu\n", i, requests_count, value.size);
M_ERROR("Unexpected value");
}
ybc_item_release(item);
}
}
}
static void m_open(struct ybc *const cache, const size_t items_count,
const size_t hot_items_count, const size_t max_item_size)
{
char config_buf[ybc_config_get_size()];
struct ybc_config *const config = (struct ybc_config *)config_buf;
const size_t data_file_size = max_item_size * items_count;
const size_t hot_data_size = max_item_size * hot_items_count;
ybc_config_init(config);
ybc_config_set_max_items_count(config, items_count);
ybc_config_set_hot_items_count(config, hot_items_count);
ybc_config_set_data_file_size(config, data_file_size);
ybc_config_set_hot_data_size(config, hot_data_size);
if (!ybc_open(cache, config, 1)) {
M_ERROR("Cannot create a cache");
}
ybc_config_destroy(config);
}
static void measure_simple_ops(struct ybc *const cache,
const size_t requests_count, const size_t items_count,
const size_t hot_items_count, const size_t max_item_size)
{
double start_time, end_time;
double qps;
m_open(cache, items_count, hot_items_count, max_item_size);
printf("simple_ops(requests=%zu, items=%zu, "
"hot_items=%zu, max_item_size=%zu)\n",
requests_count, items_count, hot_items_count, max_item_size);
start_time = p_get_current_time();
simple_get_miss(cache, requests_count, items_count);
end_time = p_get_current_time();
qps = requests_count / (end_time - start_time) * 1000;
printf(" get_miss: %.02f qps\n", qps);
start_time = p_get_current_time();
simple_set(cache, requests_count, items_count, max_item_size);
end_time = p_get_current_time();
qps = requests_count / (end_time - start_time) * 1000;
printf(" set : %.02f qps\n", qps);
const size_t get_items_count = hot_items_count ? hot_items_count :
items_count;
start_time = p_get_current_time();
simple_get_hit(cache, requests_count, get_items_count, max_item_size);
end_time = p_get_current_time();
qps = requests_count / (end_time - start_time) * 1000;
printf(" get_hit : %.02f qps\n", qps);
ybc_close(cache);
}
void matscan(FILE *fs, int maxVarsToSearch, int returnString)
{
int info[5];
long i;
long fileptr,tfileptr,tfp;
long nbyt=0,nelem,skip;
int type;
int mrows,mcols;
int imagf;
int namelen;
long varNumber = 0;
char varname[80];
char *stype="";
int varnum=0;
Array *a= PushDataBlock(NewArray(&stringStruct, (Dimension *)0));
long extra=1;
fileptr = ftell(fs);
if (DEBUG) printf("Entering matscan\n");
while (1) {
tfileptr = ftell(fs);
if (DEBUG) printf("at address %ld \n",tfileptr);
if (fread(info,4,5,fs)==5) {
if (info[4] & 0xffff0000) { // convert header from little endian to big indian
// info[0] changed to info[4] 2006/3/15 as double type can be 0, hence
// no way to know big from little endian info[0] for doubles.
if (DEBUG) printf("swapping!\n");
for (i=0;i<5;i++) SWAP_INT(info[i]);
}
info[0] = info[0]%1000;
tfp = ftell(fs);
if (DEBUG) printf("at address %ld \n",tfp);
if (DEBUG) printf("info = %d %d %d %d %d\n",info[0],info[1],info[2],info[3],info[4]);
type = info[0]%1000;
if ((namelen = info[4])<80L) {
if (fread(varname,1,info[4],fs)==(int)info[4]) {
if (type==0) {
// 8-byte doubles
stype=p_strcpy("double*8"); nbyt=8;
} else if (type==10) {
// 4-byte reals
stype=p_strcpy("real*4 "); nbyt=4;
} else if ((type==120) || (type==20)) {
// 4-byte int
stype=p_strcpy("int*4 "); nbyt=4;
} else if (type==30) {
// 2-byte signed (30) shorts
stype=p_strcpy("short*2 "); nbyt=2;
} else if (type==40) {
// 2-byte unsigned (40) shorts
stype=p_strcpy("ushort*2"); nbyt=2;
} else if ((type==50) || (type==51)) {
// 1-byte signed or unsigned chars (50) or text (51)
stype=p_strcpy("char*1 "); nbyt=1;
} else {
sprintf(message,"Unknown data type %d",type);
YError(message);
}
if (returnString) {
if (varnum!=0) a= PushDataBlock((void *)GrowArray(a, extra));
a->value.q[varnum] = p_malloc(81);
sprintf(a->value.q[varnum],"%30s %s array [%d,%d]",varname, \
stype,info[1],info[2]);
varnum++;
} else {
printf("%30s %s array [%d,%d]\n",varname,stype,info[1],info[2]);
}
mrows=info[1];
mcols=info[2];
nelem=mrows*mcols;
imagf=info[3];
if (imagf) nbyt=2*nbyt;
skip = nbyt*nelem;
if (DEBUG) printf("skiping data part: %ld bytes\n",skip);
if (skip) fseek(fs,nbyt*nelem,SEEK_CUR);
}
}
} else {
break;
}
if (maxVarsToSearch) {
if (++varNumber >= maxVarsToSearch) {
break;
}
}
}
}
Engine *GpNewEngine(long size, char *name, g_callbacks *on,
GpTransform *transform, int landscape,
void (*Kill)(Engine*), int (*Clear)(Engine*,int), int (*Flush)(Engine*),
void (*ChangeMap)(Engine*), int (*ChangePalette)(Engine*),
int (*DrawLines)(Engine*,long,const GpReal*,const GpReal*,int,int),
int (*DrawMarkers)(Engine*,long,const GpReal*,const GpReal *),
int (*DrwText)(Engine*e,GpReal,GpReal,const char*),
int (*DrawFill)(Engine*,long,const GpReal*,const GpReal*),
int (*DrawCells)(Engine*,GpReal,GpReal,GpReal,GpReal,
long,long,long,const GpColor*),
int (*DrawDisjoint)(Engine*,long,const GpReal*,const GpReal*,
const GpReal*,const GpReal*))
{
long lname= name? strlen(name) : 0;
Engine *engine;
/* For Electric Fence package and maybe others, it is nice to ensure
that size of block allocated for Engine is a multiple of the size
of the most restrictively aligned object which can be in any
Engine; assume this is a double. */
lname= (lname/sizeof(double) + 1)*sizeof(double); /* >= lname+1 */
engine= (Engine *)p_malloc(size+lname);
if (!engine) return 0;
/* Fill in Engine properties, link into gistEngines list */
engine->next= gistEngines;
gistEngines= engine;
engine->nextActive= 0;
engine->name= (char *)engine + size;
strcpy(name? engine->name : "", name);
engine->on = on;
engine->active= 0;
engine->marked= 0;
engine->transform= *transform;
engine->landscape= landscape? 1 : 0;
GpDeviceMap(engine);
/* (a proper map will be installed when the engine is activated) */
engine->map.x.scale= engine->map.y.scale= 1.0;
engine->map.x.offset= engine->map.y.offset= 0.0;
/* No pseudocolor map initially */
engine->colorChange= 0;
engine->colorMode= 0;
engine->nColors= 0;
engine->palette= 0;
/* No associated drawing initially */
engine->drawing= 0;
engine->lastDrawn= -1;
engine->systemsSeen[0]= engine->systemsSeen[1]= 0;
engine->inhibit= 0;
engine->damaged= 0; /* causes Clear if no ClearArea virtual function */
engine->damage.xmin= engine->damage.xmax=
engine->damage.ymin= engine->damage.ymax= 0.0;
/* Fill in virtual function table */
engine->Kill= Kill;
engine->Clear= Clear;
engine->Flush= Flush;
engine->ChangeMap= ChangeMap;
engine->ChangePalette= ChangePalette;
engine->DrawLines= DrawLines;
engine->DrawMarkers= DrawMarkers;
engine->DrwText= DrwText;
engine->DrawFill= DrawFill;
engine->DrawCells= DrawCells;
engine->DrawDisjoint= DrawDisjoint;
engine->ClearArea= &DefaultClearArea; /* damage causes complete redraw */
return engine;
}
bool auth_request_handler_auth_begin(struct auth_request_handler *handler,
const char *args)
{
const struct mech_module *mech;
struct auth_request *request;
const char *const *list, *name, *arg, *initial_resp;
void *initial_resp_data;
unsigned int id;
buffer_t *buf;
i_assert(!handler->destroyed);
/* <id> <mechanism> [...] */
list = t_strsplit_tab(args);
if (list[0] == NULL || list[1] == NULL ||
str_to_uint(list[0], &id) < 0) {
i_error("BUG: Authentication client %u "
"sent broken AUTH request", handler->client_pid);
return FALSE;
}
if (handler->token_auth) {
mech = &mech_dovecot_token;
if (strcmp(list[1], mech->mech_name) != 0) {
/* unsupported mechanism */
i_error("BUG: Authentication client %u requested invalid "
"authentication mechanism %s (DOVECOT-TOKEN required)",
handler->client_pid, str_sanitize(list[1], MAX_MECH_NAME_LEN));
return FALSE;
}
} else {
mech = mech_module_find(list[1]);
if (mech == NULL) {
/* unsupported mechanism */
i_error("BUG: Authentication client %u requested unsupported "
"authentication mechanism %s", handler->client_pid,
str_sanitize(list[1], MAX_MECH_NAME_LEN));
return FALSE;
}
}
request = auth_request_new(mech);
request->handler = handler;
request->connect_uid = handler->connect_uid;
request->client_pid = handler->client_pid;
request->id = id;
request->auth_only = handler->master_callback == NULL;
/* parse optional parameters */
initial_resp = NULL;
for (list += 2; *list != NULL; list++) {
arg = strchr(*list, '=');
if (arg == NULL) {
name = *list;
arg = "";
} else {
name = t_strdup_until(*list, arg);
arg++;
}
if (auth_request_import_auth(request, name, arg))
;
else if (strcmp(name, "resp") == 0) {
initial_resp = arg;
/* this must be the last parameter */
list++;
break;
}
}
if (*list != NULL) {
i_error("BUG: Authentication client %u "
"sent AUTH parameters after 'resp'",
handler->client_pid);
auth_request_unref(&request);
return FALSE;
}
if (request->service == NULL) {
i_error("BUG: Authentication client %u "
"didn't specify service in request",
handler->client_pid);
auth_request_unref(&request);
return FALSE;
}
if (hash_table_lookup(handler->requests, POINTER_CAST(id)) != NULL) {
i_error("BUG: Authentication client %u "
"sent a duplicate ID %u", handler->client_pid, id);
auth_request_unref(&request);
return FALSE;
}
auth_request_init(request);
request->to_abort = timeout_add(MASTER_AUTH_SERVER_TIMEOUT_SECS * 1000,
auth_request_timeout, request);
hash_table_insert(handler->requests, POINTER_CAST(id), request);
if (request->set->ssl_require_client_cert &&
!request->valid_client_cert) {
/* we fail without valid certificate */
auth_request_handler_auth_fail(handler, request,
"Client didn't present valid SSL certificate");
return TRUE;
}
/* Empty initial response is a "=" base64 string. Completely empty
string shouldn't really be sent, but at least Exim does it,
so just allow it for backwards compatibility.. */
if (initial_resp != NULL && *initial_resp != '\0') {
size_t len = strlen(initial_resp);
buf = buffer_create_dynamic(pool_datastack_create(),
MAX_BASE64_DECODED_SIZE(len));
if (base64_decode(initial_resp, len, NULL, buf) < 0) {
auth_request_handler_auth_fail(handler, request,
"Invalid base64 data in initial response");
return TRUE;
}
initial_resp_data =
p_malloc(request->pool, I_MAX(buf->used, 1));
memcpy(initial_resp_data, buf->data, buf->used);
request->initial_response = initial_resp_data;
request->initial_response_len = buf->used;
}
/* handler is referenced until auth_request_handler_reply()
is called. */
handler->refcount++;
/* before we start authenticating, see if we need to wait first */
auth_penalty_lookup(auth_penalty, request, auth_penalty_callback);
return TRUE;
}
void Y_edit_times(int nArgs)
{
Symbol *stack= sp-nArgs+1;
IOStream *file;
HistoryInfo *history;
int *ifiles;
long *offsets;
Dimension *dims;
long nKeep, *keepList= 0;
double *newTimes= 0, *times, bad= 0.0;
long *newNcycs= 0, *ncycs, nbad= 0;
long i, nRecs, j;
if (nArgs<1) YError("edit_times needs at least one argument");
file= YGetFile(stack);
history= file->history;
if (!history)
YError("binary file in edit_times has no history records");
ifiles= history->ifile;
offsets= history->offset;
times= history->time;
ncycs= history->ncyc;
nRecs= history->nRecords;
/* collect keepList, newTimes, and newNcycs arguments */
nKeep= 0;
if (nArgs>1) {
long lastKept= -1;
stack++;
keepList= YGet_L(stack, 1, &dims);
if (keepList) nKeep= TotalNumber(dims);
else nKeep= 0;
for (i=0 ; i<nKeep ; i++) {
if (keepList[i]<0 || keepList[i]>=nRecs || keepList[i]<=lastKept)
YError("keep_list out of range or not increasing in edit_times");
lastKept= keepList[i];
}
if (nArgs>2) {
stack++;
newTimes= YGet_D(stack, 1, &dims);
i= TotalNumber(dims);
if (newTimes && (nKeep? (i!=nKeep) : (i!=nRecs)))
YError("new_times has wrong length in edit_times");
if (nArgs>3) {
stack++;
newNcycs= YGet_L(stack, 1, &dims);
i= TotalNumber(dims);
if (newNcycs && (nKeep? (i!=nKeep) : (i!=nRecs)))
YError("new_ncycs has wrong length in edit_times");
}
}
}
/* find appropriate bad value to mark records to be deleted */
if (times) {
bad= times[0];
for (i=1 ; i<nRecs ; i++) if (times[i]>bad) bad= times[i];
if (bad>0.5) bad*= 1.001;
else if (bad<-0.5) bad*= 0.999;
else bad= 1.0;
} else if (ncycs) {
nbad= ncycs[0];
for (i=1 ; i<nRecs ; i++) if (ncycs[i]>nbad) nbad= ncycs[i];
nbad++;
}
/* mark the records to be deleted (if any) */
if (!keepList) nKeep= nRecs;
if (!keepList && !newTimes && !newNcycs) {
/* filter to record lists to force strictly increasing times or ncycs */
if (times) {
double downTo;
downTo= times[nRecs-1];
for (i=nRecs-2 ; i>=0 ; i--) {
if (times[i]<downTo) downTo= times[i];
else { times[i]= bad; nKeep--; } /* mark records to be deleted */
}
} else if (ncycs) {
long downTo;
downTo= ncycs[nRecs-1];
for (i=nRecs-2 ; i>=0 ; i--) {
if (ncycs[i]<downTo) downTo= ncycs[i];
else { ncycs[i]= nbad; nKeep--; } /* mark records to be deleted */
}
}
}
/* delete requested records */
if (nKeep && nKeep<nRecs) {
if (keepList) {
for (i=0 ; i<nKeep ; i++) {
j= keepList[i];
ifiles[i]= ifiles[j];
offsets[i]= offsets[j];
if (times) times[i]= times[j];
if (ncycs) ncycs[i]= ncycs[j];
}
history->nRecords= nRecs= nKeep;
} else if (times) {
for (i=j=0 ; i<nRecs ; i++) {
if (times[i]!=bad) {
ifiles[j]= ifiles[i];
offsets[j]= offsets[i];
if (ncycs) ncycs[j]= ncycs[i];
times[j++]= times[i];
}
}
history->nRecords= nRecs= j;
} else if (ncycs) {
for (i=j=0 ; i<nRecs ; i++) {
if (ncycs[i]!=nbad) {
ifiles[j]= ifiles[i];
offsets[j]= offsets[i];
ncycs[j++]= ncycs[i];
}
}
history->nRecords= nRecs= j;
}
}
/* install new times */
if (newTimes && nRecs>0) {
if (!times) times = history->time =
p_malloc(sizeof(double)*(16+((nRecs-1)&~0xf)));
for (i=0 ; i<nRecs ; i++) times[i]= newTimes[i];
}
/* install new ncycs */
if (newNcycs && nRecs>0) {
if (!ncycs) ncycs = history->ncyc =
p_malloc(sizeof(long)*(16+((nRecs-1)&~0xf)));
for (i=0 ; i<nRecs ; i++) ncycs[i]= newNcycs[i];
}
}
void
p_text(p_win *w, int x0, int y0, const char *text, int n)
{
HDC dc = w_getdc(w, 1);
if (dc) {
int i;
if (n <= 0) n = 16350;
for (i=0 ; i<n ; i++) if (!text[i]) break;
n = i;
if (!w->orient) {
TextOut(dc, x0, y0, text, n);
} else {
/* only do this with opaque background,
* since only 8x8 patterned brushes supported under Win95
* (could read screen, rotate, TextOut, rotate back) */
TEXTMETRIC tm;
SIZE sz;
if (n>0 && GetTextMetrics(dc, &tm) &&
GetTextExtentPoint32(dc, text, n, &sz) &&
sz.cx>0) {
int width = sz.cx;
int height = tm.tmHeight + 1;
int base = tm.tmAscent;
int scandx = ((width-1)/(8*sizeof(DWORD)) + 1)*(8*sizeof(DWORD));
int scandy = ((height-1)/(8*sizeof(DWORD)) + 1)*(8*sizeof(DWORD));
HDC offdc = CreateCompatibleDC(dc);
HBITMAP offbm = CreateCompatibleBitmap(dc, scandx, scandy);
HFONT font = GetCurrentObject(dc, OBJ_FONT);
void *monodat = p_malloc(scandx*scandy/4);
void *monorot = (char *)monodat + (scandx*scandy/8);
struct {
BITMAPINFOHEADER bmiHeader;
DWORD bmiColors[2];
} mono;
mono.bmiHeader.biSize = sizeof(BITMAPINFOHEADER);
mono.bmiHeader.biWidth = scandx;
mono.bmiHeader.biHeight = -scandy;
mono.bmiHeader.biPlanes = 1;
mono.bmiHeader.biBitCount = 1;
mono.bmiHeader.biCompression = BI_RGB;
mono.bmiHeader.biSizeImage = 0;
mono.bmiHeader.biXPelsPerMeter = 0;
mono.bmiHeader.biYPelsPerMeter = 0;
mono.bmiHeader.biClrUsed = 0;
mono.bmiHeader.biClrImportant = 0;
/* apparently, bmiColors ignored on read
* -white bits are set, black bits reset no matter what */
mono.bmiColors[0] = RGB(0,0,0);
mono.bmiColors[1] = RGB(255,255,255);
if (offdc && offbm && font && monodat) {
RECT r;
SelectObject(offdc, offbm);
SelectObject(offdc, font);
SetBkMode(offdc, TRANSPARENT);
SetTextColor(offdc, RGB(255,255,255));
SetTextAlign(offdc, TA_LEFT | TA_BASELINE | TA_NOUPDATECP);
r.left = r.top = 0;
r.right = scandx;
r.bottom = scandy;
FillRect(offdc, &r, GetStockObject(BLACK_BRUSH));
if (w->orient == 2) {
TextOut(offdc, scandx-width, base, text, n);
} else if (w->orient == 1) {
TextOut(offdc, scandx-width, base, text, n);
} else {
TextOut(offdc, 0, scandy-height+base, text, n);
}
GetDIBits(offdc, offbm, 0, scandy, monodat,
(BITMAPINFO *)&mono, DIB_RGB_COLORS);
}
if (offdc) DeleteDC(offdc);
if (offbm) DeleteObject(offbm);
if (monodat) {
COLORREF bg = w_color(w,w->bg);
COLORREF fg = w_color(w,w->color);
if (w->orient == 2) {
p_mrot180(monodat, monorot, scandx, height);
x0 -= width-1;
y0 -= (height-base)-1;
scandx = width, width = height, height = scandx;
} else if (w->orient == 1) {
p_mrot090(monodat, monorot, scandx, scandy);
x0 -= base;
y0 -= width-1;
} else {
p_mrot270(monodat, monorot, scandx, scandy);
x0 -= (height-base)-1;
}
mono.bmiColors[0] = W_SWAPRGB(bg);
mono.bmiColors[1] = W_SWAPRGB(fg);
mono.bmiHeader.biWidth = height;
mono.bmiHeader.biHeight = -width;
/* prefer this to SetDIBitsToDevice? */
StretchDIBits(dc, x0, y0,
height, width,
0, 0, height, width, monorot,
(BITMAPINFO *)&mono, DIB_RGB_COLORS, SRCCOPY);
p_free(monodat);
}
}
}
}
}
struct maildir_filename *
maildir_save_add(struct mail_save_context *_ctx, const char *tmp_fname,
struct mail *src_mail)
{
struct maildir_save_context *ctx = (struct maildir_save_context *)_ctx;
struct mail_save_data *mdata = &_ctx->data;
struct maildir_filename *mf;
struct istream *input;
unsigned int keyword_count;
i_assert(*tmp_fname != '\0');
/* allow caller to specify recent flag only when uid is specified
(we're replicating, converting, etc.). */
if (mdata->uid == 0)
mdata->flags |= MAIL_RECENT;
else if ((mdata->flags & MAIL_RECENT) == 0 &&
ctx->last_nonrecent_uid < mdata->uid)
ctx->last_nonrecent_uid = mdata->uid;
/* now, we want to be able to rollback the whole append session,
so we'll just store the name of this temp file and move it later
into new/ or cur/. */
/* @UNSAFE */
keyword_count = mdata->keywords == NULL ? 0 : mdata->keywords->count;
mf = p_malloc(ctx->pool, sizeof(*mf) +
sizeof(unsigned int) * keyword_count);
mf->tmp_name = mf->dest_basename = p_strdup(ctx->pool, tmp_fname);
mf->flags = mdata->flags;
mf->size = (uoff_t)-1;
mf->vsize = (uoff_t)-1;
ctx->file_last = mf;
i_assert(*ctx->files_tail == NULL);
*ctx->files_tail = mf;
ctx->files_tail = &mf->next;
ctx->files_count++;
if (mdata->keywords != NULL) {
/* @UNSAFE */
mf->keywords_count = keyword_count;
memcpy(mf + 1, mdata->keywords->idx,
sizeof(unsigned int) * keyword_count);
ctx->have_keywords = TRUE;
}
if (mdata->pop3_uidl != NULL)
mf->pop3_uidl = p_strdup(ctx->pool, mdata->pop3_uidl);
mf->pop3_order = mdata->pop3_order;
/* insert into index */
mail_index_append(ctx->trans, mdata->uid, &ctx->seq);
mail_index_update_flags(ctx->trans, ctx->seq,
MODIFY_REPLACE, mdata->flags & ~MAIL_RECENT);
if (mdata->keywords != NULL) {
mail_index_update_keywords(ctx->trans, ctx->seq,
MODIFY_REPLACE, mdata->keywords);
}
if (mdata->min_modseq != 0) {
mail_index_update_modseq(ctx->trans, ctx->seq,
mdata->min_modseq);
}
if (ctx->first_seq == 0) {
ctx->first_seq = ctx->seq;
i_assert(ctx->files->next == NULL);
}
if (_ctx->dest_mail == NULL) {
if (ctx->mail == NULL)
ctx->mail = mail_alloc(_ctx->transaction, 0, NULL);
_ctx->dest_mail = ctx->mail;
}
mail_set_seq_saving(_ctx->dest_mail, ctx->seq);
if (ctx->input == NULL) {
/* copying with hardlinking. */
i_assert(src_mail != NULL);
index_copy_cache_fields(_ctx, src_mail, ctx->seq);
ctx->cur_dest_mail = NULL;
} else {
input = index_mail_cache_parse_init(_ctx->dest_mail,
ctx->input);
i_stream_unref(&ctx->input);
ctx->input = input;
ctx->cur_dest_mail = _ctx->dest_mail;
}
return mf;
}
static void test_set_txn_ops(struct ybc *const cache)
{
m_open_anonymous(cache);
char set_txn_buf[ybc_set_txn_get_size()];
struct ybc_set_txn *const txn = (struct ybc_set_txn *)set_txn_buf;
struct ybc_key key = {
.ptr = "abc",
.size = 3,
};
struct ybc_value value = {
.ptr = "qwerty",
.size = 6,
.ttl = YBC_MAX_TTL,
};
test_set_txn_rollback(cache, txn, &key, value.size);
test_set_txn_commit(cache, txn, &key, &value);
test_set_txn_commit_item(cache, txn, &key, &value);
/* Test zero-length key. */
key.size = 0;
test_set_txn_commit(cache, txn, &key, &value);
test_set_txn_commit_item(cache, txn, &key, &value);
/* Test zero-length value. */
value.size = 0;
test_set_txn_commit(cache, txn, &key, &value);
test_set_txn_commit_item(cache, txn, &key, &value);
/* Test too large key. */
value.size = 6;
key.size = SIZE_MAX;
test_set_txn_failure(cache, txn, &key, value.size);
/* Test too large value. */
key.size = 3;
test_set_txn_failure(cache, txn, &key, SIZE_MAX);
test_set_txn_failure(cache, txn, &key, SIZE_MAX / 2);
ybc_close(cache);
}
static void test_item_ops(struct ybc *const cache,
const size_t iterations_count)
{
m_open_anonymous(cache);
struct ybc_key key;
struct ybc_value value;
value.ttl = YBC_MAX_TTL;
for (size_t i = 0; i < iterations_count; ++i) {
key.ptr = &i;
key.size = sizeof(i);
expect_item_miss(cache, &key);
}
for (size_t i = 0; i < iterations_count; ++i) {
key.ptr = &i;
key.size = sizeof(i);
value.ptr = &i;
value.size = sizeof(i);
expect_item_set_no_acquire(cache, &key, &value);
expect_item_set(cache, &key, &value);
expect_item_remove(cache, &key);
}
for (size_t i = 0; i < iterations_count; ++i) {
key.ptr = &i;
key.size = sizeof(i);
expect_item_miss(cache, &key);
}
ybc_close(cache);
}
static void test_expiration(struct ybc *const cache)
{
m_open_anonymous(cache);
const struct ybc_key key = {
.ptr = "aaa",
.size = 3,
};
const struct ybc_value value = {
.ptr = "1234",
.size = 4,
.ttl = 200,
};
expect_item_set(cache, &key, &value);
p_sleep(300);
/* The item should expire now. */
expect_item_miss(cache, &key);
ybc_close(cache);
}
static void test_dogpile_effect_ops(struct ybc *const cache)
{
m_open_anonymous(cache);
struct ybc_key key = {
.ptr = "foo",
.size = 3,
};
const struct ybc_value value = {
.ptr = "bar",
.size = 3,
.ttl = 2 * 1000,
};
/*
* De-aware method should return an empty item on the first try
* for non-existing item. The second try for the same non-existing item
* will result in waiting for up to grace ttl period of time.
*/
expect_item_miss_de(cache, &key, 200);
/* Will wait for 200 milliseconds. */
expect_item_miss_de(cache, &key, 10 * 1000);
key.ptr = "bar";
expect_item_set(cache, &key, &value);
/*
* If grace ttl is smaller than item's ttl, then the item should be returned.
*/
expect_item_hit_de(cache, &key, &value, value.ttl / 10);
/*
* If grace ttl is larger than item's ttl, then an empty item
* should be returned on the first try and the item itself should be returned
* on subsequent tries irregardless of grace ttl value.
*/
expect_item_miss_de(cache, &key, value.ttl * 10);
expect_item_hit_de(cache, &key, &value, value.ttl * 10);
expect_item_hit_de(cache, &key, &value, value.ttl / 10);
ybc_close(cache);
}
static void test_dogpile_effect_ops_async(struct ybc *const cache)
{
char item_buf[ybc_item_get_size()];
struct ybc_item *const item = (struct ybc_item *)item_buf;
m_open_anonymous(cache);
struct ybc_key key = {
.ptr = "foo",
.size = 3,
};
const struct ybc_value value = {
.ptr = "bar",
.size = 3,
.ttl = 2 * 1000,
};
/*
* De-aware method should return an empty item on the first try
* for non-existing item. The second try for the same non-existing item
* should result in YBC_DE_WOULDBLOCK.
*/
if (ybc_item_get_de_async(cache, item, &key, 10 * 1000) != YBC_DE_NOTFOUND) {
M_ERROR("unexpected status returned from ybc_item_get_de_async()");
}
/* Should return immediately instead of waiting for 10 seconds. */
if (ybc_item_get_de_async(cache, item, &key, 5 * 1000) !=
YBC_DE_WOULDBLOCK) {
M_ERROR("unexpected status returned from ybc_item_get_de_async()");
}
key.ptr = "bar";
expect_item_set(cache, &key, &value);
if (ybc_item_get_de_async(cache, item, &key, value.ttl / 10) !=
YBC_DE_SUCCESS) {
M_ERROR("unexpected status returned from ybc_item_get_de_async()");
}
ybc_item_release(item);
/*
* If grace ttl is larger than item's ttl, then an empty item
* should be returned on the first try and the item itself should be returned
* on subsequent tries irregardless of grace ttl value.
*/
if (ybc_item_get_de_async(cache, item, &key, value.ttl * 10) !=
YBC_DE_NOTFOUND) {
M_ERROR("unexpected status returned from ybc_item_get_de_async()");
}
if (ybc_item_get_de_async(cache, item, &key, value.ttl * 10) !=
YBC_DE_SUCCESS) {
M_ERROR("unexpected status returned from ybc_item_get_de_async()");
}
ybc_item_release(item);
if (ybc_item_get_de_async(cache, item, &key, value.ttl / 10) !=
YBC_DE_SUCCESS) {
M_ERROR("unexpected status returned from ybc_item_get_de_async()");
}
ybc_item_release(item);
ybc_close(cache);
}
static void m_test_de_hashtable(struct ybc *const cache,
const size_t hashtable_size, const size_t pending_items_count)
{
char config_buf[ybc_config_get_size()];
struct ybc_config *const config = (struct ybc_config *)config_buf;
ybc_config_init(config);
ybc_config_set_de_hashtable_size(config, hashtable_size);
if (!ybc_open(cache, config, 1)) {
M_ERROR("cannot create an anonymous cache");
}
ybc_config_destroy(config);
char item_buf[ybc_item_get_size()];
struct ybc_item *const item = (struct ybc_item *)item_buf;
size_t i;
struct ybc_key key = {
.ptr = &i,
.size = sizeof(i),
};
for (i = 0; i < pending_items_count; ++i) {
if (ybc_item_get_de_async(cache, item, &key, 1000) != YBC_DE_NOTFOUND) {
M_ERROR("unexpected status returned from ybc_item_get_de_async()");
}
if (ybc_item_get_de_async(cache, item, &key, 1000) != YBC_DE_WOULDBLOCK) {
M_ERROR("unexpected status returned from ybc_item_get_de_async()");
}
}
ybc_close(cache);
}
static void test_dogpile_effect_hashtable(struct ybc *const cache)
{
for (size_t hashtable_size = 1; hashtable_size <= 1000;
hashtable_size *= 10) {
for (size_t pending_items_count = 1; pending_items_count <= 10000;
pending_items_count *= 100) {
m_test_de_hashtable(cache, hashtable_size, pending_items_count);
}
}
}
static void test_cluster_ops(const size_t cluster_size,
const size_t iterations_count)
{
char configs_buf[ybc_config_get_size() * cluster_size];
struct ybc_config *const configs = (struct ybc_config *)configs_buf;
for (size_t i = 0; i < cluster_size; ++i) {
ybc_config_init(YBC_CONFIG_GET(configs, i));
}
char cluster_buf[ybc_cluster_get_size(cluster_size)];
struct ybc_cluster *const cluster = (struct ybc_cluster *)cluster_buf;
/* Unfored open must fail. */
if (ybc_cluster_open(cluster, configs, cluster_size, 0)) {
M_ERROR("cache cluster shouldn't be opened without force");
}
/* Forced open must succeed. */
if (!ybc_cluster_open(cluster, configs, cluster_size, 1)) {
M_ERROR("failed opening cache cluster");
}
/* Configs are no longer needed, so they can be destroyed. */
for (size_t i = 0; i < cluster_size; ++i) {
ybc_config_destroy(YBC_CONFIG_GET(configs, i));
}
struct ybc_key key;
struct ybc_value value;
value.ttl = YBC_MAX_TTL;
for (size_t i = 0; i < iterations_count; ++i) {
key.ptr = &i;
key.size = sizeof(i);
value.ptr = &i;
value.size = sizeof(i);
struct ybc *const cache = ybc_cluster_get_cache(cluster, &key);
expect_item_set(cache, &key, &value);
}
ybc_cluster_clear(cluster);
for (size_t i = 0; i < iterations_count; ++i) {
key.ptr = &i;
key.size = sizeof(i);
value.ptr = &i;
value.size = sizeof(i);
struct ybc *const cache = ybc_cluster_get_cache(cluster, &key);
expect_item_miss(cache, &key);
}
ybc_cluster_close(cluster);
}
static struct ybc_item *m_get_item(struct ybc_item *const items, const size_t i)
{
return (struct ybc_item *)(((char *)items) + ybc_item_get_size() * i);
}
static void test_overlapped_acquirements(struct ybc *const cache,
const size_t items_count)
{
m_open_anonymous(cache);
char added_items_buf[ybc_item_get_size() * items_count];
struct ybc_item *const added_items = (struct ybc_item *)added_items_buf;
char obtained_items_buf[ybc_item_get_size() * items_count];
struct ybc_item *const obtained_items = (struct ybc_item *)obtained_items_buf;
size_t i;
const struct ybc_key key = {
.ptr = &i,
.size = sizeof(i),
};
const struct ybc_value value = {
.ptr = &i,
.size = sizeof(i),
.ttl = YBC_MAX_TTL,
};
const struct ybc_key static_key = {
.ptr = "aaaabbb",
.size = 7,
};
for (i = 0; i < items_count; ++i) {
ybc_item_set_item(cache, m_get_item(added_items, i), &static_key, &value);
}
for (i = 0; i < items_count; ++i) {
ybc_item_get(cache, m_get_item(obtained_items, i), &static_key);
}
for (i = 0; i < items_count; ++i) {
ybc_item_release(m_get_item(obtained_items, i));
ybc_item_release(m_get_item(added_items, i));
}
for (i = 0; i < items_count; ++i) {
ybc_item_set_item(cache, m_get_item(added_items, i), &key, &value);
}
for (i = 0; i < items_count; ++i) {
ybc_item_get(cache, m_get_item(obtained_items, i), &key);
expect_value(m_get_item(obtained_items, i), &value);
}
for (i = 0; i < items_count; ++i) {
ybc_item_release(m_get_item(obtained_items, items_count - i - 1));
}
for (i = 0; i < items_count; ++i) {
ybc_item_release(m_get_item(added_items, i));
}
ybc_close(cache);
}
static void test_interleaved_sets(struct ybc *const cache)
{
m_open_anonymous(cache);
char set_txn1_buf[ybc_set_txn_get_size()];
char set_txn2_buf[ybc_set_txn_get_size()];
struct ybc_set_txn *const txn1 = (struct ybc_set_txn *)set_txn1_buf;
struct ybc_set_txn *const txn2 = (struct ybc_set_txn *)set_txn2_buf;
char item1_buf[ybc_item_get_size()];
char item2_buf[ybc_item_get_size()];
struct ybc_item *const item1 = (struct ybc_item *)item1_buf;
struct ybc_item *const item2 = (struct ybc_item *)item2_buf;
const struct ybc_key key1 = {
.ptr = "foo",
.size = 3,
};
const struct ybc_key key2 = {
.ptr = "barz",
.size = 4,
};
const struct ybc_value value1 = {
.ptr = "123456",
.size = 6,
.ttl = YBC_MAX_TTL,
};
const struct ybc_value value2 = {
.ptr = "qwert",
.size = 4,
.ttl = YBC_MAX_TTL,
};
if (!ybc_set_txn_begin(cache, txn1, &key1, value1.size, value1.ttl)) {
M_ERROR("Cannot start the first set transaction");
}
if (!ybc_set_txn_begin(cache, txn2, &key2, value2.size, value2.ttl)) {
M_ERROR("Cannot start the second set transaction");
}
struct ybc_set_txn_value txn_value;
ybc_set_txn_get_value(txn1, &txn_value);
assert(txn_value.size == value1.size);
memcpy(txn_value.ptr, value1.ptr, value1.size);
ybc_set_txn_get_value(txn2, &txn_value);
assert(txn_value.size == value2.size);
memcpy(txn_value.ptr, value2.ptr, value2.size);
expect_item_miss(cache, &key1);
expect_item_miss(cache, &key2);
ybc_set_txn_commit_item(txn1, item1);
ybc_set_txn_commit_item(txn2, item2);
expect_value(item1, &value1);
expect_value(item2, &value2);
ybc_item_release(item1);
ybc_item_release(item2);
expect_item_hit(cache, &key1, &value1);
expect_item_hit(cache, &key2, &value2);
ybc_close(cache);
}
static void test_instant_clear(struct ybc *const cache)
{
char config_buf[ybc_config_get_size()];
struct ybc_config *const config = (struct ybc_config *)config_buf;
ybc_config_init(config);
ybc_config_set_max_items_count(config, 1000);
ybc_config_set_data_file_size(config, 128 * 1024);
if (!ybc_open(cache, config, 1)) {
M_ERROR("cannot create anonymous cache");
}
ybc_config_destroy(config);
struct ybc_key key;
struct ybc_value value;
/* Add a lot of items to the cache */
value.ttl = YBC_MAX_TTL;
for (size_t i = 0; i < 1000; ++i) {
key.ptr = &i;
key.size = sizeof(i);
value.ptr = &i;
value.size = sizeof(i);
expect_item_set(cache, &key, &value);
}
ybc_clear(cache);
/* Test that the cache doesn't contain any items after the clearance. */
for (size_t i = 0; i < 1000; ++i) {
key.ptr = &i;
key.size = sizeof(i);
expect_item_miss(cache, &key);
}
ybc_close(cache);
}
static void expect_persistent_survival(struct ybc *const cache,
const uint64_t sync_interval)
{
char config_buf[ybc_config_get_size()];
struct ybc_config *const config = (struct ybc_config *)config_buf;
ybc_config_init(config);
ybc_config_set_index_file(config, "./tmp_cache.index");
ybc_config_set_data_file(config, "./tmp_cache.data");
ybc_config_set_max_items_count(config, 10);
ybc_config_set_data_file_size(config, 1024);
ybc_config_set_sync_interval(config, sync_interval);
if (!ybc_open(cache, config, 1)) {
M_ERROR("cannot create persistent cache");
}
const struct ybc_key key = {
.ptr = "foobar",
.size = 6.
};
const struct ybc_value value = {
.ptr = "qwert",
.size = 5,
.ttl = YBC_MAX_TTL,
};
expect_item_set(cache, &key, &value);
ybc_close(cache);
/* Re-open the same cache and make sure the item exists there. */
if (!ybc_open(cache, config, 0)) {
M_ERROR("cannot open persistent cache");
}
expect_item_hit(cache, &key, &value);
ybc_close(cache);
ybc_remove(config);
ybc_config_destroy(config);
}
static void test_persistent_survival(struct ybc *const cache)
{
/*
* Test persistence with enabled data syncing.
*/
expect_persistent_survival(cache, 10 * 1000);
/*
* Test persistence with disabled data syncing.
*/
expect_persistent_survival(cache, 0);
}
static void test_broken_index_handling(struct ybc *const cache)
{
char config_buf[ybc_config_get_size()];
struct ybc_config *const config = (struct ybc_config *)config_buf;
ybc_config_init(config);
ybc_config_set_index_file(config, "./tmp_cache.index");
ybc_config_set_data_file(config, "./tmp_cache.data");
ybc_config_set_max_items_count(config, 1000);
ybc_config_set_data_file_size(config, 64 * 1024);
/* Create index and data files. */
if (!ybc_open(cache, config, 1)) {
M_ERROR("cannot create persistent cache");
}
struct ybc_key key;
struct ybc_value value = {
.ptr = "foobar",
.size = 6,
.ttl = YBC_MAX_TTL,
};
/* Add some data to cache. */
for (size_t i = 0; i < 1000; i++) {
key.ptr = &i;
key.size = sizeof(i);
expect_item_set(cache, &key, &value);
}
ybc_close(cache);
/* Corrupt index file. */
FILE *const fp = fopen("./tmp_cache.index", "r+");
int rv = fseek(fp, 0, SEEK_END);
if (rv != 0) {
M_ERROR("fseek(SEEK_END, 0) failed");
}
const long pos = ftell(fp);
if (pos < 0) {
M_ERROR("ftell failed");
}
const size_t file_size = (size_t)pos;
rv = fseek(fp, 0, SEEK_SET);
if (rv != 0) {
M_ERROR("fseek(SEEK_SET, 0) failed");
}
for (size_t i = 0; i < file_size; ++i) {
if (fputc((unsigned char)i, fp) == EOF) {
M_ERROR("cannot write data");
}
}
fclose(fp);
/* Try reading index file. It must become "empty". */
if (!ybc_open(cache, config, 0)) {
M_ERROR("cannot open persistent cache");
}
for (size_t i = 0; i < 1000; ++i) {
key.ptr = &i;
key.size = sizeof(i);
expect_item_miss(cache, &key);
}
ybc_close(cache);
/* Remove index and data files. */
ybc_remove(config);
ybc_config_destroy(config);
}
static void test_large_cache(struct ybc *const cache)
{
char config_buf[ybc_config_get_size()];
struct ybc_config *const config = (struct ybc_config *)config_buf;
ybc_config_init(config);
ybc_config_set_max_items_count(config, 10 * 1000);
ybc_config_set_data_file_size(config, 32 * 1024 * 1024);
if (!ybc_open(cache, config, 1)) {
M_ERROR("cannot create anonymous cache");
}
ybc_config_destroy(config);
const size_t value_buf_size = 13 * 3457;
char *const value_buf = p_malloc(value_buf_size);
memset(value_buf, 'q', value_buf_size);
struct ybc_key key;
const struct ybc_value value = {
.ptr = value_buf,
.size = value_buf_size,
.ttl = YBC_MAX_TTL,
};
/* Test handling of cache data size wrapping. */
for (size_t i = 0; i < 10 * 1000; ++i) {
key.ptr = &i;
key.size = sizeof(i);
expect_item_set(cache, &key, &value);
}
p_free(value_buf);
ybc_close(cache);
}
static void test_out_of_memory(struct ybc *const cache)
{
char config_buf[ybc_config_get_size()];
struct ybc_config *const config = (struct ybc_config *)config_buf;
ybc_config_init(config);
ybc_config_set_data_file_size(config, 1024 * 1024);
if (!ybc_open(cache, config, 1)) {
M_ERROR("cannot create anonymous cache");
}
ybc_config_destroy(config);
const size_t value_buf_size = 1024 * 1024 + 1;
void *const value_buf = p_malloc(value_buf_size);
memset(value_buf, 0, value_buf_size);
char item_buf[ybc_item_get_size()];
struct ybc_item *const item = (struct ybc_item *)item_buf;
struct ybc_key key = {
.ptr = "foobar",
.size = 6,
};
struct ybc_value value = {
.ptr = value_buf,
.size = value_buf_size,
.ttl = YBC_MAX_TTL,
};
/*
* The value size exceeds cache size.
*/
if (ybc_item_set(cache, &key, &value)) {
M_ERROR("unexpected item addition");
}
/*
* The acquired item should prevent from adding new item into the cache.
*/
value.size -= 1000;
if (!ybc_item_set_item(cache, item, &key, &value)) {
M_ERROR("cannot store item to cache");
}
char item2_buf[ybc_item_get_size()];
struct ybc_item *const item2 =(struct ybc_item *)item2_buf;
key.ptr = "abcdef";
value.size = 1000;
if (ybc_item_set_item(cache, item2, &key, &value)) {
M_ERROR("unexpected item addition");
}
ybc_item_release(item);
/*
* Now the item2 should be added, since the item is released
* and the cache has enough room for the item2.
*/
if (!ybc_item_set_item(cache, item2, &key, &value)) {
M_ERROR("cannot store item to cache");
}
ybc_item_release(item2);
p_free(value_buf);
ybc_close(cache);
}
static int is_item_exists(struct ybc *const cache,
const struct ybc_key *const key)
{
char item_buf[ybc_item_get_size()];
struct ybc_item *const item = (struct ybc_item *)item_buf;
if (!ybc_item_get(cache, item, key)) {
return 0;
}
ybc_item_release(item);
return 1;
}
static void expect_cache_with_data(struct ybc *const cache,
const size_t items_count, const size_t expected_hits_count,
struct ybc_key *const key, struct ybc_value *const value)
{
size_t hits_count = 0;
for (size_t i = 0; i < items_count; ++i) {
key->ptr = &i;
key->size = sizeof(i);
if (is_item_exists(cache, key)) {
value->ptr = &i;
value->size = sizeof(i);
expect_item_hit(cache, key, value);
++hits_count;
}
}
assert(hits_count > expected_hits_count);
}
static void expect_cache_works(struct ybc *const cache,
const size_t items_count, const size_t expected_hits_count,
const size_t hot_items_count, const size_t hot_data_size,
const uint64_t sync_interval)
{
char config_buf[ybc_config_get_size()];
struct ybc_config *const config = (struct ybc_config *)config_buf;
ybc_config_init(config);
assert(items_count <= SIZE_MAX / 100);
ybc_config_set_max_items_count(config, items_count * 2);
ybc_config_set_data_file_size(config, items_count * 100);
ybc_config_set_hot_items_count(config, hot_items_count);
ybc_config_set_hot_data_size(config, hot_data_size);
ybc_config_set_sync_interval(config, sync_interval);
if (!ybc_open(cache, config, 1)) {
M_ERROR("cannot create anonymous cache");
}
ybc_config_destroy(config);
struct ybc_key key;
struct ybc_value value;
value.ttl = YBC_MAX_TTL;
for (size_t i = 0; i < items_count; ++i) {
key.ptr = &i;
key.size = sizeof(i);
value.ptr = &i;
value.size = sizeof(i);
expect_item_set(cache, &key, &value);
}
/*
* Verify twice that the cache contains added data.
* The second verification checks correctness of internal cache algorithms,
* which may re-arrange data when reading it during the first check
* (for instance, cache compaction algorithms).
*/
expect_cache_with_data(cache, items_count, expected_hits_count, &key, &value);
expect_cache_with_data(cache, items_count, expected_hits_count, &key, &value);
ybc_close(cache);
}
static void test_data_compaction(struct ybc *const cache)
{
/*
* The cache will compact data on items' retrieval,
* because items_count * item_size is greater than hot_data_size.
* It is assumed that item_size is equal to 2*sizeof(size_t)
* (8 bytes on 32-bit builds and 16 bytes on 64-bit builds).
* See expect_cache_works() sources for details.
*/
const size_t items_count = 1000;
const size_t expected_hits_count = 900;
const size_t hot_items_count = 1000;
const size_t hot_data_size = items_count * sizeof(size_t) * 3;
const uint64_t sync_interval = 10 * 1000;
expect_cache_works(cache, items_count, expected_hits_count, hot_items_count,
hot_data_size, sync_interval);
}
static void test_small_sync_interval(struct ybc *const cache)
{
char config_buf[ybc_config_get_size()];
struct ybc_config *const config = (struct ybc_config *)config_buf;
ybc_config_init(config);
ybc_config_set_max_items_count(config, 100);
ybc_config_set_data_file_size(config, 4000);
ybc_config_set_sync_interval(config, 100);
if (!ybc_open(cache, config, 1)) {
M_ERROR("cannot create anonymous cache");
}
ybc_config_destroy(config);
struct ybc_key key;
const struct ybc_value value = {
.ptr = "1234567890a",
.size = 11,
.ttl = YBC_MAX_TTL,
};
for (size_t i = 0; i < 10; ++i) {
for (size_t j = 0; j < 100; ++j) {
key.ptr = &j;
key.size = sizeof(j);
expect_item_set(cache, &key, &value);
}
p_sleep(31);
}
ybc_close(cache);
}
static void test_disabled_hot_items_cache(struct ybc *const cache)
{
const size_t items_count = 1000;
const size_t expected_hits_count = 900;
const size_t hot_items_count = 0;
const size_t hot_data_size = 100 * 1024;
const uint64_t sync_interval = 10 * 1000;
expect_cache_works(cache, items_count, expected_hits_count, hot_items_count,
hot_data_size, sync_interval);
}
static void test_disabled_data_compaction(struct ybc *const cache)
{
const size_t items_count = 1000;
const size_t expected_hits_count = 900;
const size_t hot_items_count = 100;
const size_t hot_data_size = 0;
const uint64_t sync_interval = 10 * 1000;
expect_cache_works(cache, items_count, expected_hits_count, hot_items_count,
hot_data_size, sync_interval);
}
static void test_disabled_syncing(struct ybc *const cache)
{
const size_t items_count = 1000;
const size_t expected_hits_count = 900;
const size_t hot_items_count = 100;
const size_t hot_data_size = 10 * 1024;
const uint64_t sync_interval = 0;
expect_cache_works(cache, items_count, expected_hits_count, hot_items_count,
hot_data_size, sync_interval);
}
struct thread_task
{
struct ybc *const cache;
int should_exit;
};
static void thread_func(void *const ctx)
{
struct thread_task *const task = ctx;
char item_buf[ybc_item_get_size()];
struct ybc_item *const item = (struct ybc_item *)item_buf;
struct ybc_key key;
struct ybc_value value;
int tmp;
key.size = sizeof(tmp);
value.size = sizeof(tmp);
value.ttl = YBC_MAX_TTL;
while (!task->should_exit) {
/*
* It is OK using non-threadsafe rand() function here.
*/
tmp = rand() % 100;
key.ptr = &tmp;
value.ptr = &tmp;
switch (rand() % 5) {
case 0: case 1:
if (!ybc_item_set_item(task->cache, item, &key, &value)) {
M_ERROR("error when storing item in the cache");
}
expect_value(item, &value);
ybc_item_release(item);
break;
case 2:
(void)ybc_item_remove(task->cache, &key);
break;
default:
if (ybc_item_get(task->cache, item, &key)) {
expect_value(item, &value);
ybc_item_release(item);
}
}
}
}
static void test_multithreaded_access(struct ybc *const cache,
const size_t threads_count)
{
m_open_anonymous(cache);
struct p_thread threads[threads_count];
struct thread_task task = {
.cache = cache,
.should_exit = 0,
};
for (size_t i = 0; i < threads_count; ++i) {
p_thread_init_and_start(&threads[i], thread_func, &task);
}
p_sleep(300);
task.should_exit = 1;
for (size_t i = 0; i < threads_count; ++i) {
p_thread_join_and_destroy(&threads[i]);
}
ybc_close(cache);
}
int main(void)
{
char cache_buf[ybc_get_size()];
struct ybc *const cache = (struct ybc *)cache_buf;
test_anonymous_cache_create(cache);
test_persistent_cache_create(cache);
test_set_txn_ops(cache);
test_item_ops(cache, 1000);
test_expiration(cache);
test_dogpile_effect_ops_async(cache);
test_dogpile_effect_ops(cache);
test_dogpile_effect_hashtable(cache);
test_cluster_ops(5, 1000);
test_overlapped_acquirements(cache, 1000);
test_interleaved_sets(cache);
test_instant_clear(cache);
test_persistent_survival(cache);
test_broken_index_handling(cache);
test_large_cache(cache);
test_out_of_memory(cache);
test_data_compaction(cache);
test_small_sync_interval(cache);
test_disabled_hot_items_cache(cache);
test_disabled_data_compaction(cache);
test_disabled_syncing(cache);
test_multithreaded_access(cache, 100);
printf("All functional tests done\n");
return 0;
}
static void simple_set(struct ybc *const cache, const size_t requests_count,
const size_t items_count, const size_t max_item_size)
{
struct m_rand_state rand_state;
uint64_t tmp;
char *const buf = p_malloc(max_item_size);
const struct ybc_key key = {
.ptr = &tmp,
.size = sizeof(tmp),
};
struct ybc_value value = {
.ptr = buf,
.size = 0,
.ttl = YBC_MAX_TTL,
};
m_rand_init(&rand_state);
for (size_t i = 0; i < requests_count; ++i) {
tmp = m_rand_next(&rand_state) % items_count;
value.size = m_rand_next(&rand_state) % (max_item_size + 1);
m_memset(buf, (char)value.size, value.size);
if (!ybc_item_set(cache, &key, &value)) {
M_ERROR("Cannot store item in the cache");
}
}
p_free(buf);
}
static void simple_set_simple(struct ybc *const cache,
const size_t requests_count, const size_t items_count,
const size_t max_item_size)
{
struct m_rand_state rand_state;
uint64_t tmp;
char *const buf = p_malloc(max_item_size);
const struct ybc_key key = {
.ptr = &tmp,
.size = sizeof(tmp),
};
struct ybc_value value = {
.ptr = buf,
.size = 0,
.ttl = YBC_MAX_TTL,
};
m_rand_init(&rand_state);
for (size_t i = 0; i < requests_count; ++i) {
tmp = m_rand_next(&rand_state) % items_count;
value.size = m_rand_next(&rand_state) % (max_item_size + 1);
m_memset(buf, (char)value.size, value.size);
if (!ybc_simple_set(cache, &key, &value)) {
M_ERROR("Cannot store item in the cache");
}
}
p_free(buf);
}
static void simple_get_miss(struct ybc *const cache,
const size_t requests_count, const size_t items_count)
{
char item_buf[ybc_item_get_size()];
struct ybc_item *const item = (struct ybc_item *)item_buf;
struct m_rand_state rand_state;
uint64_t tmp;
const struct ybc_key key = {
.ptr = &tmp,
.size = sizeof(tmp),
};
m_rand_init(&rand_state);
for (size_t i = 0; i < requests_count; ++i) {
tmp = m_rand_next(&rand_state) % items_count;
if (ybc_item_get(cache, item, &key)) {
M_ERROR("Unexpected item found");
}
}
}
static void simple_get_hit(struct ybc *const cache,
const size_t requests_count, const size_t items_count,
const size_t max_item_size)
{
char item_buf[ybc_item_get_size()];
struct ybc_item *const item = (struct ybc_item *)item_buf;
struct m_rand_state rand_state;
uint64_t tmp;
const struct ybc_key key = {
.ptr = &tmp,
.size = sizeof(tmp),
};
struct ybc_value value;
m_rand_init(&rand_state);
for (size_t i = 0; i < requests_count; ++i) {
tmp = m_rand_next(&rand_state) % items_count;
if (ybc_item_get(cache, item, &key)) {
/* Emulate access to the item */
ybc_item_get_value(item, &value);
if (value.size > max_item_size) {
M_ERROR("Unexpected value size");
}
if (!m_memset_check(value.ptr, (char)value.size, value.size)) {
fprintf(stderr, "i=%zu, requests_count=%zu, value.size=%zu\n", i, requests_count, value.size);
M_ERROR("Unexpected value");
}
ybc_item_release(item);
}
}
}
static void simple_get_simple_hit(struct ybc *const cache,
const size_t requests_count, const size_t items_count,
const size_t max_item_size)
{
struct m_rand_state rand_state;
uint64_t tmp;
const struct ybc_key key = {
.ptr = &tmp,
.size = sizeof(tmp),
};
struct ybc_value value;
value.size = max_item_size;
value.ptr = p_malloc(value.size);
m_rand_init(&rand_state);
for (size_t i = 0; i < requests_count; ++i) {
tmp = m_rand_next(&rand_state) % items_count;
value.size = max_item_size;
int rv = ybc_simple_get(cache, &key, &value);
if (rv == 0) {
continue;
}
assert(rv == 1);
if (value.size > max_item_size) {
M_ERROR("Unexpected value size");
}
if (!m_memset_check(value.ptr, (char)value.size, value.size)) {
fprintf(stderr, "i=%zu, requests_count=%zu, value.size=%zu\n", i, requests_count, value.size);
M_ERROR("Unexpected value");
}
}
p_free((void *)value.ptr);
}
static void m_open(struct ybc *const cache, const int use_shm,
const size_t items_count, const size_t hot_items_count,
const size_t max_item_size, const int has_overwrite_protection)
{
char config_buf[ybc_config_get_size()];
struct ybc_config *const config = (struct ybc_config *)config_buf;
const size_t data_file_size = max_item_size * items_count;
const size_t hot_data_size = max_item_size * hot_items_count;
ybc_config_init(config);
if (use_shm) {
ybc_config_set_data_file(config, "/dev/shm/ybc-perftest-cache.data");
ybc_config_set_index_file(config, "/dev/shm/ybc-perftest-cache.index");
}
ybc_config_set_max_items_count(config, items_count);
ybc_config_set_hot_items_count(config, hot_items_count);
ybc_config_set_data_file_size(config, data_file_size);
ybc_config_set_hot_data_size(config, hot_data_size);
if (!has_overwrite_protection) {
ybc_config_disable_overwrite_protection(config);
}
if (!ybc_open(cache, config, 1)) {
M_ERROR("Cannot create a cache");
}
ybc_config_destroy(config);
if (use_shm) {
ybc_clear(cache);
}
}
static void m_close(struct ybc *const cache, const int use_shm)
{
ybc_close(cache);
if (!use_shm) {
return;
}
char config_buf[ybc_config_get_size()];
struct ybc_config *const config = (struct ybc_config *)config_buf;
ybc_config_init(config);
ybc_config_set_data_file(config, "/dev/shm/ybc-perftest-cache.data");
ybc_config_set_index_file(config, "/dev/shm/ybc-perftest-cache.index");
ybc_remove(config);
ybc_config_destroy(config);
}
static void measure_simple_ops(struct ybc *const cache, const int use_shm,
const size_t requests_count, const size_t items_count,
const size_t hot_items_count, const size_t max_item_size,
const int has_overwrite_protection)
{
double start_time, end_time;
double qps;
m_open(cache, use_shm, items_count, hot_items_count, max_item_size,
has_overwrite_protection);
printf("simple_ops(requests=%zu, items=%zu, "
"hot_items=%zu, max_item_size=%zu, has_overwrite_protection=%d, use_shm=%d)\n",
requests_count, items_count, hot_items_count, max_item_size,
has_overwrite_protection, use_shm);
start_time = p_get_current_time();
simple_get_miss(cache, requests_count, items_count);
end_time = p_get_current_time();
qps = requests_count / (end_time - start_time) * 1000;
printf(" get_miss : %.02f qps\n", qps);
start_time = p_get_current_time();
simple_set(cache, requests_count, items_count, max_item_size);
end_time = p_get_current_time();
qps = requests_count / (end_time - start_time) * 1000;
printf(" set : %.02f qps\n", qps);
const size_t get_items_count = hot_items_count ? hot_items_count :
items_count;
if (has_overwrite_protection) {
start_time = p_get_current_time();
simple_get_hit(cache, requests_count, get_items_count, max_item_size);
end_time = p_get_current_time();
qps = requests_count / (end_time - start_time) * 1000;
printf(" get_hit : %.02f qps\n", qps);
}
ybc_clear(cache);
start_time = p_get_current_time();
simple_set_simple(cache, requests_count, items_count, max_item_size);
end_time = p_get_current_time();
qps = requests_count / (end_time - start_time) * 1000;
printf(" set_simple : %.02f qps\n", qps);
start_time = p_get_current_time();
simple_get_simple_hit(cache, requests_count, get_items_count, max_item_size);
end_time = p_get_current_time();
qps = requests_count / (end_time - start_time) * 1000;
printf(" get_simple_hit : %.02f qps\n", qps);
m_close(cache, use_shm);
}
struct thread_task
{
struct p_lock lock;
struct ybc *cache;
size_t requests_count;
size_t items_count;
size_t get_items_count;
size_t max_item_size;
};
static size_t get_batch_requests_count(struct thread_task *const task)
{
static const size_t batch_requests_count = 10000;
size_t requests_count = batch_requests_count;
p_lock_lock(&task->lock);
if (task->requests_count < batch_requests_count) {
requests_count = task->requests_count;
}
task->requests_count -= requests_count;
p_lock_unlock(&task->lock);
return requests_count;
}
static void thread_func_set(void *const ctx)
{
struct thread_task *const task = ctx;
for (;;) {
const size_t requests_count = get_batch_requests_count(task);
if (requests_count == 0) {
break;
}
simple_set(task->cache, requests_count, task->items_count,
task->max_item_size);
}
}
static void thread_func_get_miss(void *const ctx)
{
struct thread_task *const task = ctx;
for (;;) {
const size_t requests_count = get_batch_requests_count(task);
if (requests_count == 0) {
break;
}
simple_get_miss(task->cache, requests_count, task->get_items_count);
}
}
static void thread_func_get_hit(void *const ctx)
{
struct thread_task *const task = ctx;
for (;;) {
const size_t requests_count = get_batch_requests_count(task);
if (requests_count == 0) {
break;
}
simple_get_hit(task->cache, requests_count, task->get_items_count,
task->max_item_size);
}
}
static void thread_func_set_get(void *const ctx)
{
struct thread_task *const task = ctx;
for (;;) {
const size_t requests_count = get_batch_requests_count(task);
if (requests_count == 0) {
break;
}
const size_t set_requests_count = (size_t)(requests_count * 0.1);
const size_t get_requests_count = requests_count - set_requests_count;
simple_set(task->cache, set_requests_count, task->items_count,
task->max_item_size);
simple_get_hit(task->cache, get_requests_count, task->get_items_count,
task->max_item_size);
}
}
static void thread_func_set_simple(void *const ctx)
{
struct thread_task *const task = ctx;
for (;;) {
const size_t requests_count = get_batch_requests_count(task);
if (requests_count == 0) {
break;
}
simple_set_simple(task->cache, requests_count, task->items_count,
task->max_item_size);
}
}
static void thread_func_get_simple_hit(void *const ctx)
{
struct thread_task *const task = ctx;
for (;;) {
const size_t requests_count = get_batch_requests_count(task);
if (requests_count == 0) {
break;
}
simple_get_simple_hit(task->cache, requests_count, task->get_items_count,
task->max_item_size);
}
}
static double measure_qps(struct thread_task *const task,
const p_thread_func thread_func, const size_t threads_count,
const size_t requests_count)
{
struct p_thread threads[threads_count];
task->requests_count = requests_count;
double start_time = p_get_current_time();
for (size_t i = 0; i < threads_count; ++i) {
p_thread_init_and_start(&threads[i], thread_func, task);
}
for (size_t i = 0; i < threads_count; ++i) {
p_thread_join_and_destroy(&threads[i]);
}
double end_time = p_get_current_time();
return requests_count / (end_time - start_time) * 1000;
}
static void measure_multithreaded_ops(struct ybc *const cache,
const int use_shm,
const size_t threads_count, const size_t requests_count,
const size_t items_count, const size_t hot_items_count,
const size_t max_item_size, const int has_overwrite_protection)
{
double qps;
m_open(cache, use_shm, items_count, hot_items_count, max_item_size,
has_overwrite_protection);
struct thread_task task = {
.cache = cache,
.items_count = items_count,
.get_items_count = hot_items_count ? hot_items_count : items_count,
.max_item_size = max_item_size,
};
p_lock_init(&task.lock);
printf("multithreaded_ops(requests=%zu, items=%zu, hot_items=%zu, "
"max_item_size=%zu, threads=%zu, has_overwrite_protection=%d, use_shm=%d)\n",
requests_count, items_count, hot_items_count, max_item_size,
threads_count, has_overwrite_protection, use_shm);
qps = measure_qps(&task, thread_func_get_miss, threads_count, requests_count);
printf(" get_miss : %.2f qps\n", qps);
qps = measure_qps(&task, thread_func_set, threads_count, requests_count);
printf(" set : %.2f qps\n", qps);
if (has_overwrite_protection) {
qps = measure_qps(&task, thread_func_get_hit, threads_count,
requests_count);
printf(" get_hit : %.2f qps\n", qps);
qps = measure_qps(&task, thread_func_set_get, threads_count,
requests_count);
printf(" get_set : %.2f qps\n", qps);
}
ybc_clear(cache);
qps = measure_qps(&task, thread_func_set_simple, threads_count,
requests_count);
printf(" set_simple : %.2f qps\n", qps);
qps = measure_qps(&task, thread_func_get_simple_hit, threads_count,
requests_count);
printf(" get_simple_hit : %.2f qps\n", qps);
p_lock_destroy(&task.lock);
m_close(cache, use_shm);
}
int main(void)
{
char cache_buf[ybc_get_size()];
struct ybc *const cache = (struct ybc *)cache_buf;
const size_t requests_count = 4 * 1000 * 1000;
const size_t items_count = 200 * 1000;
for (size_t max_item_size = 8; max_item_size <= 4096; max_item_size *= 2) {
measure_simple_ops(cache, 0, requests_count, items_count, 0, max_item_size, 0);
measure_simple_ops(cache, 0, requests_count, items_count, 0, max_item_size, 1);
measure_simple_ops(cache, 1, requests_count, items_count, 0, max_item_size, 0);
measure_simple_ops(cache, 1, requests_count, items_count, 0, max_item_size, 1);
for (size_t hot_items_count = 1000; hot_items_count <= items_count;
hot_items_count *= 10) {
measure_simple_ops(cache, 0, requests_count, items_count, hot_items_count, max_item_size, 0);
measure_simple_ops(cache, 0, requests_count, items_count, hot_items_count, max_item_size, 1);
measure_simple_ops(cache, 1, requests_count, items_count, hot_items_count, max_item_size, 0);
measure_simple_ops(cache, 1, requests_count, items_count, hot_items_count, max_item_size, 1);
}
for (size_t threads_count = 1; threads_count <= 16; threads_count *= 2) {
measure_multithreaded_ops(cache, 0, threads_count, requests_count, items_count, 10 * 1000, max_item_size, 0);
measure_multithreaded_ops(cache, 0, threads_count, requests_count, items_count, 10 * 1000, max_item_size, 1);
measure_multithreaded_ops(cache, 1, threads_count, requests_count, items_count, 10 * 1000, max_item_size, 0);
measure_multithreaded_ops(cache, 1, threads_count, requests_count, items_count, 10 * 1000, max_item_size, 1);
}
}
printf("All performance tests done\n");
return 0;
}
