// 清理所有数据
void CFileQueue::remove()
{
// m_head.write_seqno ++;
m_head.read_seqno = m_head.write_seqno;
m_head.read_offset = 0;
m_head.write_filesize = 0;
m_head.queue_size = 0;
deleteReadFile(m_head.read_seqno);
openWriteFile();
openReadFile();
}
/**
* ����һ��¼, ���ó���Ҫ�����ͷ�
*/
queue_item *CFileQueue::pop(uint32_t index)
{
if (m_readFd == -1) {
openReadFile();
if (m_readFd == -1) {
TBSYS_LOG(WARN, "�ļ�û����: %s:%d", m_queuePath, m_head.read_seqno);
return NULL;
}
}
int ret, size = 0;
queue_item *item = NULL;
int retryReadCount = 0;
index %= TBFQ_MAX_THREAD_COUNT;
while(retryReadCount<3) {
int retSize = read(m_readFd, &size, sizeof(int));
if (retSize < 0) {
TBSYS_LOG(ERROR, "������, m_readFd:%d, %s(%d)", m_readFd, strerror(errno), errno);
break;
}
if (retSize == sizeof(int)) {
size -= sizeof(int);
// ����size
if (size < (int)sizeof(queue_item) || size > (int)TBFQ_MAX_FILE_SIZE) {
int curPos = (int)lseek(m_readFd, 0-retSize, SEEK_CUR);
TBSYS_LOG(WARN, "������size����ȷ:%d,curPos:%d,readOff:%d,retry:%d,seqno:%u,m_readFd:%d",
size, curPos, m_head.read_offset, retryReadCount, m_head.read_seqno,m_readFd);
retryReadCount ++;
if (m_writeFd != -1 && m_head.read_seqno == m_head.write_seqno) {
fsync(m_writeFd);
}
continue;
}
// �����ڴ�
item = (queue_item*) malloc(size);
assert(item != NULL);
if ((ret = read(m_readFd, item, size)) != size) {
// ���¶�
int64_t curPos = lseek(m_readFd, 0-ret-retSize, SEEK_CUR);
TBSYS_LOG(WARN, "���ļ�����ȷ:%d<>%d,curPos:%ld,readOff:%d,retry:%d,seqno:%u,m_readFd:%d",
ret, size, curPos, m_head.read_offset, retryReadCount, m_head.read_seqno,m_readFd);
retryReadCount ++;
free(item);
item = NULL;
if (m_writeFd != -1 && m_head.read_seqno == m_head.write_seqno) {
fsync(m_writeFd);
}
continue;
}
// ����flag
if (item->flag != TBFQ_FILE_QUEUE_FLAG) {
// ���¶�
int64_t curPos = lseek(m_readFd, 0-ret-retSize, SEEK_CUR);
TBSYS_LOG(WARN, "flag����ȷ:item->flag(%d)<>FLAG(%d),curPos:%ld,readOff:%d,retry:%d,seqno:%u,m_readFd:%d",
item->flag, TBFQ_FILE_QUEUE_FLAG, curPos, m_head.read_offset,
retryReadCount, m_head.read_seqno,m_readFd);
retryReadCount ++;
free(item);
item = NULL;
if (m_writeFd != -1 && m_head.read_seqno == m_head.write_seqno) {
fsync(m_writeFd);
}
continue;
}
// ����len
if (item->len + (int)sizeof(queue_item) != size) {
// ���¶�
int64_t curPos = lseek(m_readFd, 0-ret-retSize, SEEK_CUR);
TBSYS_LOG(WARN, "������len����ȷ:%ld<>%d,curPos:%ld,readOff:%d,retry:%d,seqno:%u,m_readFd:%d",
item->len + sizeof(queue_item), size, curPos, m_head.read_offset,
retryReadCount, m_head.read_seqno,m_readFd);
retryReadCount ++;
free(item);
item = NULL;
if (m_writeFd != -1 && m_head.read_seqno == m_head.write_seqno) {
fsync(m_writeFd);
}
continue;
}
retryReadCount = 0;
m_head.pos[index].seqno = m_head.read_seqno;
m_head.pos[index].offset = m_head.read_offset;
m_head.read_offset += (size + sizeof(int));
break;
} else if (m_head.write_seqno > m_head.read_seqno) {
// ɾ�����������ļ�
deleteReadFile();
m_head.read_seqno ++;
m_head.read_offset = 0;
openReadFile();
if (m_readFd == -1) {
m_head.queue_size = 0;
break;
}
} else {
if (retSize > 0) {
TBSYS_LOG(WARN, "IOæ,retSize:%d,seqno:%u", retSize, m_head.read_seqno);
lseek(m_readFd, 0-retSize, SEEK_CUR);
}
m_head.queue_size = 0;
break;
}
}
if (retryReadCount>=3) { // ����ʧ��������,�Ƶ�����
backup(index);
if (m_head.write_seqno > m_head.read_seqno)
{
deleteReadFile();
m_head.read_seqno ++;
m_head.read_offset = 0;
openReadFile();
} else {
clear();
}
}
writeHead();
return item;
}
/**
* 读出一记录, 调用程序要负责释放
*/
queue_item *CFileQueue::pop(uint32_t index)
{
if (m_readFd == -1) {
openReadFile();
if (m_readFd == -1) {
SYS_LOG(WARN, "open file failed: %s:%d", m_queuePath, m_head.read_seqno);
return NULL;
}
}
int ret, size = 0;
queue_item *item = NULL;
int retryReadCount = 0;
index %= CTFO_MAX_THREAD_COUNT;
while(retryReadCount<3) {
int retSize = read(m_readFd, &size, sizeof(int));
if (retSize < 0) {
SYS_LOG(ERROR, "read error, m_readFd:%d, %s(%d)", m_readFd, strerror(errno), errno);
break;
}
if (retSize == sizeof(int)) {
size -= sizeof(int);
// 检查size
if (size < (int)sizeof(queue_item) || size > (int)CTFO_MAX_FILE_SIZE) {
int curPos = (int)lseek(m_readFd, 0-retSize, SEEK_CUR);
SYS_LOG(WARN, "read size error:%d,curPos:%d,readOff:%d,retry:%d,seqno:%u,m_readFd:%d",
size, curPos, m_head.read_offset, retryReadCount, m_head.read_seqno,m_readFd);
retryReadCount ++;
if (m_writeFd != -1 && m_head.read_seqno == m_head.write_seqno) {
fsync(m_writeFd);
}
continue;
}
// 分配内存
item = (queue_item*) malloc(size);
assert(item != NULL);
if ((ret = read(m_readFd, item, size)) != size) {
// 重新读
int64_t curPos = lseek(m_readFd, 0-ret-retSize, SEEK_CUR);
SYS_LOG(WARN, "read file error:%d<>%d,curPos:%d,readOff:%d,retry:%d,seqno:%u,m_readFd:%d",
ret, size, (int)curPos, m_head.read_offset, retryReadCount, m_head.read_seqno,m_readFd);
retryReadCount ++;
free(item);
item = NULL;
if (m_writeFd != -1 && m_head.read_seqno == m_head.write_seqno) {
fsync(m_writeFd);
}
continue;
}
// 检查flag
if (item->flag != CTFO_FILE_QUEUE_FLAG) {
// 重新读
int64_t curPos = lseek(m_readFd, 0-ret-retSize, SEEK_CUR);
SYS_LOG(WARN, "flag error:item->flag(%d)<>FLAG(%d),curPos:%d,readOff:%d,retry:%d,seqno:%u,m_readFd:%d",
item->flag, CTFO_FILE_QUEUE_FLAG, (int)curPos, m_head.read_offset,
retryReadCount, m_head.read_seqno,m_readFd);
retryReadCount ++;
free(item);
item = NULL;
if (m_writeFd != -1 && m_head.read_seqno == m_head.write_seqno) {
fsync(m_writeFd);
}
continue;
}
// 检查len
if (item->len + (int)sizeof(queue_item) != size) {
// 重新读
int64_t curPos = lseek(m_readFd, 0-ret-retSize, SEEK_CUR);
SYS_LOG(WARN, "read len error:%d<>%d,curPos:%d,readOff:%d,retry:%d,seqno:%u,m_readFd:%d",
(int)(item->len + sizeof(queue_item)), size, (int)curPos, m_head.read_offset,
retryReadCount, m_head.read_seqno,m_readFd);
retryReadCount ++;
free(item);
item = NULL;
if (m_writeFd != -1 && m_head.read_seqno == m_head.write_seqno) {
fsync(m_writeFd);
}
continue;
}
retryReadCount = 0;
m_head.pos[index].seqno = m_head.read_seqno;
m_head.pos[index].offset = m_head.read_offset;
m_head.read_offset += (size + sizeof(int));
break;
} else if (m_head.write_seqno > m_head.read_seqno) {
// 删除处理完的文件
deleteReadFile();
m_head.read_seqno ++;
m_head.read_offset = 0;
openReadFile();
if (m_readFd == -1) {
m_head.queue_size = 0;
break;
}
} else {
if (retSize > 0) {
SYS_LOG(WARN, "IO busy,retSize:%d,seqno:%u", retSize, m_head.read_seqno);
lseek(m_readFd, 0-retSize, SEEK_CUR);
}
m_head.queue_size = 0;
break;
}
}
if (retryReadCount>=3) { // 连接失败了三次,移到最后
backup(index);
if (m_head.write_seqno > m_head.read_seqno)
{
deleteReadFile();
m_head.read_seqno ++;
m_head.read_offset = 0;
openReadFile();
} else {
clear();
}
}
writeHead();
return item;
}