Пример #1
0
// 清理所有数据
void CFileQueue::remove()
{
   // m_head.write_seqno ++;
   m_head.read_seqno = m_head.write_seqno;
   m_head.read_offset = 0;
   m_head.write_filesize = 0;
   m_head.queue_size = 0;
   deleteReadFile(m_head.read_seqno);
   openWriteFile();
   openReadFile();
}
Пример #2
0
 /**
  * ����һ��¼, ���ó���Ҫ�����ͷ�
  */
 queue_item *CFileQueue::pop(uint32_t index)
 {
     if (m_readFd == -1) {
         openReadFile();
         if (m_readFd == -1) {
             TBSYS_LOG(WARN, "�ļ�û����: %s:%d", m_queuePath, m_head.read_seqno);
             return NULL;
         }
     }
     int ret, size = 0;
     queue_item *item = NULL;
     int retryReadCount = 0;
     index %= TBFQ_MAX_THREAD_COUNT;
     
     while(retryReadCount<3) {
         int retSize = read(m_readFd, &size, sizeof(int));
         if (retSize < 0) {
             TBSYS_LOG(ERROR, "������, m_readFd:%d, %s(%d)", m_readFd, strerror(errno), errno);
             break;
         }
         if (retSize == sizeof(int)) {
             size -= sizeof(int);
             // ����size
             if (size < (int)sizeof(queue_item) || size > (int)TBFQ_MAX_FILE_SIZE) {
                 int curPos = (int)lseek(m_readFd, 0-retSize, SEEK_CUR);
                 TBSYS_LOG(WARN, "������size����ȷ:%d,curPos:%d,readOff:%d,retry:%d,seqno:%u,m_readFd:%d", 
                     size, curPos, m_head.read_offset, retryReadCount, m_head.read_seqno,m_readFd);
                 retryReadCount ++;
                 if (m_writeFd != -1 && m_head.read_seqno == m_head.write_seqno) {
                     fsync(m_writeFd);
                 }
                 continue;
             }
             // �����ڴ�
             item = (queue_item*) malloc(size);
             assert(item != NULL);
             if ((ret = read(m_readFd, item, size)) != size) {
                 // ���¶�
                 int64_t curPos = lseek(m_readFd, 0-ret-retSize, SEEK_CUR);
                 TBSYS_LOG(WARN, "���ļ�����ȷ:%d<>%d,curPos:%ld,readOff:%d,retry:%d,seqno:%u,m_readFd:%d", 
                     ret, size, curPos, m_head.read_offset, retryReadCount, m_head.read_seqno,m_readFd);
                 retryReadCount ++;
                 free(item);
                 item = NULL;
                 if (m_writeFd != -1 && m_head.read_seqno == m_head.write_seqno) {
                     fsync(m_writeFd);
                 }
                 continue;
             }
             // ����flag
             if (item->flag != TBFQ_FILE_QUEUE_FLAG) {
                  // ���¶�
                 int64_t curPos = lseek(m_readFd, 0-ret-retSize, SEEK_CUR);
                 TBSYS_LOG(WARN, "flag����ȷ:item->flag(%d)<>FLAG(%d),curPos:%ld,readOff:%d,retry:%d,seqno:%u,m_readFd:%d", 
                     item->flag, TBFQ_FILE_QUEUE_FLAG, curPos, m_head.read_offset, 
                     retryReadCount, m_head.read_seqno,m_readFd);
                 retryReadCount ++;
                 free(item);
                 item = NULL;                    
                 if (m_writeFd != -1 && m_head.read_seqno == m_head.write_seqno) {
                     fsync(m_writeFd);
                 }
                 continue;
             }
             // ����len
             if (item->len + (int)sizeof(queue_item) != size) {
                 // ���¶�
                 int64_t curPos = lseek(m_readFd, 0-ret-retSize, SEEK_CUR);
                 TBSYS_LOG(WARN, "������len����ȷ:%ld<>%d,curPos:%ld,readOff:%d,retry:%d,seqno:%u,m_readFd:%d", 
                     item->len + sizeof(queue_item), size, curPos, m_head.read_offset, 
                     retryReadCount, m_head.read_seqno,m_readFd);
                 retryReadCount ++;
                 free(item);
                 item = NULL;
                 if (m_writeFd != -1 && m_head.read_seqno == m_head.write_seqno) {
                     fsync(m_writeFd);
                 }
                 continue;
             }
             
             retryReadCount = 0;
             m_head.pos[index].seqno = m_head.read_seqno;
             m_head.pos[index].offset = m_head.read_offset;
             m_head.read_offset += (size + sizeof(int));
             break;
         } else if (m_head.write_seqno > m_head.read_seqno) {
             // ɾ�����������ļ�
             deleteReadFile();
             m_head.read_seqno ++;
             m_head.read_offset = 0;
             openReadFile();
             if (m_readFd == -1) {
                 m_head.queue_size = 0;
                 break;
             }
         } else {
             if (retSize > 0) {
                 TBSYS_LOG(WARN, "IOæ,retSize:%d,seqno:%u", retSize, m_head.read_seqno);
                 lseek(m_readFd, 0-retSize, SEEK_CUR); 
             }
             m_head.queue_size = 0;
             break;
         }
     }
     if (retryReadCount>=3) { // ����ʧ��������,�Ƶ�����
         backup(index);
         if (m_head.write_seqno > m_head.read_seqno)
         {
             deleteReadFile();
             m_head.read_seqno ++;
             m_head.read_offset = 0;
             openReadFile();
         } else {
             clear();
         }
     }
     writeHead();
     return item;
 }
Пример #3
0
/**
 * 读出一记录, 调用程序要负责释放
 */
queue_item *CFileQueue::pop(uint32_t index)
{
	if (m_readFd == -1) {
		openReadFile();
		if (m_readFd == -1) {
			SYS_LOG(WARN, "open file failed: %s:%d", m_queuePath, m_head.read_seqno);
			return NULL;
		}
	}
	int ret, size = 0;
	queue_item *item = NULL;
	int retryReadCount = 0;
	index %= CTFO_MAX_THREAD_COUNT;

	while(retryReadCount<3) {
		int retSize = read(m_readFd, &size, sizeof(int));
		if (retSize < 0) {
			SYS_LOG(ERROR, "read error, m_readFd:%d, %s(%d)", m_readFd, strerror(errno), errno);
			break;
		}
		if (retSize == sizeof(int)) {
			size -= sizeof(int);
			// 检查size
			if (size < (int)sizeof(queue_item) || size > (int)CTFO_MAX_FILE_SIZE) {
				int curPos = (int)lseek(m_readFd, 0-retSize, SEEK_CUR);
				SYS_LOG(WARN, "read size error:%d,curPos:%d,readOff:%d,retry:%d,seqno:%u,m_readFd:%d",
					size, curPos, m_head.read_offset, retryReadCount, m_head.read_seqno,m_readFd);
				retryReadCount ++;
				if (m_writeFd != -1 && m_head.read_seqno == m_head.write_seqno) {
					fsync(m_writeFd);
				}
				continue;
			}
			// 分配内存
			item = (queue_item*) malloc(size);
			assert(item != NULL);
			if ((ret = read(m_readFd, item, size)) != size) {
				// 重新读
				int64_t curPos = lseek(m_readFd, 0-ret-retSize, SEEK_CUR);
				SYS_LOG(WARN, "read file error:%d<>%d,curPos:%d,readOff:%d,retry:%d,seqno:%u,m_readFd:%d",
					ret, size, (int)curPos, m_head.read_offset, retryReadCount, m_head.read_seqno,m_readFd);
				retryReadCount ++;
				free(item);
				item = NULL;
				if (m_writeFd != -1 && m_head.read_seqno == m_head.write_seqno) {
					fsync(m_writeFd);
				}
				continue;
			}
			// 检查flag
			if (item->flag != CTFO_FILE_QUEUE_FLAG) {
				 // 重新读
				int64_t curPos = lseek(m_readFd, 0-ret-retSize, SEEK_CUR);
				SYS_LOG(WARN, "flag error:item->flag(%d)<>FLAG(%d),curPos:%d,readOff:%d,retry:%d,seqno:%u,m_readFd:%d",
					item->flag, CTFO_FILE_QUEUE_FLAG, (int)curPos, m_head.read_offset,
					retryReadCount, m_head.read_seqno,m_readFd);
				retryReadCount ++;
				free(item);
				item = NULL;
				if (m_writeFd != -1 && m_head.read_seqno == m_head.write_seqno) {
					fsync(m_writeFd);
				}
				continue;
			}
			// 检查len
			if (item->len + (int)sizeof(queue_item) != size) {
				// 重新读
				int64_t curPos = lseek(m_readFd, 0-ret-retSize, SEEK_CUR);
				SYS_LOG(WARN, "read len error:%d<>%d,curPos:%d,readOff:%d,retry:%d,seqno:%u,m_readFd:%d",
					(int)(item->len + sizeof(queue_item)), size, (int)curPos, m_head.read_offset,
					retryReadCount, m_head.read_seqno,m_readFd);
				retryReadCount ++;
				free(item);
				item = NULL;
				if (m_writeFd != -1 && m_head.read_seqno == m_head.write_seqno) {
					fsync(m_writeFd);
				}
				continue;
			}

			retryReadCount = 0;
			m_head.pos[index].seqno = m_head.read_seqno;
			m_head.pos[index].offset = m_head.read_offset;
			m_head.read_offset += (size + sizeof(int));
			break;
		} else if (m_head.write_seqno > m_head.read_seqno) {
			// 删除处理完的文件
			deleteReadFile();
			m_head.read_seqno ++;
			m_head.read_offset = 0;
			openReadFile();
			if (m_readFd == -1) {
				m_head.queue_size = 0;
				break;
			}
		} else {
			if (retSize > 0) {
				SYS_LOG(WARN, "IO busy,retSize:%d,seqno:%u", retSize, m_head.read_seqno);
				lseek(m_readFd, 0-retSize, SEEK_CUR);
			}
			m_head.queue_size = 0;
			break;
		}
	}
	if (retryReadCount>=3) { // 连接失败了三次,移到最后
		backup(index);
		if (m_head.write_seqno > m_head.read_seqno)
		{
			deleteReadFile();
			m_head.read_seqno ++;
			m_head.read_offset = 0;
			openReadFile();
		} else {
			clear();
		}
	}
	writeHead();
	return item;
}