static void
storeCleanup(void *datanotused)
{
static int bucketnum = -1;
static int validnum = 0;
static int store_errors = 0;
int validnum_start;
StoreEntry *e;
hash_link *link_ptr = NULL;
hash_link *link_next = NULL;
int limit = opt_foreground_rebuild ? 1 << 30 : 500;
validnum_start = validnum;
while (validnum - validnum_start < limit) {
if (++bucketnum >= store_hash_buckets) {
debug(20, 1) (" Completed Validation Procedure\n");
debug(20, 1) (" Validated %d Entries\n", validnum);
debug(20, 1) (" store_swap_size = %dk\n", store_swap_size);
store_dirs_rebuilding--;
assert(0 == store_dirs_rebuilding);
if (opt_store_doublecheck)
assert(store_errors == 0);
if (store_digest)
storeDigestNoteStoreReady();
return;
}
link_next = hash_get_bucket(store_table, bucketnum);
while (NULL != (link_ptr = link_next)) {
link_next = link_ptr->next;
e = (StoreEntry *) link_ptr;
if (EBIT_TEST(e->flags, ENTRY_VALIDATED))
continue;
/*
* Calling storeRelease() has no effect because we're
* still in 'store_rebuilding' state
*/
if (e->swap_filen < 0)
continue;
if (opt_store_doublecheck)
if (storeCleanupDoubleCheck(e))
store_errors++;
EBIT_SET(e->flags, ENTRY_VALIDATED);
/*
* Only set the file bit if we know its a valid entry
* otherwise, set it in the validation procedure
*/
storeDirUpdateSwapSize(&Config.cacheSwap.swapDirs[e->swap_dirn], e->swap_file_sz, 1);
/* Get rid of private objects. Not useful */
if (EBIT_TEST(e->flags, KEY_PRIVATE))
storeRelease(e);
if ((++validnum & 0x3FFFF) == 0)
debug(20, 1) (" %7d Entries Validated so far.\n", validnum);
}
}
eventAdd("storeCleanup", storeCleanup, NULL, 0.0, 1);
}
/* should we digest this entry? used by storeDigestAdd() */
static int
storeDigestAddable(const StoreEntry * e)
{
/* add some stats! XXX */
debug(71, 6) ("storeDigestAddable: checking entry, key: %s\n",
storeKeyText(e->hash.key));
/* check various entry flags (mimics storeCheckCachable XXX) */
if (!EBIT_TEST(e->flags, ENTRY_CACHABLE)) {
debug(71, 6) ("storeDigestAddable: NO: not cachable\n");
return 0;
}
if (EBIT_TEST(e->flags, KEY_PRIVATE)) {
debug(71, 6) ("storeDigestAddable: NO: private key\n");
return 0;
}
if (EBIT_TEST(e->flags, ENTRY_NEGCACHED)) {
debug(71, 6) ("storeDigestAddable: NO: negative cached\n");
return 0;
}
if (EBIT_TEST(e->flags, RELEASE_REQUEST)) {
debug(71, 6) ("storeDigestAddable: NO: release requested\n");
return 0;
}
if (e->store_status == STORE_OK && EBIT_TEST(e->flags, ENTRY_BAD_LENGTH)) {
debug(71, 6) ("storeDigestAddable: NO: wrong content-length\n");
return 0;
}
/* do not digest huge objects */
if (e->swap_file_sz > Config.Store.maxObjectSize) {
debug(71, 6) ("storeDigestAddable: NO: too big\n");
return 0;
}
/* still here? check staleness */
/* Note: We should use the time of the next rebuild, not (cur_time+period) */
if (refreshCheckDigest(e, Config.digest.rebuild_period)) {
debug(71, 6) ("storeDigestAddable: entry expires within %d secs, ignoring\n",
(int) Config.digest.rebuild_period);
return 0;
}
/*
* idea: how about also skipping very fresh (thus, potentially
* unstable) entries? Should be configurable through
* cd_refresh_pattern, of course.
*/
/*
* idea: skip objects that are going to be purged before the next
* update.
*/
#if OLD_UNUSED_CODE /* This code isn't applicable anymore, we can't fix it atm either :( */
if ((squid_curtime + Config.digest.rebuild_period) - e->lastref > storeExpiredReferenceAge())
return 0;
#endif
return 1;
}
/* copy bytes requested by the client */
void
storeClientCopy(StoreEntry * e,
off_t seen_offset,
off_t copy_offset,
size_t size,
char *buf,
STCB * callback,
void *data)
{
store_client *sc;
assert(!EBIT_TEST(e->flags, ENTRY_ABORTED));
debug(20, 3) ("storeClientCopy: %s, seen %d, want %d, size %d, cb %p, cbdata %p\n",
storeKeyText(e->key),
(int) seen_offset,
(int) copy_offset,
(int) size,
callback,
data);
sc = storeClientListSearch(e->mem_obj, data);
assert(sc != NULL);
assert(sc->callback == NULL);
sc->copy_offset = copy_offset;
sc->seen_offset = seen_offset;
sc->callback = callback;
sc->copy_buf = buf;
sc->copy_size = size;
sc->copy_offset = copy_offset;
storeClientCopy2(e, sc);
}
/* copy bytes requested by the client */
void
storeClientCopy(store_client * sc,
StoreEntry * e,
squid_off_t seen_offset,
squid_off_t copy_offset,
size_t size,
char *buf,
STCB * callback,
void *data)
{
assert(!EBIT_TEST(e->flags, ENTRY_ABORTED));
debug(20, 3) ("storeClientCopy: %s, seen %" PRINTF_OFF_T ", want %" PRINTF_OFF_T ", size %d, cb %p, cbdata %p\n",
storeKeyText(e->hash.key),
seen_offset,
copy_offset,
(int) size,
callback,
data);
assert(sc != NULL);
#if STORE_CLIENT_LIST_DEBUG
assert(sc == storeClientListSearch(e->mem_obj, data));
#endif
assert(sc->callback == NULL);
assert(sc->entry == e);
sc->seen_offset = seen_offset;
sc->callback = callback;
sc->copy_buf = buf;
sc->copy_size = size;
sc->copy_offset = copy_offset;
storeClientCopy2(e, sc);
}
static void
peerMonitorFetchReplyHeaders(void *data, char *buf, ssize_t size)
{
PeerMonitor *pm = data;
HttpReply *reply;
http_status status;
if (EBIT_TEST(pm->running.e->flags, ENTRY_ABORTED))
goto completed;
if (size <= 0)
goto completed;
if (!cbdataValid(pm->peer))
goto completed;
reply = pm->running.e->mem_obj->reply;
assert(reply);
status = reply->sline.status;
pm->running.status = status;
if (status != HTTP_OK)
goto completed;
if (size > reply->hdr_sz) {
pm->running.size = size - reply->hdr_sz;
pm->running.offset = size;
} else {
pm->running.size = 0;
pm->running.offset = reply->hdr_sz;
}
storeClientCopy(pm->running.sc, pm->running.e, pm->running.offset, pm->running.offset, 4096, buf, peerMonitorFetchReply, pm);
return;
completed:
/* We are fully done with this monitoring request. Clean up */
peerMonitorCompleted(pm);
return;
}
static void
errorMapFetchHeaders(void *data, mem_node_ref nr, ssize_t size)
{
ErrorMapState *state = data;
HttpReply *reply;
http_status status;
if (EBIT_TEST(state->e->flags, ENTRY_ABORTED))
goto abort;
if (size == 0)
goto abort;
if (!cbdataValid(state->callback_data))
goto abort;
reply = state->e->mem_obj->reply;
status = reply->sline.status;
if (status != HTTP_OK)
goto abort;
/* Send object to caller (cbdataValid verified above) */
state->callback(state->e, reply->hdr_sz, httpHeaderGetSize(&reply->header, HDR_CONTENT_LENGTH), state->callback_data);
errorMapFetchComplete(state);
stmemNodeUnref(&nr);
return;
abort:
errorMapFetchAbort(state);
stmemNodeUnref(&nr);
return;
}
void
storeSwapInStart(store_client * sc)
{
StoreEntry *e = sc->entry;
assert(e->mem_status == NOT_IN_MEMORY);
if (!EBIT_TEST(e->flags, ENTRY_VALIDATED)) {
/* We're still reloading and haven't validated this entry yet */
return;
}
debug(20, 3) ("storeSwapInStart: called for %08X %s \n",
e->swap_file_number, storeKeyText(e->key));
if (e->swap_status != SWAPOUT_WRITING && e->swap_status != SWAPOUT_DONE) {
debug(20, 1) ("storeSwapInStart: bad swap_status (%s)\n",
swapStatusStr[e->swap_status]);
return;
}
if (e->swap_file_number < 0) {
debug(20, 1) ("storeSwapInStart: swap_file_number < 0\n");
return;
}
assert(e->mem_obj != NULL);
debug(20, 3) ("storeSwapInStart: Opening fileno %08X\n",
e->swap_file_number);
sc->swapin_sio = storeOpen(e->swap_file_number,
O_RDONLY,
storeSwapInFileClosed,
sc);
cbdataLock(sc->swapin_sio);
}
static void
storeClientCopy2(StoreEntry * e, store_client * sc)
{
if (sc->flags.copy_event_pending)
return;
if (EBIT_TEST(e->flags, ENTRY_FWD_HDR_WAIT)) {
debug(20, 5) ("storeClientCopy2: returning because ENTRY_FWD_HDR_WAIT set\n");
return;
}
if (sc->flags.store_copying) {
sc->flags.copy_event_pending = 1;
debug(20, 3) ("storeClientCopy2: Queueing storeClientCopyEvent()\n");
eventAdd("storeClientCopyEvent", storeClientCopyEvent, sc, 0.0, 0);
return;
}
cbdataLock(sc); /* ick, prevent sc from getting freed */
sc->flags.store_copying = 1;
debug(20, 3) ("storeClientCopy2: %s\n", storeKeyText(e->hash.key));
assert(sc->callback != NULL);
/*
* We used to check for ENTRY_ABORTED here. But there were some
* problems. For example, we might have a slow client (or two) and
* the server-side is reading far ahead and swapping to disk. Even
* if the server-side aborts, we want to give the client(s)
* everything we got before the abort condition occurred.
*/
storeClientCopy3(e, sc);
sc->flags.store_copying = 0;
cbdataUnlock(sc); /* ick, allow sc to be freed */
}
void
storeLog(int tag, const StoreEntry * e)
{
MemBuf mb;
MemObject *mem = e->mem_obj;
HttpReply *reply;
if (storelog_fd < 0)
return;
if (mem == NULL)
return;
if (EBIT_TEST(e->flags, ENTRY_DONT_LOG))
return;
if (mem->log_url == NULL) {
debug(20, 1) ("storeLog: NULL log_url for %s\n", mem->url);
storeMemObjectDump(mem);
mem->log_url = xstrdup(mem->url);
}
memBufDefInit(&mb);
reply = mem->reply;
memBufPrintf(&mb, "%9d.%03d %-7s %08X %4d %9d %9d %9d %s %d/%d %s %s\n",
(int) current_time.tv_sec,
(int) current_time.tv_usec / 1000,
storeLogTags[tag],
e->swap_file_number,
reply->sline.status,
(int) reply->date,
(int) reply->last_modified,
(int) reply->expires,
strLen(reply->content_type) ? strBuf(reply->content_type) : "unknown",
reply->content_length,
(int) (mem->inmem_hi - mem->reply->hdr_sz),
RequestMethodStr[mem->method],
mem->log_url);
file_write_mbuf(storelog_fd, -1, mb, NULL, NULL);
}
/* create MIME Header for Gopher Data */
static void
gopherMimeCreate(GopherStateData * gopherState)
{
StoreEntry *e = gopherState->entry;
HttpReply *reply = e->mem_obj->reply;
http_version_t version;
const char *mime_type = NULL;
const char *mime_enc = NULL;
switch (gopherState->type_id) {
case GOPHER_DIRECTORY:
case GOPHER_INDEX:
case GOPHER_HTML:
case GOPHER_WWW:
case GOPHER_CSO:
mime_type = "text/html";
break;
case GOPHER_GIF:
case GOPHER_IMAGE:
case GOPHER_PLUS_IMAGE:
mime_type = "image/gif";
break;
case GOPHER_SOUND:
case GOPHER_PLUS_SOUND:
mime_type = "audio/basic";
break;
case GOPHER_PLUS_MOVIE:
mime_type = "video/mpeg";
break;
case GOPHER_MACBINHEX:
mime_type = "application/macbinary";
break;
case GOPHER_DOSBIN:
case GOPHER_UUENCODED:
case GOPHER_BIN:
case GOPHER_FILE:
default:
/* Rightnow We have no idea what it is. */
mime_type = mimeGetContentType(gopherState->request);
mime_enc = mimeGetContentEncoding(gopherState->request);
break;
}
storeBuffer(e);
httpReplyReset(reply);
EBIT_CLR(gopherState->entry->flags, ENTRY_FWD_HDR_WAIT);
httpBuildVersion(&version, 1, 0);
httpReplySetHeaders(reply, version, HTTP_OK, "Gatewaying", mime_type, -1, -1, -1);
if (mime_enc)
httpHeaderPutStr(&reply->header, HDR_CONTENT_ENCODING, mime_enc);
httpReplySwapOut(reply, e);
reply->hdr_sz = e->mem_obj->inmem_hi;
storeTimestampsSet(e);
if (EBIT_TEST(e->flags, ENTRY_CACHABLE)) {
storeSetPublicKey(e);
} else {
storeRelease(e);
}
}
void
neighborsHtcpReply(const cache_key * key, htcpReplyData * htcp, const struct sockaddr_in *from)
{
StoreEntry *e = storeGet(key);
MemObject *mem = NULL;
peer *p;
peer_t ntype = PEER_NONE;
debug(15, 6) ("neighborsHtcpReply: %s %s\n",
htcp->hit ? "HIT" : "MISS", storeKeyText(key));
if (NULL != (e = storeGet(key)))
mem = e->mem_obj;
if ((p = whichPeer(from)))
neighborAliveHtcp(p, mem, htcp);
/* Does the entry exist? */
if (NULL == e) {
debug(12, 3) ("neighborsHtcpReply: Cache key '%s' not found\n",
storeKeyText(key));
neighborCountIgnored(p);
return;
}
/* check if someone is already fetching it */
if (EBIT_TEST(e->flags, ENTRY_DISPATCHED)) {
debug(15, 3) ("neighborsHtcpReply: '%s' already being fetched.\n",
storeKeyText(key));
neighborCountIgnored(p);
return;
}
if (mem == NULL) {
debug(15, 2) ("Ignoring reply for missing mem_obj: %s\n",
storeKeyText(key));
neighborCountIgnored(p);
return;
}
if (e->ping_status != PING_WAITING) {
debug(15, 2) ("neighborsHtcpReply: Entry %s is not PING_WAITING\n",
storeKeyText(key));
neighborCountIgnored(p);
return;
}
if (e->lock_count == 0) {
debug(12, 1) ("neighborsHtcpReply: '%s' has no locks\n",
storeKeyText(key));
neighborCountIgnored(p);
return;
}
if (p) {
ntype = neighborType(p, mem->request);
neighborUpdateRtt(p, mem);
}
if (ignoreMulticastReply(p, mem)) {
neighborCountIgnored(p);
return;
}
debug(15, 3) ("neighborsHtcpReply: e = %p\n", e);
mem->ping_reply_callback(p, ntype, PROTO_HTCP, htcp, mem->ircb_data);
}
/*
* copied from httpReadReply()
*
* by the time this is called, the ICAP headers have already
* been read.
*/
void
icapReadReply(int fd, void *data)
{
IcapStateData *icap = data;
StoreEntry *entry = icap->respmod.entry;
const request_t *request = icap->request;
int len;
debug(81, 5) ("icapReadReply: FD %d: icap %p.\n", fd, data);
if (icap->flags.no_content && !icap->flags.http_server_eof) { //AI
return;
}
if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
comm_close(fd);
return;
}
errno = 0;
statCounter.syscalls.sock.reads++;
len = memBufRead(fd, &icap->chunk_buf);
debug(81, 5) ("icapReadReply: FD %d: len %d.\n", fd, len);
if (len > 0) {
fd_bytes(fd, len, FD_READ);
kb_incr(&statCounter.icap.all.kbytes_in, len);
commSetTimeout(fd, Config.Timeout.read, icapReadTimeout, icap);
if (icap->chunk_buf.size < icap->chunk_buf.capacity) {
*(icap->chunk_buf.buf + icap->chunk_buf.size) = '\0';
debug(81, 9) ("{%s}\n", icap->chunk_buf.buf);
}
}
if (len <= 0) {
debug(81, 2) ("icapReadReply: FD %d: read failure: %s.\n",
fd, xstrerror());
if (ignoreErrno(errno)) {
debug(81, 2) ("icapReadReply: FD %d: ignored errno\n", fd);
commSetSelect(fd, COMM_SELECT_READ, icapReadReply, icap, 0);
} else if (entry->mem_obj->inmem_hi == 0) {
ErrorState *err;
debug(81, 2) ("icapReadReply: FD %d: generating error page\n", fd);
err = errorCon(ERR_ICAP_FAILURE, HTTP_INTERNAL_SERVER_ERROR);
err->request = requestLink((request_t *) request);
err->xerrno = errno;
errorAppendEntry(entry, err);
comm_close(fd);
} else {
debug(81, 2) ("icapReadReply: FD %d: just calling comm_close()\n",
fd);
comm_close(fd);
}
return;
}
if (icapReadReply2(icap) < 0)
comm_close(fd);
}
/*
* An entry written to the swap log MUST have the following
* properties.
* 1. It MUST be a public key. It does no good to log
* a public ADD, change the key, then log a private
* DEL. So we need to log a DEL before we change a
* key from public to private.
* 2. It MUST have a valid (> -1) swap_file_number.
*/
void
storeDirSwapLog(const StoreEntry * e, int op)
{
int dirn = e->swap_file_number >> SWAP_DIR_SHIFT;
SwapDir *sd;
assert(dirn < Config.cacheSwap.n_configured);
assert(!EBIT_TEST(e->flags, KEY_PRIVATE));
assert(e->swap_file_number >= 0);
/*
* icons and such; don't write them to the swap log
*/
if (EBIT_TEST(e->flags, ENTRY_SPECIAL))
return;
assert(op > SWAP_LOG_NOP && op < SWAP_LOG_MAX);
debug(20, 3) ("storeDirSwapLog: %s %s %08X\n",
swap_log_op_str[op],
storeKeyText(e->key),
e->swap_file_number);
sd = &Config.cacheSwap.swapDirs[dirn];
sd->log.write(sd, e, op);
}
static void
whoisReadReply(int fd, void *data)
{
WhoisState *p = data;
StoreEntry *entry = p->entry;
char *buf = memAllocate(MEM_4K_BUF);
MemObject *mem = entry->mem_obj;
int len;
statCounter.syscalls.sock.reads++;
len = FD_READ_METHOD(fd, buf, 4095);
buf[len] = '\0';
debug(75, 3) ("whoisReadReply: FD %d read %d bytes\n", fd, len);
debug(75, 5) ("{%s}\n", buf);
if (len > 0) {
if (0 == mem->inmem_hi) {
http_reply *reply = mem->reply;
http_version_t version;
storeBuffer(entry);
httpBuildVersion(&version, 1, 0);
httpReplySetHeaders(reply, version, HTTP_OK, "Gatewaying", "text/plain", -1, -1, -2);
httpReplySwapOut(reply, entry);
}
fd_bytes(fd, len, FD_READ);
kb_incr(&statCounter.server.all.kbytes_in, len);
kb_incr(&statCounter.server.http.kbytes_in, len);
storeAppend(entry, buf, len);
storeBufferFlush(entry);
commSetSelect(fd, COMM_SELECT_READ, whoisReadReply, p, Config.Timeout.read);
} else if (len < 0) {
debug(50, 2) ("whoisReadReply: FD %d: read failure: %s.\n",
fd, xstrerror());
if (ignoreErrno(errno)) {
commSetSelect(fd, COMM_SELECT_READ, whoisReadReply, p, Config.Timeout.read);
} else {
ErrorState *err;
err = errorCon(ERR_READ_ERROR, HTTP_INTERNAL_SERVER_ERROR, p->fwd->request);
err->xerrno = errno;
fwdFail(p->fwd, err);
comm_close(fd);
}
} else {
storeTimestampsSet(entry);
storeBufferFlush(entry);
if (!EBIT_TEST(entry->flags, RELEASE_REQUEST))
storeSetPublicKey(entry);
fwdComplete(p->fwd);
debug(75, 3) ("whoisReadReply: Done: %s\n", storeUrl(entry));
comm_close(fd);
}
memFree(buf, MEM_4K_BUF);
}
/* return 1 if the request should be aborted */
static int
CheckQuickAbort2(StoreEntry * entry)
{
squid_off_t curlen;
squid_off_t minlen;
squid_off_t expectlen;
MemObject *mem = entry->mem_obj;
assert(mem);
debug(20, 3) ("CheckQuickAbort2: entry=%p, mem=%p\n", entry, mem);
if (mem->request && !mem->request->flags.cachable) {
debug(20, 3) ("CheckQuickAbort2: YES !mem->request->flags.cachable\n");
return 1;
}
expectlen = httpReplyBodySize(mem->method, mem->reply) + mem->reply->hdr_sz;
curlen = mem->inmem_hi;
if (expectlen == curlen) {
debug(20, 3) ("CheckQuickAbort2: NO already finished\n");
return 0;
}
if (EBIT_TEST(entry->flags, KEY_PRIVATE)) {
debug(20, 3) ("CheckQuickAbort2: YES KEY_PRIVATE\n");
return 1;
}
minlen = Config.quickAbort.min << 10;
if (minlen < 0) {
debug(20, 3) ("CheckQuickAbort2: NO disabled\n");
return 0;
}
if (curlen > expectlen) {
debug(20, 3) ("CheckQuickAbort2: YES bad content length\n");
return 1;
}
if ((expectlen - curlen) < minlen) {
debug(20, 3) ("CheckQuickAbort2: NO only little more left\n");
return 0;
}
if ((expectlen - curlen) > (Config.quickAbort.max << 10)) {
debug(20, 3) ("CheckQuickAbort2: YES too much left to go\n");
return 1;
}
if (expectlen < 100) {
debug(20, 3) ("CheckQuickAbort2: NO avoid FPE\n");
return 0;
}
if ((curlen / (expectlen / 100)) > Config.quickAbort.pct) {
debug(20, 3) ("CheckQuickAbort2: NO past point of no return\n");
return 0;
}
debug(20, 3) ("CheckQuickAbort2: YES default, returning 1\n");
return 1;
}
static int
hookSwdaeSwapRegiestMemPd(HttpStateData *hs, char *buf, ssize_t len)
{
StoreEntry *e = hs ? hs->entry : NULL;
if (0 == g_CheckReply_Flags && e->mem_obj && e && EBIT_TEST(e->flags, ENTRY_CACHABLE))
{
SD_MemPd_t *ssPd = cc_get_mod_private_data(MEMOBJECT_PRIVATE_DATA, e->mem_obj , mod);
if (NULL == ssPd) ssPd = p_Mod_AllocMemPd();
ssPd->fd = hs->fd;
cc_register_mod_private_data(MEMOBJECT_PRIVATE_DATA, e->mem_obj, ssPd, p_Mod_FreeMemPd, mod);
debug(172,5)("%s %s %s %s\n", PMOD_NAME,__func__, ssPd ? "cache":"no-cache", storeUrl(e));
}
return 0;
}
static void
heap_add(RemovalPolicy * policy, StoreEntry * entry, RemovalPolicyNode * node)
{
HeapPolicyData *heap = policy->_data;
assert(!node->data);
if (EBIT_TEST(entry->flags, ENTRY_SPECIAL))
return; /* We won't manage these.. they messes things up */
node->data = heap_insert(heap->heap, entry);
heap->count += 1;
if (!heap->type)
heap->type = heap_guessType(entry, node);
/* Add a little more variance to the aging factor */
heap->heap->age += heap->heap->age / 100000000;
}
void
storeLog(int tag, const StoreEntry * e)
{
MemObject *mem = e->mem_obj;
HttpReply *reply;
if (NULL == storelog)
return;
#if UNUSED_CODE
if (EBIT_TEST(e->flags, ENTRY_DONT_LOG))
return;
#endif
if (mem != NULL) {
if (mem->log_url == NULL) {
debug(20, 1) ("storeLog: NULL log_url for %s\n", mem->url);
storeMemObjectDump(mem);
mem->log_url = xstrdup(mem->url);
}
reply = mem->reply;
/*
* XXX Ok, where should we print the dir number here?
* Because if we print it before the swap file number, it'll break
* the existing log format.
*/
logfilePrintf(storelog, "%9d.%03d %-7s %02d %08X %s %4d %9d %9d %9d %s %d/%d %s %s\n",
(int) current_time.tv_sec,
(int) current_time.tv_usec / 1000,
storeLogTags[tag],
e->swap_dirn,
e->swap_filen,
storeKeyText(e->hash.key),
reply->sline.status,
(int) reply->date,
(int) reply->last_modified,
(int) reply->expires,
strLen(reply->content_type) ? strBuf(reply->content_type) : "unknown",
reply->content_length,
(int) (mem->inmem_hi - mem->reply->hdr_sz),
RequestMethodStr[mem->method],
mem->log_url);
} else {
/* no mem object. Most RELEASE cases */
logfilePrintf(storelog, "%9d.%03d %-7s %02d %08X %s ? ? ? ? ?/? ?/? ? ?\n",
(int) current_time.tv_sec,
(int) current_time.tv_usec / 1000,
storeLogTags[tag],
e->swap_dirn,
e->swap_filen,
storeKeyText(e->hash.key));
}
}
void
storeLog(int tag, const StoreEntry * e)
{
MemObject *mem = e->mem_obj;
HttpReply *reply;
if (NULL == storelog)
return;
#if UNUSED_CODE
if (EBIT_TEST(e->flags, ENTRY_DONT_LOG))
return;
#endif
if (mem != NULL) {
reply = mem->reply;
/*
* XXX Ok, where should we print the dir number here?
* Because if we print it before the swap file number, it'll break
* the existing log format.
*/
logfileLineStart(storelog);
logfilePrintf(storelog, "%9ld.%03d %-7s %02d %08X %s %4d %9ld %9ld %9ld %s %" PRINTF_OFF_T "/%" PRINTF_OFF_T " %s %s\n",
(long int) current_time.tv_sec,
(int) current_time.tv_usec / 1000,
storeLogTags[tag],
e->swap_dirn,
e->swap_filen,
storeKeyText(e->hash.key),
reply->sline.status,
(long int) reply->date,
(long int) reply->last_modified,
(long int) reply->expires,
strLen(reply->content_type) ? strBuf(reply->content_type) : "unknown",
reply->content_length,
mem->inmem_hi - mem->reply->hdr_sz,
RequestMethods[mem->method].str,
rfc1738_escape_unescaped(mem->url));
logfileLineEnd(storelog);
} else {
/* no mem object. Most RELEASE cases */
logfileLineStart(storelog);
logfilePrintf(storelog, "%9ld.%03d %-7s %02d %08X %s ? ? ? ? ?/? ?/? ? ?\n",
(long int) current_time.tv_sec,
(int) current_time.tv_usec / 1000,
storeLogTags[tag],
e->swap_dirn,
e->swap_filen,
storeKeyText(e->hash.key));
logfileLineEnd(storelog);
}
}
static void
storeUpdateCopy(void *data, char *buf, ssize_t size)
{
StoreUpdateState *state = data;
if (EBIT_TEST(state->newentry->flags, ENTRY_ABORTED)) {
debug(20, 1) ("storeUpdateCopy: Aborted at %d (%d)\n", (int) state->offset, (int) size);
/* the abort callback deals with the needed cleanup */
return;
}
if (EBIT_TEST(state->newentry->flags, KEY_PRIVATE) && state->newentry->mem_obj->nclients == 0) {
debug(20, 2) ("storeUpdateCopy: Gone stale with no clients, skip copying of the rest\n");
storeUpdateDone(state);
return;
}
if (size < 0) {
debug(20, 1) ("storeUpdateCopy: Error at %d (%d)\n", (int) state->offset, (int) size);
storeUpdateDone(state);
return;
}
if (size > 0) {
storeAppend(state->newentry, buf, size);
if (EBIT_TEST(state->newentry->flags, ENTRY_ABORTED)) {
debug(20, 1) ("storeUpdateCopy: Aborted on write at %d (%d)\n", (int) state->offset, (int) size);
return;
}
state->offset += size;
storeClientCopy(state->sc, state->oldentry, state->offset, state->offset, sizeof(state->buf), state->buf, storeUpdateCopy, state);
return;
} else {
storeComplete(state->newentry);
storeUnlockObject(state->newentry);
state->newentry = NULL;
storeUpdateDone(state);
}
}
static void
CheckQuickAbort(StoreEntry * entry)
{
if (entry == NULL)
return;
if (storePendingNClients(entry) > 0)
return;
if (entry->store_status != STORE_PENDING)
return;
if (EBIT_TEST(entry->flags, ENTRY_SPECIAL))
return;
if (CheckQuickAbort2(entry) == 0)
return;
statCounter.aborted_requests++;
storeAbort(entry);
}
static int
peerSelectIcpPing(request_t * request, int direct, StoreEntry * entry)
{
int n;
assert(entry);
assert(entry->ping_status == PING_NONE);
assert(direct != DIRECT_YES);
debug(44, 3) ("peerSelectIcpPing: %s\n", storeUrl(entry));
if (!request->flags.hierarchical && direct != DIRECT_NO)
return 0;
if (EBIT_TEST(entry->flags, KEY_PRIVATE) && !neighbors_do_private_keys)
if (direct != DIRECT_NO)
return 0;
n = neighborsCount(request);
debug(44, 3) ("peerSelectIcpPing: counted %d neighbors\n", n);
return n;
}
/*
* Function: errorAppendEntry
*
* Arguments: err - This object is destroyed after use in this function.
*
* Abstract: This function generates a error page from the info contained
* by 'err' and then stores the text in the specified store
* entry. This function should only be called by ``server
* side routines'' which need to communicate errors to the
* client side. It should also be called from client_side.c
* because we now support persistent connections, and
* cannot assume that we can immediately write to the socket
* for an error.
*/
void
errorAppendEntry(StoreEntry * entry, ErrorState * err)
{
HttpReply *rep;
MemObject *mem = entry->mem_obj;
assert(mem != NULL);
assert(mem->inmem_hi == 0);
if (entry->store_status != STORE_PENDING) {
/*
* If the entry is not STORE_PENDING, then no clients
* care about it, and we don't need to generate an
* error message
*/
assert(EBIT_TEST(entry->flags, ENTRY_ABORTED));
assert(mem->nclients == 0);
errorStateFree(err);
return;
}
if (err->page_id == TCP_RESET) {
if (err->request) {
debug(4, 2) ("RSTing this reply\n");
err->request->flags.reset_tcp = 1;
}
}
storeLockObject(entry);
storeBuffer(entry);
rep = errorBuildReply(err);
/* Add authentication header */
/* TODO: alter errorstate to be accel on|off aware. The 0 on the next line
* depends on authenticate behaviour: all schemes to date send no extra data
* on 407/401 responses, and do not check the accel state on 401/407 responses
*/
authenticateFixHeader(rep, err->auth_user_request, err->request, 0, 1);
httpReplySwapOut(rep, entry);
httpReplyAbsorb(mem->reply, rep);
EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT);
storeBufferFlush(entry);
storeComplete(entry);
storeNegativeCache(entry);
storeReleaseRequest(entry);
storeUnlockObject(entry);
errorStateFree(err);
}
/* This will be called when data is ready to be read from fd. Read until
* error or connection closed. */
static void
gopherReadReply(int fd, void *data)
{
GopherStateData *gopherState = data;
StoreEntry *entry = gopherState->entry;
char *buf = NULL;
int len;
int clen;
int bin;
size_t read_sz;
#if DELAY_POOLS
delay_id delay_id;
#endif
if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
comm_close(fd);
return;
}
errno = 0;
buf = memAllocate(MEM_4K_BUF);
read_sz = 4096 - 1; /* leave room for termination */
#if DELAY_POOLS
delay_id = delayMostBytesAllowed(entry->mem_obj, &read_sz);
#endif
/* leave one space for \0 in gopherToHTML */
statCounter.syscalls.sock.reads++;
len = FD_READ_METHOD(fd, buf, read_sz);
if (len > 0) {
fd_bytes(fd, len, FD_READ);
#if DELAY_POOLS
delayBytesIn(delay_id, len);
#endif
kb_incr(&statCounter.server.all.kbytes_in, len);
kb_incr(&statCounter.server.other.kbytes_in, len);
}
debug(10, 5) ("gopherReadReply: FD %d read len=%d\n", fd, len);
if (len > 0) {
commSetTimeout(fd, Config.Timeout.read, NULL, NULL);
IOStats.Gopher.reads++;
for (clen = len - 1, bin = 0; clen; bin++)
clen >>= 1;
IOStats.Gopher.read_hist[bin]++;
}
static void
peerMonitorFetchReplyHeaders(void *data, mem_node_ref nr, ssize_t size)
{
PeerMonitor *pm = data;
const char *buf = NULL;
http_status status;
HttpReply *reply;
if (EBIT_TEST(pm->running.e->flags, ENTRY_ABORTED))
goto completed;
if (size <= 0)
goto completed;
if (!cbdataValid(pm->peer))
goto completed;
buf = nr.node->data + nr.offset;
reply = pm->running.e->mem_obj->reply;
assert(reply);
status = reply->sline.status;
pm->running.status = status;
if (status != HTTP_OK)
goto completed;
if (size > reply->hdr_sz) {
pm->running.size = size - reply->hdr_sz;
pm->running.offset = size;
} else {
pm->running.size = 0;
pm->running.offset = reply->hdr_sz;
}
storeClientRef(pm->running.sc, pm->running.e, pm->running.offset, pm->running.offset, SM_PAGE_SIZE, peerMonitorFetchReply, pm);
stmemNodeUnref(&nr);
return;
completed:
/* We are fully done with this monitoring request. Clean up */
stmemNodeUnref(&nr);
peerMonitorCompleted(pm);
return;
}
/* This will be called when data is ready to be read from fd. Read until
* error or connection closed. */
static void
waisReadReply(int fd, void *data)
{
WaisStateData *waisState = data;
LOCAL_ARRAY(char, buf, 4096);
StoreEntry *entry = waisState->entry;
int len;
int clen;
int bin;
size_t read_sz;
#if DELAY_POOLS
delay_id delay_id;
#endif
if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
comm_close(fd);
return;
}
errno = 0;
read_sz = 4096;
#if DELAY_POOLS
delay_id = delayMostBytesAllowed(entry->mem_obj, &read_sz);
#endif
statCounter.syscalls.sock.reads++;
len = FD_READ_METHOD(fd, buf, read_sz);
if (len > 0) {
fd_bytes(fd, len, FD_READ);
#if DELAY_POOLS
delayBytesIn(delay_id, len);
#endif
kb_incr(&statCounter.server.all.kbytes_in, len);
kb_incr(&statCounter.server.other.kbytes_in, len);
}
debug(24, 5) ("waisReadReply: FD %d read len:%d\n", fd, len);
if (len > 0) {
commSetTimeout(fd, Config.Timeout.read, NULL, NULL);
IOStats.Wais.reads++;
for (clen = len - 1, bin = 0; clen; bin++)
clen >>= 1;
IOStats.Wais.read_hist[bin]++;
}
static void
icapReadReply3(IcapStateData * icap)
{
StoreEntry *entry = icap->respmod.entry;
int fd = icap->icap_fd;
debug(81, 3) ("icapReadReply3\n");
if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
debug(81, 3) ("icapReadReply3: Entry Aborded\n");
comm_close(fd);
} else if (icapPconnTransferDone(fd, icap)) {
storeComplete(entry);
icapRespModKeepAliveOrClose(icap);
} else if (!icap->flags.no_content) {
/* Wait for EOF condition */
commSetSelect(fd, COMM_SELECT_READ, icapReadReply, icap, 0);
debug(81,
3)
("icapReadReply3: Going to read mode data throught icapReadReply\n");
} else {
debug(81, 3) ("icapReadReply3: Nothing\n");
}
}
static store_client_t
storeClientType(StoreEntry * e)
{
MemObject *mem = e->mem_obj;
if (mem->inmem_lo)
return STORE_DISK_CLIENT;
if (EBIT_TEST(e->flags, ENTRY_ABORTED)) {
/* I don't think we should be adding clients to aborted entries */
debug(20, 1) ("storeClientType: adding to ENTRY_ABORTED entry\n");
return STORE_MEM_CLIENT;
}
if (e->store_status == STORE_OK) {
if (mem->inmem_lo == 0 && mem->inmem_hi > 0)
return STORE_MEM_CLIENT;
else
return STORE_DISK_CLIENT;
}
/* here and past, entry is STORE_PENDING */
/*
* If this is the first client, let it be the mem client
*/
else if (mem->nclients == 1)
return STORE_MEM_CLIENT;
/*
* If there is no disk file to open yet, we must make this a
* mem client. If we can't open the swapin file before writing
* to the client, there is no guarantee that we will be able
* to open it later when we really need it.
*/
else if (e->swap_status == SWAPOUT_NONE)
return STORE_MEM_CLIENT;
/*
* otherwise, make subsequent clients read from disk so they
* can not delay the first, and vice-versa.
*/
else
return STORE_DISK_CLIENT;
}
/* copy bytes requested by the client */
void
storeClientCopy(store_client * sc,
StoreEntry * e,
squid_off_t seen_offset,
squid_off_t copy_offset,
size_t size,
char *buf,
STCB * callback,
void *data)
{
debug(20, 3) ("storeClientCopy: %s, seen %" PRINTF_OFF_T ", want %" PRINTF_OFF_T ", size %d, cb %p, cbdata %p\n",
storeKeyText(e->hash.key),
seen_offset,
copy_offset,
(int) size,
callback,
data);
assert(sc != NULL);
#if STORE_CLIENT_LIST_DEBUG
assert(sc == storeClientListSearch(e->mem_obj, data));
#endif
assert(sc->callback == NULL);
assert(sc->entry == e);
sc->seen_offset = seen_offset;
sc->callback = callback;
sc->callback_data = data;
cbdataLock(sc->callback_data);
sc->copy_buf = buf;
sc->copy_size = size;
sc->copy_offset = copy_offset;
/* If the read is being deferred, run swapout in case this client has the
* lowest seen_offset. storeSwapOut() frees the memory and clears the
* ENTRY_DEFER_READ bit if necessary */
if (EBIT_TEST(e->flags, ENTRY_DEFER_READ)) {
storeSwapOut(e);
}
storeClientCopy2(e, sc);
}
void
storeDigestDel(const StoreEntry * entry)
{
#if USE_CACHE_DIGESTS
if (!Config.onoff.digest_generation) {
return;
}
assert(entry && store_digest);
debug(71, 6) ("storeDigestDel: checking entry, key: %s\n",
storeKeyText(entry->hash.key));
if (!EBIT_TEST(entry->flags, KEY_PRIVATE)) {
if (!cacheDigestTest(store_digest, entry->hash.key)) {
sd_stats.del_lost_count++;
debug(71, 6) ("storeDigestDel: lost entry, key: %s url: %s\n",
storeKeyText(entry->hash.key), storeUrl(entry));
} else {
sd_stats.del_count++;
cacheDigestDel(store_digest, entry->hash.key);
debug(71, 6) ("storeDigestDel: deled entry, key: %s\n",
storeKeyText(entry->hash.key));
}
}
#endif
}
