/*
* ShmemAllocNoError -- allocate max-aligned chunk from shared memory
*
* As ShmemAlloc, but returns NULL if out of space, rather than erroring.
*/
void *
ShmemAllocNoError(Size size)
{
Size newStart;
Size newFree;
void *newSpace;
/*
* Ensure all space is adequately aligned. We used to only MAXALIGN this
* space but experience has proved that on modern systems that is not good
* enough. Many parts of the system are very sensitive to critical data
* structures getting split across cache line boundaries. To avoid that,
* attempt to align the beginning of the allocation to a cache line
* boundary. The calling code will still need to be careful about how it
* uses the allocated space - e.g. by padding each element in an array of
* structures out to a power-of-two size - but without this, even that
* won't be sufficient.
*/
size = CACHELINEALIGN(size);
Assert(ShmemSegHdr != NULL);
SpinLockAcquire(ShmemLock);
newStart = ShmemSegHdr->freeoffset;
newFree = newStart + size;
if (newFree <= ShmemSegHdr->totalsize)
{
newSpace = (void *) ((char *) ShmemBase + newStart);
ShmemSegHdr->freeoffset = newFree;
}
else
newSpace = NULL;
SpinLockRelease(ShmemLock);
/* note this assert is okay with newSpace == NULL */
Assert(newSpace == (void *) CACHELINEALIGN(newSpace));
return newSpace;
}
/*
* InitShmemAllocation() --- set up shared-memory space allocation.
*
* This should be called only in the postmaster or a standalone backend.
*/
void
InitShmemAllocation(void)
{
PGShmemHeader *shmhdr = ShmemSegHdr;
char *aligned;
Assert(shmhdr != NULL);
/*
* If spinlocks are disabled, initialize emulation layer. We have to do
* the space allocation the hard way, since obviously ShmemAlloc can't be
* called yet.
*/
#ifndef HAVE_SPINLOCKS
{
PGSemaphore spinsemas;
spinsemas = (PGSemaphore) (((char *) shmhdr) + shmhdr->freeoffset);
shmhdr->freeoffset += MAXALIGN(SpinlockSemaSize());
SpinlockSemaInit(spinsemas);
Assert(shmhdr->freeoffset <= shmhdr->totalsize);
}
#endif
/*
* Initialize the spinlock used by ShmemAlloc; we have to do this the hard
* way, too, for the same reasons as above.
*/
ShmemLock = (slock_t *) (((char *) shmhdr) + shmhdr->freeoffset);
shmhdr->freeoffset += MAXALIGN(sizeof(slock_t));
Assert(shmhdr->freeoffset <= shmhdr->totalsize);
/* Make sure the first allocation begins on a cache line boundary. */
aligned = (char *)
(CACHELINEALIGN((((char *) shmhdr) + shmhdr->freeoffset)));
shmhdr->freeoffset = aligned - (char *) shmhdr;
SpinLockInit(ShmemLock);
/* ShmemIndex can't be set up yet (need LWLocks first) */
shmhdr->index = NULL;
ShmemIndex = (HTAB *) NULL;
/*
* Initialize ShmemVariableCache for transaction manager. (This doesn't
* really belong here, but not worth moving.)
*/
ShmemVariableCache = (VariableCache)
ShmemAlloc(sizeof(*ShmemVariableCache));
memset(ShmemVariableCache, 0, sizeof(*ShmemVariableCache));
}
/*
* ShmemAlloc -- allocate max-aligned chunk from shared memory
*
* Assumes ShmemLock and ShmemSegHdr are initialized.
*
* Returns: real pointer to memory or NULL if we are out
* of space. Has to return a real pointer in order
* to be compatible with malloc().
*/
void *
ShmemAlloc(Size size)
{
Size newStart;
Size newFree;
void *newSpace;
/*
* Ensure all space is adequately aligned. We used to only MAXALIGN this
* space but experience has proved that on modern systems that is not good
* enough. Many parts of the system are very sensitive to critical data
* structures getting split across cache line boundaries. To avoid that,
* attempt to align the beginning of the allocation to a cache line
* boundary. The calling code will still need to be careful about how it
* uses the allocated space - e.g. by padding each element in an array of
* structures out to a power-of-two size - but without this, even that
* won't be sufficient.
*/
size = CACHELINEALIGN(size);
Assert(ShmemSegHdr != NULL);
SpinLockAcquire(ShmemLock);
newStart = ShmemSegHdr->freeoffset;
/* extra alignment for large requests, since they are probably buffers */
if (size >= BLCKSZ)
newStart = BUFFERALIGN(newStart);
newFree = newStart + size;
if (newFree <= ShmemSegHdr->totalsize)
{
newSpace = (void *) ((char *) ShmemBase + newStart);
ShmemSegHdr->freeoffset = newFree;
}
else
newSpace = NULL;
SpinLockRelease(ShmemLock);
if (!newSpace)
ereport(WARNING,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of shared memory")));
return newSpace;
}
/*
* InitShmemAllocation() --- set up shared-memory space allocation.
*
* This should be called only in the postmaster or a standalone backend.
*/
void
InitShmemAllocation(void)
{
PGShmemHeader *shmhdr = ShmemSegHdr;
char *aligned;
Assert(shmhdr != NULL);
/*
* Initialize the spinlock used by ShmemAlloc. We must use
* ShmemAllocUnlocked, since obviously ShmemAlloc can't be called yet.
*/
ShmemLock = (slock_t *) ShmemAllocUnlocked(sizeof(slock_t));
SpinLockInit(ShmemLock);
/*
* Allocations after this point should go through ShmemAlloc, which
* expects to allocate everything on cache line boundaries. Make sure the
* first allocation begins on a cache line boundary.
*/
aligned = (char *)
(CACHELINEALIGN((((char *) shmhdr) + shmhdr->freeoffset)));
shmhdr->freeoffset = aligned - (char *) shmhdr;
/* ShmemIndex can't be set up yet (need LWLocks first) */
shmhdr->index = NULL;
ShmemIndex = (HTAB *) NULL;
/*
* Initialize ShmemVariableCache for transaction manager. (This doesn't
* really belong here, but not worth moving.)
*/
ShmemVariableCache = (VariableCache)
ShmemAlloc(sizeof(*ShmemVariableCache));
memset(ShmemVariableCache, 0, sizeof(*ShmemVariableCache));
}
/*
* Initialize shared buffer pool
*
* This is called once during shared-memory initialization (either in the
* postmaster, or in a standalone backend).
*/
void
InitBufferPool(void)
{
bool foundBufs,
foundDescs,
foundIOLocks;
/* Align descriptors to a cacheline boundary. */
BufferDescriptors = (BufferDescPadded *)
CACHELINEALIGN(
ShmemInitStruct("Buffer Descriptors",
NBuffers * sizeof(BufferDescPadded)
+ PG_CACHE_LINE_SIZE,
&foundDescs));
BufferBlocks = (char *)
ShmemInitStruct("Buffer Blocks",
NBuffers * (Size) BLCKSZ, &foundBufs);
/* Align lwlocks to cacheline boundary */
BufferIOLWLockArray = (LWLockMinimallyPadded *)
CACHELINEALIGN(ShmemInitStruct("Buffer IO Locks",
NBuffers * (Size) sizeof(LWLockMinimallyPadded)
+ PG_CACHE_LINE_SIZE,
&foundIOLocks));
BufferIOLWLockTranche.name = "buffer_io";
BufferIOLWLockTranche.array_base = BufferIOLWLockArray;
BufferIOLWLockTranche.array_stride = sizeof(LWLockMinimallyPadded);
LWLockRegisterTranche(LWTRANCHE_BUFFER_IO_IN_PROGRESS,
&BufferIOLWLockTranche);
BufferContentLWLockTranche.name = "buffer_content";
BufferContentLWLockTranche.array_base =
((char *) BufferDescriptors) + offsetof(BufferDesc, content_lock);
BufferContentLWLockTranche.array_stride = sizeof(BufferDescPadded);
LWLockRegisterTranche(LWTRANCHE_BUFFER_CONTENT,
&BufferContentLWLockTranche);
if (foundDescs || foundBufs || foundIOLocks)
{
/* should find all of these, or none of them */
Assert(foundDescs && foundBufs && foundIOLocks);
/* note: this path is only taken in EXEC_BACKEND case */
}
else
{
int i;
/*
* Initialize all the buffer headers.
*/
for (i = 0; i < NBuffers; i++)
{
BufferDesc *buf = GetBufferDescriptor(i);
CLEAR_BUFFERTAG(buf->tag);
buf->flags = 0;
buf->usage_count = 0;
buf->refcount = 0;
buf->wait_backend_pid = 0;
SpinLockInit(&buf->buf_hdr_lock);
buf->buf_id = i;
/*
* Initially link all the buffers together as unused. Subsequent
* management of this list is done by freelist.c.
*/
buf->freeNext = i + 1;
LWLockInitialize(BufferDescriptorGetContentLock(buf),
LWTRANCHE_BUFFER_CONTENT);
LWLockInitialize(BufferDescriptorGetIOLock(buf),
LWTRANCHE_BUFFER_IO_IN_PROGRESS);
}
/* Correct last entry of linked list */
GetBufferDescriptor(NBuffers - 1)->freeNext = FREENEXT_END_OF_LIST;
}
/* Init other shared buffer-management stuff */
StrategyInitialize(!foundDescs);
}
/*
* ShmemInitStruct -- Create/attach to a structure in shared memory.
*
* This is called during initialization to find or allocate
* a data structure in shared memory. If no other process
* has created the structure, this routine allocates space
* for it. If it exists already, a pointer to the existing
* structure is returned.
*
* Returns: pointer to the object. *foundPtr is set TRUE if the object was
* already in the shmem index (hence, already initialized).
*
* Note: before Postgres 9.0, this function returned NULL for some failure
* cases. Now, it always throws error instead, so callers need not check
* for NULL.
*/
void *
ShmemInitStruct(const char *name, Size size, bool *foundPtr)
{
ShmemIndexEnt *result;
void *structPtr;
LWLockAcquire(ShmemIndexLock, LW_EXCLUSIVE);
if (!ShmemIndex)
{
PGShmemHeader *shmemseghdr = ShmemSegHdr;
/* Must be trying to create/attach to ShmemIndex itself */
Assert(strcmp(name, "ShmemIndex") == 0);
if (IsUnderPostmaster)
{
/* Must be initializing a (non-standalone) backend */
Assert(shmemseghdr->index != NULL);
structPtr = shmemseghdr->index;
*foundPtr = TRUE;
}
else
{
/*
* If the shmem index doesn't exist, we are bootstrapping: we must
* be trying to init the shmem index itself.
*
* Notice that the ShmemIndexLock is released before the shmem
* index has been initialized. This should be OK because no other
* process can be accessing shared memory yet.
*/
Assert(shmemseghdr->index == NULL);
structPtr = ShmemAlloc(size);
shmemseghdr->index = structPtr;
*foundPtr = FALSE;
}
LWLockRelease(ShmemIndexLock);
return structPtr;
}
/* look it up in the shmem index */
result = (ShmemIndexEnt *)
hash_search(ShmemIndex, name, HASH_ENTER_NULL, foundPtr);
if (!result)
{
LWLockRelease(ShmemIndexLock);
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("could not create ShmemIndex entry for data structure \"%s\"",
name)));
}
if (*foundPtr)
{
/*
* Structure is in the shmem index so someone else has allocated it
* already. The size better be the same as the size we are trying to
* initialize to, or there is a name conflict (or worse).
*/
if (result->size != size)
{
LWLockRelease(ShmemIndexLock);
ereport(ERROR,
(errmsg("ShmemIndex entry size is wrong for data structure"
" \"%s\": expected %zu, actual %zu",
name, size, result->size)));
}
structPtr = result->location;
}
else
{
/* It isn't in the table yet. allocate and initialize it */
structPtr = ShmemAllocNoError(size);
if (structPtr == NULL)
{
/* out of memory; remove the failed ShmemIndex entry */
hash_search(ShmemIndex, name, HASH_REMOVE, NULL);
LWLockRelease(ShmemIndexLock);
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("not enough shared memory for data structure"
" \"%s\" (%zu bytes requested)",
name, size)));
}
result->size = size;
result->location = structPtr;
}
LWLockRelease(ShmemIndexLock);
Assert(ShmemAddrIsValid(structPtr));
Assert(structPtr == (void *) CACHELINEALIGN(structPtr));
return structPtr;
}
