void
__go_builtin_close (struct __go_channel *channel)
{
int i;
i = pthread_mutex_lock (&channel->lock);
__go_assert (i == 0);
while (channel->selected_for_send)
{
i = pthread_cond_wait (&channel->cond, &channel->lock);
__go_assert (i == 0);
}
if (channel->is_closed)
{
i = pthread_mutex_unlock (&channel->lock);
__go_assert (i == 0);
__go_panic_msg ("close of closed channel");
}
channel->is_closed = 1;
i = pthread_cond_broadcast (&channel->cond);
__go_assert (i == 0);
__go_unlock_and_notify_selects (channel);
}
void
runtime_semrelease (uint32 *addr)
{
int32_t val;
val = __sync_fetch_and_add (addr, 1);
/* VAL is the old value. It should never be negative. If it is
negative, that implies that Semacquire somehow decremented a zero
value, or that the count has overflowed. */
__go_assert (val >= 0);
/* If the old value was zero, then we have now released a count, and
we signal the condition variable. If the old value was positive,
then nobody can be waiting. We have to use
pthread_cond_broadcast, not pthread_cond_signal, because
otherwise there would be a race condition when the count is
incremented twice before any locker manages to decrement it. */
if (val == 0)
{
int i;
i = pthread_mutex_lock (&sem_lock);
__go_assert (i == 0);
i = pthread_cond_broadcast (&sem_cond);
__go_assert (i == 0);
i = pthread_mutex_unlock (&sem_lock);
__go_assert (i == 0);
}
}
void
__go_receive_release (struct __go_channel *channel)
{
int i;
if (channel->num_entries != 0)
channel->next_fetch = (channel->next_fetch + 1) % channel->num_entries;
else
{
/* For a synchronous receiver, we tell the sender that we picked
up the value by setting the next_store field back to 0.
Using the mutexes should implement a memory barrier. */
__go_assert (channel->next_store == 1);
channel->next_store = 0;
channel->waiting_to_receive = 0;
}
channel->selected_for_receive = 0;
/* This is a broadcast to make sure that a synchronous sender sees
it. */
i = pthread_cond_broadcast (&channel->cond);
__go_assert (i == 0);
__go_unlock_and_notify_selects (channel);
}
static _Bool
mark_all_channels_waiting (struct select_channel* channels, size_t count,
struct __go_channel **selected_pointer,
_Bool *selected_for_read_pointer)
{
_Bool ret;
int x;
size_t i;
ret = 0;
for (i = 0; i < count; ++i)
{
struct __go_channel *channel = channels[i].channel;
_Bool is_send = channels[i].is_send;
if (channel == NULL)
continue;
if (channels[i].dup_index != (size_t) -1UL)
{
size_t j;
/* A channel may be selected for both read and write. */
if (channels[channels[i].dup_index].is_send != is_send)
{
for (j = channels[i].dup_index + 1; j < i; ++j)
{
if (channels[j].channel == channel
&& channels[j].is_send == is_send)
break;
}
if (j < i)
continue;
}
}
x = pthread_mutex_lock (&channel->lock);
__go_assert (x == 0);
/* To avoid a race condition, we have to check again whether the
channel is ready. It may have become ready since we did the
first set of checks but before we acquired the select mutex.
If we don't check here, we could sleep forever on the select
condition variable. */
if (is_channel_ready (channel, is_send))
ret = 1;
/* If SELECTED_POINTER is NULL, then we have already marked the
channel as waiting. */
if (selected_pointer != NULL)
mark_select_waiting (&channels[i], selected_pointer,
selected_for_read_pointer);
x = pthread_mutex_unlock (&channel->lock);
__go_assert (x == 0);
}
return ret;
}
void *
__go_allocate_trampoline (uintptr_t size, void *closure)
{
uintptr_t ptr_size;
uintptr_t full_size;
unsigned char *ret;
/* Because the garbage collector only looks at aligned addresses, we
need to store the closure at an aligned address to ensure that it
sees it. */
ptr_size = sizeof (void *);
full_size = (((size + ptr_size - 1) / ptr_size) * ptr_size);
full_size += ptr_size;
runtime_lock (&trampoline_lock);
if (full_size < trampoline_page_size - trampoline_page_used)
trampoline_page = NULL;
if (trampoline_page == NULL)
{
uintptr_t page_size;
unsigned char *page;
page_size = getpagesize ();
__go_assert (page_size >= full_size);
page = (unsigned char *) runtime_mallocgc (2 * page_size - 1, 0, 0, 0);
page = (unsigned char *) (((uintptr_t) page + page_size - 1)
& ~ (page_size - 1));
#ifdef HAVE_SYS_MMAN_H
{
int i;
i = mprotect (page, page_size, PROT_READ | PROT_WRITE | PROT_EXEC);
__go_assert (i == 0);
}
#endif
trampoline_page = page;
trampoline_page_size = page_size;
trampoline_page_used = 0;
}
ret = trampoline_page + trampoline_page_used;
trampoline_page_used += full_size;
runtime_unlock (&trampoline_lock);
__builtin_memcpy (ret + full_size - ptr_size, &closure, ptr_size);
return (void *) ret;
}
void
__go_send_acquire (struct __go_channel *channel, _Bool for_select)
{
int i;
i = pthread_mutex_lock (&channel->lock);
__go_assert (i == 0);
while (1)
{
if (channel->is_closed)
{
if (for_select)
channel->selected_for_send = 0;
i = pthread_mutex_unlock (&channel->lock);
__go_assert (i == 0);
__go_panic_msg ("send on closed channel");
}
/* If somebody else has the channel locked for sending, we have
to wait. If FOR_SELECT is true, then we are the one with the
lock. */
if (!channel->selected_for_send || for_select)
{
if (channel->num_entries == 0)
{
/* This is a synchronous channel. If nobody else is
waiting to send, we grab the channel and tell the
caller to send the data. We will then wait for a
receiver. */
if (!channel->waiting_to_send)
{
__go_assert (channel->next_store == 0);
return;
}
}
else
{
/* If there is room on the channel, we are OK. */
if ((channel->next_store + 1) % channel->num_entries
!= channel->next_fetch)
return;
}
}
/* Wait for something to change, then loop around and try
again. */
i = pthread_cond_wait (&channel->cond, &channel->lock);
__go_assert (i == 0);
}
}
void
runtime_noteclear (Note* n)
{
int32 i;
i = pthread_mutex_lock (¬e_lock);
__go_assert (i == 0);
n->woken = 0;
i = pthread_mutex_unlock (¬e_lock);
__go_assert (i == 0);
}
uint64_t
__go_receive_small_closed (struct __go_channel *channel, _Bool for_select,
_Bool *received)
{
uintptr_t element_size;
uint64_t ret;
if (channel == NULL)
__go_panic_msg ("receive from nil channel");
element_size = channel->element_type->__size;
__go_assert (element_size <= sizeof (uint64_t));
if (!__go_receive_acquire (channel, for_select))
{
if (received != NULL)
*received = 0;
return 0;
}
ret = channel->data[channel->next_fetch];
__go_receive_release (channel);
if (received != NULL)
*received = 1;
return ret;
}
void
runtime_notewakeup (Note *n)
{
int32 i;
i = pthread_mutex_lock (¬e_lock);
__go_assert (i == 0);
n->woken = 1;
i = pthread_cond_broadcast (¬e_cond);
__go_assert (i == 0);
i = pthread_mutex_unlock (¬e_lock);
__go_assert (i == 0);
}
size_t
force_selected_channel_ready (struct select_channel *channels, size_t count,
struct __go_channel *selected_channel,
_Bool selected_for_read)
{
size_t ready_count;
size_t i;
ready_count = 0;
for (i = 0; i < count; ++i)
{
struct __go_channel *channel = channels[i].channel;
_Bool is_send = channels[i].is_send;
if (channel == NULL)
continue;
if (channel != selected_channel
|| (is_send ? !selected_for_read : selected_for_read))
channels[i].is_ready = 0;
else
{
channels[i].is_ready = 1;
++ready_count;
}
}
__go_assert (ready_count > 0);
return ready_count;
}
uint64_t
__go_receive_small_closed (struct __go_channel *channel, _Bool for_select,
_Bool *received)
{
uintptr_t element_size;
uint64_t ret;
if (channel == NULL)
{
/* Block forever. */
__go_select (0, 0, NULL, NULL);
}
element_size = channel->element_type->__size;
__go_assert (element_size <= sizeof (uint64_t));
if (!__go_receive_acquire (channel, for_select))
{
if (received != NULL)
*received = 0;
return 0;
}
ret = channel->data[channel->next_fetch];
__go_receive_release (channel);
if (received != NULL)
*received = 1;
return ret;
}
void *
__go_map_index (struct __go_map *map, const void *key, _Bool insert)
{
const struct __go_map_descriptor *descriptor;
const struct __go_type_descriptor *key_descriptor;
uintptr_t key_offset;
_Bool (*equalfn) (const void*, const void*, uintptr_t);
size_t key_hash;
size_t key_size;
size_t bucket_index;
char *entry;
if (map == NULL)
{
if (insert)
runtime_panicstring ("assignment to entry in nil map");
return NULL;
}
descriptor = map->__descriptor;
key_descriptor = descriptor->__map_descriptor->__key_type;
key_offset = descriptor->__key_offset;
key_size = key_descriptor->__size;
__go_assert (key_size != 0 && key_size != -1UL);
equalfn = key_descriptor->__equalfn;
key_hash = key_descriptor->__hashfn (key, key_size);
bucket_index = key_hash % map->__bucket_count;
entry = (char *) map->__buckets[bucket_index];
while (entry != NULL)
{
if (equalfn (key, entry + key_offset, key_size))
return entry + descriptor->__val_offset;
entry = *(char **) entry;
}
if (!insert)
return NULL;
if (map->__element_count >= map->__bucket_count)
{
__go_map_rehash (map);
bucket_index = key_hash % map->__bucket_count;
}
entry = (char *) __go_alloc (descriptor->__entry_size);
__builtin_memset (entry, 0, descriptor->__entry_size);
__builtin_memcpy (entry + key_offset, key, key_size);
*(char **) entry = map->__buckets[bucket_index];
map->__buckets[bucket_index] = entry;
map->__element_count += 1;
return entry + descriptor->__val_offset;
}
uint32
__sync_fetch_and_add_4(uint32* ptr, uint32 add)
{
int i;
uint32 ret;
i = pthread_mutex_lock(&sync_lock);
__go_assert(i == 0);
ret = *ptr;
*ptr += add;
i = pthread_mutex_unlock(&sync_lock);
__go_assert(i == 0);
return ret;
}
void
runtime_notesleep (Note* n)
{
int32 i;
i = pthread_mutex_lock (¬e_lock);
__go_assert (i == 0);
while (!n->woken)
{
i = pthread_cond_wait (¬e_cond, ¬e_lock);
__go_assert (i == 0);
}
i = pthread_mutex_unlock (¬e_lock);
__go_assert (i == 0);
}
int
__go_map_len (struct __go_map *map)
{
if (map == NULL)
return 0;
__go_assert (map->__element_count == (uintptr_t) (int) map->__element_count);
return map->__element_count;
}
static void
sig_panic_leadin (int sig)
{
int i;
sigset_t clear;
if (runtime_m ()->mallocing)
{
runtime_printf ("caught signal while mallocing: %d\n", sig);
runtime_throw ("caught signal while mallocing");
}
/* The signal handler blocked signals; unblock them. */
i = sigfillset (&clear);
__go_assert (i == 0);
i = sigprocmask (SIG_UNBLOCK, &clear, NULL);
__go_assert (i == 0);
}
static void
sig_handler (int sig)
{
int i;
#ifdef SIGPROF
if (sig == SIGPROF)
{
/* FIXME. */
runtime_sigprof (0, 0, nil, nil);
return;
}
#endif
for (i = 0; runtime_sigtab[i].sig != -1; ++i)
{
struct sigaction sa;
SigTab *t;
t = &runtime_sigtab[i];
if (t->sig != sig)
continue;
if ((t->flags & SigNotify) != 0)
{
if (__go_sigsend (sig))
return;
}
if ((t->flags & SigKill) != 0)
runtime_exit (2);
if ((t->flags & SigThrow) == 0)
return;
runtime_startpanic ();
/* We should do a stack backtrace here. Until we can do that,
we reraise the signal in order to get a slightly better
report from the shell. */
memset (&sa, 0, sizeof sa);
sa.sa_handler = SIG_DFL;
i = sigemptyset (&sa.sa_mask);
__go_assert (i == 0);
if (sigaction (sig, &sa, NULL) != 0)
abort ();
raise (sig);
runtime_exit (2);
}
__builtin_unreachable ();
}
unsigned char *
mapiterinit (struct __go_map_type *mt, uintptr_t m)
{
struct __go_hash_iter *it;
__go_assert (mt->__common.__code == GO_MAP);
it = __go_alloc (sizeof (struct __go_hash_iter));
__go_mapiterinit ((struct __go_map *) m, it);
return (unsigned char *) it;
}
void
__go_broadcast_to_select (struct __go_channel *channel)
{
pthread_mutex_t *select_mutex;
pthread_cond_t *select_cond;
int i;
select_mutex = channel->select_mutex;
select_cond = channel->select_cond;
i = pthread_mutex_unlock (&channel->lock);
__go_assert (i == 0);
__go_assert (select_mutex != NULL && select_cond != NULL);
i = pthread_mutex_lock (select_mutex);
__go_assert (i == 0);
i = pthread_cond_broadcast (select_cond);
__go_assert (i == 0);
i = pthread_mutex_unlock (select_mutex);
__go_assert (i == 0);
i = pthread_mutex_lock (&channel->lock);
__go_assert (i == 0);
}
_Bool
__go_synch_with_select (struct __go_channel *channel, _Bool is_send)
{
struct __go_channel_select *p;
int i;
__go_assert (channel->num_entries == 0);
i = pthread_mutex_lock (&__go_select_data_mutex);
__go_assert (i == 0);
for (p = (is_send
? channel->select_receive_queue
: channel->select_send_queue);
p != NULL;
p = p->next)
{
if (*p->selected == NULL)
{
*p->selected = channel;
*p->is_read = !is_send;
if (is_send)
channel->selected_for_receive = 1;
else
channel->selected_for_send = 1;
break;
}
}
i = pthread_mutex_unlock (&__go_select_data_mutex);
__go_assert (i == 0);
/* The caller is responsible for signalling the select condition
variable so that the other select knows that something has
changed. We can't signal it here because we can't acquire the
select mutex while we hold a channel lock. */
return p != NULL;
}
void
__initsig ()
{
struct sigaction sa;
int i;
siginit ();
memset (&sa, 0, sizeof sa);
sa.sa_handler = sighandler;
i = sigfillset (&sa.sa_mask);
__go_assert (i == 0);
for (i = 0; signals[i].sig != -1; ++i)
{
sa.sa_flags = signals[i].restart ? SA_RESTART : 0;
if (sigaction (signals[i].sig, &sa, NULL) != 0)
__go_assert (0);
}
}
void
runtime_resetcpuprofiler(int32 hz)
{
struct itimerval it;
struct sigaction sa;
int i;
memset (&it, 0, sizeof it);
memset (&sa, 0, sizeof sa);
i = sigfillset (&sa.sa_mask);
__go_assert (i == 0);
if (hz == 0)
{
i = setitimer (ITIMER_PROF, &it, NULL);
__go_assert (i == 0);
sa.sa_handler = SIG_IGN;
i = sigaction (SIGPROF, &sa, NULL);
__go_assert (i == 0);
}
else
{
sa.sa_handler = sighandler;
sa.sa_flags = SA_RESTART;
i = sigaction (SIGPROF, &sa, NULL);
__go_assert (i == 0);
it.it_interval.tv_sec = 0;
it.it_interval.tv_usec = 1000000 / hz;
it.it_value = it.it_interval;
i = setitimer (ITIMER_PROF, &it, NULL);
__go_assert (i == 0);
}
m->profilehz = hz;
}
void
__go_unlock_and_notify_selects (struct __go_channel *channel)
{
pthread_mutex_t* select_mutex;
pthread_cond_t* select_cond;
int i;
select_mutex = channel->select_mutex;
select_cond = channel->select_cond;
i = pthread_mutex_unlock (&channel->lock);
__go_assert (i == 0);
if (select_mutex != NULL)
{
i = pthread_mutex_lock (select_mutex);
__go_assert (i == 0);
i = pthread_cond_broadcast (select_cond);
__go_assert (i == 0);
i = pthread_mutex_unlock (select_mutex);
__go_assert (i == 0);
}
}
static _Bool
is_queue_ready (struct __go_channel_select *queue)
{
int x;
if (queue == NULL)
return 0;
x = pthread_mutex_lock (&__go_select_data_mutex);
__go_assert (x == 0);
while (queue != NULL)
{
if (*queue->selected == NULL)
break;
queue = queue->next;
}
x = pthread_mutex_unlock (&__go_select_data_mutex);
__go_assert (x == 0);
return queue != NULL;
}
void
__go_send_small (struct __go_channel *channel, uint64_t val, _Bool for_select)
{
if (channel == NULL)
__go_panic_msg ("send to nil channel");
__go_assert (channel->element_size <= sizeof (uint64_t));
__go_send_acquire (channel, for_select);
channel->data[channel->next_store] = val;
__go_send_release (channel);
}
void
__go_mapiter1 (struct __go_hash_iter *it, unsigned char *key)
{
const struct __go_map *map;
const struct __go_map_descriptor *descriptor;
const struct __go_type_descriptor *key_descriptor;
const char *p;
map = it->map;
descriptor = map->__descriptor;
key_descriptor = descriptor->__map_descriptor->__key_type;
p = it->entry;
__go_assert (p != NULL);
__builtin_memcpy (key, p + descriptor->__key_offset, key_descriptor->__size);
}
void
runtime_semacquire (uint32 *addr)
{
while (1)
{
int i;
/* If the current count is positive, and we are able to atomically
decrement it, then we have acquired the semaphore. */
if (acquire (addr))
return;
/* Lock the mutex. */
i = pthread_mutex_lock (&sem_lock);
__go_assert (i == 0);
/* Check the count again with the mutex locked. */
if (acquire (addr))
{
i = pthread_mutex_unlock (&sem_lock);
__go_assert (i == 0);
return;
}
/* The count is zero. Even if a call to runtime.Semrelease
increments it to become positive, that call will try to
acquire the mutex and block, so we are sure to see the signal
of the condition variable. */
i = pthread_cond_wait (&sem_cond, &sem_lock);
__go_assert (i == 0);
/* Unlock the mutex and try again. */
i = pthread_mutex_unlock (&sem_lock);
__go_assert (i == 0);
}
}
struct mapaccess_ret
mapaccess (struct __go_map_type *mt, uintptr_t m, uintptr_t key_i)
{
struct __go_map *map = (struct __go_map *) m;
void *key;
const struct __go_type_descriptor *key_descriptor;
void *p;
const struct __go_type_descriptor *val_descriptor;
struct mapaccess_ret ret;
void *val;
void *pv;
__go_assert (mt->__common.__code == GO_MAP);
key_descriptor = mt->__key_type;
if (__go_is_pointer_type (key_descriptor))
key = &key_i;
else
key = (void *) key_i;
if (map == NULL)
p = NULL;
else
p = __go_map_index (map, key, 0);
val_descriptor = mt->__val_type;
if (__go_is_pointer_type (val_descriptor))
{
val = NULL;
pv = &val;
}
else
{
val = __go_alloc (val_descriptor->__size);
pv = val;
}
if (p == NULL)
ret.pres = 0;
else
{
__builtin_memcpy (pv, p, val_descriptor->__size);
ret.pres = 1;
}
ret.val = (uintptr_t) val;
return ret;
}
_Bool
__go_send_nonblocking_small (struct __go_channel *channel, uint64_t val)
{
if (channel == NULL)
return 0;
__go_assert (channel->element_type->__size <= sizeof (uint64_t));
if (!__go_send_nonblocking_acquire (channel))
return 0;
channel->data[channel->next_store] = val;
__go_send_release (channel);
return 1;
}
void
__go_send_small (struct __go_channel *channel, uint64_t val, _Bool for_select)
{
if (channel == NULL)
{
// Block forever.
__go_select (0, 0, NULL, NULL);
}
__go_assert (channel->element_type->__size <= sizeof (uint64_t));
__go_send_acquire (channel, for_select);
channel->data[channel->next_store] = val;
__go_send_release (channel);
}
