554 lines
15 KiB
C
554 lines
15 KiB
C
#include "epoch_cache.h"
|
|
#include <errno.h>
|
|
#include <pthread.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
struct reader_slot {
|
|
uint64_t generation;
|
|
uint64_t seen_max;
|
|
uint64_t completed;
|
|
uint64_t serial;
|
|
uint64_t snapshot_epoch;
|
|
pthread_t thread;
|
|
int registered;
|
|
int busy;
|
|
int active;
|
|
};
|
|
|
|
struct epoch_cache {
|
|
pthread_mutex_t mutex;
|
|
pthread_cond_t changed;
|
|
pthread_rwlock_t data_lock;
|
|
uint64_t owner;
|
|
uint64_t epoch;
|
|
uint64_t next_ticket;
|
|
uint64_t serving_ticket;
|
|
uint64_t next_serial;
|
|
unsigned registered;
|
|
unsigned active_readers;
|
|
unsigned admitting_readers;
|
|
unsigned queued_writers;
|
|
int writer_active;
|
|
int closed;
|
|
pthread_t writer_thread;
|
|
struct reader_slot readers[EPOCH_READERS];
|
|
struct epoch_record published[EPOCH_KEYS];
|
|
struct epoch_record staged[EPOCH_KEYS];
|
|
unsigned char dirty[EPOCH_KEYS];
|
|
};
|
|
|
|
static int lock_cache(struct epoch_cache *cache)
|
|
{
|
|
if (cache == NULL) {
|
|
return EINVAL;
|
|
}
|
|
return pthread_mutex_lock(&cache->mutex);
|
|
}
|
|
|
|
static int finish(struct epoch_cache *cache, int result)
|
|
{
|
|
int error = pthread_mutex_unlock(&cache->mutex);
|
|
return result != 0 ? result : error;
|
|
}
|
|
|
|
static int reader_valid(struct epoch_cache *cache, struct epoch_reader reader)
|
|
{
|
|
if (reader.owner != cache->owner || reader.slot >= EPOCH_READERS) {
|
|
return 0;
|
|
}
|
|
struct reader_slot *slot = &cache->readers[reader.slot];
|
|
return slot->registered && slot->generation == reader.generation;
|
|
}
|
|
|
|
static int lease_valid(struct epoch_cache *cache, struct epoch_lease lease)
|
|
{
|
|
struct epoch_reader reader;
|
|
reader.owner = lease.owner;
|
|
reader.generation = lease.generation;
|
|
reader.slot = lease.slot;
|
|
if (!reader_valid(cache, reader)) {
|
|
return 0;
|
|
}
|
|
struct reader_slot *slot = &cache->readers[reader.slot];
|
|
return slot->active && slot->busy && slot->serial == lease.serial &&
|
|
slot->snapshot_epoch == lease.epoch &&
|
|
pthread_equal(slot->thread, pthread_self());
|
|
}
|
|
|
|
static int transaction_valid(struct epoch_cache *cache, struct epoch_tx tx)
|
|
{
|
|
return tx.owner == cache->owner && cache->writer_active &&
|
|
tx.ticket == cache->serving_ticket &&
|
|
pthread_equal(cache->writer_thread, pthread_self());
|
|
}
|
|
|
|
static int own_reader(struct epoch_cache *cache)
|
|
{
|
|
for (unsigned i = 0; i < EPOCH_READERS; ++i) {
|
|
struct reader_slot *slot = &cache->readers[i];
|
|
if (slot->busy && pthread_equal(slot->thread, pthread_self())) {
|
|
return 1;
|
|
}
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static int idle(struct epoch_cache *cache)
|
|
{
|
|
return cache->registered == 0 && cache->active_readers == 0 &&
|
|
cache->queued_writers == 0 && !cache->writer_active;
|
|
}
|
|
|
|
struct epoch_cache *epoch_create(uint64_t owner, uint64_t initial_epoch)
|
|
{
|
|
if (owner == 0) {
|
|
errno = EINVAL;
|
|
return NULL;
|
|
}
|
|
struct epoch_cache *cache = calloc(1, sizeof *cache);
|
|
if (cache == NULL) {
|
|
return NULL;
|
|
}
|
|
int error = pthread_mutex_init(&cache->mutex, NULL);
|
|
if (error != 0) {
|
|
free(cache);
|
|
errno = error;
|
|
return NULL;
|
|
}
|
|
error = pthread_cond_init(&cache->changed, NULL);
|
|
if (error != 0) {
|
|
pthread_mutex_destroy(&cache->mutex);
|
|
free(cache);
|
|
errno = error;
|
|
return NULL;
|
|
}
|
|
error = pthread_rwlock_init(&cache->data_lock, NULL);
|
|
if (error != 0) {
|
|
pthread_cond_destroy(&cache->changed);
|
|
pthread_mutex_destroy(&cache->mutex);
|
|
free(cache);
|
|
errno = error;
|
|
return NULL;
|
|
}
|
|
cache->owner = owner;
|
|
cache->epoch = initial_epoch;
|
|
cache->next_serial = 1;
|
|
return cache;
|
|
}
|
|
|
|
int epoch_register(struct epoch_cache *cache, struct epoch_reader *out)
|
|
{
|
|
if (out == NULL) {
|
|
return EINVAL;
|
|
}
|
|
int error = lock_cache(cache);
|
|
if (error != 0) {
|
|
return error;
|
|
}
|
|
if (cache->closed) {
|
|
return finish(cache, ESHUTDOWN);
|
|
}
|
|
unsigned selected = EPOCH_READERS;
|
|
for (unsigned i = 0; i < EPOCH_READERS; ++i) {
|
|
if (!cache->readers[i].registered) {
|
|
selected = i;
|
|
break;
|
|
}
|
|
}
|
|
if (selected == EPOCH_READERS) {
|
|
return finish(cache, ENOSPC);
|
|
}
|
|
struct reader_slot *slot = &cache->readers[selected];
|
|
slot->generation++;
|
|
slot->seen_max = cache->epoch;
|
|
slot->completed = 0;
|
|
slot->serial = 0;
|
|
slot->snapshot_epoch = 0;
|
|
slot->registered = 1;
|
|
slot->busy = 0;
|
|
slot->active = 0;
|
|
cache->registered++;
|
|
struct epoch_reader result;
|
|
memset(&result, 0, sizeof result);
|
|
result.owner = cache->owner;
|
|
result.generation = slot->generation;
|
|
result.slot = selected;
|
|
*out = result;
|
|
return finish(cache, 0);
|
|
}
|
|
|
|
int epoch_unregister(struct epoch_cache *cache, struct epoch_reader reader)
|
|
{
|
|
int error = lock_cache(cache);
|
|
if (error != 0) {
|
|
return error;
|
|
}
|
|
if (cache->closed) {
|
|
return finish(cache, ESHUTDOWN);
|
|
}
|
|
if (!reader_valid(cache, reader)) {
|
|
return finish(cache, EINVAL);
|
|
}
|
|
struct reader_slot *slot = &cache->readers[reader.slot];
|
|
if (slot->busy) {
|
|
return finish(cache, EBUSY);
|
|
}
|
|
slot->registered = 0;
|
|
cache->registered--;
|
|
return finish(cache, 0);
|
|
}
|
|
|
|
int epoch_reader_status(struct epoch_cache *cache, struct epoch_reader reader,
|
|
struct epoch_reader_info *out)
|
|
{
|
|
if (out == NULL) {
|
|
return EINVAL;
|
|
}
|
|
int error = lock_cache(cache);
|
|
if (error != 0) {
|
|
return error;
|
|
}
|
|
if (cache->closed) {
|
|
return finish(cache, ESHUTDOWN);
|
|
}
|
|
if (!reader_valid(cache, reader)) {
|
|
return finish(cache, EINVAL);
|
|
}
|
|
struct reader_slot *slot = &cache->readers[reader.slot];
|
|
struct epoch_reader_info result;
|
|
memset(&result, 0, sizeof result);
|
|
result.seen_max = slot->seen_max;
|
|
result.completed = slot->completed;
|
|
result.busy = slot->busy;
|
|
*out = result;
|
|
return finish(cache, 0);
|
|
}
|
|
|
|
int epoch_read_begin(struct epoch_cache *cache, struct epoch_reader reader,
|
|
struct epoch_lease *out)
|
|
{
|
|
if (out == NULL) {
|
|
return EINVAL;
|
|
}
|
|
int error = lock_cache(cache);
|
|
if (error != 0) {
|
|
return error;
|
|
}
|
|
if (cache->closed) {
|
|
return finish(cache, ESHUTDOWN);
|
|
}
|
|
if (!reader_valid(cache, reader)) {
|
|
return finish(cache, EINVAL);
|
|
}
|
|
struct reader_slot *slot = &cache->readers[reader.slot];
|
|
if (slot->busy) {
|
|
return finish(cache, EBUSY);
|
|
}
|
|
slot->busy = 1;
|
|
slot->thread = pthread_self();
|
|
while (cache->writer_active || cache->queued_writers != 0) {
|
|
error = pthread_cond_wait(&cache->changed, &cache->mutex);
|
|
if (error != 0) {
|
|
slot->busy = 0;
|
|
return finish(cache, error);
|
|
}
|
|
}
|
|
cache->admitting_readers++;
|
|
error = pthread_mutex_unlock(&cache->mutex);
|
|
if (error != 0) {
|
|
return error;
|
|
}
|
|
error = pthread_rwlock_rdlock(&cache->data_lock);
|
|
if (error != 0) {
|
|
pthread_mutex_lock(&cache->mutex);
|
|
cache->admitting_readers--;
|
|
slot->busy = 0;
|
|
pthread_cond_broadcast(&cache->changed);
|
|
return finish(cache, error);
|
|
}
|
|
error = pthread_mutex_lock(&cache->mutex);
|
|
if (error != 0) {
|
|
pthread_rwlock_unlock(&cache->data_lock);
|
|
return error;
|
|
}
|
|
cache->admitting_readers--;
|
|
cache->active_readers++;
|
|
slot->active = 1;
|
|
slot->serial = cache->next_serial++;
|
|
slot->snapshot_epoch = cache->epoch;
|
|
struct epoch_lease result;
|
|
memset(&result, 0, sizeof result);
|
|
result.owner = cache->owner;
|
|
result.generation = slot->generation;
|
|
result.serial = slot->serial;
|
|
result.epoch = slot->snapshot_epoch;
|
|
result.slot = reader.slot;
|
|
*out = result;
|
|
return finish(cache, 0);
|
|
}
|
|
|
|
int epoch_read_value(struct epoch_cache *cache, struct epoch_lease lease,
|
|
unsigned key, struct epoch_sample *out)
|
|
{
|
|
if (out == NULL) {
|
|
return EINVAL;
|
|
}
|
|
int error = lock_cache(cache);
|
|
if (error != 0) {
|
|
return error;
|
|
}
|
|
if (cache->closed) {
|
|
return finish(cache, ESHUTDOWN);
|
|
}
|
|
if (key >= EPOCH_KEYS || !lease_valid(cache, lease)) {
|
|
return finish(cache, EINVAL);
|
|
}
|
|
struct epoch_sample result;
|
|
memset(&result, 0, sizeof result);
|
|
result.snapshot_epoch = lease.epoch;
|
|
result.record = cache->published[key];
|
|
*out = result;
|
|
return finish(cache, 0);
|
|
}
|
|
|
|
int epoch_read_end(struct epoch_cache *cache, struct epoch_lease lease)
|
|
{
|
|
int error = lock_cache(cache);
|
|
if (error != 0) {
|
|
return error;
|
|
}
|
|
if (cache->closed) {
|
|
return finish(cache, ESHUTDOWN);
|
|
}
|
|
if (!lease_valid(cache, lease)) {
|
|
return finish(cache, EINVAL);
|
|
}
|
|
struct reader_slot *slot = &cache->readers[lease.slot];
|
|
if (slot->seen_max < lease.epoch) {
|
|
slot->seen_max = lease.epoch;
|
|
}
|
|
slot->completed++;
|
|
slot->active = 0;
|
|
slot->busy = 0;
|
|
cache->active_readers--;
|
|
error = pthread_rwlock_unlock(&cache->data_lock);
|
|
pthread_cond_broadcast(&cache->changed);
|
|
return finish(cache, error);
|
|
}
|
|
|
|
int epoch_write_begin(struct epoch_cache *cache, struct epoch_tx *out)
|
|
{
|
|
if (out == NULL) {
|
|
return EINVAL;
|
|
}
|
|
int error = lock_cache(cache);
|
|
if (error != 0) {
|
|
return error;
|
|
}
|
|
if (cache->closed) {
|
|
return finish(cache, ESHUTDOWN);
|
|
}
|
|
if (own_reader(cache) || (cache->writer_active &&
|
|
pthread_equal(cache->writer_thread, pthread_self()))) {
|
|
return finish(cache, EDEADLK);
|
|
}
|
|
uint64_t ticket = cache->next_ticket++;
|
|
cache->queued_writers++;
|
|
while (cache->writer_active || cache->active_readers != 0 ||
|
|
cache->admitting_readers != 0 ||
|
|
ticket != cache->serving_ticket) {
|
|
error = pthread_cond_wait(&cache->changed, &cache->mutex);
|
|
if (error != 0) {
|
|
return finish(cache, error);
|
|
}
|
|
}
|
|
cache->queued_writers--;
|
|
cache->writer_active = 1;
|
|
cache->writer_thread = pthread_self();
|
|
pthread_mutex_unlock(&cache->mutex);
|
|
error = pthread_rwlock_wrlock(&cache->data_lock);
|
|
if (error != 0) {
|
|
pthread_mutex_lock(&cache->mutex);
|
|
cache->writer_active = 0;
|
|
cache->serving_ticket++;
|
|
pthread_cond_broadcast(&cache->changed);
|
|
return finish(cache, error);
|
|
}
|
|
pthread_mutex_lock(&cache->mutex);
|
|
memcpy(cache->staged, cache->published, sizeof cache->staged);
|
|
memset(cache->dirty, 0, sizeof cache->dirty);
|
|
struct epoch_tx result;
|
|
memset(&result, 0, sizeof result);
|
|
result.owner = cache->owner;
|
|
result.ticket = ticket;
|
|
*out = result;
|
|
return finish(cache, 0);
|
|
}
|
|
|
|
int epoch_write_put(struct epoch_cache *cache, struct epoch_tx tx,
|
|
unsigned key, int64_t value, uint64_t tag)
|
|
{
|
|
int error = lock_cache(cache);
|
|
if (error != 0) {
|
|
return error;
|
|
}
|
|
if (cache->closed) {
|
|
return finish(cache, ESHUTDOWN);
|
|
}
|
|
if (key >= EPOCH_KEYS || !transaction_valid(cache, tx)) {
|
|
return finish(cache, EINVAL);
|
|
}
|
|
struct epoch_record record;
|
|
memset(&record, 0, sizeof record);
|
|
record.present = 1;
|
|
record.value = value;
|
|
record.tag = tag;
|
|
cache->staged[key] = record;
|
|
cache->dirty[key] = 1;
|
|
return finish(cache, 0);
|
|
}
|
|
|
|
int epoch_write_erase(struct epoch_cache *cache, struct epoch_tx tx,
|
|
unsigned key)
|
|
{
|
|
int error = lock_cache(cache);
|
|
if (error != 0) {
|
|
return error;
|
|
}
|
|
if (cache->closed) {
|
|
return finish(cache, ESHUTDOWN);
|
|
}
|
|
if (key >= EPOCH_KEYS || !transaction_valid(cache, tx)) {
|
|
return finish(cache, EINVAL);
|
|
}
|
|
memset(&cache->staged[key], 0, sizeof cache->staged[key]);
|
|
cache->dirty[key] = 1;
|
|
return finish(cache, 0);
|
|
}
|
|
|
|
static int end_writer(struct epoch_cache *cache)
|
|
{
|
|
int error = pthread_rwlock_unlock(&cache->data_lock);
|
|
cache->writer_active = 0;
|
|
cache->serving_ticket++;
|
|
pthread_cond_broadcast(&cache->changed);
|
|
return finish(cache, error);
|
|
}
|
|
|
|
int epoch_write_commit(struct epoch_cache *cache, struct epoch_tx tx)
|
|
{
|
|
int error = lock_cache(cache);
|
|
if (error != 0) {
|
|
return error;
|
|
}
|
|
if (cache->closed) {
|
|
return finish(cache, ESHUTDOWN);
|
|
}
|
|
if (!transaction_valid(cache, tx)) {
|
|
return finish(cache, EINVAL);
|
|
}
|
|
if (cache->epoch == UINT64_MAX) {
|
|
return finish(cache, EOVERFLOW);
|
|
}
|
|
uint64_t next = cache->epoch + 1;
|
|
for (unsigned key = 0; key < EPOCH_KEYS; ++key) {
|
|
if (cache->dirty[key] && cache->staged[key].present) {
|
|
cache->staged[key].version = next;
|
|
}
|
|
}
|
|
/* No publication: only the counter advances. */
|
|
cache->epoch = next;
|
|
return end_writer(cache);
|
|
}
|
|
|
|
int epoch_write_abort(struct epoch_cache *cache, struct epoch_tx tx)
|
|
{
|
|
int error = lock_cache(cache);
|
|
if (error != 0) {
|
|
return error;
|
|
}
|
|
if (cache->closed) {
|
|
return finish(cache, ESHUTDOWN);
|
|
}
|
|
if (!transaction_valid(cache, tx)) {
|
|
return finish(cache, EINVAL);
|
|
}
|
|
return end_writer(cache);
|
|
}
|
|
|
|
int epoch_inspect(struct epoch_cache *cache, struct epoch_state *out)
|
|
{
|
|
if (out == NULL) {
|
|
return EINVAL;
|
|
}
|
|
int error = lock_cache(cache);
|
|
if (error != 0) {
|
|
return error;
|
|
}
|
|
struct epoch_state state;
|
|
memset(&state, 0, sizeof state);
|
|
state.epoch = cache->epoch;
|
|
state.next_ticket = cache->next_ticket;
|
|
state.serving_ticket = cache->serving_ticket;
|
|
state.registered = cache->registered;
|
|
state.active_readers = cache->active_readers;
|
|
state.queued_writers = cache->queued_writers;
|
|
state.writer_active = cache->writer_active;
|
|
state.closed = cache->closed;
|
|
*out = state;
|
|
return finish(cache, 0);
|
|
}
|
|
|
|
int epoch_reseed(struct epoch_cache *cache, uint64_t epoch)
|
|
{
|
|
int error = lock_cache(cache);
|
|
if (error != 0) {
|
|
return error;
|
|
}
|
|
if (cache->closed) {
|
|
return finish(cache, ESHUTDOWN);
|
|
}
|
|
if (!idle(cache)) {
|
|
return finish(cache, EBUSY);
|
|
}
|
|
memset(cache->published, 0, sizeof cache->published);
|
|
memset(cache->staged, 0, sizeof cache->staged);
|
|
memset(cache->dirty, 0, sizeof cache->dirty);
|
|
cache->epoch = epoch;
|
|
cache->next_ticket = 0;
|
|
cache->serving_ticket = 0;
|
|
cache->next_serial = 1;
|
|
return finish(cache, 0);
|
|
}
|
|
|
|
int epoch_close(struct epoch_cache *cache)
|
|
{
|
|
int error = lock_cache(cache);
|
|
if (error != 0) {
|
|
return error;
|
|
}
|
|
if (!idle(cache)) {
|
|
return finish(cache, EBUSY);
|
|
}
|
|
cache->closed = 1;
|
|
return finish(cache, 0);
|
|
}
|
|
|
|
int epoch_destroy(struct epoch_cache *cache)
|
|
{
|
|
int error = lock_cache(cache);
|
|
if (error != 0) {
|
|
return error;
|
|
}
|
|
if (!cache->closed || !idle(cache)) {
|
|
return finish(cache, EBUSY);
|
|
}
|
|
pthread_mutex_unlock(&cache->mutex);
|
|
int first = pthread_rwlock_destroy(&cache->data_lock);
|
|
int second = pthread_cond_destroy(&cache->changed);
|
|
int third = pthread_mutex_destroy(&cache->mutex);
|
|
free(cache);
|
|
return first != 0 ? first : second != 0 ? second : third;
|
|
}
|