556 lines
15 KiB
C

#include "epoch_cache.h"
#include <errno.h>
#include <pthread.h>
#include <stdlib.h>
#include <string.h>
struct reader_slot {
uint64_t generation;
uint64_t seen_max;
uint64_t completed;
uint64_t serial;
uint64_t snapshot_epoch;
pthread_t thread;
int registered;
int busy;
int active;
};
struct epoch_cache {
pthread_mutex_t mutex;
pthread_cond_t changed;
pthread_rwlock_t data_lock;
uint64_t owner;
uint64_t epoch;
uint64_t next_ticket;
uint64_t serving_ticket;
uint64_t next_serial;
unsigned registered;
unsigned active_readers;
unsigned admitting_readers;
unsigned queued_writers;
int writer_active;
int closed;
pthread_t writer_thread;
struct reader_slot readers[EPOCH_READERS];
struct epoch_record published[EPOCH_KEYS];
struct epoch_record staged[EPOCH_KEYS];
unsigned char dirty[EPOCH_KEYS];
};
static int lock_cache(struct epoch_cache *cache)
{
if (cache == NULL) {
return EINVAL;
}
return pthread_mutex_lock(&cache->mutex);
}
static int finish(struct epoch_cache *cache, int result)
{
int error = pthread_mutex_unlock(&cache->mutex);
return result != 0 ? result : error;
}
static int reader_valid(struct epoch_cache *cache, struct epoch_reader reader)
{
if (reader.owner != cache->owner || reader.slot >= EPOCH_READERS) {
return 0;
}
struct reader_slot *slot = &cache->readers[reader.slot];
return slot->registered && slot->generation == reader.generation;
}
static int lease_valid(struct epoch_cache *cache, struct epoch_lease lease)
{
struct epoch_reader reader;
reader.owner = lease.owner;
reader.generation = lease.generation;
reader.slot = lease.slot;
if (!reader_valid(cache, reader)) {
return 0;
}
struct reader_slot *slot = &cache->readers[reader.slot];
return slot->active && slot->busy && slot->serial == lease.serial &&
slot->snapshot_epoch == lease.epoch &&
pthread_equal(slot->thread, pthread_self());
}
static int transaction_valid(struct epoch_cache *cache, struct epoch_tx tx)
{
return tx.owner == cache->owner && cache->writer_active &&
tx.ticket == cache->serving_ticket &&
pthread_equal(cache->writer_thread, pthread_self());
}
static int own_reader(struct epoch_cache *cache)
{
for (unsigned i = 0; i < EPOCH_READERS; ++i) {
struct reader_slot *slot = &cache->readers[i];
if (slot->busy && pthread_equal(slot->thread, pthread_self())) {
return 1;
}
}
return 0;
}
static int idle(struct epoch_cache *cache)
{
return cache->registered == 0 && cache->active_readers == 0 &&
cache->queued_writers == 0 && !cache->writer_active;
}
struct epoch_cache *epoch_create(uint64_t owner, uint64_t initial_epoch)
{
if (owner == 0) {
errno = EINVAL;
return NULL;
}
struct epoch_cache *cache = calloc(1, sizeof *cache);
if (cache == NULL) {
return NULL;
}
int error = pthread_mutex_init(&cache->mutex, NULL);
if (error != 0) {
free(cache);
errno = error;
return NULL;
}
error = pthread_cond_init(&cache->changed, NULL);
if (error != 0) {
pthread_mutex_destroy(&cache->mutex);
free(cache);
errno = error;
return NULL;
}
error = pthread_rwlock_init(&cache->data_lock, NULL);
if (error != 0) {
pthread_cond_destroy(&cache->changed);
pthread_mutex_destroy(&cache->mutex);
free(cache);
errno = error;
return NULL;
}
cache->owner = owner;
cache->epoch = initial_epoch;
cache->next_serial = 1;
return cache;
}
int epoch_register(struct epoch_cache *cache, struct epoch_reader *out)
{
if (out == NULL) {
return EINVAL;
}
int error = lock_cache(cache);
if (error != 0) {
return error;
}
if (cache->closed) {
return finish(cache, ESHUTDOWN);
}
unsigned selected = EPOCH_READERS;
for (unsigned i = 0; i < EPOCH_READERS; ++i) {
if (!cache->readers[i].registered) {
selected = i;
break;
}
}
if (selected == EPOCH_READERS) {
return finish(cache, ENOSPC);
}
struct reader_slot *slot = &cache->readers[selected];
slot->generation++;
slot->seen_max = cache->epoch;
slot->completed = 0;
slot->serial = 0;
slot->snapshot_epoch = 0;
slot->registered = 1;
slot->busy = 0;
slot->active = 0;
cache->registered++;
struct epoch_reader result;
memset(&result, 0, sizeof result);
result.owner = cache->owner;
result.generation = slot->generation;
result.slot = selected;
*out = result;
return finish(cache, 0);
}
int epoch_unregister(struct epoch_cache *cache, struct epoch_reader reader)
{
int error = lock_cache(cache);
if (error != 0) {
return error;
}
if (cache->closed) {
return finish(cache, ESHUTDOWN);
}
if (!reader_valid(cache, reader)) {
return finish(cache, EINVAL);
}
struct reader_slot *slot = &cache->readers[reader.slot];
if (slot->busy) {
return finish(cache, EBUSY);
}
slot->registered = 0;
cache->registered--;
return finish(cache, 0);
}
int epoch_reader_status(struct epoch_cache *cache, struct epoch_reader reader,
struct epoch_reader_info *out)
{
if (out == NULL) {
return EINVAL;
}
int error = lock_cache(cache);
if (error != 0) {
return error;
}
if (cache->closed) {
return finish(cache, ESHUTDOWN);
}
if (!reader_valid(cache, reader)) {
return finish(cache, EINVAL);
}
struct reader_slot *slot = &cache->readers[reader.slot];
struct epoch_reader_info result;
memset(&result, 0, sizeof result);
result.seen_max = slot->seen_max;
result.completed = slot->completed;
result.busy = slot->busy;
*out = result;
return finish(cache, 0);
}
int epoch_read_begin(struct epoch_cache *cache, struct epoch_reader reader,
struct epoch_lease *out)
{
if (out == NULL) {
return EINVAL;
}
int error = lock_cache(cache);
if (error != 0) {
return error;
}
if (cache->closed) {
return finish(cache, ESHUTDOWN);
}
if (!reader_valid(cache, reader)) {
return finish(cache, EINVAL);
}
struct reader_slot *slot = &cache->readers[reader.slot];
if (slot->busy) {
return finish(cache, EBUSY);
}
slot->busy = 1;
slot->thread = pthread_self();
while (cache->writer_active || cache->queued_writers != 0) {
error = pthread_cond_wait(&cache->changed, &cache->mutex);
if (error != 0) {
slot->busy = 0;
return finish(cache, error);
}
}
cache->admitting_readers++;
error = pthread_mutex_unlock(&cache->mutex);
if (error != 0) {
return error;
}
error = pthread_rwlock_rdlock(&cache->data_lock);
if (error != 0) {
pthread_mutex_lock(&cache->mutex);
cache->admitting_readers--;
slot->busy = 0;
pthread_cond_broadcast(&cache->changed);
return finish(cache, error);
}
error = pthread_mutex_lock(&cache->mutex);
if (error != 0) {
pthread_rwlock_unlock(&cache->data_lock);
return error;
}
cache->admitting_readers--;
cache->active_readers++;
slot->active = 1;
slot->serial = cache->next_serial++;
slot->snapshot_epoch = cache->epoch;
struct epoch_lease result;
memset(&result, 0, sizeof result);
result.owner = cache->owner;
result.generation = slot->generation;
result.serial = slot->serial;
result.epoch = slot->snapshot_epoch;
result.slot = reader.slot;
*out = result;
return finish(cache, 0);
}
int epoch_read_value(struct epoch_cache *cache, struct epoch_lease lease,
unsigned key, struct epoch_sample *out)
{
if (out == NULL) {
return EINVAL;
}
int error = lock_cache(cache);
if (error != 0) {
return error;
}
if (cache->closed) {
return finish(cache, ESHUTDOWN);
}
if (key >= EPOCH_KEYS || !lease_valid(cache, lease)) {
return finish(cache, EINVAL);
}
struct epoch_sample result;
memset(&result, 0, sizeof result);
result.snapshot_epoch = lease.epoch;
result.record = cache->published[key];
*out = result;
return finish(cache, 0);
}
int epoch_read_end(struct epoch_cache *cache, struct epoch_lease lease)
{
int error = lock_cache(cache);
if (error != 0) {
return error;
}
if (cache->closed) {
return finish(cache, ESHUTDOWN);
}
if (!lease_valid(cache, lease)) {
return finish(cache, EINVAL);
}
struct reader_slot *slot = &cache->readers[lease.slot];
if (slot->seen_max < lease.epoch) {
slot->seen_max = lease.epoch;
}
slot->completed++;
slot->active = 0;
slot->busy = 0;
cache->active_readers--;
error = pthread_rwlock_unlock(&cache->data_lock);
pthread_cond_broadcast(&cache->changed);
return finish(cache, error);
}
int epoch_write_begin(struct epoch_cache *cache, struct epoch_tx *out)
{
if (out == NULL) {
return EINVAL;
}
int error = lock_cache(cache);
if (error != 0) {
return error;
}
if (cache->closed) {
return finish(cache, ESHUTDOWN);
}
if (own_reader(cache) || (cache->writer_active &&
pthread_equal(cache->writer_thread, pthread_self()))) {
return finish(cache, EDEADLK);
}
uint64_t ticket = cache->next_ticket++;
cache->queued_writers++;
while (cache->writer_active || cache->active_readers != 0 ||
cache->admitting_readers != 0 ||
ticket != cache->serving_ticket) {
error = pthread_cond_wait(&cache->changed, &cache->mutex);
if (error != 0) {
return finish(cache, error);
}
}
cache->queued_writers--;
cache->writer_active = 1;
cache->writer_thread = pthread_self();
pthread_mutex_unlock(&cache->mutex);
error = pthread_rwlock_wrlock(&cache->data_lock);
if (error != 0) {
pthread_mutex_lock(&cache->mutex);
cache->writer_active = 0;
cache->serving_ticket++;
pthread_cond_broadcast(&cache->changed);
return finish(cache, error);
}
pthread_mutex_lock(&cache->mutex);
memset(cache->staged, 0, sizeof cache->staged);
memset(cache->dirty, 0, sizeof cache->dirty);
struct epoch_tx result;
memset(&result, 0, sizeof result);
result.owner = cache->owner;
result.ticket = ticket;
*out = result;
return finish(cache, 0);
}
int epoch_write_put(struct epoch_cache *cache, struct epoch_tx tx,
unsigned key, int64_t value, uint64_t tag)
{
int error = lock_cache(cache);
if (error != 0) {
return error;
}
if (cache->closed) {
return finish(cache, ESHUTDOWN);
}
if (key >= EPOCH_KEYS || !transaction_valid(cache, tx)) {
return finish(cache, EINVAL);
}
struct epoch_record record;
memset(&record, 0, sizeof record);
record.present = 1;
record.value = value;
record.tag = tag;
cache->staged[key] = record;
cache->dirty[key] = 1;
return finish(cache, 0);
}
int epoch_write_erase(struct epoch_cache *cache, struct epoch_tx tx,
unsigned key)
{
int error = lock_cache(cache);
if (error != 0) {
return error;
}
if (cache->closed) {
return finish(cache, ESHUTDOWN);
}
if (key >= EPOCH_KEYS || !transaction_valid(cache, tx)) {
return finish(cache, EINVAL);
}
memset(&cache->staged[key], 0, sizeof cache->staged[key]);
cache->dirty[key] = 1;
return finish(cache, 0);
}
static int end_writer(struct epoch_cache *cache)
{
int error = pthread_rwlock_unlock(&cache->data_lock);
cache->writer_active = 0;
cache->serving_ticket++;
pthread_cond_broadcast(&cache->changed);
return finish(cache, error);
}
int epoch_write_commit(struct epoch_cache *cache, struct epoch_tx tx)
{
int error = lock_cache(cache);
if (error != 0) {
return error;
}
if (cache->closed) {
return finish(cache, ESHUTDOWN);
}
if (!transaction_valid(cache, tx)) {
return finish(cache, EINVAL);
}
if (cache->epoch == UINT64_MAX) {
return finish(cache, EOVERFLOW);
}
uint64_t next = cache->epoch + 1;
for (unsigned key = 0; key < EPOCH_KEYS; ++key) {
if (cache->dirty[key]) {
if (cache->staged[key].present) {
cache->staged[key].version = next;
}
cache->published[key] = cache->staged[key];
}
}
cache->epoch = next;
return end_writer(cache);
}
int epoch_write_abort(struct epoch_cache *cache, struct epoch_tx tx)
{
int error = lock_cache(cache);
if (error != 0) {
return error;
}
if (cache->closed) {
return finish(cache, ESHUTDOWN);
}
if (!transaction_valid(cache, tx)) {
return finish(cache, EINVAL);
}
return end_writer(cache);
}
int epoch_inspect(struct epoch_cache *cache, struct epoch_state *out)
{
if (out == NULL) {
return EINVAL;
}
int error = lock_cache(cache);
if (error != 0) {
return error;
}
struct epoch_state state;
memset(&state, 0, sizeof state);
state.epoch = cache->epoch;
state.next_ticket = cache->next_ticket;
state.serving_ticket = cache->serving_ticket;
state.registered = cache->registered;
state.active_readers = cache->active_readers;
state.queued_writers = cache->queued_writers;
state.writer_active = cache->writer_active;
state.closed = cache->closed;
*out = state;
return finish(cache, 0);
}
int epoch_reseed(struct epoch_cache *cache, uint64_t epoch)
{
int error = lock_cache(cache);
if (error != 0) {
return error;
}
if (cache->closed) {
return finish(cache, ESHUTDOWN);
}
if (!idle(cache)) {
return finish(cache, EBUSY);
}
memset(cache->published, 0, sizeof cache->published);
memset(cache->staged, 0, sizeof cache->staged);
memset(cache->dirty, 0, sizeof cache->dirty);
cache->epoch = epoch;
cache->next_ticket = 0;
cache->serving_ticket = 0;
cache->next_serial = 1;
return finish(cache, 0);
}
int epoch_close(struct epoch_cache *cache)
{
int error = lock_cache(cache);
if (error != 0) {
return error;
}
if (!idle(cache)) {
return finish(cache, EBUSY);
}
cache->closed = 1;
return finish(cache, 0);
}
int epoch_destroy(struct epoch_cache *cache)
{
int error = lock_cache(cache);
if (error != 0) {
return error;
}
if (!cache->closed || !idle(cache)) {
return finish(cache, EBUSY);
}
pthread_mutex_unlock(&cache->mutex);
int first = pthread_rwlock_destroy(&cache->data_lock);
int second = pthread_cond_destroy(&cache->changed);
int third = pthread_mutex_destroy(&cache->mutex);
free(cache);
return first != 0 ? first : second != 0 ? second : third;
}