mako creation
This commit is contained in:
Kao Makino
parent
e1e99ce4df
commit
ae0a5388c9
|
|
@ -0,0 +1,45 @@
|
|||
cmake_minimum_required(VERSION 3.5)
|
||||
project(mako)
|
||||
|
||||
set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${PROJECT_SOURCE_DIR}/cmake")
|
||||
message (STATUS "${PROJECT_SOURCE_DIR} ${PROJECT_BINARY_DIR}")
|
||||
|
||||
if("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}")
|
||||
message(FATAL_ERROR "In-source builds are forbidden and unsupported")
|
||||
endif()
|
||||
|
||||
# default to 307
|
||||
if(NOT FDB_API_VERSION)
|
||||
message(STATUS "Setting FDB API version to 307")
|
||||
set(FDB_API_VERSION 307)
|
||||
endif()
|
||||
|
||||
# we don't want rpath to be set
|
||||
set(CMAKE_SKIP_BUILD_RPATH TRUE)
|
||||
|
||||
if(FDB_BUILD_DIR)
|
||||
set(FDB_LIB_DIR ${FDB_BUILD_DIR}/lib)
|
||||
if (FDB_SOURCE_DIR)
|
||||
set(FDB_INCLUDE_DIR ${FDB_SOURCE_DIR}/bindings/c ${FDB_BUILD_DIR}/bindings/c/foundationdb)
|
||||
endif()
|
||||
else()
|
||||
if (APPLE)
|
||||
set(FDB_LIB_DIR ${FDB_DIR}/usr/local/lib)
|
||||
set(FDB_INCLUDE_DIR ${FDB_DIR}/usr/local/include)
|
||||
else()
|
||||
set(FDB_LIB_DIR ${FDB_DIR}/usr/lib64)
|
||||
set(FDB_INCLUDE_DIR ${FDB_DIR}/usr/include)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
link_directories(${FDB_LIB_DIR})
|
||||
include_directories(${FDB_INCLUDE_DIR})
|
||||
|
||||
file(GLOB SOURCES "*.c")
|
||||
add_executable(mako ${SOURCES})
|
||||
if(APPLE)
|
||||
target_link_libraries(mako ${PROJECT_LINK_LIBS} pthread m fdb_c)
|
||||
else()
|
||||
target_link_libraries(mako ${PROJECT_LINK_LIBS} pthread rt m fdb_c)
|
||||
endif()
|
||||
target_compile_options(mako PUBLIC -g -fPIC -O2 -DFDB_API_VERSION=${FDB_API_VERSION})
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
default_target: mako
|
||||
|
||||
#FDB_API_VERSION=307
|
||||
#FDBDIR=/home/kmakino/scratch/rpms/3.2.2
|
||||
FDB_API_VERSION=610
|
||||
FDBDIR=/home/kmakino/scratch/rpms/6.1.0
|
||||
|
||||
INCS = -I. -I$(FDBDIR)/usr/include
|
||||
|
||||
CFLAGS = -g -fPIC -O2 $(INCS) -DFDB_API_VERSION=$(FDB_API_VERSION)
|
||||
LDFLAGS = -L$(FDBDIR)/usr/lib64 -lm -lpthread -lrt -lfdb_c
|
||||
|
||||
TARGET = mako
|
||||
SRCS = mako.c utils.c zipf.c
|
||||
OBJS := $(SRCS:.c=.o)
|
||||
|
||||
$(TARGET): $(OBJS)
|
||||
$(CC) -o $@ $^ $(LDFLAGS)
|
||||
|
||||
clean:
|
||||
$(RM) $(TARGET) $(OBJS)
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
* How to Build
|
||||
|
||||
- Use system-wide FDB installation
|
||||
cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release <mako_srcdir>
|
||||
make
|
||||
|
||||
- Use a production package installed somewhere else
|
||||
cmake -G "Unix Makefiles" -DFDB_DIR=<fdbdir> -DCMAKE_BUILD_TYPE=Release <mako_srcdir>
|
||||
make
|
||||
|
||||
- Use the development source/build directories
|
||||
cmake -G "Unix Makefiles" -DFDB_BUILD_DIR=<fdb_builddir> -DFDB_SOURCE_DIR=<fdb_srcdir> -DCMAKE_BUILD_TYPE=Release <mako_srcdir>
|
||||
make
|
||||
|
||||
|
||||
|
||||
* How to Run
|
||||
|
||||
1. Create FDB cluster and database
|
||||
|
||||
2. Populate initial data
|
||||
e.g.
|
||||
Populate 1 million rows with 8 processes, using 32-byte key and 16-byte value
|
||||
mako --mode build --keylen 32 --vallen 16 --rows 1000000 -p 8
|
||||
|
||||
3. Run
|
||||
e.g.
|
||||
Run a GET RANGE test for 100 seconds.
|
||||
16 GET RANGE operations with 10-key range in one transaction (not committed),
|
||||
using 8 processes and 32 threads per process, total of 256 threads.
|
||||
mako --mode run --keylen 32 --vallen 16 --rows 1000000 -p 8 -t 32 --transaction "gr16:10" --seconds 100
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,137 @@
|
|||
#ifndef MAKO_H
|
||||
#define MAKO_H
|
||||
#pragma once
|
||||
|
||||
//#define FDB_API_VERSION 307
|
||||
//#define FDB_API_VERSION 610
|
||||
|
||||
#include <foundationdb/fdb_c.h>
|
||||
#include <pthread.h>
|
||||
#include <sys/types.h>
|
||||
#if defined(__linux__)
|
||||
#include <linux/limits.h>
|
||||
#elif defined(__APPLE__)
|
||||
#include <sys/syslimits.h>
|
||||
#else
|
||||
#include <limits.h>
|
||||
#endif
|
||||
|
||||
#define VERBOSE_NONE 0
|
||||
#define VERBOSE_DEFAULT 1
|
||||
#define VERBOSE_ANNOYING 2
|
||||
#define VERBOSE_DEBUG 3
|
||||
|
||||
#define MODE_INVALID -1
|
||||
#define MODE_CLEAN 0
|
||||
#define MODE_BUILD 1
|
||||
#define MODE_RUN 2
|
||||
|
||||
/* we set mako_txn_t and mako_args_t only once in the master process,
|
||||
* and won't be touched by child processes.
|
||||
*/
|
||||
|
||||
/* transaction specification */
|
||||
#define OP_GETREADVERSION 0
|
||||
#define OP_GET 1
|
||||
#define OP_GETRANGE 2
|
||||
#define OP_SGET 3
|
||||
#define OP_SGETRANGE 4
|
||||
#define OP_UPDATE 5
|
||||
#define OP_INSERT 6
|
||||
#define OP_INSERTRANGE 7
|
||||
#define OP_CLEAR 8
|
||||
#define OP_SETCLEAR 9
|
||||
#define OP_CLEARRANGE 10
|
||||
#define OP_SETCLEARRANGE 11
|
||||
#define OP_COMMIT 12
|
||||
#define MAX_OP 13 /* update this when adding a new operation */
|
||||
|
||||
#define OP_COUNT 0
|
||||
#define OP_RANGE 1
|
||||
#define OP_REVERSE 2
|
||||
|
||||
/* for arguments */
|
||||
#define ARG_KEYLEN 1
|
||||
#define ARG_VALLEN 2
|
||||
#define ARG_TPS 3
|
||||
#define ARG_COMMITGET 4
|
||||
#define ARG_SAMPLING 5
|
||||
#define ARG_VERSION 6
|
||||
#define ARG_KNOBS 7
|
||||
#define ARG_FLATBUFFERS 8
|
||||
#define ARG_TRACE 9
|
||||
#define ARG_TRACEPATH 10
|
||||
|
||||
#define KEYPREFIX "mako"
|
||||
#define KEYPREFIXLEN 4
|
||||
|
||||
typedef struct {
|
||||
/* for each operation, it stores "count", "range" and "reverse" */
|
||||
int ops[MAX_OP][3];
|
||||
} mako_txnspec_t;
|
||||
|
||||
#define KNOB_MAX 256
|
||||
|
||||
/* benchmark parameters */
|
||||
typedef struct {
|
||||
int json;
|
||||
int num_processes;
|
||||
int num_threads;
|
||||
int mode;
|
||||
int rows; /* is 2 billion enough? */
|
||||
int seconds;
|
||||
int iteration;
|
||||
int tps;
|
||||
int sampling;
|
||||
int key_length;
|
||||
int value_length;
|
||||
int zipf;
|
||||
int commit_get;
|
||||
int verbose;
|
||||
mako_txnspec_t txnspec;
|
||||
char cluster_file[PATH_MAX];
|
||||
int trace;
|
||||
char tracepath[PATH_MAX];
|
||||
char knobs[KNOB_MAX];
|
||||
uint8_t flatbuffers;
|
||||
} mako_args_t;
|
||||
|
||||
/* shared memory */
|
||||
#define SIGNAL_RED 0
|
||||
#define SIGNAL_GREEN 1
|
||||
#define SIGNAL_OFF 2
|
||||
|
||||
typedef struct {
|
||||
int signal;
|
||||
int readycount;
|
||||
} mako_shmhdr_t;
|
||||
|
||||
typedef struct {
|
||||
uint64_t xacts;
|
||||
uint64_t conflicts;
|
||||
uint64_t ops[MAX_OP];
|
||||
uint64_t errors[MAX_OP];
|
||||
uint64_t latency_samples[MAX_OP];
|
||||
uint64_t latency_us_total[MAX_OP];
|
||||
uint64_t latency_us_min[MAX_OP];
|
||||
uint64_t latency_us_max[MAX_OP];
|
||||
} mako_stats_t;
|
||||
|
||||
/* per-process information */
|
||||
typedef struct {
|
||||
int worker_id;
|
||||
FDBDatabase *database;
|
||||
mako_args_t *args;
|
||||
mako_shmhdr_t *shm;
|
||||
} process_info_t;
|
||||
|
||||
/* args for threads */
|
||||
typedef struct {
|
||||
int thread_id;
|
||||
process_info_t *process;
|
||||
} thread_args_t;
|
||||
|
||||
/* process type */
|
||||
typedef enum { proc_master = 0, proc_worker, proc_stats } proc_type_t;
|
||||
|
||||
#endif /* MAKO_H */
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <math.h>
|
||||
#include "utils.h"
|
||||
#include "mako.h"
|
||||
|
||||
/* uniform-distribution random */
|
||||
int urand(int low, int high) {
|
||||
double r = rand() / (1.0 + RAND_MAX);
|
||||
int range = high - low + 1;
|
||||
return (int)((r * range) + low);
|
||||
}
|
||||
|
||||
/* random string */
|
||||
/* len is the buffer size, must include null */
|
||||
void randstr(char *str, int len) {
|
||||
int i;
|
||||
for (i = 0; i < len-1; i++) {
|
||||
str[i] = '!' + urand(0, 'z'-'!'); /* generage a char from '!' to 'z' */
|
||||
}
|
||||
str[len-1] = '\0';
|
||||
}
|
||||
|
||||
/* random numeric string */
|
||||
/* len is the buffer size, must include null */
|
||||
void randnumstr(char *str, int len) {
|
||||
int i;
|
||||
for (i = 0; i < len-1; i++) {
|
||||
str[i] = '0' + urand(0, 9); /* generage a char from '!' to 'z' */
|
||||
}
|
||||
str[len-1] = '\0';
|
||||
}
|
||||
|
||||
/* return the first key to be inserted */
|
||||
int insert_begin(int rows, int p_idx, int t_idx, int total_p, int total_t) {
|
||||
double interval = (double)rows / total_p / total_t;
|
||||
return (int)(round(interval * ((p_idx * total_t) + t_idx)));
|
||||
}
|
||||
|
||||
/* return the last key to be inserted */
|
||||
int insert_end(int rows, int p_idx, int t_idx, int total_p, int total_t) {
|
||||
double interval = (double)rows / total_p / total_t;
|
||||
return (int)(round(interval * ((p_idx * total_t) + t_idx + 1) - 1));
|
||||
}
|
||||
|
||||
/* devide val equally among threads */
|
||||
int compute_thread_portion(int val, int p_idx, int t_idx, int total_p, int total_t) {
|
||||
int interval = val / total_p / total_t;
|
||||
int remaining = val - (interval * total_p * total_t);
|
||||
if ((p_idx * total_t + t_idx) < remaining) {
|
||||
return interval+1;
|
||||
} else if (interval == 0) {
|
||||
return -1;
|
||||
}
|
||||
/* else */
|
||||
return interval;
|
||||
}
|
||||
|
||||
/* number of digits */
|
||||
int digits(int num) {
|
||||
int digits = 0;
|
||||
while (num > 0) {
|
||||
num /= 10;
|
||||
digits++;
|
||||
}
|
||||
return digits;
|
||||
}
|
||||
|
||||
|
||||
/* generate a key for a given key number */
|
||||
/* len is the buffer size, key length + null */
|
||||
void genkey(char *str, int num, int rows, int len) {
|
||||
int i;
|
||||
int rowdigit = digits(rows);
|
||||
sprintf(str, KEYPREFIX "%0.*d", rowdigit, num);
|
||||
for (i = (KEYPREFIXLEN + rowdigit); i < len-1; i++) {
|
||||
str[i] = 'x';
|
||||
}
|
||||
str[len-1] = '\0';
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
#ifndef UTILS_H
|
||||
#define UTILS_H
|
||||
#pragma once
|
||||
|
||||
/* uniform-distribution random */
|
||||
/* return a uniform random number between low and high, both inclusive */
|
||||
int urand(int low, int high);
|
||||
|
||||
/* write a random string of the length of (len-1) to memory pointed by str
|
||||
* with a null-termination character at str[len-1].
|
||||
*/
|
||||
void randstr(char *str, int len);
|
||||
|
||||
/* write a random numeric string of the length of (len-1) to memory pointed by str
|
||||
* with a null-termination character at str[len-1].
|
||||
*/
|
||||
void randnumstr(char *str, int len);
|
||||
|
||||
/* given the total number of rows to be inserted,
|
||||
* the worker process index p_idx and the thread index t_idx (both 0-based),
|
||||
* and the total number of processes, total_p, and threads, total_t,
|
||||
* returns the first row number assigned to this partition.
|
||||
*/
|
||||
int insert_begin(int rows, int p_idx, int t_idx, int total_p, int total_t);
|
||||
|
||||
/* similar to insert_begin, insert_end returns the last row numer */
|
||||
int insert_end(int rows, int p_idx, int t_idx, int total_p, int total_t);
|
||||
|
||||
/* devide a value equally among threads */
|
||||
int compute_thread_portion(int val, int p_idx, int t_idx, int total_p,
|
||||
int total_t);
|
||||
|
||||
/* similar to insert_begin/end, compute_thread_tps computes
|
||||
* the per-thread target TPS for given configuration.
|
||||
*/
|
||||
#define compute_thread_tps(val, p_idx, t_idx, total_p, total_t) \
|
||||
compute_thread_portion(val, p_idx, t_idx, total_p, total_t)
|
||||
|
||||
/* similar to compute_thread_tps,
|
||||
* compute_thread_iters computs the number of iterations.
|
||||
*/
|
||||
#define compute_thread_iters(val, p_idx, t_idx, total_p, total_t) \
|
||||
compute_thread_portion(val, p_idx, t_idx, total_p, total_t)
|
||||
|
||||
/* get the number of digits */
|
||||
int digits(int num);
|
||||
|
||||
/* generate a key for a given key number */
|
||||
/* len is the buffer size, key length + null */
|
||||
void genkey(char *str, int num, int rows, int len);
|
||||
|
||||
#endif /* UTILS_H */
|
||||
|
|
@ -0,0 +1,132 @@
|
|||
/*
|
||||
* zipfian distribution copied from YCSB
|
||||
* https://github.com/brianfrankcooper/YCSB
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <math.h>
|
||||
#include <time.h>
|
||||
#include "zipf.h"
|
||||
|
||||
/* global static */
|
||||
static int items;
|
||||
static int base;
|
||||
static double zipfianconstant;
|
||||
static double alpha, zetan, eta, theta, zeta2theta;
|
||||
static int countforzeta;
|
||||
static int allowitemcountdecrease = 0;
|
||||
|
||||
/* declarations */
|
||||
double zetastatic2(int st, int n, double theta, double initialsum);
|
||||
double zeta2(int st, int n, double theta_val, double initialsum);
|
||||
double zetastatic(int n, double theta);
|
||||
double zeta(int n, double theta_val);
|
||||
|
||||
|
||||
double rand_double() {
|
||||
return (double)rand() / (double)RAND_MAX;
|
||||
}
|
||||
|
||||
|
||||
int next_int(int itemcount) {
|
||||
double u, uz;
|
||||
int ret;
|
||||
|
||||
if (itemcount != countforzeta) {
|
||||
zetan = zeta2(countforzeta, itemcount, theta, zetan);
|
||||
} else if ((itemcount < countforzeta) && (allowitemcountdecrease)) {
|
||||
zetan = zeta(itemcount, theta);
|
||||
}
|
||||
eta = (1 - pow(2.0 / items, 1 - theta)) / (1 - zeta2theta / zetan);
|
||||
|
||||
u = rand_double();
|
||||
uz = u * zetan;
|
||||
|
||||
if (uz < 1.0) {
|
||||
return base;
|
||||
}
|
||||
|
||||
if (uz < 1.0 + pow(0.5, theta)) {
|
||||
return base + 1;
|
||||
}
|
||||
|
||||
ret = base + (int)(itemcount * pow(eta * u - eta + 1, alpha));
|
||||
return ret;
|
||||
}
|
||||
|
||||
int zipfian_next() {
|
||||
return next_int(items);
|
||||
}
|
||||
|
||||
double zetastatic2(int st, int n, double theta, double initialsum) {
|
||||
int i;
|
||||
double sum = initialsum;
|
||||
for (i = st; i < n; i++) {
|
||||
sum += 1 / pow(i + 1, theta);
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
double zeta2(int st, int n, double theta_val, double initialsum) {
|
||||
countforzeta = n;
|
||||
return zetastatic2(st, n, theta_val, initialsum);
|
||||
}
|
||||
|
||||
double zetastatic(int n, double theta) {
|
||||
return zetastatic2(0, n, theta, 0);
|
||||
}
|
||||
|
||||
double zeta(int n, double theta_val) {
|
||||
countforzeta = n;
|
||||
return zetastatic(n, theta_val);
|
||||
}
|
||||
|
||||
void zipfian_generator4(int min, int max, double _zipfianconstant, double _zetan) {
|
||||
items = max - min + 1;
|
||||
base = min;
|
||||
zipfianconstant = _zipfianconstant;
|
||||
|
||||
theta = zipfianconstant;
|
||||
zeta2theta = zeta(2, theta);
|
||||
alpha = 1.0 / (1.0 - theta);
|
||||
zetan = _zetan;
|
||||
countforzeta = items;
|
||||
eta = (1 - pow(2.0 / items, 1 - theta)) / (1 - zeta2theta / zetan);
|
||||
|
||||
zipfian_next();
|
||||
}
|
||||
|
||||
void zipfian_generator3(int min, int max, double zipfianconstant) {
|
||||
zipfian_generator4(min, max, zipfianconstant, zetastatic(max - min + 1, zipfianconstant));
|
||||
}
|
||||
|
||||
void zipfian_generator2(int min, int max) {
|
||||
zipfian_generator3(min, max, ZIPFIAN_CONSTANT);
|
||||
}
|
||||
|
||||
void zipfian_generator(int items) {
|
||||
zipfian_generator2(0, items - 1);
|
||||
}
|
||||
|
||||
|
||||
#if 0 /* test */
|
||||
void main() {
|
||||
int i = 0;
|
||||
int histogram[1000] = { 0 };
|
||||
|
||||
srand(time(0));
|
||||
|
||||
zipfian_generator(1000);
|
||||
|
||||
for (i = 0; i < 1000000; i++) {
|
||||
int val = next_value();
|
||||
//printf("%d\n", val);
|
||||
histogram[val]++;
|
||||
}
|
||||
|
||||
for (i = 0; i < 1000; i++) {
|
||||
printf("%d\n", histogram[i]);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
/*
|
||||
* zipfian distribution copied from YCSB
|
||||
* https://github.com/brianfrankcooper/YCSB
|
||||
*/
|
||||
|
||||
#ifndef ZIPF_H
|
||||
#define ZIPF_H
|
||||
#pragma once
|
||||
|
||||
#define ZIPFIAN_CONSTANT 0.99
|
||||
|
||||
void zipfian_generator(int items);
|
||||
int zipfian_next();
|
||||
|
||||
#endif /* ZIPF_H */
|
||||
Loading…
Reference in New Issue