fumiama/simple-dict
1
0
Fork 0
mirror of https://github.com/fumiama/simple-dict.git synced 2026-06-05 02:00:25 +08:00
Code Activity

add mutex

Browse Source
This commit is contained in:
源文雨
2022-04-04 18:03:18 +08:00
parent fdeb306c9b
commit a1af0a7e8e
6 changed files with 152 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
CMakeLists.txt
View File

@@ -3,7 +3,7 @@ project(simple-dict-server C)
SET(CMAKE_BUILD_TYPE "Release")
add_definitions(-DLISTEN_ON_IPV6)
add_definitions(-DDEBUG)
#add_definitions(-DDEBUG)
IF(CMAKE_SIZEOF_VOID_P EQUAL 8)
add_definitions("-DCPUBIT64")
ELSE()
@@ -23,12 +23,12 @@ add_executable(cfgwriter cfgwriter.c)
#add_executable(migrate migrate.c)
#add_executable(migratenew migratenew.c)
target_link_libraries(simple-dict-server scrypto spb pthread)
target_link_libraries(simple-dict-server scrypto libspb.a pthread)
target_link_libraries(simple-dict-client scrypto pthread)
target_link_libraries(cfgwriter spb)
target_link_libraries(cfgwriter libspb.a)
#target_link_libraries(migrate spb)
#target_link_libraries(migratenew spb)
#target_link_libraries(migrate libspb.a)
#target_link_libraries(migratenew libspb.a)
INSTALL(TARGETS simple-dict-server RUNTIME DESTINATION bin)
#INSTALL(TARGETS simple-dict-client RUNTIME DESTINATION bin)

14
crypto.c
View File

@@ -108,14 +108,12 @@ void cmdpacket_encrypt(CMDPACKET* p, int index, const char pwd[64]) {
TEADAT* tout = tea_encrypt_native_endian(tea, sumtable, &tin);
uint8_t* datamd5 = md5(p->data, p->datalen);
md5(p->data, p->datalen, p->md5);
#ifdef DEBUG
printf("encrypt md5: ");
for(int i = 0; i < 16; i++) printf("%02x", datamd5[i]);
for(int i = 0; i < 16; i++) printf("%02x", p->md5[i]);
putchar('\n');
#endif
memcpy(p->md5, datamd5, 16);
free(datamd5);
p->datalen = tout->len;
memcpy(p->data, tout->data, p->datalen);
@@ -156,8 +154,8 @@ int cmdpacket_decrypt(CMDPACKET* p, int index, const char pwd[64]) {
free(tout);
return 0;
}
uint8_t* datamd5 = md5(tout->data, tout->len);
uint8_t datamd5[16];
md5(tout->data, tout->len, datamd5);
#ifdef DEBUG
printf("decrypt md5: ");
for(int i = 0; i < 16; i++) printf("%02x", datamd5[i]);
@@ -166,16 +164,14 @@ int cmdpacket_decrypt(CMDPACKET* p, int index, const char pwd[64]) {
for(int i = 0; i < tout->len; i++) printf("%02x", tout->data[i]);
putchar('\n');
#endif
if(is_md5_equal(datamd5, p->md5)) {
if(is_md5_equal((uint8_t*)datamd5, p->md5)) {
seqs[index]++;
p->datalen = tout->len;
memcpy(p->data, tout->data, p->datalen);
free(datamd5);
free(tout->ptr);
free(tout);
return 1;
}
free(datamd5);
free(tout->ptr);
free(tout);
return 0;

76
dict.c
View File

@@ -3,41 +3,43 @@
#include <stdlib.h>
#include <string.h>
#include <simplecrypto.h>
#include <pthread.h>
#include "dict.h"
#include "server.h"
static uint8_t lock = 0;
static pthread_rwlock_t mu;
static int lock;
static char* filepath;
static uint8_t* dict_md5;
static uint8_t dict_md5[16];
static FILE* fp = NULL; //fp for EX
static FILE* fp5 = NULL; //fp for md5
static FILE* thread_fp[THREADCNT];
#ifdef CPUBIT64
#define _dict_md5_2 ((uint64_t*)dict_md5)
#define _dict_md5_2 ((uint64_t*)&dict_md5)
#else
#define _dict_md5_4 ((uint32_t*)dict_md5)
#define _dict_md5_4 ((uint32_t*)&dict_md5)
#endif
int fill_md5() {
size_t size = get_dict_size();
if(!size) {
if(dict_md5) free(dict_md5);
dict_md5 = malloc(16);
memset(dict_md5, 0, 16);
puts("Dict is empty, use all zero md5");
return 1;
}
uint8_t* dict_buff = (uint8_t*)malloc(size);
if(dict_buff) {
pthread_rwlock_rdlock(&mu);
rewind(fp5);
if(fread(dict_buff, size, 1, fp5) == 1) {
if(dict_md5) free(dict_md5);
dict_md5 = md5(dict_buff, size);
pthread_rwlock_unlock(&mu);
md5(dict_buff, size, dict_md5);
free(dict_buff);
return 1;
} else {
pthread_rwlock_unlock(&mu);
free(dict_buff);
puts("Read dict error");
return 0;
@@ -52,6 +54,11 @@ int init_dict(char* file_path) {
fp = fopen(file_path, "rb+");
fp5 = fopen(file_path, "rb");
if(fp) {
int err = pthread_rwlock_init(&mu, NULL);
if(err) {
puts("Init lock error");
return 0;
}
lock = DICT_LOCK_UN;
filepath = file_path;
return fill_md5();
@@ -61,52 +68,51 @@ int init_dict(char* file_path) {
}
}
#define _open_dict(p)\
if(p) {\
lock |= lock_type;\
rewind(p);\
return p;\
} else {\
puts("Open dict error");\
return NULL;\
}
FILE* open_dict(uint8_t lock_type, uint32_t index) {
if(lock & DICT_LOCK_EX) return NULL;
else if(lock_type & DICT_LOCK_EX) {
if(lock_type & DICT_LOCK_EX) {
pthread_rwlock_wrlock(&mu);
lock |= DICT_LOCK_EX;
if(!fp) fp = fopen(filepath, "rb+");
_open_dict(fp);
} else if(index < THREADCNT) {
if(!thread_fp[index]) thread_fp[index] = fopen(filepath, "rb");
_open_dict(thread_fp[index]);
} else {
puts("Index out of bounds");
else rewind(fp);
return fp;
}
if(index >= THREADCNT) {
puts("Open dict: Index out of bounds");
return NULL;
}
pthread_rwlock_rdlock(&mu);
lock |= DICT_LOCK_SH;
if(!thread_fp[index]) thread_fp[index] = fopen(filepath, "rb");
else rewind(thread_fp[index]);
return thread_fp[index];
}
FILE* get_dict_fp(uint32_t index) {
FILE* get_dict_fp_wr() {
if(lock & DICT_LOCK_EX) return fp;
else if(lock & DICT_LOCK_SH && index < THREADCNT) return thread_fp[index];
else return NULL;
return NULL;
}
FILE* get_unique_dict_fp() {
FILE* get_dict_fp_rd() {
rewind(fp5);
return fp5;
}
void close_dict(uint8_t lock_type, uint32_t index) {
puts("Close dict");
lock &= ~lock_type;
if(lock_type & DICT_LOCK_EX) fflush(fp);
else fflush(thread_fp[index]);
lock &= ~lock_type;
pthread_rwlock_unlock(&mu);
puts("Close dict");
}
off_t get_dict_size() {
struct stat statbuf;
if(stat(filepath, &statbuf)==0) return statbuf.st_size;
else return -1;
pthread_rwlock_rdlock(&mu);
if(stat(filepath, &statbuf)==0) {
pthread_rwlock_unlock(&mu);
return statbuf.st_size;
}
pthread_rwlock_unlock(&mu);
return -1;
}
int is_md5_equal(uint8_t* digest) {

4
dict.h
View File

@@ -19,9 +19,9 @@ typedef struct DICT DICT;
int init_dict(char* file_path);
void close_dict(uint8_t lock_type, uint32_t index);
int fill_md5();
FILE* get_dict_fp(uint32_t index);
FILE* get_dict_fp_wr();
off_t get_dict_size();
FILE* get_unique_dict_fp();
FILE* get_dict_fp_rd();
int is_md5_equal(uint8_t* digest);
FILE *open_dict(uint8_t lock_type, uint32_t index);

192
server.c
View File

@@ -50,8 +50,6 @@ static uint32_t* items_len;
static CONFIG* cfg;
static pthread_attr_t attr;
// DICTPOOLBYTE must lower than 4*8 = 32
#define DICTPOOLBIT 16
#define DICTPOOLSZ (((uint32_t)-1)>>((sizeof(uint32_t)*8-DICTPOOLBIT)))
static DICT* dict_pool[DICTPOOLSZ+1];
@@ -142,87 +140,87 @@ static int send_data(int accept_fd, int index, char *data, size_t length) {
static int send_all(THREADTIMER *timer) {
int re = 1;
FILE *fp = open_dict(DICT_LOCK_SH, timer->index);
if(fp) {
timer->lock_type = DICT_LOCK_SH;
off_t len = 0, file_size = get_dict_size();
char* buf = (char*)malloc(file_size);
if(buf) {
if(fread(buf, file_size, 1, fp) == 1) {
#ifdef DEBUG
printf("Get dict file size: %u\n", (unsigned int)file_size);
#endif
char* encbuf = raw_encrypt(buf, &file_size, timer->index, cfg->pwd);
sprintf(timer->dat, "%u$", (unsigned int)file_size);
//printf("Get encrypted file size: %s\n", timer->dat);
//FILE* fp = fopen("raw_after_enc", "wb+");
//fwrite(encbuf, file_size, 1, fp);
//fclose(fp);
if(send(timer->accept_fd, timer->dat, strlen(timer->dat), 0) > 0) {
re = send(timer->accept_fd, encbuf, file_size, 0);
printf("Send %u bytes.\n", re);
close_dict(DICT_LOCK_SH, timer->index);
} else re = 0;
free(encbuf);
}
free(buf);
if(!fp) return 1;
timer->lock_type = DICT_LOCK_SH;
off_t len = 0, file_size = get_dict_size();
char* buf = (char*)malloc(file_size);
if(buf) {
if(fread(buf, file_size, 1, fp) == 1) {
#ifdef DEBUG
printf("Get dict file size: %u\n", (unsigned int)file_size);
#endif
char* encbuf = raw_encrypt(buf, &file_size, timer->index, cfg->pwd);
sprintf(timer->dat, "%u$", (unsigned int)file_size);
//printf("Get encrypted file size: %s\n", timer->dat);
//FILE* fp = fopen("raw_after_enc", "wb+");
//fwrite(encbuf, file_size, 1, fp);
//fclose(fp);
if(send(timer->accept_fd, timer->dat, strlen(timer->dat), 0) > 0) {
re = send(timer->accept_fd, encbuf, file_size, 0);
printf("Send %u bytes.\n", re);
} else re = 0;
free(encbuf);
}
free(buf);
}
close_dict(DICT_LOCK_SH, timer->index);
return re;
}
#define has_next(fp, ch) ((ch=getc(fp)),(feof(fp)?0:(ch?ungetc(ch,fp):1)))
static void init_dict_pool(FILE *fp) {
uint8_t digest[16];
int ch;
while(has_next(fp, ch)) {
if(!ch) continue; // skip null bytes
SIMPLE_PB* spb = get_pb(fp);
if(!spb) continue; // skip error bytes
DICT* d = (DICT*)spb->target;
DICT* dnew = (DICT*)malloc(sizeof(DICT));
memcpy(dnew, d, sizeof(DICT));
char* digest = (char*)md5((uint8_t *)d->key, strlen(d->key)+1);
char* dp = digest;
md5((uint8_t *)d->key, strlen(d->key)+1, digest);
uint8_t* dp = digest;
int p = ((*((uint32_t*)digest))>>(8*sizeof(uint32_t)-DICTPOOLBIT))&DICTPOOLSZ;
uint32_t c = 16-DICTPOOLSZ/8;
int c = 16-4;
DICT* slot;
while((slot=dict_pool[p]) && c--) {
while((slot=dict_pool[p]) && c-->0) {
#ifdef DEBUG
printf("digest of %s: %08x got conflicted.\n",d->key, p);
printf("digest of %s: %08x got conflicted, remaining chance: %d.\n", d->key, p, c);
#endif
p = ((*((uint32_t*)++dp))>>(8*sizeof(uint32_t)-DICTPOOLBIT))&DICTPOOLSZ; // 哈希碰撞
p = ((*((uint32_t*)(++dp)))>>(8*sizeof(uint32_t)-DICTPOOLBIT))&DICTPOOLSZ; // 哈希碰撞
#ifdef DEBUG
printf("skip digest of %s to %08x.\n",d->key, p);
printf("skip digest of %s to %08x.\n", d->key, p);
#endif
}
#ifdef DEBUG
if(slot) printf("digest of %s: %08x is invalid, drop it.\n",d->key, p);
if(slot) printf("cannot find any empty slot for digest of %s: %08x, drop it.\n", d->key, p);
#endif
if(!slot) dict_pool[p] = dnew; // 解决哈希冲突
else free(dnew); // 未解决哈希冲突
if(!slot) {
DICT* dnew = (DICT*)malloc(sizeof(DICT));
memcpy(dnew, d, sizeof(DICT));
dict_pool[p] = dnew; // 解决哈希冲突
}
free(spb);
free(digest);
}
}
static int s1_get(THREADTIMER *timer) {
uint8_t digest[16];
FILE *fp = open_dict(DICT_LOCK_SH, timer->index);
//timer->status = 0;
if(fp) {
int ch;
timer->lock_type = DICT_LOCK_SH;
char* digest = (char*)md5((uint8_t*)timer->dat, strlen(timer->dat)+1);
char* dp = digest;
md5((uint8_t*)timer->dat, strlen(timer->dat)+1, digest);
uint8_t* dp = digest;
int p = ((*((uint32_t*)digest))>>(8*sizeof(uint32_t)-DICTPOOLBIT))&DICTPOOLSZ;
if(!dict_pool[p]) return close_and_send(timer, "null", 4);
uint32_t c = 16-DICTPOOLSZ/8;
int c = 16-4;
int notok = 1;
while(dict_pool[p] && (notok=strcmp(timer->dat, dict_pool[p]->key)) && c--) p = ((*((uint32_t*)++dp))>>(8*sizeof(uint32_t)-DICTPOOLBIT))&DICTPOOLSZ; // 哈希碰撞
free(digest);
while(dict_pool[p] && (notok=strcmp(timer->dat, dict_pool[p]->key)) && c-->0) p = ((*((uint32_t*)(++dp)))>>(8*sizeof(uint32_t)-DICTPOOLBIT))&DICTPOOLSZ; // 哈希碰撞
if(!notok) {
return close_and_send(timer, dict_pool[p]->data, last_nonnull(dict_pool[p]->data, DICTDATSZ));
}
@@ -242,20 +240,19 @@ static int s1_get(THREADTIMER *timer) {
}
static int s2_set(THREADTIMER *timer) {
uint8_t digest[16];
FILE *fp = open_dict(DICT_LOCK_EX, timer->index);
if(fp) {
timer->lock_type = DICT_LOCK_EX;
char* digest = (char*)md5((uint8_t*)timer->dat, strlen(timer->dat)+1);
char* dp = digest;
md5((uint8_t*)timer->dat, strlen(timer->dat)+1, digest);
uint8_t* dp = digest;
int p = ((*((uint32_t*)digest))>>(8*sizeof(uint32_t)-DICTPOOLBIT))&DICTPOOLSZ;
if(!dict_pool[p]) setdict = dict_pool[p] = (DICT*)malloc(sizeof(DICT));
else {
uint32_t c = 16-DICTPOOLSZ/8;
int c = 16-4;
int notok;
while(dict_pool[p] && (notok=strcmp(timer->dat, dict_pool[p]->key)) && c--) p = ((*((uint32_t*)++dp))>>(8*sizeof(uint32_t)-DICTPOOLBIT))&DICTPOOLSZ; // 哈希碰撞
free(digest);
while(dict_pool[p] && (notok=strcmp(timer->dat, dict_pool[p]->key)) && c-->0) p = ((*((uint32_t*)(++dp)))>>(8*sizeof(uint32_t)-DICTPOOLBIT))&DICTPOOLSZ; // 哈希碰撞
if(!dict_pool[p]) setdict = dict_pool[p] = (DICT*)malloc(sizeof(DICT)); // 无值
else if(notok) setdict = &d; // 全部冲突
else { // 已有值
@@ -289,7 +286,7 @@ static int s3_set_data(THREADTIMER *timer) {
#endif
memcpy(setdict->data, timer->dat, datasize);
if(!set_pb(get_dict_fp(timer->index), items_len, sizeof(DICT), setdict)) {
if(!set_pb(get_dict_fp_wr(), items_len, sizeof(DICT), setdict)) {
printf("Error set data: dict[%s]=%s\n", setdict->key, timer->dat);
return close_and_send(timer, "erro", 4);
} else {
@@ -304,66 +301,65 @@ static void del(FILE *fp, char* key, int len, char ret[4]) {
if(!ch) continue; // skip null bytes
SIMPLE_PB* spb = get_pb(fp);
DICT* d = (DICT*)spb->target;
if(!memcmp(key, d->key, len)) {
uint32_t next = ftell(fp);
uint32_t this = next - spb->real_len;
fseek(fp, 0, SEEK_END);
uint32_t end = ftell(fp);
if(next == end) {
if(!ftruncate(fileno(fp), end - spb->real_len)) {
free(spb);
*(uint32_t*)ret = *(uint32_t*)"succ";
return;
} else {
free(spb);
*(uint32_t*)ret = *(uint32_t*)"erro";
return;
}
if(memcmp(key, d->key, len)) {
free(spb);
continue;
}
uint32_t next = ftell(fp);
uint32_t this = next - spb->real_len;
fseek(fp, 0, SEEK_END);
uint32_t end = ftell(fp);
if(next == end) {
if(!ftruncate(fileno(fp), end - spb->real_len)) {
free(spb);
*(uint32_t*)ret = *(uint32_t*)"succ";
} else {
uint32_t cap = end - next;
#ifdef DEBUG
printf("this: %u, next: %u, end: %u, cap: %u\n", this, next, end, cap);
#endif
char* data = malloc(cap);
if(data) {
fseek(fp, next, SEEK_SET);
if(fread(data, cap, 1, fp) == 1) {
if(!ftruncate(fileno(fp), end - spb->real_len)) {
fseek(fp, this, SEEK_SET);
if(fwrite(data, cap, 1, fp) == 1) {
free(data);
free(spb);
*(uint32_t*)ret = *(uint32_t*)"succ";
return;
}
}
}
free(data);
}
free(spb);
*(uint32_t*)ret = *(uint32_t*)"erro";
return;
}
} else free(spb);
return;
}
uint32_t cap = end - next;
#ifdef DEBUG
printf("this: %u, next: %u, end: %u, cap: %u\n", this, next, end, cap);
#endif
char* data = malloc(cap);
if(data) {
fseek(fp, next, SEEK_SET);
if(fread(data, cap, 1, fp) == 1) {
if(!ftruncate(fileno(fp), end - spb->real_len)) {
fseek(fp, this, SEEK_SET);
if(fwrite(data, cap, 1, fp) == 1) {
free(data);
free(spb);
*(uint32_t*)ret = *(uint32_t*)"succ";
return;
}
}
}
free(data);
}
free(spb);
*(uint32_t*)ret = *(uint32_t*)"erro";
return;
}
*(uint32_t*)ret = *(uint32_t*)"null";
return;
}
static int s4_del(THREADTIMER *timer) {
uint8_t digest[16];
char ret[4];
FILE *fp = open_dict(DICT_LOCK_EX, timer->index);
//timer->status = 0;
if(fp) {
char ret[4];
timer->lock_type = DICT_LOCK_EX;
char* digest = (char*)md5((uint8_t*)timer->dat, strlen(timer->dat)+1);
char* dp = digest;
md5((uint8_t*)timer->dat, strlen(timer->dat)+1, digest);
uint8_t* dp = digest;
int p = ((*((uint32_t*)digest))>>(8*sizeof(uint32_t)-DICTPOOLBIT))&DICTPOOLSZ;
uint32_t c = 16-DICTPOOLSZ/8;
int c = 16-4;
int notok = 1;
while(dict_pool[p] && (notok=strcmp(timer->dat, dict_pool[p]->key)) && c--) p = ((*((uint32_t*)++dp))>>(8*sizeof(uint32_t)-DICTPOOLBIT))&DICTPOOLSZ; // 哈希碰撞
free(digest);
while(dict_pool[p] && (notok=strcmp(timer->dat, dict_pool[p]->key)) && c-->0) p = ((*((uint32_t*)(++dp)))>>(8*sizeof(uint32_t)-DICTPOOLBIT))&DICTPOOLSZ; // 哈希碰撞
if(notok) return close_and_send(timer, "null", 4);
free(dict_pool[p]);
dict_pool[p] = NULL;
@@ -443,7 +439,7 @@ static void handle_accept(void *p) {
//send_data(accept_fd, "Welcome to simple dict server.", 31);
//timer_pointer_of(p)->status = -1;
uint32_t index = timer_pointer_of(p)->index;
char *buff = calloc(BUFSIZ, sizeof(char));
char *buff = malloc(BUFSIZ*sizeof(char));
if(buff) {
timer_pointer_of(p)->ptr = buff;
CMDPACKET* cp = (CMDPACKET*)buff;
@@ -561,7 +557,7 @@ static void accept_client() {
pthread_attr_init(&attr);
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
init_crypto();
init_dict_pool(get_unique_dict_fp());
init_dict_pool(get_dict_fp_rd());
if(pid < 0) puts("Error when forking a subprocess.");
else while(1) {
puts("Ready for accept, waitting...");

5
server.h
View File

@@ -1,9 +1,14 @@
#ifndef _SERVER_H_
#define _SERVER_H_
#include <stdint.h>
#define THREADCNT 16
#define MAXWAITSEC 10
// DICTPOOLBIT must be lower than 4*8 = 32
#define DICTPOOLBIT 16
enum SERVERCMD {CMDGET, CMDCAT, CMDMD5, CMDACK, CMDEND, CMDSET, CMDDEL, CMDDAT};
struct CMDPACKET {