diff --git a/CMakeLists.txt b/CMakeLists.txt index 37c4fbc..d38ca98 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ project(simple-dict-server C) SET(CMAKE_BUILD_TYPE "Release") add_definitions(-DLISTEN_ON_IPV6) -add_definitions(-DDEBUG) +#add_definitions(-DDEBUG) IF(CMAKE_SIZEOF_VOID_P EQUAL 8) add_definitions("-DCPUBIT64") ELSE() @@ -23,12 +23,12 @@ add_executable(cfgwriter cfgwriter.c) #add_executable(migrate migrate.c) #add_executable(migratenew migratenew.c) -target_link_libraries(simple-dict-server scrypto spb pthread) +target_link_libraries(simple-dict-server scrypto libspb.a pthread) target_link_libraries(simple-dict-client scrypto pthread) -target_link_libraries(cfgwriter spb) +target_link_libraries(cfgwriter libspb.a) -#target_link_libraries(migrate spb) -#target_link_libraries(migratenew spb) +#target_link_libraries(migrate libspb.a) +#target_link_libraries(migratenew libspb.a) INSTALL(TARGETS simple-dict-server RUNTIME DESTINATION bin) #INSTALL(TARGETS simple-dict-client RUNTIME DESTINATION bin) diff --git a/crypto.c b/crypto.c index a7c8fbf..fb854c4 100644 --- a/crypto.c +++ b/crypto.c @@ -108,14 +108,12 @@ void cmdpacket_encrypt(CMDPACKET* p, int index, const char pwd[64]) { TEADAT* tout = tea_encrypt_native_endian(tea, sumtable, &tin); - uint8_t* datamd5 = md5(p->data, p->datalen); + md5(p->data, p->datalen, p->md5); #ifdef DEBUG printf("encrypt md5: "); - for(int i = 0; i < 16; i++) printf("%02x", datamd5[i]); + for(int i = 0; i < 16; i++) printf("%02x", p->md5[i]); putchar('\n'); #endif - memcpy(p->md5, datamd5, 16); - free(datamd5); p->datalen = tout->len; memcpy(p->data, tout->data, p->datalen); @@ -156,8 +154,8 @@ int cmdpacket_decrypt(CMDPACKET* p, int index, const char pwd[64]) { free(tout); return 0; } - - uint8_t* datamd5 = md5(tout->data, tout->len); + uint8_t datamd5[16]; + md5(tout->data, tout->len, datamd5); #ifdef DEBUG printf("decrypt md5: "); for(int i = 0; i < 16; i++) printf("%02x", datamd5[i]); @@ -166,16 +164,14 @@ int cmdpacket_decrypt(CMDPACKET* p, int index, const char pwd[64]) { for(int i = 0; i < tout->len; i++) printf("%02x", tout->data[i]); putchar('\n'); #endif - if(is_md5_equal(datamd5, p->md5)) { + if(is_md5_equal((uint8_t*)datamd5, p->md5)) { seqs[index]++; p->datalen = tout->len; memcpy(p->data, tout->data, p->datalen); - free(datamd5); free(tout->ptr); free(tout); return 1; } - free(datamd5); free(tout->ptr); free(tout); return 0; diff --git a/dict.c b/dict.c index 0e7be99..2a3aa84 100644 --- a/dict.c +++ b/dict.c @@ -3,41 +3,43 @@ #include #include #include +#include #include "dict.h" #include "server.h" -static uint8_t lock = 0; +static pthread_rwlock_t mu; +static int lock; static char* filepath; -static uint8_t* dict_md5; +static uint8_t dict_md5[16]; static FILE* fp = NULL; //fp for EX static FILE* fp5 = NULL; //fp for md5 static FILE* thread_fp[THREADCNT]; #ifdef CPUBIT64 - #define _dict_md5_2 ((uint64_t*)dict_md5) + #define _dict_md5_2 ((uint64_t*)&dict_md5) #else - #define _dict_md5_4 ((uint32_t*)dict_md5) + #define _dict_md5_4 ((uint32_t*)&dict_md5) #endif int fill_md5() { size_t size = get_dict_size(); if(!size) { - if(dict_md5) free(dict_md5); - dict_md5 = malloc(16); memset(dict_md5, 0, 16); puts("Dict is empty, use all zero md5"); return 1; } uint8_t* dict_buff = (uint8_t*)malloc(size); if(dict_buff) { + pthread_rwlock_rdlock(&mu); rewind(fp5); if(fread(dict_buff, size, 1, fp5) == 1) { - if(dict_md5) free(dict_md5); - dict_md5 = md5(dict_buff, size); + pthread_rwlock_unlock(&mu); + md5(dict_buff, size, dict_md5); free(dict_buff); return 1; } else { + pthread_rwlock_unlock(&mu); free(dict_buff); puts("Read dict error"); return 0; @@ -52,6 +54,11 @@ int init_dict(char* file_path) { fp = fopen(file_path, "rb+"); fp5 = fopen(file_path, "rb"); if(fp) { + int err = pthread_rwlock_init(&mu, NULL); + if(err) { + puts("Init lock error"); + return 0; + } lock = DICT_LOCK_UN; filepath = file_path; return fill_md5(); @@ -61,52 +68,51 @@ int init_dict(char* file_path) { } } -#define _open_dict(p)\ - if(p) {\ - lock |= lock_type;\ - rewind(p);\ - return p;\ - } else {\ - puts("Open dict error");\ - return NULL;\ - } - FILE* open_dict(uint8_t lock_type, uint32_t index) { - if(lock & DICT_LOCK_EX) return NULL; - else if(lock_type & DICT_LOCK_EX) { + if(lock_type & DICT_LOCK_EX) { + pthread_rwlock_wrlock(&mu); + lock |= DICT_LOCK_EX; if(!fp) fp = fopen(filepath, "rb+"); - _open_dict(fp); - } else if(index < THREADCNT) { - if(!thread_fp[index]) thread_fp[index] = fopen(filepath, "rb"); - _open_dict(thread_fp[index]); - } else { - puts("Index out of bounds"); + else rewind(fp); + return fp; + } + if(index >= THREADCNT) { + puts("Open dict: Index out of bounds"); return NULL; } + pthread_rwlock_rdlock(&mu); + lock |= DICT_LOCK_SH; + if(!thread_fp[index]) thread_fp[index] = fopen(filepath, "rb"); + else rewind(thread_fp[index]); + return thread_fp[index]; } -FILE* get_dict_fp(uint32_t index) { +FILE* get_dict_fp_wr() { if(lock & DICT_LOCK_EX) return fp; - else if(lock & DICT_LOCK_SH && index < THREADCNT) return thread_fp[index]; - else return NULL; + return NULL; } -FILE* get_unique_dict_fp() { +FILE* get_dict_fp_rd() { rewind(fp5); return fp5; } void close_dict(uint8_t lock_type, uint32_t index) { - puts("Close dict"); - lock &= ~lock_type; if(lock_type & DICT_LOCK_EX) fflush(fp); - else fflush(thread_fp[index]); + lock &= ~lock_type; + pthread_rwlock_unlock(&mu); + puts("Close dict"); } off_t get_dict_size() { struct stat statbuf; - if(stat(filepath, &statbuf)==0) return statbuf.st_size; - else return -1; + pthread_rwlock_rdlock(&mu); + if(stat(filepath, &statbuf)==0) { + pthread_rwlock_unlock(&mu); + return statbuf.st_size; + } + pthread_rwlock_unlock(&mu); + return -1; } int is_md5_equal(uint8_t* digest) { diff --git a/dict.h b/dict.h index 6c20248..b02c482 100644 --- a/dict.h +++ b/dict.h @@ -19,9 +19,9 @@ typedef struct DICT DICT; int init_dict(char* file_path); void close_dict(uint8_t lock_type, uint32_t index); int fill_md5(); -FILE* get_dict_fp(uint32_t index); +FILE* get_dict_fp_wr(); off_t get_dict_size(); -FILE* get_unique_dict_fp(); +FILE* get_dict_fp_rd(); int is_md5_equal(uint8_t* digest); FILE *open_dict(uint8_t lock_type, uint32_t index); diff --git a/server.c b/server.c index 4b33a53..4e9cf62 100644 --- a/server.c +++ b/server.c @@ -50,8 +50,6 @@ static uint32_t* items_len; static CONFIG* cfg; static pthread_attr_t attr; -// DICTPOOLBYTE must lower than 4*8 = 32 -#define DICTPOOLBIT 16 #define DICTPOOLSZ (((uint32_t)-1)>>((sizeof(uint32_t)*8-DICTPOOLBIT))) static DICT* dict_pool[DICTPOOLSZ+1]; @@ -142,87 +140,87 @@ static int send_data(int accept_fd, int index, char *data, size_t length) { static int send_all(THREADTIMER *timer) { int re = 1; FILE *fp = open_dict(DICT_LOCK_SH, timer->index); - if(fp) { - timer->lock_type = DICT_LOCK_SH; - off_t len = 0, file_size = get_dict_size(); - char* buf = (char*)malloc(file_size); - if(buf) { - if(fread(buf, file_size, 1, fp) == 1) { - #ifdef DEBUG - printf("Get dict file size: %u\n", (unsigned int)file_size); - #endif - char* encbuf = raw_encrypt(buf, &file_size, timer->index, cfg->pwd); - sprintf(timer->dat, "%u$", (unsigned int)file_size); - //printf("Get encrypted file size: %s\n", timer->dat); - //FILE* fp = fopen("raw_after_enc", "wb+"); - //fwrite(encbuf, file_size, 1, fp); - //fclose(fp); - if(send(timer->accept_fd, timer->dat, strlen(timer->dat), 0) > 0) { - re = send(timer->accept_fd, encbuf, file_size, 0); - printf("Send %u bytes.\n", re); - close_dict(DICT_LOCK_SH, timer->index); - } else re = 0; - free(encbuf); - } - free(buf); + if(!fp) return 1; + timer->lock_type = DICT_LOCK_SH; + off_t len = 0, file_size = get_dict_size(); + char* buf = (char*)malloc(file_size); + if(buf) { + if(fread(buf, file_size, 1, fp) == 1) { + #ifdef DEBUG + printf("Get dict file size: %u\n", (unsigned int)file_size); + #endif + char* encbuf = raw_encrypt(buf, &file_size, timer->index, cfg->pwd); + sprintf(timer->dat, "%u$", (unsigned int)file_size); + //printf("Get encrypted file size: %s\n", timer->dat); + //FILE* fp = fopen("raw_after_enc", "wb+"); + //fwrite(encbuf, file_size, 1, fp); + //fclose(fp); + if(send(timer->accept_fd, timer->dat, strlen(timer->dat), 0) > 0) { + re = send(timer->accept_fd, encbuf, file_size, 0); + printf("Send %u bytes.\n", re); + } else re = 0; + free(encbuf); } + free(buf); } + close_dict(DICT_LOCK_SH, timer->index); return re; } #define has_next(fp, ch) ((ch=getc(fp)),(feof(fp)?0:(ch?ungetc(ch,fp):1))) static void init_dict_pool(FILE *fp) { + uint8_t digest[16]; int ch; while(has_next(fp, ch)) { if(!ch) continue; // skip null bytes SIMPLE_PB* spb = get_pb(fp); + if(!spb) continue; // skip error bytes DICT* d = (DICT*)spb->target; - DICT* dnew = (DICT*)malloc(sizeof(DICT)); - memcpy(dnew, d, sizeof(DICT)); - - char* digest = (char*)md5((uint8_t *)d->key, strlen(d->key)+1); - char* dp = digest; + md5((uint8_t *)d->key, strlen(d->key)+1, digest); + uint8_t* dp = digest; int p = ((*((uint32_t*)digest))>>(8*sizeof(uint32_t)-DICTPOOLBIT))&DICTPOOLSZ; - uint32_t c = 16-DICTPOOLSZ/8; + int c = 16-4; DICT* slot; - while((slot=dict_pool[p]) && c--) { + while((slot=dict_pool[p]) && c-->0) { #ifdef DEBUG - printf("digest of %s: %08x got conflicted.\n",d->key, p); + printf("digest of %s: %08x got conflicted, remaining chance: %d.\n", d->key, p, c); #endif - p = ((*((uint32_t*)++dp))>>(8*sizeof(uint32_t)-DICTPOOLBIT))&DICTPOOLSZ; // 哈希碰撞 + p = ((*((uint32_t*)(++dp)))>>(8*sizeof(uint32_t)-DICTPOOLBIT))&DICTPOOLSZ; // 哈希碰撞 #ifdef DEBUG - printf("skip digest of %s to %08x.\n",d->key, p); + printf("skip digest of %s to %08x.\n", d->key, p); #endif } #ifdef DEBUG - if(slot) printf("digest of %s: %08x is invalid, drop it.\n",d->key, p); + if(slot) printf("cannot find any empty slot for digest of %s: %08x, drop it.\n", d->key, p); #endif - if(!slot) dict_pool[p] = dnew; // 解决哈希冲突 - else free(dnew); // 未解决哈希冲突 + + if(!slot) { + DICT* dnew = (DICT*)malloc(sizeof(DICT)); + memcpy(dnew, d, sizeof(DICT)); + dict_pool[p] = dnew; // 解决哈希冲突 + } free(spb); - free(digest); } } static int s1_get(THREADTIMER *timer) { + uint8_t digest[16]; FILE *fp = open_dict(DICT_LOCK_SH, timer->index); //timer->status = 0; if(fp) { int ch; timer->lock_type = DICT_LOCK_SH; - - char* digest = (char*)md5((uint8_t*)timer->dat, strlen(timer->dat)+1); - char* dp = digest; + md5((uint8_t*)timer->dat, strlen(timer->dat)+1, digest); + uint8_t* dp = digest; int p = ((*((uint32_t*)digest))>>(8*sizeof(uint32_t)-DICTPOOLBIT))&DICTPOOLSZ; if(!dict_pool[p]) return close_and_send(timer, "null", 4); - uint32_t c = 16-DICTPOOLSZ/8; + int c = 16-4; int notok = 1; - while(dict_pool[p] && (notok=strcmp(timer->dat, dict_pool[p]->key)) && c--) p = ((*((uint32_t*)++dp))>>(8*sizeof(uint32_t)-DICTPOOLBIT))&DICTPOOLSZ; // 哈希碰撞 - free(digest); + while(dict_pool[p] && (notok=strcmp(timer->dat, dict_pool[p]->key)) && c-->0) p = ((*((uint32_t*)(++dp)))>>(8*sizeof(uint32_t)-DICTPOOLBIT))&DICTPOOLSZ; // 哈希碰撞 if(!notok) { return close_and_send(timer, dict_pool[p]->data, last_nonnull(dict_pool[p]->data, DICTDATSZ)); } @@ -242,20 +240,19 @@ static int s1_get(THREADTIMER *timer) { } static int s2_set(THREADTIMER *timer) { + uint8_t digest[16]; FILE *fp = open_dict(DICT_LOCK_EX, timer->index); if(fp) { timer->lock_type = DICT_LOCK_EX; - - char* digest = (char*)md5((uint8_t*)timer->dat, strlen(timer->dat)+1); - char* dp = digest; + md5((uint8_t*)timer->dat, strlen(timer->dat)+1, digest); + uint8_t* dp = digest; int p = ((*((uint32_t*)digest))>>(8*sizeof(uint32_t)-DICTPOOLBIT))&DICTPOOLSZ; if(!dict_pool[p]) setdict = dict_pool[p] = (DICT*)malloc(sizeof(DICT)); else { - uint32_t c = 16-DICTPOOLSZ/8; + int c = 16-4; int notok; - while(dict_pool[p] && (notok=strcmp(timer->dat, dict_pool[p]->key)) && c--) p = ((*((uint32_t*)++dp))>>(8*sizeof(uint32_t)-DICTPOOLBIT))&DICTPOOLSZ; // 哈希碰撞 - free(digest); + while(dict_pool[p] && (notok=strcmp(timer->dat, dict_pool[p]->key)) && c-->0) p = ((*((uint32_t*)(++dp)))>>(8*sizeof(uint32_t)-DICTPOOLBIT))&DICTPOOLSZ; // 哈希碰撞 if(!dict_pool[p]) setdict = dict_pool[p] = (DICT*)malloc(sizeof(DICT)); // 无值 else if(notok) setdict = &d; // 全部冲突 else { // 已有值 @@ -289,7 +286,7 @@ static int s3_set_data(THREADTIMER *timer) { #endif memcpy(setdict->data, timer->dat, datasize); - if(!set_pb(get_dict_fp(timer->index), items_len, sizeof(DICT), setdict)) { + if(!set_pb(get_dict_fp_wr(), items_len, sizeof(DICT), setdict)) { printf("Error set data: dict[%s]=%s\n", setdict->key, timer->dat); return close_and_send(timer, "erro", 4); } else { @@ -304,66 +301,65 @@ static void del(FILE *fp, char* key, int len, char ret[4]) { if(!ch) continue; // skip null bytes SIMPLE_PB* spb = get_pb(fp); DICT* d = (DICT*)spb->target; - if(!memcmp(key, d->key, len)) { - uint32_t next = ftell(fp); - uint32_t this = next - spb->real_len; - fseek(fp, 0, SEEK_END); - uint32_t end = ftell(fp); - if(next == end) { - if(!ftruncate(fileno(fp), end - spb->real_len)) { - free(spb); - *(uint32_t*)ret = *(uint32_t*)"succ"; - return; - } else { - free(spb); - *(uint32_t*)ret = *(uint32_t*)"erro"; - return; - } + if(memcmp(key, d->key, len)) { + free(spb); + continue; + } + uint32_t next = ftell(fp); + uint32_t this = next - spb->real_len; + fseek(fp, 0, SEEK_END); + uint32_t end = ftell(fp); + if(next == end) { + if(!ftruncate(fileno(fp), end - spb->real_len)) { + free(spb); + *(uint32_t*)ret = *(uint32_t*)"succ"; } else { - uint32_t cap = end - next; - #ifdef DEBUG - printf("this: %u, next: %u, end: %u, cap: %u\n", this, next, end, cap); - #endif - char* data = malloc(cap); - if(data) { - fseek(fp, next, SEEK_SET); - if(fread(data, cap, 1, fp) == 1) { - if(!ftruncate(fileno(fp), end - spb->real_len)) { - fseek(fp, this, SEEK_SET); - if(fwrite(data, cap, 1, fp) == 1) { - free(data); - free(spb); - *(uint32_t*)ret = *(uint32_t*)"succ"; - return; - } - } - } - free(data); - } free(spb); *(uint32_t*)ret = *(uint32_t*)"erro"; - return; } - } else free(spb); + return; + } + uint32_t cap = end - next; + #ifdef DEBUG + printf("this: %u, next: %u, end: %u, cap: %u\n", this, next, end, cap); + #endif + char* data = malloc(cap); + if(data) { + fseek(fp, next, SEEK_SET); + if(fread(data, cap, 1, fp) == 1) { + if(!ftruncate(fileno(fp), end - spb->real_len)) { + fseek(fp, this, SEEK_SET); + if(fwrite(data, cap, 1, fp) == 1) { + free(data); + free(spb); + *(uint32_t*)ret = *(uint32_t*)"succ"; + return; + } + } + } + free(data); + } + free(spb); + *(uint32_t*)ret = *(uint32_t*)"erro"; + return; } *(uint32_t*)ret = *(uint32_t*)"null"; return; } static int s4_del(THREADTIMER *timer) { + uint8_t digest[16]; + char ret[4]; FILE *fp = open_dict(DICT_LOCK_EX, timer->index); //timer->status = 0; if(fp) { - char ret[4]; timer->lock_type = DICT_LOCK_EX; - - char* digest = (char*)md5((uint8_t*)timer->dat, strlen(timer->dat)+1); - char* dp = digest; + md5((uint8_t*)timer->dat, strlen(timer->dat)+1, digest); + uint8_t* dp = digest; int p = ((*((uint32_t*)digest))>>(8*sizeof(uint32_t)-DICTPOOLBIT))&DICTPOOLSZ; - uint32_t c = 16-DICTPOOLSZ/8; + int c = 16-4; int notok = 1; - while(dict_pool[p] && (notok=strcmp(timer->dat, dict_pool[p]->key)) && c--) p = ((*((uint32_t*)++dp))>>(8*sizeof(uint32_t)-DICTPOOLBIT))&DICTPOOLSZ; // 哈希碰撞 - free(digest); + while(dict_pool[p] && (notok=strcmp(timer->dat, dict_pool[p]->key)) && c-->0) p = ((*((uint32_t*)(++dp)))>>(8*sizeof(uint32_t)-DICTPOOLBIT))&DICTPOOLSZ; // 哈希碰撞 if(notok) return close_and_send(timer, "null", 4); free(dict_pool[p]); dict_pool[p] = NULL; @@ -443,7 +439,7 @@ static void handle_accept(void *p) { //send_data(accept_fd, "Welcome to simple dict server.", 31); //timer_pointer_of(p)->status = -1; uint32_t index = timer_pointer_of(p)->index; - char *buff = calloc(BUFSIZ, sizeof(char)); + char *buff = malloc(BUFSIZ*sizeof(char)); if(buff) { timer_pointer_of(p)->ptr = buff; CMDPACKET* cp = (CMDPACKET*)buff; @@ -561,7 +557,7 @@ static void accept_client() { pthread_attr_init(&attr); pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); init_crypto(); - init_dict_pool(get_unique_dict_fp()); + init_dict_pool(get_dict_fp_rd()); if(pid < 0) puts("Error when forking a subprocess."); else while(1) { puts("Ready for accept, waitting..."); diff --git a/server.h b/server.h index a1e6463..d661f54 100644 --- a/server.h +++ b/server.h @@ -1,9 +1,14 @@ #ifndef _SERVER_H_ #define _SERVER_H_ +#include + #define THREADCNT 16 #define MAXWAITSEC 10 +// DICTPOOLBIT must be lower than 4*8 = 32 +#define DICTPOOLBIT 16 + enum SERVERCMD {CMDGET, CMDCAT, CMDMD5, CMDACK, CMDEND, CMDSET, CMDDEL, CMDDAT}; struct CMDPACKET {