diff --git a/.gitignore b/.gitignore index 800f757..889a442 100644 --- a/.gitignore +++ b/.gitignore @@ -52,4 +52,5 @@ Mkfile.old dkms.conf build -.vscode \ No newline at end of file +.vscode +.DS_Store diff --git a/api/index.md b/api/index.md index 602a0aa..f889917 100644 --- a/api/index.md +++ b/api/index.md @@ -72,6 +72,7 @@ > 备选方案 - 使用哈希桶作索引，允许溢出桶，同样每个桶大小为`4096`字节，可装`340`个值。共`256`个桶，初始占用`1M`空间，后续由于溢出可能会增加。 +- 哈希算法: hash = (n*(n+11454191981))%256 ``` 0 8 12 20 ┌───────────────────┬─────────┬───────────────────┬─────────┐ @@ -114,6 +115,7 @@ > 备选方案 - 使用哈希桶作索引，允许溢出桶，同样每个桶大小为`4096`字节，可装`255`个值。共`1024`个桶，初始占用`4M`空间，后续由于溢出可能会增加。 +- 哈希算法: hash = (n*(n+18446744073709551557))%1024 ``` 0 8 ┌───────────────────┬───────────────────┐ diff --git a/include/types.h b/include/types.h index 3809984..7e0a76b 100644 --- a/include/types.h +++ b/include/types.h @@ -4,6 +4,10 @@ #include #include +#ifndef uint8_t +typedef unsigned char uint8_t; +#endif + typedef uint8_t type_t; typedef uint64_t key_t; diff --git a/include/types/int32.h b/include/types/int32.h index d588d7f..4c6600c 100644 --- a/include/types/int32.h +++ b/include/types/int32.h @@ -4,11 +4,13 @@ #include #include "../types.h" -#define INT32_BUCKET_SZ (256) -#define INT32_INDEX_SZ ((INT32_BUCKET_SZ+1)*(PAGESZ+8)) +#define INT32_BUCKET_SZ (256ull) +#define INT32_BUCKET_MAX_ITEM_SZ (340ull) +#define INT32_INDEX_SZ ((INT32_BUCKET_SZ+2)*(PAGESZ+8)) +#define INT32_DIGEST_FACTOR (11454191981ull) -// len(buf) >= INT32_INDEX_SZ = (PAGESZ+8)*(256+1) = 4104*257 = 1054728 (1M) -// 256个哈希桶, 多出来一个是机动空间, 用于加载溢出桶 +// len(buf) >= INT32_INDEX_SZ = (PAGESZ+8)*(256+2) = 4104*258 +// 256个哈希桶, 多出来2个是机动空间, 用于加载溢出桶 void* create_int32_index(int fd, void* buf); void* load_int32_index(int fd, uint64_t ptr, void* buf); diff --git a/include/types/int64.h b/include/types/int64.h new file mode 100644 index 0000000..c21b6f7 --- /dev/null +++ b/include/types/int64.h @@ -0,0 +1,28 @@ +#ifndef _TYPE_INT64_H_ +#define _TYPE_INT64_H_ + +#include +#include "../types.h" + +#define INT64_BUCKET_SZ (1024ull) +#define INT64_BUCKET_MAX_ITEM_SZ (255ull) +#define INT64_INDEX_SZ ((INT64_BUCKET_SZ+2)*(PAGESZ+8)) +#define INT64_DIGEST_FACTOR (18446744073709551557ull) + +// len(buf) >= INT64_INDEX_SZ +// 1024个哈希桶, 多出来2个是机动空间, 用于加载溢出桶 +void* create_int64_index(int fd, void* buf); + +void* load_int64_index(int fd, uint64_t ptr, void* buf); + +int remove_int64_index(int fd, void* index); + +uint64_t count_int64_items(int fd, void* index); + +int insert_int64_item(int fd, void* index, key_t k, uint64_t ptr); + +uint64_t find_item_by_int64_key(int fd, void* index, key_t k); + +uint64_t remove_item_by_int64_key(int fd, void* index, key_t k); + +#endif \ No newline at end of file diff --git a/src/types.c b/src/types.c index 8ad06d0..a78daf4 100644 --- a/src/types.c +++ b/src/types.c @@ -2,6 +2,8 @@ #include "../include/types.h" #include "../include/types/int8.h" #include "../include/types/int16.h" +#include "../include/types/int32.h" +#include "../include/types/int64.h" // ptr = init(fd) typedef void* (*_type_init_t)(int, void*); @@ -63,70 +65,70 @@ static uint64_t remove_item_by_not_impl_key(int fd, void* index, key_t k) { static _type_init_t _types_init[] = { create_int8_index, create_int16_index, - create_not_impl_index, - create_not_impl_index, - create_not_impl_index, - create_not_impl_index, + create_int32_index, + create_int64_index, + create_int32_index, // float + create_int64_index, // double create_not_impl_index }; static _type_load_t _types_load[] = { load_int8_index, load_int16_index, - load_not_impl_index, - load_not_impl_index, - load_not_impl_index, - load_not_impl_index, + load_int32_index, + load_int64_index, + load_int32_index, // float + load_int64_index, // double load_not_impl_index }; static _type_remove_t _types_remove[] = { remove_int8_index, remove_int16_index, - remove_not_impl_index, - remove_not_impl_index, - remove_not_impl_index, - remove_not_impl_index, + remove_int32_index, + remove_int64_index, + remove_int32_index, // float + remove_int64_index, // double remove_not_impl_index }; static _type_count_t _types_count[] = { count_int8_items, count_int16_items, - count_not_impl_items, - count_not_impl_items, - count_not_impl_items, - count_not_impl_items, + count_int32_items, + count_int64_items, + count_int32_items, // float + count_int64_items, // double count_not_impl_items }; static _insert_item_t _insert_item[] = { insert_int8_item, insert_int16_item, - insert_not_impl_item, - insert_not_impl_item, - insert_not_impl_item, - insert_not_impl_item, + insert_int32_item, + insert_int64_item, + insert_int32_item, // float + insert_int64_item, // double insert_not_impl_item }; static _find_by_key_t _find_item_by_key[] = { find_item_by_int8_key, find_item_by_int16_key, - find_item_by_not_impl_key, - find_item_by_not_impl_key, - find_item_by_not_impl_key, - find_item_by_not_impl_key, + find_item_by_int32_key, + find_item_by_int64_key, + find_item_by_int32_key, // float + find_item_by_int64_key, // double find_item_by_not_impl_key }; static _remove_by_key_t _remove_item_by_key[] = { remove_item_by_int8_key, remove_item_by_int16_key, - remove_item_by_not_impl_key, - remove_item_by_not_impl_key, - remove_item_by_not_impl_key, - remove_item_by_not_impl_key, + remove_item_by_int32_key, + remove_item_by_int64_key, + remove_item_by_int32_key, // float + remove_item_by_int64_key, // double remove_item_by_not_impl_key }; diff --git a/src/types/int32.c b/src/types/int32.c index 6d20744..3c78e1a 100644 --- a/src/types/int32.c +++ b/src/types/int32.c @@ -9,13 +9,15 @@ #include "../../include/page.h" #include "../../include/types/int32.h" +//#define DEBUG + void* create_int32_index(int fd, void* buf) { void* page = alloc_page(fd, buf); if(unlikely(page == NULL)) return NULL; memset(page, 0, PAGESZ); void* prev_page = page; for(int i = 1; i < INT32_BUCKET_SZ; i++) { - page = alloc_page(fd, buf); + page = alloc_page(fd, buf+i*(PAGESZ+8)); if(unlikely(page == NULL)) { for(int j = 0; j < i; j++) free_page(fd, buf+j*(PAGESZ+8)+8); return NULL; @@ -34,13 +36,16 @@ void* create_int32_index(int fd, void* buf) { for(int i = 0; i < INT32_BUCKET_SZ; i++) free_page(fd, buf+i*(PAGESZ+8)+8); return NULL; } + #ifdef DEBUG + puts("alloc finish"); + #endif return buf+8; } void* load_int32_index(int fd, uint64_t ptr, void* buf) { void* page = get_page(fd, ptr, buf); if(unlikely(page == NULL)) return NULL; - for(int i = 1; i < INT32_BUCKET_SZ-1; i++) { + for(int i = 1; i < INT32_BUCKET_SZ; i++) { page = get_page(fd, le64(page+PAGESZ-8), buf+i*(PAGESZ+8)); if(unlikely(page == NULL)) return NULL; } @@ -52,27 +57,213 @@ int remove_int32_index(int fd, void* index) { void* page = index+i*(PAGESZ+8); uint64_t ptr = le64(page+PAGESZ-16); while(unlikely(ptr)) { - void* p = get_page(fd, ptr, index+INT32_INDEX_SZ-BUFSIZ-8); + void* p = get_page(fd, ptr, index-8+INT32_INDEX_SZ-PAGESZ-8); if(unlikely(p == NULL)) return EOF; ptr = le64(p+PAGESZ-16); if(unlikely(free_page(fd, p))) return EOF; } if(unlikely(free_page(fd, page))) return EOF; } + return 0; } uint64_t count_int32_items(int fd, void* index) { - + uint64_t sum = 0; + for(int i = 0; i < INT32_BUCKET_SZ; i++) { + void* page = index+i*(PAGESZ+8); + uint64_t ptr = 1; + for(int j = 0; j < INT32_BUCKET_MAX_ITEM_SZ && ptr; j++) { + ptr = le64(page+j*(8+4)); + sum += !!ptr; + } + ptr = le64(page+PAGESZ-16); // 查看是否有溢出桶 + while(ptr) { + #ifdef DEBUG + //printf("%x ", ptr); + //fflush(stdout); + #endif + page = get_page(fd, ptr, index-8+INT32_INDEX_SZ-PAGESZ-8); + for(int j = 0; j < INT32_BUCKET_MAX_ITEM_SZ && ptr; j++) { + ptr = le64(page+j*(8+4)); + sum += !!ptr; + } + ptr = le64(page+PAGESZ-16); // 查看是否有溢出桶 + } + } + return sum; } int insert_int32_item(int fd, void* index, key_t k, uint64_t ptr) { - + uint32_t key = (uint32_t)k; + uint64_t digest = (k*(k+INT32_DIGEST_FACTOR))%256; + // 访问digest指向的存储桶 + void* page = index+digest*(PAGESZ+8); + uint64_t p = 1; + while(1) { + for(int j = 0; j < INT32_BUCKET_MAX_ITEM_SZ && p; j++) { + void* slot = page+j*(8+4); + p = le64(slot); + if(p && key == le32(slot+8)) { // 已存在值, 替换之 + putle64(slot, ptr); + return sync_page(fd, page); + } else if(!p) { // 不存在值, 新建 + putle64(slot, ptr); + putle32(slot+8, key); + return sync_page(fd, page); + } + } + // 溢出 + p = le64(page+PAGESZ-16); // 查看是否有溢出桶 + if(!p) { // 无溢出桶, 新建 + void* ovpage = alloc_page(fd, index-8+INT32_INDEX_SZ-PAGESZ-8); + if(unlikely(ovpage == NULL)) return EOF; + memset(ovpage, 0, PAGESZ); + putle64(page+PAGESZ-16, le64(ovpage-8)); + #ifdef DEBUG + printf("new ovpage: %x, this page: %x\n", le64(page+PAGESZ-16), le64(page-8)); + #endif + int r = sync_page(fd, page); + if(unlikely(r)) return r; + putle64(ovpage, ptr); + putle32(ovpage+8, key); + #ifdef DEBUG + printf("write ptr: %x, key: %x\n", ptr, key); + #endif + return sync_page(fd, ovpage); + } + // 有溢出桶, 进入下一轮搜索 + page = get_page(fd, p, index-8+INT32_INDEX_SZ-PAGESZ-8); + } } uint64_t find_item_by_int32_key(int fd, void* index, key_t k) { - + uint32_t key = (uint32_t)k; + uint64_t digest = (k*(k+INT32_DIGEST_FACTOR))%256; + // 访问digest指向的存储桶 + void* page = index+digest*(PAGESZ+8); + uint64_t p = 1; + while(1) { + for(int j = 0; j < INT32_BUCKET_MAX_ITEM_SZ && p; j++) { + void* slot = page+j*(8+4); + p = le64(slot); + if(!p || (p && key == le32(slot+8))) { // 找到项目 or 不存在值, 无该项目 + return p; + } + } + // 桶已满 + p = le64(page+PAGESZ-16); // 查看是否有溢出桶 + if(!p) { // 无溢出桶, 无该项目 + return 0; + } + // 有溢出桶, 进入下一轮搜索 + page = get_page(fd, p, index-8+INT32_INDEX_SZ-PAGESZ-8); + } } uint64_t remove_item_by_int32_key(int fd, void* index, key_t k) { - + uint32_t key = (uint32_t)k; + uint64_t digest = (k*(k+INT32_DIGEST_FACTOR))%256; + // 访问digest指向的存储桶 + void* page = index+digest*(PAGESZ+8); + uint64_t p = 1; + while(1) { + for(int j = 0; j < INT32_BUCKET_MAX_ITEM_SZ && p; j++) { + void* slot = page+j*(8+4); + p = le64(slot); + if(!p) { // 不存在值, 无该项目 + return 0; + } + if(p && key == le32(slot+8)) { // 找到项 + int i = j; + for(; i < INT32_BUCKET_MAX_ITEM_SZ && le64(page+i*(8+4)); i++) { + #ifdef DEBUG + //printf("lookup %03d: %x\n", i, le64(page+i*(8+4))); + #endif + } + #ifdef DEBUG + printf("page: %x(%x), i: %d, p: %x\n", page, le64(page-8), i, p); + #endif + void *next_page = page, *prev_page; + while(i == INT32_BUCKET_MAX_ITEM_SZ) { // 本桶已满 + int next = le64(next_page+PAGESZ-16); // 查看是否有溢出桶 + if(!next) { // 无溢出桶, 将最后一个值搬移至slot并为其补0 + // 不是最后一项, 搬移 + if(j+1 != INT32_BUCKET_MAX_ITEM_SZ || next_page!=page) memcpy(slot, next_page-8-8-8-4, 8+4); + memset(next_page-8-8-8-4, 0, 8+4); + int r = sync_page(fd, next_page); + if(unlikely(r)) return r; + if(next_page!=page) { + r = sync_page(fd, page); + if(unlikely(r)) return r; + } + return p; + } + // 有溢出桶, 加载 + prev_page = next_page; + #ifdef DEBUG + printf("%x have next bucket at: %x, ", le64(prev_page-8), next); + #endif + void* buf = index-8+INT32_INDEX_SZ-PAGESZ-8-PAGESZ-8; + #ifdef DEBUG + printf("buf: %p, ", buf); + fflush(stdout); + #endif + next_page = get_page(fd, next, buf); + #ifdef DEBUG + printf("next page: %p\n", next_page); + #endif + for(i = 0; i < INT32_BUCKET_MAX_ITEM_SZ && le64(next_page+i*(8+4)); i++); + #ifdef DEBUG + printf("new i: %d\n", i); + #endif + } + // 转至新桶第一个, 但发现为空, 即最后一项在上个桶末尾 + if(!i) { + #ifdef DEBUG + puts("i == 0"); + #endif + memcpy(slot, prev_page+PAGESZ-8-8-8-4, 8+4); + memset(prev_page+PAGESZ-8-8-8-4, 0, 8+4); + #ifdef DEBUG + printf("prev_page: %x(%x), page: %x(%x)\n", prev_page, le64(prev_page-8), page, le64(page-8)); + printf("slot@%016x+%03d, last@%016x+339, ptr: %x, key: %x\n", le64(page-8), (slot-page)/12, le64(prev_page-8), le64(slot), le64(slot+8)); + #endif + int r = sync_page(fd, prev_page); + if(unlikely(r)) return r; + if(prev_page!=page) { + r = sync_page(fd, page); + if(unlikely(r)) return r; + } + return p; + } + // 本桶未满 + if(--i == j && page == next_page) { // 是最后一项, 直接删除 + memset(slot, 0, 8+4); + int r = sync_page(fd, page); + if(unlikely(r)) return r; + return p; + } + // 不是最后一项, 搬移 + memcpy(slot, next_page+i*(8+4), 8+4); + memset(next_page+i*(8+4), 0, 8+4); + int r = sync_page(fd, next_page); + if(unlikely(r)) return r; + if(next_page!=page) { + r = sync_page(fd, page); + if(unlikely(r)) return r; + } + #ifdef DEBUG + printf("slot@%016x+%03d, last@%016x+%03d\n", le64(page-8), (slot-page)/12, le64(next_page-8), i); + #endif + return p; + } + } + // 桶已满 + p = le64(page+PAGESZ-16); // 查看是否有溢出桶 + if(!p) { // 无溢出桶, 无该项目 + return 0; + } + // 有溢出桶, 进入下一轮搜索 + page = get_page(fd, p, index-8+INT32_INDEX_SZ-PAGESZ-8); + } } diff --git a/src/types/int64.c b/src/types/int64.c new file mode 100644 index 0000000..52b8db0 --- /dev/null +++ b/src/types/int64.c @@ -0,0 +1,269 @@ +#include +#include +#include +#include +#include +#include + +#include "../../include/binary.h" +#include "../../include/page.h" +#include "../../include/types/int64.h" + +//#define DEBUG + +void* create_int64_index(int fd, void* buf) { + void* page = alloc_page(fd, buf); + if(unlikely(page == NULL)) return NULL; + memset(page, 0, PAGESZ); + void* prev_page = page; + for(int i = 1; i < INT64_BUCKET_SZ; i++) { + page = alloc_page(fd, buf+i*(PAGESZ+8)); + if(unlikely(page == NULL)) { + for(int j = 0; j < i; j++) free_page(fd, buf+j*(PAGESZ+8)+8); + return NULL; + } + memset(page, 0, PAGESZ); + putle64(prev_page+PAGESZ-8, le64(page-8)); + if(unlikely(sync_page(fd, prev_page))) { + free_page(fd, page); + for(int j = 0; j < i; j++) free_page(fd, buf+j*(PAGESZ+8)+8); + return NULL; + } + prev_page = page; + } + if(unlikely(sync_page(fd, page))) { + free_page(fd, page); + for(int i = 0; i < INT64_BUCKET_SZ; i++) free_page(fd, buf+i*(PAGESZ+8)+8); + return NULL; + } + #ifdef DEBUG + puts("alloc finish"); + #endif + return buf+8; +} + +void* load_int64_index(int fd, uint64_t ptr, void* buf) { + void* page = get_page(fd, ptr, buf); + if(unlikely(page == NULL)) return NULL; + for(int i = 1; i < INT64_BUCKET_SZ; i++) { + page = get_page(fd, le64(page+PAGESZ-8), buf+i*(PAGESZ+8)); + if(unlikely(page == NULL)) return NULL; + } + return buf+8; +} + +int remove_int64_index(int fd, void* index) { + for(int i = 0; i < INT64_BUCKET_SZ; i++) { + void* page = index+i*(PAGESZ+8); + uint64_t ptr = le64(page+PAGESZ-16); + while(unlikely(ptr)) { + void* p = get_page(fd, ptr, index-8+INT64_INDEX_SZ-PAGESZ-8); + if(unlikely(p == NULL)) return EOF; + ptr = le64(p+PAGESZ-16); + if(unlikely(free_page(fd, p))) return EOF; + } + if(unlikely(free_page(fd, page))) return EOF; + } + return 0; +} + +uint64_t count_int64_items(int fd, void* index) { + uint64_t sum = 0; + for(int i = 0; i < INT64_BUCKET_SZ; i++) { + void* page = index+i*(PAGESZ+8); + uint64_t ptr = 1; + for(int j = 0; j < INT64_BUCKET_MAX_ITEM_SZ && ptr; j++) { + ptr = le64(page+j*(8+8)); + sum += !!ptr; + } + ptr = le64(page+PAGESZ-16); // 查看是否有溢出桶 + while(ptr) { + #ifdef DEBUG + //printf("%x ", ptr); + //fflush(stdout); + #endif + page = get_page(fd, ptr, index-8+INT64_INDEX_SZ-PAGESZ-8); + for(int j = 0; j < INT64_BUCKET_MAX_ITEM_SZ && ptr; j++) { + ptr = le64(page+j*(8+8)); + sum += !!ptr; + } + ptr = le64(page+PAGESZ-16); // 查看是否有溢出桶 + } + } + return sum; +} + +int insert_int64_item(int fd, void* index, key_t k, uint64_t ptr) { + uint64_t key = (uint64_t)k; + uint64_t digest = (k*(k+INT64_DIGEST_FACTOR))%256; + // 访问digest指向的存储桶 + void* page = index+digest*(PAGESZ+8); + uint64_t p = 1; + while(1) { + for(int j = 0; j < INT64_BUCKET_MAX_ITEM_SZ && p; j++) { + void* slot = page+j*(8+8); + p = le64(slot); + if(p && key == le64(slot+8)) { // 已存在值, 替换之 + putle64(slot, ptr); + return sync_page(fd, page); + } else if(!p) { // 不存在值, 新建 + putle64(slot, ptr); + putle64(slot+8, key); + return sync_page(fd, page); + } + } + // 溢出 + p = le64(page+PAGESZ-16); // 查看是否有溢出桶 + if(!p) { // 无溢出桶, 新建 + void* ovpage = alloc_page(fd, index-8+INT64_INDEX_SZ-PAGESZ-8); + if(unlikely(ovpage == NULL)) return EOF; + memset(ovpage, 0, PAGESZ); + putle64(page+PAGESZ-16, le64(ovpage-8)); + #ifdef DEBUG + printf("new ovpage: %x, this page: %x\n", le64(page+PAGESZ-16), le64(page-8)); + #endif + int r = sync_page(fd, page); + if(unlikely(r)) return r; + putle64(ovpage, ptr); + putle64(ovpage+8, key); + #ifdef DEBUG + printf("write ptr: %x, key: %x\n", ptr, key); + #endif + return sync_page(fd, ovpage); + } + // 有溢出桶, 进入下一轮搜索 + page = get_page(fd, p, index-8+INT64_INDEX_SZ-PAGESZ-8); + } +} + +uint64_t find_item_by_int64_key(int fd, void* index, key_t k) { + uint64_t key = (uint64_t)k; + uint64_t digest = (k*(k+INT64_DIGEST_FACTOR))%256; + // 访问digest指向的存储桶 + void* page = index+digest*(PAGESZ+8); + uint64_t p = 1; + while(1) { + for(int j = 0; j < INT64_BUCKET_MAX_ITEM_SZ && p; j++) { + void* slot = page+j*(8+8); + p = le64(slot); + if(!p || (p && key == le64(slot+8))) { // 找到项目 or 不存在值, 无该项目 + return p; + } + } + // 桶已满 + p = le64(page+PAGESZ-16); // 查看是否有溢出桶 + if(!p) { // 无溢出桶, 无该项目 + return 0; + } + // 有溢出桶, 进入下一轮搜索 + page = get_page(fd, p, index-8+INT64_INDEX_SZ-PAGESZ-8); + } +} + +uint64_t remove_item_by_int64_key(int fd, void* index, key_t k) { + uint64_t key = (uint64_t)k; + uint64_t digest = (k*(k+INT64_DIGEST_FACTOR))%256; + // 访问digest指向的存储桶 + void* page = index+digest*(PAGESZ+8); + uint64_t p = 1; + while(1) { + for(int j = 0; j < INT64_BUCKET_MAX_ITEM_SZ && p; j++) { + void* slot = page+j*(8+8); + p = le64(slot); + if(!p) { // 不存在值, 无该项目 + return 0; + } + if(p && key == le64(slot+8)) { // 找到项 + int i = j; + for(; i < INT64_BUCKET_MAX_ITEM_SZ && le64(page+i*(8+8)); i++) { + #ifdef DEBUG + //printf("lookup %03d: %x\n", i, le64(page+i*(8+8))); + #endif + } + #ifdef DEBUG + printf("page: %x(%x), i: %d, p: %x\n", page, le64(page-8), i, p); + #endif + void *next_page = page, *prev_page; + while(i == INT64_BUCKET_MAX_ITEM_SZ) { // 本桶已满 + int next = le64(next_page+PAGESZ-16); // 查看是否有溢出桶 + if(!next) { // 无溢出桶, 将最后一个值搬移至slot并为其补0 + // 不是最后一项, 搬移 + if(j+1 != INT64_BUCKET_MAX_ITEM_SZ || next_page!=page) memcpy(slot, next_page-8-8-8-8, 8+8); + memset(next_page-8-8-8-8, 0, 8+8); + int r = sync_page(fd, next_page); + if(unlikely(r)) return r; + if(next_page!=page) { + r = sync_page(fd, page); + if(unlikely(r)) return r; + } + return p; + } + // 有溢出桶, 加载 + prev_page = next_page; + #ifdef DEBUG + printf("%x have next bucket at: %x, ", le64(prev_page-8), next); + #endif + void* buf = index-8+INT64_INDEX_SZ-PAGESZ-8-PAGESZ-8; + #ifdef DEBUG + printf("buf: %p, ", buf); + fflush(stdout); + #endif + next_page = get_page(fd, next, buf); + #ifdef DEBUG + printf("next page: %p\n", next_page); + #endif + for(i = 0; i < INT64_BUCKET_MAX_ITEM_SZ && le64(next_page+i*(8+8)); i++); + #ifdef DEBUG + printf("new i: %d\n", i); + #endif + } + // 转至新桶第一个, 但发现为空, 即最后一项在上个桶末尾 + if(!i) { + #ifdef DEBUG + puts("i == 0"); + #endif + memcpy(slot, prev_page+PAGESZ-8-8-8-8, 8+8); + memset(prev_page+PAGESZ-8-8-8-8, 0, 8+8); + #ifdef DEBUG + printf("prev_page: %x(%x), page: %x(%x)\n", prev_page, le64(prev_page-8), page, le64(page-8)); + printf("slot@%016x+%03d, last@%016x+339, ptr: %x, key: %x\n", le64(page-8), (slot-page)/12, le64(prev_page-8), le64(slot), le64(slot+8)); + #endif + int r = sync_page(fd, prev_page); + if(unlikely(r)) return r; + if(prev_page!=page) { + r = sync_page(fd, page); + if(unlikely(r)) return r; + } + return p; + } + // 本桶未满 + if(--i == j && page == next_page) { // 是最后一项, 直接删除 + memset(slot, 0, 8+8); + int r = sync_page(fd, page); + if(unlikely(r)) return r; + return p; + } + // 不是最后一项, 搬移 + memcpy(slot, next_page+i*(8+8), 8+8); + memset(next_page+i*(8+8), 0, 8+8); + int r = sync_page(fd, next_page); + if(unlikely(r)) return r; + if(next_page!=page) { + r = sync_page(fd, page); + if(unlikely(r)) return r; + } + #ifdef DEBUG + printf("slot@%016x+%03d, last@%016x+%03d\n", le64(page-8), (slot-page)/12, le64(next_page-8), i); + #endif + return p; + } + } + // 桶已满 + p = le64(page+PAGESZ-16); // 查看是否有溢出桶 + if(!p) { // 无溢出桶, 无该项目 + return 0; + } + // 有溢出桶, 进入下一轮搜索 + page = get_page(fd, p, index-8+INT64_INDEX_SZ-PAGESZ-8); + } +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4f56159..3addf49 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,12 +1,16 @@ cmake_minimum_required(VERSION 3.0.0) project(fumidb_test VERSION 1.0) -add_executable(binary_test binary_test.c) -add_executable(page_test page_test.c ../src/page.c ../src/file.c) -add_executable(types_test types_test.c ../src/types.c ../src/types/int8.c ../src/types/int16.c ../src/page.c ../src/file.c) -add_executable(table_test table_test.c ../src/table.c ../src/types.c ../src/types/int8.c ../src/types/int16.c ../src/page.c ../src/file.c) +add_executable(binary_test binary_test.c) +add_executable(page_test page_test.c ../src/page.c ../src/file.c) +#add_executable(types816_test types816_test.c ../src/types.c ../src/types/int8.c ../src/types/int16.c ../src/types/int32.c ../src/types/int64.c ../src/page.c ../src/file.c) +add_executable(types32_test types32_test.c ../src/types.c ../src/types/int8.c ../src/types/int16.c ../src/types/int32.c ../src/types/int64.c ../src/page.c ../src/file.c) +add_executable(types64_test types64_test.c ../src/types.c ../src/types/int8.c ../src/types/int16.c ../src/types/int32.c ../src/types/int64.c ../src/page.c ../src/file.c) +add_executable(table_test table_test.c ../src/table.c ../src/types.c ../src/types/int8.c ../src/types/int16.c ../src/types/int32.c ../src/types/int64.c ../src/page.c ../src/file.c) -add_test(test_binary binary_test COMMAND binary_test) -add_test(test_page page_test COMMAND page_test) -add_test(test_types types_test COMMAND types_test) -add_test(test_table table_test COMMAND table_test) +add_test(test_binary binary_test COMMAND binary_test) +add_test(test_page page_test COMMAND page_test) +#add_test(test_types816 types816_test COMMAND types816_test) +add_test(test_types32 types32_test COMMAND types32_test) +add_test(test_types64 types64_test COMMAND types64_test) +add_test(test_table table_test COMMAND table_test) diff --git a/tests/types32_test.c b/tests/types32_test.c new file mode 100644 index 0000000..c927c61 --- /dev/null +++ b/tests/types32_test.c @@ -0,0 +1,115 @@ +#include +#include +#include +#include +#include "../include/binary.h" +#include "../include/file.h" +#include "../include/page.h" +#include "../include/types.h" +#include "../include/types/int32.h" + +uint8_t buf[INT32_INDEX_SZ]; + +int main() { + /* test int32 */ + int fd = open("types_test_tmp.bin", O_RDWR | O_CREAT | O_TRUNC, 0644); + if(fd < 0) { + perror("create"); + return 1; + } + if(init_file_header_page(fd) < 0) return 2; + void* index = create_index(fd, TYPE_INT32, buf); + if(!index) { + perror("create_int32_index"); + return 3; + } + for(int i = 0, j = 8192; i < INT32_BUCKET_SZ-1; i++, j+=PAGESZ) { + if(le64(index+i*(PAGESZ+8)+PAGESZ-8) != j) { + printf("index: %d, ptr: %d!=%d\n", i, le64(index+PAGESZ-8), j); + return 3; + } + } + if(le64(index+255*(PAGESZ+8)+PAGESZ-8) != 0) { + printf("index: 255, ptr: %d!=%d\n", le64(index+PAGESZ-8), 0); + return 3; + } + int cnt = 0; + for(int i = 123456; i < 123456+8192; i++, cnt++) { + int n; + if((n=count_items(fd, TYPE_INT32, index)) != cnt) { + printf("%d != %d\n", cnt, n); + return 4; + } + // printf("c", i); + if(insert_item(fd, TYPE_INT32, index, (key_t)i, i)) { + printf("%u ", (uint32_t)i); + fflush(stdout); + perror("insert_int32_item"); + return 4; + } + // printf("i ", i); + } + + for(int i = 123456; i < 123456+8192; i++) { + if((int)find_item_by_key(fd, TYPE_INT32, index, (key_t)i) != i) { + printf("%u ", (uint32_t)i); + fflush(stdout); + perror("find_item_by_key"); + return 5; + } + } + + close(fd); + fd = open("types_test_tmp.bin", O_RDWR, 0644); + memset(buf, 0, sizeof(buf)); + index = load_index(fd, TYPE_INT32, PAGESZ, buf); + + for(int i = 0, j = 8192; i < INT32_BUCKET_SZ-1; i++, j+=PAGESZ) { + if(le64(index+i*(PAGESZ+8)+PAGESZ-8) != j) { + printf("index: %d, ptr: %d!=%d\n", i, le64(index+PAGESZ-8), j); + return 3; + } + } + + if(le64(index+255*(PAGESZ+8)+PAGESZ-8) != 0) { + printf("index: 255, ptr: %d!=%d\n", le64(index+PAGESZ-8), 0); + return 3; + } + + for(int i = 123456; i < 123456+4098; i++, cnt--) { + int n; + if((n=count_items(fd, TYPE_INT32, index)) != cnt) { + printf("%d != %d\n", cnt, n); + return 6; + } + if(remove_item_by_key(fd, TYPE_INT32, index, (key_t)i) != i) { + printf("%u ", (uint32_t)i); + fflush(stdout); + perror("remove_item_by_key"); + return 6; + } + } + + for(int i = 123456; i < 123456+4098; i++) { + if((int)find_item_by_key(fd, TYPE_INT32, index, (key_t)i) != 0) { + printf("%u ", (uint32_t)i); + fflush(stdout); + perror("find_item_by_key"); + return 7; + } + } + + for(int i = 123456+4098; i < 123456+8192; i++) { + if((int)find_item_by_key(fd, TYPE_INT32, index, (key_t)i) != i) { + printf("%u ", (uint32_t)i); + fflush(stdout); + perror("find_item_by_key"); + return 8; + } + } + + close(fd); + /* end test int32 */ + // remove("types_test_tmp.bin"); + return 0; +} diff --git a/tests/types64_test.c b/tests/types64_test.c new file mode 100644 index 0000000..ef08aca --- /dev/null +++ b/tests/types64_test.c @@ -0,0 +1,115 @@ +#include +#include +#include +#include +#include "../include/binary.h" +#include "../include/file.h" +#include "../include/page.h" +#include "../include/types.h" +#include "../include/types/int64.h" + +uint8_t buf[INT64_INDEX_SZ]; + +int main() { + /* test int64 */ + int fd = open("types_test_tmp.bin", O_RDWR | O_CREAT | O_TRUNC, 0644); + if(fd < 0) { + perror("create"); + return 1; + } + if(init_file_header_page(fd) < 0) return 2; + void* index = create_index(fd, TYPE_INT64, buf); + if(!index) { + perror("create_int64_index"); + return 3; + } + for(int64_t i = 0, j = 8192; i < INT64_BUCKET_SZ-1; i++, j+=PAGESZ) { + if(le64(index+i*(PAGESZ+8)+PAGESZ-8) != j) { + printf("index: %d, ptr: %d!=%d\n", i, le64(index+PAGESZ-8), j); + return 3; + } + } + if(le64(index+1023*(PAGESZ+8)+PAGESZ-8) != 0) { + printf("index: 1023, ptr: %d!=%d\n", le64(index+PAGESZ-8), 0); + return 3; + } + int64_t cnt = 0; + for(int64_t i = 1234567890; i < 1234567890+8192; i++, cnt++) { + int n; + if((n=count_items(fd, TYPE_INT64, index)) != cnt) { + printf("%d != %d\n", cnt, n); + return 4; + } + // printf("c", i); + if(insert_item(fd, TYPE_INT64, index, (key_t)i, i)) { + printf("%u ", (uint64_t)i); + fflush(stdout); + perror("insert_int64_item"); + return 4; + } + // printf("i ", i); + } + + for(int64_t i = 1234567890; i < 1234567890+8192; i++) { + if((int)find_item_by_key(fd, TYPE_INT64, index, (key_t)i) != i) { + printf("%u ", (uint64_t)i); + fflush(stdout); + perror("find_item_by_key"); + return 5; + } + } + + close(fd); + fd = open("types_test_tmp.bin", O_RDWR, 0644); + memset(buf, 0, sizeof(buf)); + index = load_index(fd, TYPE_INT64, PAGESZ, buf); + + for(int64_t i = 0, j = 8192; i < INT64_BUCKET_SZ-1; i++, j+=PAGESZ) { + if(le64(index+i*(PAGESZ+8)+PAGESZ-8) != j) { + printf("index: %d, ptr: %d!=%d\n", i, le64(index+PAGESZ-8), j); + return 3; + } + } + + if(le64(index+1023*(PAGESZ+8)+PAGESZ-8) != 0) { + printf("index: 1023, ptr: %d!=%d\n", le64(index+PAGESZ-8), 0); + return 3; + } + + for(int64_t i = 1234567890; i < 1234567890+4098; i++, cnt--) { + int n; + if((n=count_items(fd, TYPE_INT64, index)) != cnt) { + printf("%d != %d\n", cnt, n); + return 6; + } + if(remove_item_by_key(fd, TYPE_INT64, index, (key_t)i) != i) { + printf("%u ", (uint64_t)i); + fflush(stdout); + perror("remove_item_by_key"); + return 6; + } + } + + for(int64_t i = 1234567890; i < 1234567890+4098; i++) { + if((int)find_item_by_key(fd, TYPE_INT64, index, (key_t)i) != 0) { + printf("%u ", (uint64_t)i); + fflush(stdout); + perror("find_item_by_key"); + return 7; + } + } + + for(int64_t i = 1234567890+4098; i < 1234567890+8192; i++) { + if((int)find_item_by_key(fd, TYPE_INT64, index, (key_t)i) != i) { + printf("%u ", (uint64_t)i); + fflush(stdout); + perror("find_item_by_key"); + return 8; + } + } + + close(fd); + /* end test int64 */ + // remove("types_test_tmp.bin"); + return 0; +} diff --git a/tests/types_test.c b/tests/types816_test.c similarity index 99% rename from tests/types_test.c rename to tests/types816_test.c index 4331a4d..e774e36 100644 --- a/tests/types_test.c +++ b/tests/types816_test.c @@ -7,6 +7,7 @@ #include "../include/page.h" #include "../include/types.h" #include "../include/types/int8.h" +#include "../include/types/int16.h" uint8_t buf[10290];