fumiama/fumidb
1
0
Fork 0
mirror of https://github.com/fumiama/fumidb.git synced 2026-06-05 00:32:44 +08:00
Code Activity

finish int16 index

Browse Source
This commit is contained in:
源文雨
2022-05-03 13:13:54 +08:00
parent bd6ce1d778
commit 34700361c2
3 changed files with 217 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
src/page.c
View File

@@ -121,12 +121,21 @@ void* alloc_block(int fd, uint16_t size, void* blk) {
uint8_t buf[8];
uint16_t blksz;
if(blk == NULL) return NULL;
if(size > PAGESZ) return NULL;
if(unlikely(blk == NULL)) {
errno = EINVAL;
return NULL;
}
if(unlikely(size > PAGESZ)) {
errno = EFBIG;
return NULL;
}
// 对于 page，只关心位于第一页 8~15 字节的 ptr of unused blk
while(ptr) {
if(unlikely(lseek(fd, ptr, SEEK_SET) < 0)) return NULL;
if(unlikely(read(fd, buf, 8) != 8)) return NULL;
if(unlikely(read(fd, buf, 8) != 8)) {
errno = ENOSR;
return NULL;
}
if(unlikely(ptr == prev_ptr)) { // 文件损坏
errno = ESPIPE;
return NULL;
@@ -145,7 +154,10 @@ void* alloc_block(int fd, uint16_t size, void* blk) {
putle16(blk+8, size);
blk += 10;
if(lseek(fd, prev_ptr, SEEK_SET) < 0) return NULL;
if(write(fd, buf, 8) != 8) return NULL; // 从空闲块链表移除本块
if(write(fd, buf, 8) != 8) { // 从空闲块链表移除本块
errno = ENOSR;
return NULL;
}
return blk;
}
if(prev_prev_ptr && ptr < prev_ptr) { // 不符合顺序，进行一次调整
@@ -174,7 +186,10 @@ void* alloc_block(int fd, uint16_t size, void* blk) {
errno = ESPIPE;
return NULL;
}
if(write(fd, nullpage, PAGESZ) != PAGESZ) return NULL;
if(write(fd, nullpage, PAGESZ) != PAGESZ) {
errno = ENOSR;
return NULL;
}
if(PAGESZ-size > 10) { // 回收冗余
if(lseek(fd, prev_ptr, SEEK_SET) < 0) return NULL;
putle64(buf, ptr+size);

186
src/types/int16.c
View File

@@ -87,6 +87,28 @@ uint64_t count_int16_items(int fd, void* index) {
return total;
}
#define set_bitmap(fd, index, key) {\
/* 写入位图 */\
if(key<32768) {\
((uint8_t*)(index+INT16_INDEX_SZ+8))[key/8] |= 128>>(key%8);\
if(unlikely(sync_page(fd, index+INT16_INDEX_SZ+8))) { /* 失败，撤销更改 */\
((uint8_t*)(index+INT16_INDEX_SZ+8))[key/8] &= ~(128>>(key%8));\
return EOF;\
}\
} else {\
((uint8_t*)(index+INT16_INDEX_SZ+8+8))[key/8] |= 128>>(key%8);\
if(unlikely(sync_page(fd, index+INT16_INDEX_SZ+8+PAGESZ+8))) { /* 失败，撤销更改 */\
((uint8_t*)(index+INT16_INDEX_SZ+8+8))[key/8] &= ~(128>>(key%8));\
return EOF;\
}\
}\
((uint8_t*)(index+24))[key/256]++; /* 写入位图索引 */\
if(unlikely(sync_block(fd, index))) { /* 失败，撤销更改 */\
((uint8_t*)(index+24))[key/256]--;\
return EOF;\
}\
}
int insert_int16_item(int fd, void* index, key_t k, uint64_t ptr) {
int isexist, sum = 0, total = count_int16_items(fd, index);
uint16_t key = (uint16_t)k;
@@ -153,25 +175,7 @@ int insert_int16_item(int fd, void* index, key_t k, uint64_t ptr) {
}
if(!isexist) {
// 写入位图
if(key<32768) {
((uint8_t*)(index+INT16_INDEX_SZ+8))[key/8] |= 128>>(key%8);
if(unlikely(sync_page(fd, index+INT16_INDEX_SZ+8))) { // 失败，撤销更改
((uint8_t*)(index+INT16_INDEX_SZ+8))[key/8] &= ~(128>>(key%8));
return EOF;
}
} else {
((uint8_t*)(index+INT16_INDEX_SZ+8+8))[key/8] |= 128>>(key%8);
if(unlikely(sync_page(fd, index+INT16_INDEX_SZ+8+PAGESZ+8))) { // 失败，撤销更改
((uint8_t*)(index+INT16_INDEX_SZ+8+8))[key/8] &= ~(128>>(key%8));
return EOF;
}
}
((uint8_t*)(index+24))[key/256]++; // 写入位图索引
if(unlikely(sync_block(fd, index))) { // 失败，撤销更改
((uint8_t*)(index+24))[key/256]--;
return EOF;
}
set_bitmap(fd, index, key);
#ifdef DEBUG
printf("i: %d, sumblk: %d, map: %02x, ", key/256, ((uint8_t*)(index+24))[key/256], ((uint8_t*)(index+INT16_INDEX_SZ+8+((key<32768)?0:8)))[key/8]);
#endif
@@ -239,8 +243,11 @@ int insert_int16_item(int fd, void* index, key_t k, uint64_t ptr) {
return EOF;
}
}
#ifdef DEBUG
printf("set: %016llx, ", prev_ptr+8*((sum+1)%256+1));
#endif
// 定位到最后一个未满块或第一个空块上的最后
lseek(fd, prev_ptr+8*(sum%256+1), SEEK_SET);
lseek(fd, prev_ptr+8*((sum+1)%256+1), SEEK_SET);
#ifdef DEBUG
puts("append");
#endif
@@ -273,7 +280,7 @@ int insert_int16_item(int fd, void* index, key_t k, uint64_t ptr) {
lseek(fd, -8, SEEK_CUR); // 返回
write(fd, buf, 8); // 插入
while(prev_ptr) { // 一直搬移到末尾
if(unlikely(offset && !(offset++%256))) { // 进入新的块
if(!(++offset%256)) { // 进入新的块
lseek(fd, ptr, SEEK_SET);
readle64(fd, first_ptr); // 下一个块指针
if(unlikely(first_ptr == ptr)) { // 文件损坏
@@ -292,12 +299,20 @@ int insert_int16_item(int fd, void* index, key_t k, uint64_t ptr) {
uint64_t find_item_by_int16_key(int fd, void* index, key_t k) {
uint64_t ptr;
int isexist, sum = 0;
uint16_t key = (uint16_t)k;
uint16_t sum;
if(key < 32768) {
int isexist = ((uint8_t*)(index+INT16_INDEX_SZ+8))[key/8] & (128>>(key%8));
// key是否已存在
isexist = ((uint8_t*)(index+INT16_INDEX_SZ+8))[key/8] & (128>>(key%8));
if(!isexist) return 0;
sum = __builtin_popcount(((uint8_t*)(index+INT16_INDEX_SZ+8))[key/8] & ~(0xff>>(key%8)));
// 查找 key 之前共有多少索引
for(int i = key/256*32; i < key/8; i++) { // 从未计算的32位组开始算起
sum += __builtin_popcount(((uint8_t*)(index+INT16_INDEX_SZ+8))[i]);
}
sum += __builtin_popcount(((uint8_t*)(index+INT16_INDEX_SZ+8))[key/8] & ~(0xff>>(key%8)));
#ifdef DEBUG
printf("popc: %d, ", sum);
#endif
for(int i = 0; i < key/256; i++) {
int s = ((uint8_t*)(index+24))[i];
if(unlikely(!s && ((uint8_t*)(index+INT16_INDEX_SZ+8))[i*32])) {
@@ -306,10 +321,21 @@ uint64_t find_item_by_int16_key(int fd, void* index, key_t k) {
}
sum += s;
}
#ifdef DEBUG
printf("sum: %d, ", sum);
#endif
} else {
int isexist = ((uint8_t*)(index+INT16_INDEX_SZ+8+8))[key/8] & (128>>(key%8));
// key是否已存在
isexist = ((uint8_t*)(index+INT16_INDEX_SZ+8+8))[key/8] & (128>>(key%8));
if(!isexist) return 0;
sum = __builtin_popcount(((uint8_t*)(index+INT16_INDEX_SZ+8+8))[key/8] & ~(0xff>>(key%8)));
// 查找 key 之前共有多少索引
for(int i = key/256*32; i < key/8; i++) { // 从未计算的32位组开始算起
sum += __builtin_popcount(((uint8_t*)(index+INT16_INDEX_SZ+8+8))[i]);
}
sum += __builtin_popcount(((uint8_t*)(index+INT16_INDEX_SZ+8+8))[key/8] & ~(0xff>>(key%8)));
#ifdef DEBUG
printf("popc: %d, ", sum);
#endif
for(int i = 0; i < 128; i++) {
int s = ((uint8_t*)(index+24))[i];
if(unlikely(!s && ((uint8_t*)(index+INT16_INDEX_SZ+8))[i*32])) {
@@ -326,28 +352,70 @@ uint64_t find_item_by_int16_key(int fd, void* index, key_t k) {
}
sum += s;
}
#ifdef DEBUG
printf("sum: %d, ", sum);
#endif
}
ptr = le64(index+16);
for(int i = 0; i < sum/256 && ptr; i++) {
for(int i = 0; i < sum/256; i++) {
if(lseek(fd, ptr, SEEK_SET) < 0) return EOF;
readle64(fd, ptr);
}
if(!ptr) return EOF;
ptr += 8*(sum%256+1);
if(lseek(fd, ptr, SEEK_SET) < 0) return EOF;
#ifdef DEBUG
printf("seek: %016llx, ", ptr);
#endif
if(lseek(fd, ptr+8*(sum%256+1), SEEK_SET) < 0) return EOF;
readle64(fd, ptr);
#ifdef DEBUG
printf("ptr: %d\n", (int)ptr);
#endif
return ptr;
}
#define clear_bitmap(fd, index, key) {\
/* 写入位图 */\
if(key<32768) {\
((uint8_t*)(index+INT16_INDEX_SZ+8))[key/8] &= ~(128>>(key%8));\
if(unlikely(sync_page(fd, index+INT16_INDEX_SZ+8))) { /* 失败，撤销更改 */\
((uint8_t*)(index+INT16_INDEX_SZ+8))[key/8] |= 128>>(key%8);\
return EOF;\
}\
} else {\
((uint8_t*)(index+INT16_INDEX_SZ+8+8))[key/8] &= ~(128>>(key%8));\
if(unlikely(sync_page(fd, index+INT16_INDEX_SZ+8+PAGESZ+8))) { /* 失败，撤销更改 */\
((uint8_t*)(index+INT16_INDEX_SZ+8+8))[key/8] |= 128>>(key%8);\
return EOF;\
}\
}\
((uint8_t*)(index+24))[key/256]--; /* 写入位图索引 */\
if(unlikely(sync_block(fd, index))) { /* 失败，撤销更改 */\
((uint8_t*)(index+24))[key/256]++;\
return EOF;\
}\
}
uint64_t remove_item_by_int16_key(int fd, void* index, key_t k) {
uint64_t ptr;
int isexist, sum = 0;
uint16_t key = (uint16_t)k;
uint16_t sum;
char buf[8];
#ifdef DEBUG
printf("No.%u: ", (int)key);
#endif
if(key < 32768) {
int isexist = ((uint8_t*)(index+INT16_INDEX_SZ+8))[key/8] & (128>>(key%8));
// key是否已存在
isexist = ((uint8_t*)(index+INT16_INDEX_SZ+8))[key/8] & (128>>(key%8));
if(!isexist) return 0;
sum = __builtin_popcount(((uint8_t*)(index+INT16_INDEX_SZ+8))[key/8] & ~(0xff>>(key%8)));
// 查找 key 之前共有多少索引
for(int i = key/256*32; i < key/8; i++) { // 从未计算的32位组开始算起
sum += __builtin_popcount(((uint8_t*)(index+INT16_INDEX_SZ+8))[i]);
}
sum += __builtin_popcount(((uint8_t*)(index+INT16_INDEX_SZ+8))[key/8] & ~(0xff>>(key%8)));
#ifdef DEBUG
printf("popc: %d, ", sum);
#endif
for(int i = 0; i < key/256; i++) {
int s = ((uint8_t*)(index+24))[i];
if(unlikely(!s && ((uint8_t*)(index+INT16_INDEX_SZ+8))[i*32])) {
@@ -356,10 +424,21 @@ uint64_t remove_item_by_int16_key(int fd, void* index, key_t k) {
}
sum += s;
}
#ifdef DEBUG
printf("sum: %d, ", sum);
#endif
} else {
int isexist = ((uint8_t*)(index+INT16_INDEX_SZ+8+8))[key/8] & (128>>(key%8));
// key是否已存在
isexist = ((uint8_t*)(index+INT16_INDEX_SZ+8+8))[key/8] & (128>>(key%8));
if(!isexist) return 0;
sum = __builtin_popcount(((uint8_t*)(index+INT16_INDEX_SZ+8+8))[key/8] & ~(0xff>>(key%8)));
// 查找 key 之前共有多少索引
for(int i = key/256*32; i < key/8; i++) { // 从未计算的32位组开始算起
sum += __builtin_popcount(((uint8_t*)(index+INT16_INDEX_SZ+8+8))[i]);
}
sum += __builtin_popcount(((uint8_t*)(index+INT16_INDEX_SZ+8+8))[key/8] & ~(0xff>>(key%8)));
#ifdef DEBUG
printf("popc: %d, ", sum);
#endif
for(int i = 0; i < 128; i++) {
int s = ((uint8_t*)(index+24))[i];
if(unlikely(!s && ((uint8_t*)(index+INT16_INDEX_SZ+8))[i*32])) {
@@ -376,24 +455,38 @@ uint64_t remove_item_by_int16_key(int fd, void* index, key_t k) {
}
sum += s;
}
#ifdef DEBUG
printf("sum: %d, ", sum);
#endif
}
clear_bitmap(fd, index, key);
#ifdef DEBUG
printf("i: %d, sumblk: %d, map: %02x, ", key/256, ((uint8_t*)(index+24))[key/256], ((uint8_t*)(index+INT16_INDEX_SZ+8+((key<32768)?0:8)))[key/8]);
#endif
uint64_t cur_ptr, next_ptr, first_ptr = 0;
int offset = sum%256;
ptr = le64(index+16);
for(int i = 0; i < sum/256 && ptr; i++) {
for(int i = 0; i < sum/256; i++) {
if(lseek(fd, ptr, SEEK_SET) < 0) return EOF;
readle64(fd, ptr);
}
if(!ptr) return EOF;
int offset = sum%256;
uint64_t cur_ptr = ptr+8*(offset+1), next_ptr, first_ptr = 0;
if(lseek(fd, ptr, SEEK_SET) < 0) return EOF;
readle64(fd, next_ptr);
lseek(fd, cur_ptr, SEEK_SET); // 回到开头
readle64(fd, ptr); // 返回值
cur_ptr += 8;
if(lseek(fd, 8*offset, SEEK_CUR) < 0) return EOF;
readle64(fd, ptr);
#ifdef DEBUG
printf("ptr: %d, ", (int)ptr);
#endif
cur_ptr = lseek(fd, 0, SEEK_CUR); // 回到开头
#ifdef DEBUG
printf("cur: %016llx, next: %016llx\n", cur_ptr, next_ptr);
#endif
do { // 一直循环到末尾
if(unlikely(offset++%256 == 255)) { // 当前位于末尾，需要从下一页取值
if(!(++offset%256)) { // 当前位于末尾，需要从下一页取值
// 换下一页
if(lseek(fd, next_ptr+8, SEEK_SET) < 0) return EOF;
readle64(fd, first_ptr); // 读后一个值
@@ -402,7 +495,7 @@ uint64_t remove_item_by_int16_key(int fd, void* index, key_t k) {
putle64(buf, first_ptr);
write(fd, buf, 8); // 覆盖
// 换下一页
cur_ptr = lseek(fd, next_ptr, SEEK_SET)+8;
cur_ptr = lseek(fd, next_ptr, SEEK_SET);
readle64(fd, next_ptr);
lseek(fd, 8, SEEK_CUR);
continue;
@@ -412,8 +505,7 @@ uint64_t remove_item_by_int16_key(int fd, void* index, key_t k) {
lseek(fd, -16, SEEK_CUR); // 回原处
putle64(buf, first_ptr);
write(fd, buf, 8); // 覆盖
cur_ptr += 8;
lseek(fd, 8, SEEK_CUR);
cur_ptr = lseek(fd, 8, SEEK_CUR)-8;
} while(first_ptr);
return ptr;

60
tests/types_test.c
View File

@@ -79,8 +79,8 @@ int main() {
perror("create_int16_index");
return 3;
}
for(int i = 57344, cnt = 0; i < 65536+4099; i++, cnt++) {
int cnt = 0;
for(int i = 57344; i < 65536+257; i++, cnt++) {
int n;
if((n=count_items(fd, TYPE_INT16, index)) != cnt) {
printf("%d != %d\n", cnt, n);
@@ -93,6 +93,62 @@ int main() {
return 4;
}
}
for(int i = 57344; i < 65536+257; i++) {
if((int)find_item_by_key(fd, TYPE_INT16, index, (key_t)i) != i) {
printf("%u ", (uint16_t)i);
fflush(stdout);
perror("find_item_by_key");
return 5;
}
}
close(fd);
fd = open("types_test_tmp.bin", O_RDWR, 0644);
memset(buf, 0, sizeof(buf));
index = load_index(fd, TYPE_INT16, HEADERSZ, buf);
for(int i = 65500; i < 65536+10; i++, cnt--) {
int n;
if((n=count_items(fd, TYPE_INT16, index)) != cnt) {
printf("%d != %d\n", cnt, n);
return 6;
}
if(remove_item_by_key(fd, TYPE_INT16, index, (key_t)i) != i) {
printf("%u ", (uint16_t)i);
fflush(stdout);
perror("remove_item_by_key");
return 6;
}
}
for(int i = 65500; i < 65536+10; i++) {
if((int)find_item_by_key(fd, TYPE_INT16, index, (key_t)i) != 0) {
printf("%u ", (uint16_t)i);
fflush(stdout);
perror("find_item_by_key");
return 7;
}
}
for(int i = 57344; i < 65500; i++) {
if((int)find_item_by_key(fd, TYPE_INT16, index, (key_t)i) != i) {
printf("%u ", (uint16_t)i);
fflush(stdout);
perror("find_item_by_key");
return 8;
}
}
for(int i = 65536+10; i < 65536+257; i++) {
if((int)find_item_by_key(fd, TYPE_INT16, index, (key_t)i) != i) {
printf("%u ", (uint16_t)i);
fflush(stdout);
perror("find_item_by_key");
return 9;
}
}
close(fd);
/* end test int16 */
// remove("types_test_tmp.bin");