From 22837fee6c5fe56104ca3a6085014b841842ed4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=BA=90=E6=96=87=E9=9B=A8?= <41315874+fumiama@users.noreply.github.com> Date: Fri, 1 Apr 2022 21:14:19 +0800 Subject: [PATCH] feat: add mmap --- CMakeLists.txt | 8 +- base14.c | 30 -------- base14.h | 37 ++++++++- base1432.c | 1 - base1464.c | 1 - base16384.c | 198 +++++++++++++++++++++++++++++++------------------ 6 files changed, 163 insertions(+), 112 deletions(-) delete mode 100644 base14.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 5a736f4..4eaea8b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,12 +12,12 @@ if (${isBigEndian}) endif() IF(CMAKE_SIZEOF_VOID_P EQUAL 8) - add_library(b1464 STATIC base1464.c base14.c) - add_library(b14 SHARED base1464.c base14.c) + add_library(b1464 STATIC base1464.c) + add_library(b14 SHARED base1464.c) target_link_libraries(base16384 b1464) ELSE() - add_library(b1432 STATIC base1432.c base14.c) - add_library(b14 SHARED base1432.c base14.c) + add_library(b1432 STATIC base1432.c) + add_library(b14 SHARED base1432.c) target_link_libraries(base16384 b1432) ENDIF() diff --git a/base14.c b/base14.c deleted file mode 100644 index 15a85c8..0000000 --- a/base14.c +++ /dev/null @@ -1,30 +0,0 @@ -int encode_len(int dlen) { - int outlen = dlen / 7 * 8; - int offset = dlen % 7; - switch(offset) { // 算上偏移标志字符占用的2字节 - case 0: break; - case 1: outlen += 4; break; - case 2: - case 3: outlen += 6; break; - case 4: - case 5: outlen += 8; break; - case 6: outlen += 10; break; - default: break; - } - return outlen + 8; // 冗余的8B用于可能的结尾的覆盖 -} - -int decode_len(int dlen, int offset) { - int outlen = dlen; - switch(offset) { // 算上偏移标志字符占用的2字节 - case 0: break; - case 1: outlen -= 4; break; - case 2: - case 3: outlen -= 6; break; - case 4: - case 5: outlen -= 8; break; - case 6: outlen -= 10; break; - default: break; - } - return outlen / 8 * 7 + offset + 1; // 多出1字节用于循环覆盖 -} diff --git a/base14.h b/base14.h index 598f7a7..45d8e94 100644 --- a/base14.h +++ b/base14.h @@ -1,12 +1,45 @@ +#ifndef _BASE14_H_ +#define _BASE14_H_ + // base14.h // fumiama 20220319 // encode_len calc min buf size to fill encode result -int encode_len(int dlen); +int encode_len(int dlen) { + int outlen = dlen / 7 * 8; + int offset = dlen % 7; + switch(offset) { // 算上偏移标志字符占用的2字节 + case 0: break; + case 1: outlen += 4; break; + case 2: + case 3: outlen += 6; break; + case 4: + case 5: outlen += 8; break; + case 6: outlen += 10; break; + default: break; + } + return outlen + 8; // 冗余的8B用于可能的结尾的覆盖 +} + // decode_len calc min buf size to fill decode result -int decode_len(int dlen, int offset); +int decode_len(int dlen, int offset) { + int outlen = dlen; + switch(offset) { // 算上偏移标志字符占用的2字节 + case 0: break; + case 1: outlen -= 4; break; + case 2: + case 3: outlen -= 6; break; + case 4: + case 5: outlen -= 8; break; + case 6: outlen -= 10; break; + default: break; + } + return outlen / 8 * 7 + offset + 1; // 多出1字节用于循环覆盖 +} // encode data and write result into buf int encode(const char* data, int dlen, char* buf, int blen); // decode data and write result into buf int decode(const char* data, int dlen, char* buf, int blen); + +#endif \ No newline at end of file diff --git a/base1432.c b/base1432.c index c71776b..d4c9063 100644 --- a/base1432.c +++ b/base1432.c @@ -43,7 +43,6 @@ #endif #endif #endif -#include "base14.h" // #define DEBUG diff --git a/base1464.c b/base1464.c index e693ce4..7aa180f 100644 --- a/base1464.c +++ b/base1464.c @@ -43,7 +43,6 @@ # define htobe64(x) _byteswap_uint64(x) #endif #endif -#include "base14.h" // #define DEBUG diff --git a/base16384.c b/base16384.c index 2296b21..570ab5a 100644 --- a/base16384.c +++ b/base16384.c @@ -2,9 +2,12 @@ #include #include #include +#include #include -#ifdef __WINNT__ +#ifdef _WIN32 #include +#else + #include #endif #endif #include "base14.h" @@ -18,122 +21,169 @@ static off_t get_file_size(const char* filepath) { } #endif +char encbuf[BUFSIZ*1024/7*7]; +char decbuf[BUFSIZ*1024/8*8+2]; + void encode_file(const char* input, const char* output) { off_t inputsize = get_file_size(input); if(inputsize <= 0) { - puts("Get file size error!"); + perror("Get file size error: "); return; } - FILE* fp = NULL; - fp = fopen(input, "rb"); - if(!fp) { - puts("Open input file error!"); + if(inputsize > (1u<<31)-1) { + puts("Input file too big!"); return; } FILE* fpo = NULL; fpo = fopen(output, "wb"); if(!fpo) { - puts("Open output file error!"); + perror("Fopen output file error: "); return; } - if(inputsize > BUFSIZ*1024) inputsize = BUFSIZ*1024/7*7; // big file - char* bufi = (char*)malloc(inputsize); - if(!bufi) { - puts("Allocate input buffer error!"); - return; + if(inputsize > BUFSIZ*1024) { // big file, use encbuf & fread + inputsize = BUFSIZ*1024/7*7; + #ifdef _WIN32 } - int outputsize = encode_len(inputsize)+16; - char* bufo = (char*)malloc(outputsize); - if(!bufo) { - puts("Allocate output buffer error!"); - return; - } - size_t cnt = 0; - fputc(0xFE, fpo); - fputc(0xFF, fpo); - while((cnt = fread(bufi, sizeof(char), inputsize, fp))) { - int n = encode(bufi, cnt, bufo, outputsize); - if(fwrite(bufo, n, 1, fpo) <= 0) { - puts("Write file error!"); + #endif + FILE* fp = NULL; + fp = fopen(input, "rb"); + if(!fp) { + perror("Fopen input file error: "); return; } + + int outputsize = encode_len(inputsize)+16; + size_t cnt = 0; + fputc(0xFE, fpo); + fputc(0xFF, fpo); + while((cnt = fread(encbuf, sizeof(char), inputsize, fp))) { + int n = encode(encbuf, cnt, decbuf, outputsize); + if(fwrite(decbuf, n, 1, fpo) <= 0) { + perror("Write file error: "); + return; + } + } + /* 由操作系统负责释放资源 + fclose(fpo); + fclose(fp); + 以缩短程序运行时间 */ + #ifndef _WIN32 + } else { // small file, use mmap & fwrite + int fd = open(input, O_RDONLY); + if(fd <= 0) { + perror("Open input file error: "); + return; + } + char *input_file = mmap(NULL, (size_t)inputsize, PROT_READ, MAP_PRIVATE, fd, 0); + if(input_file == MAP_FAILED) { + perror("Map input file error: "); + return; + } + int outputsize = encode_len(inputsize)+16; + fputc(0xFE, fpo); + fputc(0xFF, fpo); + int n = encode(input_file, (int)inputsize, decbuf, outputsize); + if(fwrite(decbuf, n, 1, fpo) <= 0) { + perror("Write file error: "); + return; + } + munmap(input_file, (size_t)inputsize); + /* 由操作系统负责释放资源 + fclose(fpo); + close(fd); + 以缩短程序运行时间 */ } - /* 由操作系统负责释放资源 - free(bufo); - free(bufi); - fclose(fpo); - fclose(fp); - 以缩短程序运行时间 */ + #endif } -void rm_head(FILE* fp) { - int ch = fgetc(fp); - if(ch == 0xFE) fgetc(fp); - else rewind(fp); +#define rm_head(fp) {\ + int ch = fgetc(fp);\ + if(ch == 0xFE) fgetc(fp);\ + else rewind(fp);\ } +#define skip_offset(input_file) ((input_file[0]==(char)0xFE)?2:0) + static int is_next_end(FILE* fp) { int ch = fgetc(fp); if(ch == '=') return fgetc(fp); - else { - ungetc(ch, fp); - return 0; - } + ungetc(ch, fp); + return 0; } void decode_file(const char* input, const char* output) { off_t inputsize = get_file_size(input); if(inputsize <= 0) { - puts("Get file size error!"); + perror("Get file size error: "); return; } - FILE* fp = NULL; - fp = fopen(input, "rb"); - if(!fp) { - puts("Open input file error!"); + if(inputsize > (1u<<31)-1) { + puts("Input file too big!"); return; } FILE* fpo = NULL; fpo = fopen(output, "wb"); if(!fpo) { - puts("Open output file error!"); + perror("Fopen output file error: "); return; } - if(inputsize > BUFSIZ*1024) inputsize = BUFSIZ*1024/8*8; // big file - char* bufi = (char*)malloc(inputsize+2); // +2避免漏检结束偏移标志 - if(!bufi) { - puts("Allocate input buffer error!"); - return; + if(inputsize > BUFSIZ*1024) { // big file, use decbuf & fread + inputsize = BUFSIZ*1024/8*8; + #ifdef _WIN32 } - int outputsize = decode_len(inputsize, 0)+16; - char* bufo = (char*)malloc(outputsize); - if(!bufo) { - puts("Allocate output buffer error!"); - return; - } - int cnt = 0; - int end = 0; - rm_head(fp); - while((cnt = fread(bufi, sizeof(char), inputsize, fp))) { - if((end = is_next_end(fp))) { - bufi[cnt++] = '='; - bufi[cnt++] = end; - } - int n = decode(bufi, cnt, bufo, outputsize); - if(fwrite(bufo, n, 1, fpo) <= 0) { - puts("Write file error!"); + #endif + FILE* fp = NULL; + fp = fopen(input, "rb"); + if(!fp) { + perror("Fopen input file error: "); return; } + int outputsize = decode_len(inputsize, 0)+16; + int cnt = 0; + int end = 0; + rm_head(fp); + while((cnt = fread(decbuf, sizeof(char), inputsize, fp))) { + if((end = is_next_end(fp))) { + decbuf[cnt++] = '='; + decbuf[cnt++] = end; + } + if(fwrite(encbuf, decode(decbuf, cnt, encbuf, outputsize), 1, fpo) <= 0) { + perror("Write file error: "); + return; + } + } + /* 由操作系统负责释放资源 + fclose(fpo); + fclose(fp); + 以缩短程序运行时间 */ + #ifndef _WIN32 + } else { // small file, use mmap & fwrite + int fd = open(input, O_RDONLY); + if(fd <= 0) { + perror("Open input file error: "); + return; + } + char *input_file = mmap(NULL, (size_t)inputsize, PROT_READ, MAP_PRIVATE, fd, 0); + if(input_file == MAP_FAILED) { + perror("Map input file error: "); + return; + } + int outputsize = decode_len(inputsize, 0)+16; + int off = skip_offset(input_file); + if(fwrite(encbuf, decode(input_file+off, inputsize-off, encbuf, outputsize), 1, fpo) <= 0) { + perror("Write file error: "); + return; + } + munmap(input_file, (size_t)inputsize); + /* 由操作系统负责释放资源 + fclose(fpo); + close(fd); + 以缩短程序运行时间 */ } - /* 由操作系统负责释放资源 - free(bufo); - free(bufi); - fclose(fpo); - fclose(fp); - 以缩短程序运行时间 */ + #endif } -#ifndef __WINNT__ +#ifndef _WIN32 unsigned long get_start_ms() { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts);