fix(coder): incorrect hidden bits decoding in 32bit lib

2026-06-05 02:00:31 +08:00 · 2024-04-05 16:27:21 +09:00
parent cd15e63c86
commit 29e4019ed1
5 changed files with 70 additions and 38 deletions
--- a/base1432.c
+++ b/base1432.c
@@ -255,7 +255,7 @@ int base16384_decode_safe(const char* data, int dlen, char* buf) {
 		}
 	}
 	outlen = outlen / 8 * 7 + offset;
-	uint32_t* vals = (uint32_t*)data;
+	const uint32_t* vals = (const uint32_t*)data;
 	uint32_t n = 0;
 	int32_t i = 0;
 	for(; i < outlen - 7; i+=7) {	// n实际每次自增2
@@ -293,7 +293,7 @@ int base16384_decode_safe(const char* data, int dlen, char* buf) {
 		sum |= shift & 0x003fff00;
 		valbuf.val = be32toh(sum);
 		memcpy(buf+i+4, valbuf.buf, 3);
-	} else if(offset--) {
+	} else if((*(uint8_t*)(&vals[n]) != '=') && offset--) {
 		int cnt = dlen-2-(int)n*(int)sizeof(uint32_t);
 		if (cnt > 4) cnt = 4;
 		memcpy(valbuf.buf, &vals[n], cnt);
@@ -312,6 +312,7 @@ int base16384_decode_safe(const char* data, int dlen, char* buf) {
 				buf[i++] = ((sum & 0x000f0000) >> 12) | ((sum & 0xf0000000) >> 28);
 				if(offset--) {
 					buf[i] = (sum & 0x0f000000) >> 20;
+					if(*(uint8_t*)(&vals[n]) == '=') return outlen;
 					memcpy(valbuf.buf, &vals[n], dlen-2-(int)n*(int)sizeof(uint32_t));
 					#ifdef WORDS_BIGENDIAN
 						sum = __builtin_bswap32(valbuf.val);
@@ -351,7 +352,7 @@ int base16384_decode(const char* data, int dlen, char* buf) {
 		}
 	}
 	outlen = outlen / 8 * 7 + offset;
-	uint32_t* vals = (uint32_t*)data;
+	const uint32_t* vals = (const uint32_t*)data;
 	uint32_t n = 0;
 	int32_t i = 0;
 	for(; i <= outlen - 7; i+=7) {	// n实际每次自增2
@@ -371,6 +372,7 @@ int base16384_decode(const char* data, int dlen, char* buf) {
 		sum |= shift & 0x003fff00;
 		*(uint32_t*)(buf+i+4) = be32toh(sum);
 	}
+	if(*(uint8_t*)(&vals[n]) == '=') return outlen;
 	if(offset--) {
 		// 这里有读取越界
 		#ifdef WORDS_BIGENDIAN
@@ -387,6 +389,7 @@ int base16384_decode(const char* data, int dlen, char* buf) {
 				buf[i++] = ((sum & 0x000f0000) >> 12) | ((sum & 0xf0000000) >> 28);
 				if(offset--) {
 					buf[i] = (sum & 0x0f000000) >> 20;
+					if(*(uint8_t*)(&vals[n]) == '=') return outlen;
 					// 这里有读取越界
 					#ifdef WORDS_BIGENDIAN
 						sum = __builtin_bswap32(vals[n]);
@@ -426,7 +429,7 @@ int base16384_decode_unsafe(const char* data, int dlen, char* buf) {
 		}
 	}
 	outlen = outlen / 8 * 7 + offset;
-	uint32_t* vals = (uint32_t*)data;
+	const uint32_t* vals = (const uint32_t*)data;
 	uint32_t n = 0;
 	int32_t i = 0;
 	for(; i < outlen-7; i+=7) {	// n实际每次自增2
@@ -448,6 +451,7 @@ int base16384_decode_unsafe(const char* data, int dlen, char* buf) {
 	}
 	register uint32_t sum = 0;
 	register uint32_t shift = htobe32(vals[n++]);
+	if(((shift>>24)&0xff) == 0x3d) return outlen;
 	if(((shift>>24)&0xff) < 0x4e) shift |= 0xff000000;
 	if(((shift>> 8)&0xff) < 0x4e) shift |= 0x0000ff00;
 	shift -= 0x4e004e00;
@@ -455,7 +459,11 @@ int base16384_decode_unsafe(const char* data, int dlen, char* buf) {
 	sum |= shift & 0xfffc0000;
 	shift <<= 2;
 	sum |= shift & 0x0003fff0;
-	shift = htobe32(vals[n++]);
+	shift = htobe32(vals[n]);
+	if(((shift>>24)&0xff) == 0x3d) {
+		*(uint32_t*)(buf+i) = be32toh(sum);
+		return outlen;
+	}
 	if(((shift>>24)&0xff) < 0x4e) shift |= 0xff000000;
 	if(((shift>> 8)&0xff) < 0x4e) shift |= 0x0000ff00;
 	shift -= 0x4e004e00;
--- a/base1464.c
+++ b/base1464.c
@@ -226,7 +226,7 @@ int base16384_decode_safe(const char* data, int dlen, char* buf) {
 		}
 	}
 	outlen = outlen / 8 * 7 + offset;
-	uint64_t* vals = (uint64_t*)data;
+	const uint64_t* vals = (const uint64_t*)data;
 	uint64_t n = 0;
 	int64_t i = 0;
 	for(; i < outlen - 7; n++, i+=7) {
@@ -256,7 +256,7 @@ int base16384_decode_safe(const char* data, int dlen, char* buf) {
 		sum |= shift & 0x00000000003fff00;
 		valbuf.val = be64toh(sum);
 		memcpy(buf+i, valbuf.buf, 7);
-	} else if(offset--) {
+	} else if((*(uint8_t*)(&vals[n]) != '=') && offset--) {
 		memcpy(valbuf.buf, &vals[n], dlen-2-(int)n*(int)sizeof(uint64_t));
 		#ifdef WORDS_BIGENDIAN
 			register uint64_t sum = __builtin_bswap64(valbuf.val) - 0x000000000000004e;
@@ -303,7 +303,7 @@ int base16384_decode(const char* data, int dlen, char* buf) {
 		}
 	}
 	outlen = outlen / 8 * 7 + offset;
-	uint64_t* vals = (uint64_t*)data;
+	const uint64_t* vals = (const uint64_t*)data;
 	uint64_t n = 0;
 	int64_t i = 0;
 	for(; i <= outlen - 7; n++, i+=7) {
@@ -319,6 +319,7 @@ int base16384_decode(const char* data, int dlen, char* buf) {
 		sum |= shift & 0x00000000003fff00;
 		*(uint64_t*)(buf+i) = be64toh(sum);
 	}
+	if(*(uint8_t*)(&vals[n]) == '=') return outlen;
 	if(offset--) {
 		// 这里有读取越界
 		#ifdef WORDS_BIGENDIAN
@@ -366,7 +367,7 @@ int base16384_decode_unsafe(const char* data, int dlen, char* buf) {
 		}
 	}
 	outlen = outlen / 8 * 7 + offset;
-	uint64_t* vals = (uint64_t*)data;
+	const uint64_t* vals = (const uint64_t*)data;
 	uint64_t n = 0;
 	int64_t i = 0;
 	for(; i < outlen-7; n++, i+=7) {
@@ -384,6 +385,7 @@ int base16384_decode_unsafe(const char* data, int dlen, char* buf) {
 	}
 	register uint64_t sum = 0;
 	register uint64_t shift = htobe64(vals[n]);
+	if(((shift>>56)&0xff) == 0x3d) return outlen;
 	if(((shift>>56)&0xff) < 0x4e) shift |= 0xff00000000000000;
 	if(((shift>>40)&0xff) < 0x4e) shift |= 0x0000ff0000000000;
 	if(((shift>>24)&0xff) < 0x4e) shift |= 0x00000000ff000000;
--- a/base16384.h
+++ b/base16384.h
@@ -54,12 +54,10 @@ typedef enum base16384_err_t base16384_err_t;

 // disable 0xFEFF file header in encode
 #define BASE16384_FLAG_NOHEADER				(1<<0)
-// enable sum check when using stdin or inputsize > _BASE16384_ENCBUFSZ
+// enable sum check when using stdin or stdout or inputsize > _BASE16384_ENCBUFSZ
 #define BASE16384_FLAG_SUM_CHECK_ON_REMAIN	(1<<1)
-// forcely do sumcheck without checking decoded data length
+// forcely do sumcheck without checking data length
 #define BASE16384_FLAG_DO_SUM_CHECK_FORCELY	(1<<2)
-// initial sum value used in BASE16384_FLAG_SUM_CHECK_ON_REMAIN
-#define BASE16384_SIMPLE_SUM_INIT_VALUE		(0x8e29c213)

 /**
 * @brief calculate the exact encoded size
--- a/binary.h
+++ b/binary.h
@@ -70,15 +70,15 @@
 				#define htobe64(x) (x)
 			#endif
 		#else
-			#define be16toh(x) _byteswap_ushort(x)
-			#define be32toh(x) _byteswap_ulong(x)
+			#define be16toh(x) _bitswap_ushort(x)
+			#define be32toh(x) _bitswap_ulong(x)
 			#ifdef IS_64BIT_PROCESSOR
-				#define be64toh(x) _byteswap_uint64(x)
+				#define be64toh(x) _bitswap_uint64(x)
 			#endif
-			#define htobe16(x) _byteswap_ushort(x)
-			#define htobe32(x) _byteswap_ulong(x)
+			#define htobe16(x) _bitswap_ushort(x)
+			#define htobe32(x) _bitswap_ulong(x)
 			#ifdef IS_64BIT_PROCESSOR
-				#define htobe64(x) _byteswap_uint64(x)
+				#define htobe64(x) _bitswap_uint64(x)
 			#endif
 		#endif
 	#endif
@@ -87,6 +87,9 @@
 // leftrotate function definition
 #define LEFTROTATE(x, c) (((x) << (c)) | ((x) >> (sizeof(x)*8 - (c))))

+// initial sum value used in BASE16384_FLAG_SUM_CHECK_ON_REMAIN
+#define BASE16384_SIMPLE_SUM_INIT_VALUE		(0x8e29c213)
+
 static inline uint32_t calc_sum(uint32_t sum, size_t cnt, const char* encbuf) {
 	size_t i;
 	uint32_t buf;
@@ -99,20 +102,21 @@ static inline uint32_t calc_sum(uint32_t sum, size_t cnt, const char* encbuf) {
 	return sum;
 }

-static inline uint32_t calc_and_embed_sum(uint32_t sum, size_t cnt, char* encbuf) {
-	sum = calc_sum(sum, cnt, encbuf);
-	if(cnt%7) { // last encode
-		*(uint32_t*)(&encbuf[cnt]) = htobe32(sum);
-	}
-	return sum;
-}
-
 static inline int check_sum(uint32_t sum, uint32_t sum_read_raw, int offset) {
-	int shift = (int[]){0, 26, 20, 28, 22, 30, 24}[offset%7];
+	offset = offset%7;
+	if(!offset--) return 0; // no remain bits, pass
+	// offset 1: 0011 1111 1100 0000							remain: 3*2 bits
+	// offset 2: 0011 1111 1111 1111	0011 0000 0000 0000		remain: 6*2 bits
+	// offset 3: 0011 1111 1111 0000							remain: 2*2 bits
+	// offset 4: 0011 1111 1111 1111	0011 1100 0000 0000		remain: 5*2 bits
+	// offset 5: 0011 1111 1111 1100							remain: 1*2 bits
+	// offset 6: 0011 1111 1111 1111	0011 1111 0000 0000		remain: 4*2 bits
+	// encode: 0415263 (6-1) per 3bits, thus 0x021ab3
+	int shift = sizeof(uint32_t)*8 - ((0x021ab3>>(offset*3))&0x07)*2;
 	uint32_t sum_read = be32toh(sum_read_raw) >> shift;
 	sum >>= shift;
 	#ifdef DEBUG
-		fprintf(stderr, "offset: %d, mysum: %08x, sumrd: %08x\n", offset, sum, sum_read);
+		fprintf(stderr, "shift: %d, offset: %d, mysum: %08x, sumrd: %08x\n", shift, offset+1, sum, sum_read);
 	#endif
 	return sum != sum_read;
 }
--- a/file.c
+++ b/file.c
@@ -54,6 +54,8 @@ static inline off_t get_file_size(const char* filepath) {
 	goto base16384_##method##_file_detailed_cleanup; \
 }

+#define do_sum_check(flag) ((flag)&(BASE16384_FLAG_DO_SUM_CHECK_FORCELY|BASE16384_FLAG_SUM_CHECK_ON_REMAIN))
+
 base16384_err_t base16384_encode_file_detailed(const char* input, const char* output, char* encbuf, char* decbuf, int flag) {
 	off_t inputsize;
 	FILE *fp = NULL, *fpo;
@@ -98,7 +100,15 @@ base16384_err_t base16384_encode_file_detailed(const char* input, const char* ou
 				if(n > 0) cnt++;
 				else break;
 			}
-			if(flag&BASE16384_FLAG_SUM_CHECK_ON_REMAIN) sum = calc_and_embed_sum(sum, cnt, encbuf);
+			if(do_sum_check(flag)) {
+				sum = calc_sum(sum, cnt, encbuf);
+				if(cnt%7) { // last encode
+					*(uint32_t*)(&encbuf[cnt]) = htobe32(sum);
+					#ifdef DEBUG
+						fprintf(stderr, "writesum: %08x\n", sum);
+					#endif
+				}
+			}
 			n = base16384_encode_unsafe(encbuf, cnt, decbuf);
 			if(n && fwrite(decbuf, n, 1, fpo) <= 0) {
 				goto_base16384_file_detailed_cleanup(encode, base16384_err_write_file, {});
@@ -157,7 +167,12 @@ base16384_err_t base16384_encode_fp_detailed(FILE* input, FILE* output, char* en
 			if(n > 0) cnt++;
 			else break;
 		}
-		if(flag&BASE16384_FLAG_SUM_CHECK_ON_REMAIN) sum = calc_and_embed_sum(sum, cnt, encbuf);
+		if(do_sum_check(flag)) {
+			sum = calc_sum(sum, cnt, encbuf);
+			if(cnt%7) { // last encode
+				*(uint32_t*)(&encbuf[cnt]) = htobe32(sum);
+			}
+		}
 		n = base16384_encode_unsafe(encbuf, cnt, decbuf);
 		if(n && fwrite(decbuf, n, 1, output) <= 0) {
 			return base16384_err_write_file;
@@ -184,7 +199,12 @@ base16384_err_t base16384_encode_fd_detailed(int input, int output, char* encbuf
 			if(n > 0) cnt++;
 			else break;
 		}
-		if(flag&BASE16384_FLAG_SUM_CHECK_ON_REMAIN) sum = calc_and_embed_sum(sum, cnt, encbuf);
+		if(do_sum_check(flag)) {
+			sum = calc_sum(sum, cnt, encbuf);
+			if(cnt%7) { // last encode
+				*(uint32_t*)(&encbuf[cnt]) = htobe32(sum);
+			}
+		}
 		n = base16384_encode_unsafe(encbuf, cnt, decbuf);
 		if(n && write(output, decbuf, n) < n) {
 			return base16384_err_write_file;
@@ -269,10 +289,10 @@ base16384_err_t base16384_decode_file_detailed(const char* input, const char* ou
 				goto_base16384_file_detailed_cleanup(decode, base16384_err_write_file, {});
 			}
 			total_decoded_len += cnt;
-			if(flag&BASE16384_FLAG_SUM_CHECK_ON_REMAIN) sum = calc_sum(sum, cnt, encbuf);
+			if(do_sum_check(flag)) sum = calc_sum(sum, cnt, encbuf);
 			last_encbuf_cnt = cnt;
 		}
-		if(flag&BASE16384_FLAG_SUM_CHECK_ON_REMAIN
+		if(do_sum_check(flag)
 			&& (flag&BASE16384_FLAG_DO_SUM_CHECK_FORCELY || total_decoded_len >= _BASE16384_ENCBUFSZ)
 			&& last_decbuf_cnt > 2
 			&& decbuf[last_decbuf_cnt-2] == '='
@@ -346,10 +366,10 @@ base16384_err_t base16384_decode_fp_detailed(FILE* input, FILE* output, char* en
 			return base16384_err_write_file;
 		}
 		total_decoded_len += cnt;
-		if(flag&BASE16384_FLAG_SUM_CHECK_ON_REMAIN) sum = calc_sum(sum, cnt, encbuf);
+		if(do_sum_check(flag)) sum = calc_sum(sum, cnt, encbuf);
 		last_encbuf_cnt = cnt;
 	}
-	if(flag&BASE16384_FLAG_SUM_CHECK_ON_REMAIN
+	if(do_sum_check(flag)
 		&& (flag&BASE16384_FLAG_DO_SUM_CHECK_FORCELY || total_decoded_len >= _BASE16384_ENCBUFSZ)
 		&& last_decbuf_cnt > 2
 		&& decbuf[last_decbuf_cnt-2] == '='
@@ -425,10 +445,10 @@ base16384_err_t base16384_decode_fd_detailed(int input, int output, char* encbuf
 			return base16384_err_write_file;
 		}
 		total_decoded_len += n;
-		if(flag&BASE16384_FLAG_SUM_CHECK_ON_REMAIN) sum = calc_sum(sum, n, encbuf);
+		if(do_sum_check(flag)) sum = calc_sum(sum, n, encbuf);
 		last_encbuf_cnt = n;
 	}
-	if(flag&BASE16384_FLAG_SUM_CHECK_ON_REMAIN
+	if(do_sum_check(flag)
 		&& (flag&BASE16384_FLAG_DO_SUM_CHECK_FORCELY || total_decoded_len >= _BASE16384_ENCBUFSZ)
 		&& last_decbuf_cnt > 2
 		&& decbuf[last_decbuf_cnt-2] == '='