fix: possible concurrent errors

.gitignore

@@ -53,4 +53,3 @@ dkms.conf
 
 build
 .DS_Store
-.vscode

.vscode/launch.json

@@ -0,0 +1,26 @@
+{
+    // 使用 IntelliSense 了解相关属性。 
+    // 悬停以查看现有属性的描述。
+    // 欲了解更多信息，请访问: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+    {
+        "name": "Launch",
+        "type": "cppdbg",
+        "request": "launch",
+        "program": "${workspaceFolder}/build/simple-http-server",
+        "args": [],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}/build",
+        "environment": [
+            {
+                "name": "DYLD_FALLBACK_LIBRARY_PATH",
+                "value": "/usr/local/lib"
+            }
+        ],
+        "externalConsole": false,
+        "MIMode": "lldb",
+        "preLaunchTask": "Build Debug"
+    }
+    ]
+}

.vscode/tasks.json

@@ -0,0 +1,20 @@
+{
+    "tasks": [
+        {
+            "type": "shell",
+            "label": "Build Debug",
+            "command": "cmake -DCMAKE_BUILD_TYPE=Debug .. && make",
+            "options": {
+                "cwd": "${workspaceFolder}/build"
+            },
+            "problemMatcher": [
+                "$gcc"
+            ],
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            }
+        }
+    ],
+    "version": "2.0.0"
+}

CMakeLists.txt

@@ -1,6 +1,8 @@
 cmake_minimum_required(VERSION 2.6...4.1.1)
 project(simple-http-server C)
-SET(CMAKE_BUILD_TYPE "Release")
+IF(NOT CMAKE_BUILD_TYPE)
+    SET(CMAKE_BUILD_TYPE "Release")
+ENDIF()
 
 add_definitions(-DLISTEN_ON_IPV6)
 IF(CMAKE_SIZEOF_VOID_P EQUAL 8)

README.md

@@ -67,7 +67,7 @@ argv[1] = method; //request method (GET/POST)
 argv[2] = query_string;   //the query string, like "a=1&b=2&c=3"
 ```
 
-The server will read a `4 bytes` unsigned integer from pipe, indicating the `length` of the remaining content. Then it will send `length` bytes of data to the client directly with nothing being decorated, which means that you need to assemble the HTTP header by yourself.
+The server will read a `4 bytes` unsigned integer from pipe, indicating the `length` of the remaining content. Then it will send `length` bytes of data to the client directly with nothing being decorated, which means that you need to assemble the HTTP header by yourself. After all writings, please wait the server write `CGIFINRECV` to your `stdin`, showing that the server has read all data you send, then it is safe to exit.
 
 Here is a CGI example [CMoe-Counter](https://github.com/fumiama/CMoe-Counter)
 

server.c

@@ -83,7 +83,7 @@ static void serve_file(int, const char *);
 static int startup(uint16_t *, int);
 static int startupunix(char *, int);
 static void unimplemented(int);
-static void wait_pid_push(void*);
+
 /**********************************************************************/
 /* A request has caused a call to accept() on the server port to
  * return.  Process the request appropriately.
@@ -142,7 +142,7 @@ static void accept_action(tcpool_thread_timer_t *p) {
 	path -= 2;
 	path[0] = '.'; path[1] = '/';
 
-	printf("[%s] p='%s' ?='%s' ", getmethod(method_type), path, query_string);
+	printf("[%s] p='%s' ?='%s' (tid=%lu) ", getmethod(method_type), path, query_string, (unsigned long)pthread_self());
 	do {
 		// 花括号不可省略
 		if(stat(path, &st) == -1) {
@@ -245,15 +245,9 @@ static void error_die(const char *sc) {
 	exit(1);
 }
 
-static void cgi_timeout_handler(int sig) {
+
-	puts("CGI kill alarm triggered.");
+static void clear_sigmask_push(void* old_mask) {
-	pthread_kill(pthread_self(), SIGQUIT);
+	pthread_sigmask(SIG_SETMASK, (const sigset_t *)old_mask, NULL);
-}
-static void reset_sigaction_push(void* old_sa_timeout) {
-	sigaction(SIGALRM, (const struct sigaction *)old_sa_timeout, NULL);
-}
-static void alarm_push(void* sec) {
-	alarm((unsigned int)(uintptr_t)sec);
 }
 static void clear_in_cgi_push(void* index) {
 	is_in_cgi[(uintptr_t)index] = 0;
@@ -269,23 +263,22 @@ static void execute_cgi(tcpool_thread_timer_t *timer, int content_length, const
 	pid_t pid;
 	int client = timer->accept_fd;
 
-	// Set up timeout handler
+	// Block SIGPIPE to prevent termination when writing to closed pipe
-	struct sigaction sa_timeout, old_sa_timeout;
+	sigset_t new_mask, old_mask;
-	sa_timeout.sa_handler = cgi_timeout_handler;
+	sigemptyset(&new_mask);
-	sigemptyset(&sa_timeout.sa_mask);
+	sigaddset(&new_mask, SIGPIPE);
-	sa_timeout.sa_flags = 0;
+	pthread_sigmask(SIG_BLOCK, &new_mask, &old_mask);
-	sigaction(SIGALRM, &sa_timeout, &old_sa_timeout);
-	pthread_cleanup_push(reset_sigaction_push, (void*)&old_sa_timeout);
-	alarm(30); // 30 seconds timeout for CGI execution
-	pthread_cleanup_push(alarm_push, 0);
 
 	if(pipe(cgi_output) < 0) {
-		internal_error(client, "pipe(cgi_output) failed", errno);
+		int err = errno;
+		pthread_sigmask(SIG_SETMASK, &old_mask, NULL);
+		internal_error(client, "pipe(cgi_output) failed", err);
 		return;
 	}
 	if(pipe(cgi_input) < 0) {
 		int err = errno;
 		close(cgi_output[0]); close(cgi_output[1]);
+		pthread_sigmask(SIG_SETMASK, &old_mask, NULL);
 		internal_error(client, "pipe(cgi_input) failed", err);
 		return;
 	}
@@ -293,11 +286,32 @@ static void execute_cgi(tcpool_thread_timer_t *timer, int content_length, const
 		int err = errno;
 		close(cgi_output[0]); close(cgi_output[1]);
 		close(cgi_input[0]); close(cgi_input[1]);
+		pthread_sigmask(SIG_SETMASK, &old_mask, NULL);
 		internal_error(client, "fork failed", err);
 		return;
 	}
-	/* child: CGI script */
+	/* first child: fork again to create orphan process */
 	if(pid == 0) {
+		pid_t pid2 = fork();
+		if(pid2 < 0) {
+			_exit(EXIT_FAILURE);
+		}
+		if(pid2 > 0) {
+			// first child exits immediately
+			_exit(EXIT_SUCCESS);
+		}
+		// second child (grandchild): CGI script
+		for(int sig = 1; sig < NSIG; sig++) {
+			// 跳过无法捕获的信号
+			if(sig != SIGKILL && sig != SIGSTOP) {
+				signal(sig, SIG_DFL);
+			}
+		}
+
+		sigset_t empty_set;
+		sigemptyset(&empty_set);
+		sigprocmask(SIG_SETMASK, &empty_set, NULL);
+
 		dup2(cgi_output[1], STDOUT_FILENO);
 		dup2(cgi_input[0], STDIN_FILENO);
 		close(cgi_output[0]);
@@ -309,11 +323,14 @@ static void execute_cgi(tcpool_thread_timer_t *timer, int content_length, const
 		// execl failed, write error marker to pipe
 		uint32_t error_marker = 0xFFFFFFFF;
 		write(STDOUT_FILENO, &error_marker, sizeof(error_marker));
-		_exit(EXIT_FAILURE); // use _exit instead of exit to avoid flushing parent's buffers
+		_exit(EXIT_FAILURE);
 	}
+	// parent: wait for first child to exit immediately
+	waitpid(pid, NULL, 0);
+
+	pthread_cleanup_push(clear_sigmask_push, (void*)&old_mask);
 	pthread_cleanup_push((void (*)(void*))&close, (void*)(uintptr_t)cgi_output[0]);
 	pthread_cleanup_push((void (*)(void*))&close, (void*)(uintptr_t)cgi_input[1]);
-	pthread_cleanup_push((void (*)(void*))&wait_pid_push, (void*)(uintptr_t)pid);
 	pthread_cleanup_push(clear_in_cgi_push, (void*)(uintptr_t)timer->index);
 	/* parent */
 	is_in_cgi[timer->index] = 1;
@@ -321,11 +338,16 @@ static void execute_cgi(tcpool_thread_timer_t *timer, int content_length, const
 	close(cgi_input[0]);
 
 	if(request->method_type == POST) {
+		printf("CGI write %d bytes data to child.\n", content_length);
 		#if __APPLE__
 			for(int i = 0; i < content_length;) {
 				int cnt = recv(client, timer->data, sizeof(timer->data), 0);
 				if(cnt > 0) {
-					write(cgi_input[1], timer->data, cnt);
+					ssize_t w = write(cgi_input[1], timer->data, cnt);
+					if(w < 0 && errno == EPIPE) {
+						puts("write to CGI input pipe got EPIPE, CGI closed input.");
+						goto CGI_CLOSE;
+					}
 					i += cnt;
 				} else {
 					internal_error(client, "recv POST data failed", errno);
@@ -337,6 +359,10 @@ static void execute_cgi(tcpool_thread_timer_t *timer, int content_length, const
 			while(len < content_length) {
 				int delta = splice(client, NULL, cgi_input[1], NULL, content_length - len, SPLICE_F_GIFT);
 				if(delta <= 0) {
+					if(errno == EPIPE) {
+						puts("splice to CGI input pipe got EPIPE, CGI closed input.");
+						goto CGI_CLOSE;
+					}
 					internal_error(client, "splice POST data failed", errno);
 					goto CGI_CLOSE;
 				}
@@ -345,12 +371,34 @@ static void execute_cgi(tcpool_thread_timer_t *timer, int content_length, const
 		#endif
 	}
 
+	fd_set rfds;
+	struct timeval tv;
+	FD_ZERO(&rfds);
+	FD_SET(cgi_output[0], &rfds);
+	tv.tv_sec = 1;
+	tv.tv_usec = 0;
+
+	int sel = select(cgi_output[0]+1, &rfds, NULL, NULL, &tv);
+	if(sel <= 0) {
+		internal_error(client, "CGI select wait for prep timeout or error", errno);
+		goto CGI_CLOSE;
+	}
+
 	char buf[8];
-	read(cgi_output[0], buf, 8); // wait child finish prep
+	if(read(cgi_output[0], buf, 8) != 8) { // DONEPREP
+		internal_error(client, "CGI failed to send prep marker", errno);
+		goto CGI_CLOSE;
+	}
 
 	uint32_t cnt = 0;
 	char* p = (char*)&cnt;
 	while(p - (char*)&cnt < sizeof(uint32_t)) {
+		FD_ZERO(&rfds);
+		FD_SET(cgi_output[0], &rfds);
+		if(select(cgi_output[0]+1, &rfds, NULL, NULL, &tv) <= 0) {
+			internal_error(client, "CGI select wait for header timeout or error", errno);
+			goto CGI_CLOSE;
+		}
 		// Check if thread is being cleaned up
 		if(!is_in_cgi[timer->index]) {
 			puts("CGI thread cleanup detected, abort read.");
@@ -414,12 +462,13 @@ static void execute_cgi(tcpool_thread_timer_t *timer, int content_length, const
 		#if __APPLE__
 			int cap = (cnt>sizeof(timer->data))?sizeof(timer->data):cnt;
 			while(len < cnt) {
-				// Check if thread is being cleaned up
+				FD_ZERO(&rfds);
-				if(!is_in_cgi[timer->index]) {
+				FD_SET(cgi_output[0], &rfds);
-					puts("CGI thread cleanup detected during output read.");
+				int sel = select(cgi_output[0]+1, &rfds, NULL, NULL, &tv);
+				if(sel <= 0) {
+					internal_error(client, "CGI select wait for output timeout or error", errno);
 					goto CGI_CLOSE;
 				}
-				
 				int n = read(cgi_output[0], timer->data, cap);
 				if(n <= 0) {
 					if(n < 0 && errno == EBADF && !is_in_cgi[timer->index]) {
@@ -430,20 +479,29 @@ static void execute_cgi(tcpool_thread_timer_t *timer, int content_length, const
 					goto CGI_CLOSE;
 				}
 				len += n;
-				send(client, timer->data, n, 0);
+				ssize_t s = send(client, timer->data, n, 0
+				#ifdef MSG_NOSIGNAL
+					| MSG_NOSIGNAL
+				#endif
+				);
+				if(s < 0 && errno == EPIPE) {
+					puts("send to client got EPIPE, client closed connection.");
+					goto CGI_CLOSE;
+				}
 			}
 		#else
 			while(len < cnt) {
-				// Check if thread is being cleaned up
+				FD_ZERO(&rfds);
-				if(!is_in_cgi[timer->index]) {
+				FD_SET(cgi_output[0], &rfds);
-					puts("CGI thread cleanup detected during splice.");
+				int sel = select(cgi_output[0]+1, &rfds, NULL, NULL, &tv);
+				if(sel <= 0) {
+					internal_error(client, "CGI select wait for output timeout or error", errno);
 					goto CGI_CLOSE;
 				}
-				
 				int delta = splice(cgi_output[0], NULL, client, NULL, cnt - len, SPLICE_F_GIFT);
 				if(delta <= 0) {
-					if(delta < 0 && errno == EBADF && !is_in_cgi[timer->index]) {
+					if(errno == EPIPE) {
-						puts("CGI thread cleanup in progress.");
+						puts("splice to client got EPIPE, client closed connection.");
 						goto CGI_CLOSE;
 					}
 					internal_error(client, "splice CGI output failed", errno);
@@ -452,6 +510,8 @@ static void execute_cgi(tcpool_thread_timer_t *timer, int content_length, const
 				len += delta;
 			}
 		#endif
+		printf("write CGIFINRECV, ");
+		write(cgi_input[1], "CGIFINRECV", 10);
 		printf("send %d bytes.\n", len);
 	}
 CGI_CLOSE:
@@ -459,8 +519,6 @@ CGI_CLOSE:
 	pthread_cleanup_pop(1);
 	pthread_cleanup_pop(1);
 	pthread_cleanup_pop(1);
-	pthread_cleanup_pop(1);
-	pthread_cleanup_pop(1);
 }
 
 /**********************************************************************/
@@ -636,59 +694,6 @@ static void unimplemented(int client) {
 	puts("501 Method Not Implemented.");
 }
 
-#if __STDC_VERSION__ >= 201112L
-	static _Thread_local pid_t timeout_pid = 0;
-#else
-	static __thread pid_t timeout_pid = 0;
-#endif
-
-static void alarm_handler(int sig) {
-	pid_t p = timeout_pid;
-	timeout_pid = 0;
-	if(p > 0) {
-		printf("Wait pid %d > 5s, kill it.\n", p);
-		kill(p, SIGKILL);
-		timeout_pid = 0;
-	}
-}
-
-static void wait_pid_push(void* pid) {
-	pid_t p = (pid_t)((uintptr_t)pid);
-	timeout_pid = p;
-
-	struct sigaction sa, old_sa;
-	sa.sa_handler = alarm_handler;
-	sigemptyset(&sa.sa_mask);
-	sa.sa_flags = 0;
-	sigaction(SIGALRM, &sa, &old_sa);
-
-	printf("Wait pid %d start.\n", p);
-	alarm(5);
-	int status;
-	pid_t result = waitpid(p, &status, 0);
-	alarm(0);
-	
-	if(result > 0) {
-		if(WIFEXITED(status)) {
-			int exit_code = WEXITSTATUS(status);
-			printf("Wait pid %d finish: exited with code %d.\n", p, exit_code);
-		} else if(WIFSIGNALED(status)) {
-			int sig = WTERMSIG(status);
-			printf("Wait pid %d finish: killed by signal %d (%s)%s.\n", 
-				p, sig, strsignal(sig), WCOREDUMP(status) ? " (core dumped)" : "");
-		} else {
-			printf("Wait pid %d finish: terminated abnormally.\n", p);
-		}
-	} else if(result < 0) {
-		printf("Wait pid %d failed: %s\n", p, strerror(errno));
-	} else {
-		printf("Wait pid %d finish: no status change.\n", p);
-	}
-
-	timeout_pid = 0;
-	sigaction(SIGALRM, &old_sa, NULL);
-}
-
 #define argequ(arg) (*(uint16_t*)argv[i] == *(uint16_t*)(arg))
 #define USAGE "Usage:\tsimple-http-server [-d] [-h] [-n host.name.com:port] [-p <port|unix socket path>] [-q 16] [-r <rootdir>] [-u <uid>]\n   -d:\trun as daemon.\n   -h:\tdisplay this help.\n   -n:\tcheck hostname and port.\n   -p:\tif not set, we will choose a random port.\n   -q:\tlisten queue length (defalut is 16).\n   -r:\thttp root dir.\n   -u:\trun as this uid."
 int main(int argc, char **argv) {
@@ -736,11 +741,19 @@ int main(int argc, char **argv) {
 	if(port) printf("httpd running on 0.0.0.0:%d at %s\n", port, cdir);
 	else printf("httpd running on %s at %s\n", socket_path, cdir);
 
+	printf("main tid=%lu\n", (unsigned long)pthread_self());
+
 	if(uid > 0) {
 		setuid(uid);
 		setgid(uid);
 	}
 
+	// Block SIGPIPE to prevent termination when writing to closed pipe
+	sigset_t new_mask;
+	sigemptyset(&new_mask);
+	sigaddset(&new_mask, SIGPIPE);
+	pthread_sigmask(SIG_BLOCK, &new_mask, NULL);
+
 	pthread_cleanup_push((void (*)(void*))&close, (void*)((long long)server_sock));
 	accept_client(server_sock);
 	pthread_cleanup_pop(1);

tcpool.h

@@ -20,6 +20,7 @@
 #include <sys/wait.h>
 #include <time.h>
 #include <unistd.h>
+#include <execinfo.h>
 
 #ifndef TCPOOL_THREAD_TIMER_T_SZ
     #define TCPOOL_THREAD_TIMER_T_SZ 1024
@@ -245,13 +246,16 @@ static void handle_int(int signo) {
 
 static void handle_pipe(int signo) {
     uint32_t index = (uint32_t)((uintptr_t)pthread_getspecific(__tcpool_pthread_key_index));
-    printf("Pipe error@%d, break loop...\n", index-1);
+    printf("Pipe error@%d, break loop... (tid=%lu)\n", index-1, (unsigned long)pthread_self());
+    void *bt[32];
+    int bt_size = backtrace(bt, 32);
+    backtrace_symbols_fd(bt, bt_size, fileno(stderr));
     fflush(stdout);
     if(index) {
         sigaction(SIGPIPE, &(const struct sigaction){handle_pipe}, NULL);
         siglongjmp(__tcpool_jmp2convend[index-1], signo);
     }
-    else pthread_exit(NULL);
+    exit(signo);
 }
 
 static void accept_timer(void *p) {
@@ -261,6 +265,7 @@ static void accept_timer(void *p) {
     uint8_t isbusy;
     const time_t check_interval = (TCPOOL_MAXWAITSEC / 4) ? (TCPOOL_MAXWAITSEC / 4) : 1;
 
+    printf("Timer thread started for slot %d (tid=%lu)\n", index, (unsigned long)pthread_self());
     sleep(check_interval);
     while(thread && !pthread_kill(thread, 0)) {
         pthread_rwlock_rdlock(&timer->mb);
@@ -270,11 +275,11 @@ static void accept_timer(void *p) {
         TIMER_SLEEP:
             pthread_mutex_lock(&timer->tmc);
             timer->hastimerslept = 1;
-            printf("Timer@%d sleep\n", timer->index);
+            printf("Timer@%d sleep (tid=%lu)\n", timer->index, (unsigned long)pthread_self());
             pthread_cond_wait(&timer->tc, &timer->tmc);
             timer->hastimerslept = 0;
             pthread_mutex_unlock(&timer->tmc);
-            printf("Timer@%d wake up\n", timer->index);
+            printf("Timer@%d wake up (tid=%lu)\n", timer->index, (unsigned long)pthread_self());
             sleep(check_interval);
             thread = timer->thread;
         }
@@ -282,17 +287,22 @@ static void accept_timer(void *p) {
         pthread_rwlock_rdlock(&timer->mt);
         time_t waitsec = time(NULL) - timer->touch;
         pthread_rwlock_unlock(&timer->mt);
-        printf("Wait@%d sec: %u, max: %u\n", timer->index, (unsigned int)waitsec, TCPOOL_MAXWAITSEC);
+        printf("Wait@%d sec: %u, max: %u (tid=%lu)\n", timer->index, (unsigned int)waitsec, TCPOOL_MAXWAITSEC, (unsigned long)pthread_self());
         if(waitsec > TCPOOL_MAXWAITSEC) {
-            if(thread) {
+            pthread_rwlock_rdlock(&timer->mb);
+            isbusy = timer->isbusy;
+            pthread_rwlock_unlock(&timer->mb);
+            
+            if(thread && isbusy && !pthread_kill(thread, 0)) {
                 pthread_kill(thread, SIGQUIT);
-                printf("Kill thread@%d\n", timer->index);
+                printf("Kill thread@%d (tid=%lu)\n", timer->index, (unsigned long)pthread_self());
             }
             break;
         }
         sleep(check_interval);
         thread = timer->thread;
     }
+    printf("Timer@%d going to sleep after loop (tid=%lu)\n", timer->index, (unsigned long)pthread_self());
     goto TIMER_SLEEP;
 }
 
@@ -354,9 +364,7 @@ static void handle_accept(void *p) {
         pthread_rwlock_unlock(&tcpool_timer_pointer_of(p)->mb);
 
         puts("Set thread status to idle");
-        while(tcpool_timer_pointer_of(p)->isbusy == 0) {  // prevent fake wakeup
+        pthread_cond_wait(&tcpool_timer_pointer_of(p)->c, &tcpool_timer_pointer_of(p)->mc);
-            pthread_cond_wait(&tcpool_timer_pointer_of(p)->c, &tcpool_timer_pointer_of(p)->mc);
-        }
 
         pthread_mutex_unlock(&tcpool_timer_pointer_of(p)->mc);
         puts("Thread wakeup");
@@ -410,8 +418,10 @@ static void accept_client(int fd) {
             struct sockaddr_in client_addr;
         #endif
         int accept_fd;
+    RE_ACCEPT:
         if((accept_fd=accept(fd, (struct sockaddr *)&client_addr, &tcpool_struct_len))<=0) {
             perror("accept");
+            if (errno == EINTR) goto RE_ACCEPT;
             pthread_rwlock_wrlock(&timer->mb);
             timer->isbusy = 0;
             pthread_rwlock_unlock(&timer->mb);