Commit Diff

Commit:
4fb828fbc1f23ea9aeee2a1b2d369aef96cb27b3
From:
Stefan Sperling <stsp@stsp.name>
Date:
Message:
dispatch new requests to the least-busy gotwebd worker Instead of distributing requests across workers in a round-robin fashion keep track of how many requests a given worker has queued and send new requests to the worker which has the least amount of work queued for it. This makes gotwebd much more responsive than stupid round-robin, even though the new strategy is still not optimal because different types of requests carry different cost. E.g. blaming a file is usually heavier than listing trees. But it might be good enough, for now. Let's try and see. ok op@
Actions:
Patch | Tree
commit - 19545591fb064b03b85f23b7a8bf64061fa82b68
commit + 4fb828fbc1f23ea9aeee2a1b2d369aef96cb27b3
blob - ecc7fd3254c6a09eb4b56485c161763b345e14a1
blob + 17338707c2666ca04c1d07ca753f94cd50762191
--- gotwebd/gotwebd.c
+++ gotwebd/gotwebd.c
@@ -500,6 +500,13 @@ main(int argc, char **argv)
 			www_username = env->www_user;
 	}
 
+	if (proc_type == GOTWEBD_PROC_SERVER) {
+		env->worker_load = calloc(env->prefork,
+		    sizeof(env->worker_load[0]));
+		if (env->worker_load == NULL)
+			fatal("calloc");
+	}
+
 	pw = getpwnam(www_username);
 	if (pw == NULL)
 		fatalx("unknown user %s", www_username);
blob - 0719d555453958214b9038725b19d44c068a5c05
blob + 02bc544637cc043a756a9a812d12548947cd9442
--- gotwebd/gotwebd.h
+++ gotwebd/gotwebd.h
@@ -273,6 +273,7 @@ struct request {
 	struct event			*resp_event;
 	int				 sock_id;
 	uint32_t			 request_id;
+	int				 worker_idx;
 
 	uint8_t				 *buf;
 	size_t				 buf_len;
@@ -391,6 +392,7 @@ struct gotwebd {
 	uint16_t	 prefork;
 	int		 gotweb_pending;
 	int		 gotweb_cur;
+	int		 *worker_load;
 
 	int		 server_cnt;
 
blob - f2cc7a4eb8d7c88ef03073b64897e2a73685cc29
blob + 0c92e34a59abbd18afc609df8ab37edb345a7a71
--- gotwebd/sockets.c
+++ gotwebd/sockets.c
@@ -134,7 +134,15 @@ find_request(uint32_t request_id)
 static void
 cleanup_request(struct request *c)
 {
+	struct gotwebd *env = gotwebd_env;
+
 	cgi_inflight--;
+
+	if (c->worker_idx != -1) {
+		if (env->worker_load[c->worker_idx] <= 0)
+			fatalx("request in flight on worker with zero load");
+		env->worker_load[c->worker_idx]--;
+	}
 
 	del_request(c);
 
@@ -467,7 +475,30 @@ recv_gotweb_pipe(struct gotwebd *env, struct imsg *ims
 
 	env->gotweb_pending--;
 }
+
+static struct imsgev *
+select_worker(struct request *c)
+{
+	struct gotwebd *env = gotwebd_env;
+	int i, least_busy_worker_idx, min_load;
 
+	min_load = env->worker_load[0];
+	least_busy_worker_idx = 0;
+	for (i = 1; i < env->prefork; i++) {
+		if (env->worker_load[i] > min_load) 
+			continue;
+
+		min_load = env->worker_load[i];
+		least_busy_worker_idx = i;
+	}
+
+	log_debug("dispatching request %u to gotweb %d",
+	    c->request_id, least_busy_worker_idx);
+
+	c->worker_idx = least_busy_worker_idx;
+	return &env->iev_gotweb[least_busy_worker_idx];
+}
+
 static int
 process_request(struct request *c)
 {
@@ -491,19 +522,18 @@ process_request(struct request *c)
 		ic.priv_fd[i] = -1;
 	ic.fd = -1;
 
-	/* Round-robin requests across gotweb processes. */
-	iev_gotweb = &env->iev_gotweb[env->gotweb_cur];
-	env->gotweb_cur = (env->gotweb_cur + 1) % env->prefork;
-
+	iev_gotweb = select_worker(c);
 	ret = imsg_compose_event(iev_gotweb, GOTWEBD_IMSG_REQ_PROCESS,
 	    GOTWEBD_PROC_SERVER, -1, c->fd, &ic, sizeof(ic));
 	if (ret == -1) {
 		log_warn("imsg_compose_event");
+		c->worker_idx = -1;
 		return -1;
 	}
 
 	c->fd = -1;
 	c->client_status = CLIENT_REQUEST;
+	env->worker_load[c->worker_idx]++;
 	return 0;
 }
 
@@ -782,7 +812,7 @@ sockets_shutdown(void)
 	for (i = 0; i < gotwebd_env->prefork; i++)
 		imsgbuf_clear(&gotwebd_env->iev_gotweb[i].ibuf);
 	free(gotwebd_env->iev_gotweb);
-
+	free(gotwebd_env->worker_load);
 	free(gotwebd_env);
 
 	exit(0);
@@ -1165,6 +1195,7 @@ sockets_socket_accept(int fd, short event, void *arg)
 	c->buf_len = 0;
 	c->client_status = CLIENT_CONNECT;
 	c->request_id = get_request_id();
+	c->worker_idx = -1;
 
 	event_set(&c->ev, s, EV_READ, read_fcgi_records, c);
 	event_add(&c->ev, NULL);