From c216f26d3040beef46ca25f73e580153c4c59d02 Mon Sep 17 00:00:00 2001
From: Max Romanov <max.romanov@nginx.com>
Date: Mon, 17 May 2021 17:34:15 +0300
Subject: [PATCH] Fixing racing condition on listen socket close in router.

Listen socket is actually closed in the instant timer handler.  This patch moves
the "configuration has been applied" notification to the timer handler to avoid
a situation when the user gets the response from the controller, but the listen
socket is still open in the router.
---
 src/nxt_router.c | 24 ++++++++++++++----------
 src/nxt_router.h |  2 ++
 2 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/src/nxt_router.c b/src/nxt_router.c
index a20e4ede..da38aac0 100644
--- a/src/nxt_router.c
+++ b/src/nxt_router.c
@@ -3223,12 +3223,11 @@ nxt_router_listen_socket_update(nxt_task_t *task, void *obj, void *data)
 static void
 nxt_router_listen_socket_delete(nxt_task_t *task, void *obj, void *data)
 {
-    nxt_joint_job_t     *job;
-    nxt_socket_conf_t   *skcf;
-    nxt_listen_event_t  *lev;
-    nxt_event_engine_t  *engine;
+    nxt_socket_conf_t        *skcf;
+    nxt_listen_event_t       *lev;
+    nxt_event_engine_t       *engine;
+    nxt_socket_conf_joint_t  *joint;
 
-    job = obj;
     skcf = data;
 
     engine = task->thread->engine;
@@ -3240,15 +3239,13 @@ nxt_router_listen_socket_delete(nxt_task_t *task, void *obj, void *data)
     nxt_debug(task, "engine %p: listen socket delete: %d", engine,
               lev->socket.fd);
 
+    joint = lev->socket.data;
+    joint->close_job = obj;
+
     lev->timer.handler = nxt_router_listen_socket_close;
     lev->timer.work_queue = &engine->fast_work_queue;
 
     nxt_timer_add(engine, &lev->timer, 0);
-
-    job->work.next = NULL;
-    job->work.handler = nxt_router_conf_wait;
-
-    nxt_event_engine_post(job->tmcf->engine, &job->work);
 }
 
 
@@ -3273,6 +3270,7 @@ static void
 nxt_router_listen_socket_close(nxt_task_t *task, void *obj, void *data)
 {
     nxt_timer_t              *timer;
+    nxt_joint_job_t          *job;
     nxt_listen_event_t       *lev;
     nxt_socket_conf_joint_t  *joint;
 
@@ -3287,6 +3285,12 @@ nxt_router_listen_socket_close(nxt_task_t *task, void *obj, void *data)
     joint = lev->socket.data;
     lev->socket.data = NULL;
 
+    job = joint->close_job;
+    job->work.next = NULL;
+    job->work.handler = nxt_router_conf_wait;
+
+    nxt_event_engine_post(job->tmcf->engine, &job->work);
+
     /* 'task' refers to lev->task and we cannot use after nxt_free() */
     task = &task->thread->engine->task;
 
diff --git a/src/nxt_router.h b/src/nxt_router.h
index 5804840f..b1ccdf51 100644
--- a/src/nxt_router.h
+++ b/src/nxt_router.h
@@ -205,6 +205,8 @@ typedef struct {
     nxt_event_engine_t     *engine;
     nxt_socket_conf_t      *socket_conf;
 
+    nxt_joint_job_t        *close_job;
+
     nxt_upstream_t         **upstreams;
 
     /* Modules configuraitons. */