Tests: added file descriptor leak detection.

test/conftest.py

@@ -56,6 +56,12 @@ def pytest_addoption(parser):
         type=str,
         help="Default user for non-privileged processes of unitd",
     )
+    parser.addoption(
+        "--fds-threshold",
+        type=int,
+        default=0,
+        help="File descriptors threshold",
+    )
     parser.addoption(
         "--restart",
         default=False,
@@ -67,12 +73,23 @@ def pytest_addoption(parser):
 unit_instance = {}
 unit_log_copy = "unit.log.copy"
 _processes = []
+_fds_check = {
+    'main': {'fds': 0, 'skip': False},
+    'router': {'name': 'unit: router', 'pid': -1, 'fds': 0, 'skip': False},
+    'controller': {
+        'name': 'unit: controller',
+        'pid': -1,
+        'fds': 0,
+        'skip': False,
+    },
+}
 http = TestHTTP()
 
 def pytest_configure(config):
     option.config = config.option
 
     option.detailed = config.option.detailed
+    option.fds_threshold = config.option.fds_threshold
     option.print_log = config.option.print_log
     option.save_log = config.option.save_log
     option.unsafe = config.option.unsafe
@@ -257,6 +274,10 @@ def run(request):
     ]
     option.skip_sanitizer = False
 
+    _fds_check['main']['skip'] = False
+    _fds_check['router']['skip'] = False
+    _fds_check['router']['skip'] = False
+
     yield
 
     # stop unit
@@ -304,6 +325,50 @@ def run(request):
                 else:
                     shutil.rmtree(path)
 
+    # check descriptors (wait for some time before check)
+
+    def waitforfds(diff):
+        for i in range(600):
+            fds_diff = diff()
+
+            if fds_diff <= option.fds_threshold:
+                break
+
+            time.sleep(0.1)
+
+        return fds_diff
+
+    ps = _fds_check['main']
+    if not ps['skip']:
+        fds_diff = waitforfds(
+            lambda: _count_fds(unit_instance['pid']) - ps['fds']
+        )
+        ps['fds'] += fds_diff
+
+        assert (
+            fds_diff <= option.fds_threshold
+        ), 'descriptors leak main process'
+
+    else:
+        ps['fds'] = _count_fds(unit_instance['pid'])
+
+    for name in ['controller', 'router']:
+        ps = _fds_check[name]
+        ps_pid = ps['pid']
+        ps['pid'] = pid_by_name(ps['name'])
+
+        if not ps['skip']:
+            fds_diff = waitforfds(lambda: _count_fds(ps['pid']) - ps['fds'])
+            ps['fds'] += fds_diff
+
+            assert ps['pid'] == ps_pid, 'same pid %s' % name
+            assert fds_diff <= option.fds_threshold, (
+                'descriptors leak %s' % name
+            )
+
+        else:
+            ps['fds'] = _count_fds(ps['pid'])
+
     # print unit.log in case of error
 
     if hasattr(request.node, 'rep_call') and request.node.rep_call.failed:
@@ -371,6 +436,21 @@ def unit_run():
     unit_instance['control_sock'] = temp_dir + '/control.unit.sock'
     unit_instance['unitd'] = unitd
 
+    with open(temp_dir + '/unit.pid', 'r') as f:
+        unit_instance['pid'] = f.read().rstrip()
+
+    _clear_conf(unit_instance['temp_dir'] + '/control.unit.sock')
+
+    _fds_check['main']['fds'] = _count_fds(unit_instance['pid'])
+
+    router = _fds_check['router']
+    router['pid'] = pid_by_name(router['name'])
+    router['fds'] = _count_fds(router['pid'])
+
+    controller = _fds_check['controller']
+    controller['pid'] = pid_by_name(controller['name'])
+    controller['fds'] = _count_fds(controller['pid'])
+
     return unit_instance
 
 
@@ -492,6 +572,32 @@ def _clear_conf(sock, log=None):
 
         check_success(resp)
 
+def _count_fds(pid):
+    procfile = '/proc/%s/fd' % pid
+    if os.path.isdir(procfile):
+        return len(os.listdir(procfile))
+
+    try:
+        out = subprocess.check_output(
+            ['procstat', '-f', pid], stderr=subprocess.STDOUT,
+        ).decode()
+        return len(out.splitlines())
+
+    except (FileNotFoundError, subprocess.CalledProcessError):
+        pass
+
+    try:
+        out = subprocess.check_output(
+            ['lsof', '-n', '-p', pid], stderr=subprocess.STDOUT,
+        ).decode()
+        return len(out.splitlines())
+
+    except (FileNotFoundError, subprocess.CalledProcessError):
+        pass
+
+    return 0
+
+
 def run_process(target, *args):
     global _processes
 
@@ -517,6 +623,18 @@ def stop_processes():
         return 'Fail to stop process(es)'
 
 
+def pid_by_name(name):
+    output = subprocess.check_output(['ps', 'ax', '-O', 'ppid']).decode()
+    m = re.search(
+        r'\s*(\d+)\s*' + str(unit_instance['pid']) + r'.*' + name, output
+    )
+    return None if m is None else m.group(1)
+
+
+def find_proc(name, ps_output):
+    return re.findall(str(unit_instance['pid']) + r'.*' + name, ps_output)
+
+
 @pytest.fixture()
 def skip_alert():
     def _skip(*alerts):
@@ -525,6 +643,16 @@ def skip_alert():
     return _skip
 
 
+@pytest.fixture()
+def skip_fds_check():
+    def _skip(main=False, router=False, controller=False):
+        _fds_check['main']['skip'] = main
+        _fds_check['router']['skip'] = router
+        _fds_check['controller']['skip'] = controller
+
+    return _skip
+
+
 @pytest.fixture
 def temp_dir(request):
     return unit_instance['temp_dir']

test/test_respawn.py

@@ -58,7 +58,8 @@ class TestRespawn(TestApplicationPython):
         assert len(self.find_proc(self.PATTERN_CONTROLLER, unit_pid, out)) == 1
         assert len(self.find_proc(self.app_name, unit_pid, out)) == 1
 
-    def test_respawn_router(self, skip_alert, unit_pid):
+    def test_respawn_router(self, skip_alert, unit_pid, skip_fds_check):
+        skip_fds_check(router=True)
         pid = self.pid_by_name(self.PATTERN_ROUTER, unit_pid)
 
         self.kill_pids(pid)
@@ -68,7 +69,8 @@ class TestRespawn(TestApplicationPython):
 
         self.smoke_test(unit_pid)
 
-    def test_respawn_controller(self, skip_alert, unit_pid):
+    def test_respawn_controller(self, skip_alert, unit_pid, skip_fds_check):
+        skip_fds_check(controller=True)
         pid = self.pid_by_name(self.PATTERN_CONTROLLER, unit_pid)
 
         self.kill_pids(pid)