# Tests for process restarting in the pre-fork process model # # Copyright (C) Andrew Bartlett 2018 # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # """Tests process restarting in the pre-fork process model. NOTE: As this test kills samba processes it won't play nicely with other tests, so needs to be run in it's own environment. """ import os import signal import time import samba from samba.tests import TestCase, delete_force from samba.dcerpc import echo, netlogon from samba.messaging import Messaging from samba.samdb import SamDB from samba.credentials import Credentials, DONT_USE_KERBEROS from samba.common import get_string from samba.dsdb import ( UF_WORKSTATION_TRUST_ACCOUNT, UF_PASSWD_NOTREQD) from samba.dcerpc.misc import SEC_CHAN_WKSTA from samba.auth import system_session NUM_WORKERS = 4 MACHINE_NAME = "PFRS" class PreforkProcessRestartTests(TestCase): def setUp(self): super(PreforkProcessRestartTests, self).setUp() lp_ctx = self.get_loadparm() self.msg_ctx = Messaging(lp_ctx=lp_ctx) def tearDown(self): super(PreforkProcessRestartTests, self).tearDown() def get_process_data(self): services = self.msg_ctx.irpc_all_servers() processes = [] for service in services: for id in service.ids: processes.append((service.name, id.pid)) return processes def get_process(self, name): processes = self.get_process_data() for pname, pid in processes: if name == pname: return pid return None def get_worker_pids(self, name, workers): pids = [] for x in range(workers): process_name = "prefork-worker-{0}-{1}".format(name, x) pids.append(self.get_process(process_name)) self.assertIsNotNone(pids[x]) return pids def wait_for_workers(self, name, workers): num_workers = len(workers) for x in range(num_workers): process_name = "prefork-worker-{0}-{1}".format(name, x) self.wait_for_process(process_name, workers[x], 0, 1, 30) def wait_for_process(self, name, pid, initial_delay, wait, timeout): time.sleep(initial_delay) delay = initial_delay while delay < timeout: p = self.get_process(name) if p is not None and p != pid: # process has restarted return time.sleep(wait) delay += wait self.fail("Times out after {0} seconds waiting for {1} to restart". format(delay, name)) def check_for_duplicate_processes(self): processes = self.get_process_data() process_map = {} for name, p in processes: if (name.startswith("prefork-") or name.endswith("_server") or name.endswith("srv")): if name in process_map: if p != process_map[name]: self.fail( "Duplicate process for {0}, pids {1} and {2}". format(name, p, process_map[name])) def simple_bind(self): creds = self.insta_creds(template=self.get_credentials()) creds.set_bind_dn("%s\\%s" % (creds.get_domain(), creds.get_username())) self.samdb = SamDB(url="ldaps://%s" % os.environ["SERVER"], lp=self.get_loadparm(), credentials=creds) def rpc_echo(self): conn = echo.rpcecho("ncalrpc:", self.get_loadparm()) self.assertEqual([1, 2, 3], conn.EchoData([1, 2, 3])) def netlogon(self): server = os.environ["SERVER"] host = os.environ["SERVER_IP"] lp = self.get_loadparm() credentials = self.get_credentials() session = system_session() ldb = SamDB(url="ldap://%s" % host, session_info=session, credentials=credentials, lp=lp) machine_pass = samba.generate_random_password(32, 32) machine_name = MACHINE_NAME machine_dn = "cn=%s,%s" % (machine_name, ldb.domain_dn()) delete_force(ldb, machine_dn) utf16pw = ('"%s"' % get_string(machine_pass)).encode('utf-16-le') ldb.add({ "dn": machine_dn, "objectclass": "computer", "sAMAccountName": "%s$" % machine_name, "userAccountControl": str(UF_WORKSTATION_TRUST_ACCOUNT | UF_PASSWD_NOTREQD), "unicodePwd": utf16pw}) machine_creds = Credentials() machine_creds.guess(lp) machine_creds.set_secure_channel_type(SEC_CHAN_WKSTA) machine_creds.set_kerberos_state(DONT_USE_KERBEROS) machine_creds.set_password(machine_pass) machine_creds.set_username(machine_name + "$") machine_creds.set_workstation(machine_name) netlogon.netlogon( "ncacn_ip_tcp:%s[schannel,seal]" % server, lp, machine_creds) delete_force(ldb, machine_dn) def test_ldap_master_restart(self): # check ldap connection, do a simple bind self.simple_bind() # get ldap master process pid = self.get_process("prefork-master-ldap") self.assertIsNotNone(pid) # Get the worker processes workers = self.get_worker_pids("ldap", NUM_WORKERS) # kill it os.kill(pid, signal.SIGTERM) # wait for the process to restart self.wait_for_process("prefork-master-ldap", pid, 1, 1, 30) # restarting the master restarts the workers as well, so make sure # they have finished restarting self.wait_for_workers("ldap", workers) # get ldap master process new_pid = self.get_process("prefork-master-ldap") self.assertIsNotNone(new_pid) # check that the pid has changed self.assertNotEqual(pid, new_pid) # check that the worker processes have restarted new_workers = self.get_worker_pids("ldap", NUM_WORKERS) for x in range(NUM_WORKERS): self.assertNotEqual(workers[x], new_workers[x]) # check that the previous server entries have been removed. self.check_for_duplicate_processes() # check ldap connection, another simple bind self.simple_bind() def test_ldap_worker_restart(self): # check ldap connection, do a simple bind self.simple_bind() # get ldap master process pid = self.get_process("prefork-master-ldap") self.assertIsNotNone(pid) # Get the worker processes workers = self.get_worker_pids("ldap", NUM_WORKERS) # kill worker 0 os.kill(workers[0], signal.SIGTERM) # wait for the process to restart self.wait_for_process("prefork-worker-ldap-0", pid, 1, 1, 30) # get ldap master process new_pid = self.get_process("prefork-master-ldap") self.assertIsNotNone(new_pid) # check that the pid has not changed self.assertEqual(pid, new_pid) # check that the worker processes have restarted new_workers = self.get_worker_pids("ldap", NUM_WORKERS) # process 0 should have a new pid the others should be unchanged self.assertNotEqual(workers[0], new_workers[0]) self.assertEqual(workers[1], new_workers[1]) self.assertEqual(workers[2], new_workers[2]) self.assertEqual(workers[3], new_workers[3]) # check that the previous server entries have been removed. self.check_for_duplicate_processes() # check ldap connection, another simple bind self.simple_bind() # # Kill all the ldap worker processes and ensure that they are restarted # correctly # def test_ldap_all_workers_restart(self): # check ldap connection, do a simple bind self.simple_bind() # get ldap master process pid = self.get_process("prefork-master-ldap") self.assertIsNotNone(pid) # Get the worker processes workers = self.get_worker_pids("ldap", NUM_WORKERS) # kill all the worker processes for x in workers: os.kill(x, signal.SIGTERM) # wait for the worker processes to restart self.wait_for_workers("ldap", workers) # get ldap master process new_pid = self.get_process("prefork-master-ldap") self.assertIsNotNone(new_pid) # check that the pid has not changed self.assertEqual(pid, new_pid) # check that the worker processes have restarted new_workers = self.get_worker_pids("ldap", NUM_WORKERS) for x in range(NUM_WORKERS): self.assertNotEqual(workers[x], new_workers[x]) # check that the previous server entries have been removed. self.check_for_duplicate_processes() # check ldap connection, another simple bind self.simple_bind() def test_rpc_master_restart(self): # check rpc connection, make a rpc echo request self.rpc_echo() # get rpc master process pid = self.get_process("prefork-master-rpc") self.assertIsNotNone(pid) # Get the worker processes workers = self.get_worker_pids("rpc", NUM_WORKERS) # kill it os.kill(pid, signal.SIGTERM) # wait for the process to restart self.wait_for_process("prefork-master-rpc", pid, 1, 1, 30) # wait for workers to restart as well self.wait_for_workers("rpc", workers) # get ldap master process new_pid = self.get_process("prefork-master-rpc") self.assertIsNotNone(new_pid) # check that the pid has changed self.assertNotEqual(pid, new_pid) # check that the worker processes have restarted new_workers = self.get_worker_pids("rpc", NUM_WORKERS) for x in range(NUM_WORKERS): self.assertNotEqual(workers[x], new_workers[x]) # check that the previous server entries have been removed. self.check_for_duplicate_processes() # check rpc connection, another rpc echo request self.rpc_echo() def test_rpc_worker_zero_restart(self): # check rpc connection, make a rpc echo request and a netlogon request self.rpc_echo() self.netlogon() # get rpc master process pid = self.get_process("prefork-master-rpc") self.assertIsNotNone(pid) # Get the worker processes workers = self.get_worker_pids("rpc", NUM_WORKERS) # kill worker 0 os.kill(workers[0], signal.SIGTERM) # wait for the process to restart self.wait_for_process("prefork-worker-rpc-0", workers[0], 1, 1, 30) # get rpc master process new_pid = self.get_process("prefork-master-rpc") self.assertIsNotNone(new_pid) # check that the pid has not changed self.assertEqual(pid, new_pid) # check that the worker processes have restarted new_workers = self.get_worker_pids("rpc", NUM_WORKERS) # process 0 should have a new pid the others should be unchanged self.assertNotEqual(workers[0], new_workers[0]) self.assertEqual(workers[1], new_workers[1]) self.assertEqual(workers[2], new_workers[2]) self.assertEqual(workers[3], new_workers[3]) # check that the previous server entries have been removed. self.check_for_duplicate_processes() # check rpc connection, another rpc echo request, and netlogon request self.rpc_echo() self.netlogon() def test_rpc_all_workers_restart(self): # check rpc connection, make a rpc echo request, and a netlogon request self.rpc_echo() self.netlogon() # get rpc master process pid = self.get_process("prefork-master-rpc") self.assertIsNotNone(pid) # Get the worker processes workers = self.get_worker_pids("rpc", NUM_WORKERS) # kill all the worker processes for x in workers: os.kill(x, signal.SIGTERM) # wait for the worker processes to restart for x in range(NUM_WORKERS): self.wait_for_process( "prefork-worker-rpc-{0}".format(x), workers[x], 0, 1, 30) # get rpc master process new_pid = self.get_process("prefork-master-rpc") self.assertIsNotNone(new_pid) # check that the pid has not changed self.assertEqual(pid, new_pid) # check that the worker processes have restarted new_workers = self.get_worker_pids("rpc", NUM_WORKERS) for x in range(NUM_WORKERS): self.assertNotEqual(workers[x], new_workers[x]) # check that the previous server entries have been removed. self.check_for_duplicate_processes() # check rpc connection, another rpc echo request and netlogon self.rpc_echo() self.netlogon() def test_master_restart_backoff(self): # get kdc master process pid = self.get_process("prefork-master-echo") self.assertIsNotNone(pid) # # Check that the processes get backed off as expected # # have prefork backoff increment = 5 # prefork maximum backoff = 10 backoff_increment = 5 for expected in [0, 5, 10, 10]: # Get the worker processes workers = self.get_worker_pids("kdc", NUM_WORKERS) process = self.get_process("prefork-master-echo") os.kill(process, signal.SIGTERM) # wait for the process to restart start = time.time() self.wait_for_process("prefork-master-echo", process, 0, 1, 30) # wait for the workers to restart as well self.wait_for_workers("echo", workers) end = time.time() duration = end - start # process restart will take some time. Check that the elapsed # duration falls somewhere in the expected range, i.e. we haven't # taken longer than the backoff increment self.assertLess(duration, expected + backoff_increment) self.assertGreaterEqual(duration, expected) # check that the worker processes have restarted new_workers = self.get_worker_pids("echo", NUM_WORKERS) for x in range(NUM_WORKERS): self.assertNotEqual(workers[x], new_workers[x]) # check that the previous server entries have been removed. self.check_for_duplicate_processes() def test_worker_restart_backoff(self): # # Check that the processes get backed off as expected # # have prefork backoff increment = 5 # prefork maximum backoff = 10 backoff_increment = 5 for expected in [0, 5, 10, 10]: process = self.get_process("prefork-worker-echo-2") self.assertIsNotNone(process) os.kill(process, signal.SIGTERM) # wait for the process to restart start = time.time() self.wait_for_process("prefork-worker-echo-2", process, 0, 1, 30) end = time.time() duration = end - start # process restart will take some time. Check that the elapsed # duration falls somewhere in the expected range, i.e. we haven't # taken longer than the backoff increment self.assertLess(duration, expected + backoff_increment) self.assertGreaterEqual(duration, expected) self.check_for_duplicate_processes()