Commit 11990bca authored by Philippe Pepiot's avatar Philippe Pepiot
Browse files

Make test database template creation concurrent

build_db_cache() is used in tests to create test database templates, i.e.
DEFAULT_EMPTY_DB_ID (which is __default_empty_db__) and custom template
database using CubicwebTC test_db_id/pre_setup_database API.

When running tests in parallel using multiple processes, build_db_cache() may
try to build the same database twice. Avoid this by adding synchronisation of
process by using a file lock.

So when two processes require the same template database, one build the
database and others wait it to be created.

Use filelock (https://github.com/benediktschmitt/py-filelock) library to have a
portable (unix / windows) way for handling locks.
Also filelock is packaged in debian: https://packages.debian.org/source/python-filelock
parent ed486562ba7e
......@@ -32,6 +32,7 @@ from hashlib import sha1 # pylint: disable=E0611
from os.path import abspath, join, exists, split, isdir, dirname
from functools import partial
import filelock
from six import text_type
from six.moves import cPickle as pickle
......@@ -470,20 +471,23 @@ class TestDataBaseHandler(object):
``pre_setup_func`` to setup the database.
This function backup any database it build"""
if self.has_cache(test_db_id):
return # test_db_id, 'already in cache'
if test_db_id is DEFAULT_EMPTY_DB_ID:
self.init_test_database()
else:
print('Building %s for database %s' % (test_db_id, self.dbname))
self.build_db_cache(DEFAULT_EMPTY_DB_ID)
self.restore_database(DEFAULT_EMPTY_DB_ID)
self.get_repo(startup=True)
cnx = self.get_cnx()
with cnx:
pre_setup_func(cnx, self.config)
cnx.commit()
self.backup_database(test_db_id)
lockfile = join(self._ensure_test_backup_db_dir(),
'{}.lock'.format(test_db_id))
with filelock.FileLock(lockfile):
if self.has_cache(test_db_id):
return # test_db_id, 'already in cache'
if test_db_id is DEFAULT_EMPTY_DB_ID:
self.init_test_database()
else:
print('Building %s for database %s' % (test_db_id, self.dbname))
self.build_db_cache(DEFAULT_EMPTY_DB_ID)
self.restore_database(DEFAULT_EMPTY_DB_ID)
self.get_repo(startup=True)
cnx = self.get_cnx()
with cnx:
pre_setup_func(cnx, self.config)
cnx.commit()
self.backup_database(test_db_id)
class NoCreateDropDatabaseHandler(TestDataBaseHandler):
......
......@@ -28,6 +28,7 @@ Build-Depends:
python-passlib,
python-repoze.lru,
python-wsgicors,
python-filelock,
sphinx-common,
Standards-Version: 3.9.6
Homepage: https://www.cubicweb.org
......@@ -51,6 +52,7 @@ Depends:
python-passlib,
python-tz,
graphviz,
python-filelock,
gettext,
Recommends:
cubicweb-ctl (= ${source:Version}),
......
......@@ -74,6 +74,7 @@ setup(
'pytz',
'Markdown',
'unittest2 >= 0.7.0',
'filelock',
],
entry_points={
'console_scripts': [
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment