pytest.py 52.2 KB
Newer Older
Nicolas Chauvat's avatar
Nicolas Chauvat committed
1
# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
Aurelien Campeas's avatar
Aurelien Campeas committed
2
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
3
4
5
6
7
8
9
10
11
12
13
14
15
16
#
# This file is part of logilab-common.
#
# logilab-common is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option) any
# later version.
#
# logilab-common is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
17
# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
18
"""logilab-pytest is a tool that eases test running and debugging.
Adrien Di Mascio's avatar
Adrien Di Mascio committed
19

20
To be able to use logilab-pytest, you should either write tests using
Adrien Di Mascio's avatar
Adrien Di Mascio committed
21
22
23
the logilab.common.testlib's framework or the unittest module of the
Python's standard library.

24
25
You can customize logilab-pytest's behaviour by defining a ``pytestconf.py``
file somewhere in your test directory. In this file, you can add options or
Adrien Di Mascio's avatar
Adrien Di Mascio committed
26
27
28
29
30
31
32
33
change the way tests are run.

To add command line options, you must define a ``update_parser`` function in
your ``pytestconf.py`` file. The function must accept a single parameter
that will be the OptionParser's instance to customize.

If you wish to customize the tester, you'll have to define a class named
``CustomPyTester``. This class should extend the default `PyTester` class
34
35
defined in the logilab.common.pytest module. Take a look at the `PyTester` and
`DjangoTester` classes for more information about what can be done.
Adrien Di Mascio's avatar
Adrien Di Mascio committed
36
37
38
39
40
41

For instance, if you wish to add a custom -l option to specify a loglevel, you
could define the following ``pytestconf.py`` file ::

    import logging
    from logilab.common.pytest import PyTester
42

Adrien Di Mascio's avatar
Adrien Di Mascio committed
43
44
45
46
47
48
    def update_parser(parser):
        parser.add_option('-l', '--loglevel', dest='loglevel', action='store',
                          choices=('debug', 'info', 'warning', 'error', 'critical'),
                          default='critical', help="the default log level possible choices are "
                          "('debug', 'info', 'warning', 'error', 'critical')")
        return parser
49
50


Adrien Di Mascio's avatar
Adrien Di Mascio committed
51
52
53
54
55
56
57
    class CustomPyTester(PyTester):
        def __init__(self, cvg, options):
            super(CustomPyTester, self).__init__(cvg, options)
            loglevel = options.loglevel.upper()
            logger = logging.getLogger('erudi')
            logger.setLevel(logging.getLevelName(loglevel))

58
59
60

In your TestCase class you can then get the value of a specific option with
the ``optval`` method::
61

62
63
64
65
    class MyTestCase(TestCase):
        def test_foo(self):
            loglevel = self.optval('loglevel')
            # ...
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87


You can also tag your tag your test for fine filtering

With those tag::

    from logilab.common.testlib import tag, TestCase

    class Exemple(TestCase):

        @tag('rouge', 'carre')
        def toto(self):
            pass

        @tag('carre', 'vert')
        def tata(self):
            pass

        @tag('rouge')
        def titi(test):
            pass

88
you can filter the function with a simple python expression
89
90
91
92
93

 * ``toto`` and ``titi`` match ``rouge``
 * ``toto``, ``tata`` and ``titi``, match ``rouge or carre``
 * ``tata`` and ``titi`` match``rouge ^ carre``
 * ``titi`` match ``rouge and not carre``
Adrien Di Mascio's avatar
Adrien Di Mascio committed
94
"""
Rémi Cardona's avatar
Rémi Cardona committed
95
96
97

from __future__ import print_function

Laurent Peuch's avatar
Laurent Peuch committed
98
99
100
import os
import re
import sys
Adrien Di Mascio's avatar
Adrien Di Mascio committed
101
import os.path as osp
102
from time import process_time, time
103
104
105
106
107
108
109
110
111
112
113

try:
    from re import Match  # type: ignore
except ImportError:
    # Match is python > 3.6 only.
    #
    # To be compatible with python <= 3.6, and still provide some typing, we
    # manually define Match, in the same manner it is defined in the re module
    # of python > 3.7
    # cf https://github.com/python/cpython/blob/3.7/Lib/re.py#L264
    Match = type(re.sre_compile.compile("", 0).match(""))  # type: ignore
114
import warnings
115
import types
116
117
import inspect
import traceback
118
from inspect import isgeneratorfunction, isclass, FrameInfo
119
from random import shuffle
120
from itertools import dropwhile
Laurent Peuch's avatar
Laurent Peuch committed
121

122
123
124
125
126
# mypy error: Module 'unittest.runner' has no attribute '_WritelnDecorator'
# but it does
from unittest.runner import _WritelnDecorator  # type: ignore
from unittest.suite import TestSuite

Laurent Peuch's avatar
Laurent Peuch committed
127
from typing import Callable, Any, Optional, List, Tuple, Generator, Dict
128

129
from logilab.common.deprecation import callable_deprecated
Adrien Di Mascio's avatar
Adrien Di Mascio committed
130
from logilab.common.fileutils import abspath_listdir
131
from logilab.common import textutils
132
from logilab.common import testlib, STD_BLACKLIST
Laurent Peuch's avatar
Laurent Peuch committed
133

134
# use the same unittest module as testlib
135
from logilab.common.testlib import unittest, start_interactive_mode
Laurent Peuch's avatar
Laurent Peuch committed
136
137
138
139
140
from logilab.common.testlib import (  # noqa
    nocoverage,
    pause_trace,
    replace_trace,
)  # bwcompat
141
from logilab.common.debugger import Debugger, colorize_source
Adrien Di Mascio's avatar
Adrien Di Mascio committed
142
import doctest
143

144
import unittest as unittest_legacy
Laurent Peuch's avatar
Laurent Peuch committed
145

Laurent Peuch's avatar
Laurent Peuch committed
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
from .decorators import monkeypatch

__docformat__ = "restructuredtext en"

PYTEST_DOC = """%prog [OPTIONS] [testfile [testpattern]]

examples:

logilab-pytest path/to/mytests.py
logilab-pytest path/to/mytests.py TheseTests
logilab-pytest path/to/mytests.py TheseTests.test_thisone
logilab-pytest path/to/mytests.py -m '(not long and database) or regr'

logilab-pytest one (will run both test_thisone and test_thatone)
logilab-pytest path/to/mytests.py -s not (will skip test_notthisone)
"""

ENABLE_DBC = False
FILE_RESTART = ".pytest.restart"

166
if not getattr(unittest_legacy, "__package__", None):
167
168
169
170
171
    try:
        import unittest2.suite as unittest_suite
    except ImportError:
        sys.exit("You have to install python-unittest2 to use this module")
else:
172
173
    # mypy: Name 'unittest_suite' already defined (possibly by an import))
    import unittest.suite as unittest_suite  # type: ignore
174

175
try:
Laurent Peuch's avatar
Laurent Peuch committed
176
    import django  # noqa
177
    from logilab.common.modutils import modpath_from_file, load_module_from_modpath
Laurent Peuch's avatar
Laurent Peuch committed
178

179
180
181
182
    DJANGO_FOUND = True
except ImportError:
    DJANGO_FOUND = False

Laurent Peuch's avatar
Laurent Peuch committed
183
CONF_FILE = "pytestconf.py"
184

Laurent Peuch's avatar
Laurent Peuch committed
185
TESTFILE_RE = re.compile(r"^((unit)?test.*|smoketest)\.py$")
Laurent Peuch's avatar
Laurent Peuch committed
186
187


188
def this_is_a_testfile(filename: str) -> Optional[Match]:
189
    """returns True if `filename` seems to be a test file"""
190
    return TESTFILE_RE.match(osp.basename(filename))
191

Laurent Peuch's avatar
Laurent Peuch committed
192

193
TESTDIR_RE = re.compile("^(unit)?tests?$")
Laurent Peuch's avatar
Laurent Peuch committed
194
195


196
def this_is_a_testdir(dirpath: str) -> Optional[Match]:
197
    """returns True if `filename` seems to be a test directory"""
198
    return TESTDIR_RE.match(osp.basename(dirpath))
199
200


Adrien Di Mascio's avatar
Adrien Di Mascio committed
201
202
203
204
205
def load_pytest_conf(path, parser):
    """loads a ``pytestconf.py`` file and update default parser
    and / or tester.
    """
    namespace = {}
Laurent Peuch's avatar
Laurent Peuch committed
206
207
208
209
    exec(open(path, "rb").read(), namespace)
    if "update_parser" in namespace:
        namespace["update_parser"](parser)
    return namespace.get("CustomPyTester", PyTester)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
210
211


Chris Lamb's avatar
Chris Lamb committed
212
def project_root(parser, projdir=None):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
213
    """try to find project's root and add it to sys.path"""
Chris Lamb's avatar
Chris Lamb committed
214
215
    if projdir is None:
        projdir = os.getcwd()
216
    previousdir = curdir = osp.abspath(projdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
217
218
219
220
    testercls = PyTester
    conf_file_path = osp.join(curdir, CONF_FILE)
    if osp.isfile(conf_file_path):
        testercls = load_pytest_conf(conf_file_path, parser)
Laurent Peuch's avatar
Laurent Peuch committed
221
    while this_is_a_testdir(curdir) or osp.isfile(osp.join(curdir, "__init__.py")):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
222
223
224
        newdir = osp.normpath(osp.join(curdir, os.pardir))
        if newdir == curdir:
            break
225
        previousdir = curdir
Adrien Di Mascio's avatar
Adrien Di Mascio committed
226
        curdir = newdir
Adrien Di Mascio's avatar
Adrien Di Mascio committed
227
228
229
230
        conf_file_path = osp.join(curdir, CONF_FILE)
        if osp.isfile(conf_file_path):
            testercls = load_pytest_conf(conf_file_path, parser)
    return previousdir, testercls
231

Adrien Di Mascio's avatar
Adrien Di Mascio committed
232
233
234

class GlobalTestReport(object):
    """this class holds global test statistics"""
Laurent Peuch's avatar
Laurent Peuch committed
235

Adrien Di Mascio's avatar
Adrien Di Mascio committed
236
237
238
239
240
241
242
243
244
245
246
247
248
249
    def __init__(self):
        self.ran = 0
        self.skipped = 0
        self.failures = 0
        self.errors = 0
        self.ttime = 0
        self.ctime = 0
        self.modulescount = 0
        self.errmodules = []

    def feed(self, filename, testresult, ttime, ctime):
        """integrates new test information into internal statistics"""
        ran = testresult.testsRun
        self.ran += ran
Laurent Peuch's avatar
Laurent Peuch committed
250
        self.skipped += len(getattr(testresult, "skipped", ()))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
251
252
253
254
255
256
257
258
        self.failures += len(testresult.failures)
        self.errors += len(testresult.errors)
        self.ttime += ttime
        self.ctime += ctime
        self.modulescount += 1
        if not testresult.wasSuccessful():
            problems = len(testresult.failures) + len(testresult.errors)
            self.errmodules.append((filename[:-3], problems, ran))
259
260
261
262
263

    def failed_to_test_module(self, filename):
        """called when the test module could not be imported by unittest
        """
        self.errors += 1
264
265
        self.modulescount += 1
        self.ran += 1
266
        self.errmodules.append((filename[:-3], 1, 1))
267

268
269
270
271
    def skip_module(self, filename):
        self.modulescount += 1
        self.ran += 1
        self.errmodules.append((filename[:-3], 0, 0))
272

Adrien Di Mascio's avatar
Adrien Di Mascio committed
273
274
    def __str__(self):
        """this is just presentation stuff"""
Laurent Peuch's avatar
Laurent Peuch committed
275
        line1 = ["Ran %s test cases in %.2fs (%.2fs CPU)" % (self.ran, self.ttime, self.ctime)]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
276
        if self.errors:
Laurent Peuch's avatar
Laurent Peuch committed
277
            line1.append("%s errors" % self.errors)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
278
        if self.failures:
Laurent Peuch's avatar
Laurent Peuch committed
279
            line1.append("%s failures" % self.failures)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
280
        if self.skipped:
Laurent Peuch's avatar
Laurent Peuch committed
281
            line1.append("%s skipped" % self.skipped)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
282
283
        modulesok = self.modulescount - len(self.errmodules)
        if self.errors or self.failures:
Laurent Peuch's avatar
Laurent Peuch committed
284
285
286
            line2 = "%s modules OK (%s failed)" % (modulesok, len(self.errmodules))
            descr = ", ".join(["%s [%s/%s]" % info for info in self.errmodules])
            line3 = "\nfailures: %s" % descr
Adrien Di Mascio's avatar
Adrien Di Mascio committed
287
        elif modulesok:
Laurent Peuch's avatar
Laurent Peuch committed
288
289
            line2 = "All %s modules OK" % modulesok
            line3 = ""
Adrien Di Mascio's avatar
Adrien Di Mascio committed
290
        else:
Laurent Peuch's avatar
Laurent Peuch committed
291
292
            return ""
        return "%s\n%s%s" % (", ".join(line1), line2, line3)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
293
294


295
296
297
298
299
300
def remove_local_modules_from_sys(testdir):
    """remove all modules from cache that come from `testdir`

    This is used to avoid strange side-effects when using the
    testall() mode of pytest.
    For instance, if we run pytest on this tree::
301

302
303
304
305
306
307
      A/test/test_utils.py
      B/test/test_utils.py

    we **have** to clean sys.modules to make sure the correct test_utils
    module is ran in B
    """
308
    for modname, mod in list(sys.modules.items()):
309
310
        if mod is None:
            continue
Laurent Peuch's avatar
Laurent Peuch committed
311
        if not hasattr(mod, "__file__"):
312
313
314
            # this is the case of some built-in modules like sys, imp, marshal
            continue
        modfile = mod.__file__
315
        # if modfile is not an absolute path, it was probably loaded locally
316
317
318
319
320
        # during the tests
        if not osp.isabs(modfile) or modfile.startswith(testdir):
            del sys.modules[modname]


321
class PyTester(object):
322
    """encapsulates testrun logic"""
323

Adrien Di Mascio's avatar
Adrien Di Mascio committed
324
    def __init__(self, cvg, options):
325
        self.report = GlobalTestReport()
326
        self.cvg = cvg
Adrien Di Mascio's avatar
Adrien Di Mascio committed
327
        self.options = options
328
        self.firstwrite = True
329
        self._errcode = None
330
331
332
333

    def show_report(self):
        """prints the report and returns appropriate exitcode"""
        # everything has been ran, print report
Rémi Cardona's avatar
Rémi Cardona committed
334
335
        print("*" * 79)
        print(self.report)
336
337

    def get_errcode(self):
338
        # errcode set explicitly
339
340
        if self._errcode is not None:
            return self._errcode
341
        return self.report.failures + self.report.errors
342

343
344
    def set_errcode(self, errcode):
        self._errcode = errcode
Laurent Peuch's avatar
Laurent Peuch committed
345

346
    errcode = property(get_errcode, set_errcode)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
347

348
    def testall(self, exitfirst=False):
349
        """walks through current working directory, finds something
350
351
        which can be considered as a testdir and runs every test there
        """
352
353
        here = os.getcwd()
        for dirname, dirs, _ in os.walk(here):
354
            for skipped in STD_BLACKLIST:
355
356
357
                if skipped in dirs:
                    dirs.remove(skipped)
            basename = osp.basename(dirname)
358
            if this_is_a_testdir(basename):
Rémi Cardona's avatar
Rémi Cardona committed
359
                print("going into", dirname)
360
                # we found a testdir, let's explore it !
361
362
                if not self.testonedir(dirname, exitfirst):
                    break
363
                dirs[:] = []
364
        if self.report.ran == 0:
Rémi Cardona's avatar
Rémi Cardona committed
365
            print("no test dir found testing here:", here)
366
367
368
369
            # if no test was found during the visit, consider
            # the local directory as a test directory even if
            # it doesn't have a traditional test directory name
            self.testonedir(here)
370

371
    def testonedir(self, testdir, exitfirst=False):
372
373
374
375
376
        """finds each testfile in the `testdir` and runs it

        return true when all tests has been executed, false if exitfirst and
        some test has failed.
        """
377
378
379
        files = abspath_listdir(testdir)
        shuffle(files)
        for filename in files:
380
            if this_is_a_testfile(filename):
381
382
383
                if self.options.exitfirst and not self.options.restart:
                    # overwrite restart file
                    try:
384
                        restartfile = open(FILE_RESTART, "w")
385
                        restartfile.close()
386
                    except Exception:
Laurent Peuch's avatar
Laurent Peuch committed
387
388
389
390
391
                        print(
                            "Error while overwriting succeeded test file :",
                            osp.join(os.getcwd(), FILE_RESTART),
                            file=sys.__stderr__,
                        )
392
                        raise
393
394
395
                # run test and collect information
                prog = self.testfile(filename, batchmode=True)
                if exitfirst and (prog is None or not prog.result.wasSuccessful()):
396
                    return False
397
                self.firstwrite = True
398
399
        # clean local modules
        remove_local_modules_from_sys(testdir)
400
        return True
401
402
403
404
405
406
407
408
409
410

    def testfile(self, filename, batchmode=False):
        """runs every test in `filename`

        :param filename: an absolute path pointing to a unittest file
        """
        here = os.getcwd()
        dirname = osp.dirname(filename)
        if dirname:
            os.chdir(dirname)
411
412
413
        # overwrite restart file if it has not been done already
        if self.options.exitfirst and not self.options.restart and self.firstwrite:
            try:
414
                restartfile = open(FILE_RESTART, "w")
415
                restartfile.close()
416
            except Exception:
Laurent Peuch's avatar
Laurent Peuch committed
417
418
419
420
421
                print(
                    "Error while overwriting succeeded test file :",
                    osp.join(os.getcwd(), FILE_RESTART),
                    file=sys.__stderr__,
                )
422
                raise
423
        modname = osp.basename(filename)[:-3]
Laurent Peuch's avatar
Laurent Peuch committed
424
        print(("  %s  " % osp.basename(filename)).center(70, "="), file=sys.__stderr__)
425
        try:
426
            tstart, cstart = time(), process_time()
427
            try:
Laurent Peuch's avatar
Laurent Peuch committed
428
429
430
431
432
433
434
                testprog = SkipAwareTestProgram(
                    modname,
                    batchmode=batchmode,
                    cvg=self.cvg,
                    options=self.options,
                    outstream=sys.stderr,
                )
435
436
            except KeyboardInterrupt:
                raise
437
            except SystemExit as exc:
438
                self.errcode = exc.code
Adrien Di Mascio's avatar
Adrien Di Mascio committed
439
                raise
440
            except testlib.SkipTest:
Rémi Cardona's avatar
Rémi Cardona committed
441
                print("Module skipped:", filename)
442
443
                self.report.skip_module(filename)
                return None
Stephanie Marcu's avatar
Stephanie Marcu committed
444
            except Exception:
445
                self.report.failed_to_test_module(filename)
Laurent Peuch's avatar
Laurent Peuch committed
446
                print("unhandled exception occurred while testing", modname, file=sys.stderr)
Sylvain's avatar
Sylvain committed
447
                import traceback
Laurent Peuch's avatar
Laurent Peuch committed
448

449
                traceback.print_exc(file=sys.stderr)
450
451
                return None

452
            tend, cend = time(), process_time()
453
454
455
            ttime, ctime = (tend - tstart), (cend - cstart)
            self.report.feed(filename, testprog.result, ttime, ctime)
            return testprog
456
457
458
459
460
        finally:
            if dirname:
                os.chdir(here)


461
462
463
464
class DjangoTester(PyTester):
    def load_django_settings(self, dirname):
        """try to find project's setting and load it"""
        curdir = osp.abspath(dirname)
Laurent Peuch's avatar
Laurent Peuch committed
465
466
467
        while not osp.isfile(osp.join(curdir, "settings.py")) and osp.isfile(
            osp.join(curdir, "__init__.py")
        ):
468
469
            newdir = osp.normpath(osp.join(curdir, os.pardir))
            if newdir == curdir:
Laurent Peuch's avatar
Laurent Peuch committed
470
                raise AssertionError("could not find settings.py")
471
            curdir = newdir
472
        # late django initialization
Laurent Peuch's avatar
Laurent Peuch committed
473
        settings = load_module_from_modpath(modpath_from_file(osp.join(curdir, "settings.py")))
474
        from django.core.management import setup_environ
Laurent Peuch's avatar
Laurent Peuch committed
475

476
477
        setup_environ(settings)
        settings.DEBUG = False
478
        self.settings = settings
479
480
481
        # add settings dir to pythonpath since it's the project's root
        if curdir not in sys.path:
            sys.path.insert(1, curdir)
482
483
484
485
486

    def before_testfile(self):
        # Those imports must be done **after** setup_environ was called
        from django.test.utils import setup_test_environment
        from django.test.utils import create_test_db
Laurent Peuch's avatar
Laurent Peuch committed
487

488
489
        setup_test_environment()
        create_test_db(verbosity=0)
490
        self.dbname = self.settings.TEST_DATABASE_NAME
491

492
493
494
495
    def after_testfile(self):
        # Those imports must be done **after** setup_environ was called
        from django.test.utils import teardown_test_environment
        from django.test.utils import destroy_test_db
Laurent Peuch's avatar
Laurent Peuch committed
496

497
        teardown_test_environment()
Laurent Peuch's avatar
Laurent Peuch committed
498
        print("destroying", self.dbname)
499
        destroy_test_db(self.dbname, verbosity=0)
500

501
    def testall(self, exitfirst=False):
502
        """walks through current working directory, finds something
503
504
        which can be considered as a testdir and runs every test there
        """
505
        for dirname, dirs, files in os.walk(os.getcwd()):
Laurent Peuch's avatar
Laurent Peuch committed
506
            for skipped in ("CVS", ".svn", ".hg"):
507
508
                if skipped in dirs:
                    dirs.remove(skipped)
Laurent Peuch's avatar
Laurent Peuch committed
509
            if "tests.py" in files:
510
511
                if not self.testonedir(dirname, exitfirst):
                    break
512
513
514
                dirs[:] = []
            else:
                basename = osp.basename(dirname)
Laurent Peuch's avatar
Laurent Peuch committed
515
                if basename in ("test", "tests"):
Rémi Cardona's avatar
Rémi Cardona committed
516
                    print("going into", dirname)
517
                    # we found a testdir, let's explore it !
518
519
                    if not self.testonedir(dirname, exitfirst):
                        break
520
521
522
                    dirs[:] = []

    def testonedir(self, testdir, exitfirst=False):
523
524
525
526
527
        """finds each testfile in the `testdir` and runs it

        return true when all tests has been executed, false if exitfirst and
        some test has failed.
        """
528
        # special django behaviour : if tests are splitted in several files,
529
        # remove the main tests.py file and tests each test file separately
Laurent Peuch's avatar
Laurent Peuch committed
530
        testfiles = [fpath for fpath in abspath_listdir(testdir) if this_is_a_testfile(fpath)]
531
532
        if len(testfiles) > 1:
            try:
Laurent Peuch's avatar
Laurent Peuch committed
533
                testfiles.remove(osp.join(testdir, "tests.py"))
534
535
536
537
538
539
            except ValueError:
                pass
        for filename in testfiles:
            # run test and collect information
            prog = self.testfile(filename, batchmode=True)
            if exitfirst and (prog is None or not prog.result.wasSuccessful()):
540
                return False
541
542
        # clean local modules
        remove_local_modules_from_sys(testdir)
543
        return True
544

545
546
547
548
549
550
551
552
553
554
555
    def testfile(self, filename, batchmode=False):
        """runs every test in `filename`

        :param filename: an absolute path pointing to a unittest file
        """
        here = os.getcwd()
        dirname = osp.dirname(filename)
        if dirname:
            os.chdir(dirname)
        self.load_django_settings(dirname)
        modname = osp.basename(filename)[:-3]
Laurent Peuch's avatar
Laurent Peuch committed
556
        print(("  %s  " % osp.basename(filename)).center(70, "="), file=sys.stderr)
557
558
        try:
            try:
559
                tstart, cstart = time(), process_time()
560
                self.before_testfile()
561
                testprog = SkipAwareTestProgram(modname, batchmode=batchmode, cvg=self.cvg)
562
                tend, cend = time(), process_time()
563
564
565
566
567
                ttime, ctime = (tend - tstart), (cend - cstart)
                self.report.feed(filename, testprog.result, ttime, ctime)
                return testprog
            except SystemExit:
                raise
568
            except Exception as exc:
569
                import traceback
Laurent Peuch's avatar
Laurent Peuch committed
570

571
                traceback.print_exc()
572
                self.report.failed_to_test_module(filename)
Laurent Peuch's avatar
Laurent Peuch committed
573
574
                print("unhandled exception occurred while testing", modname)
                print("error: %s" % exc)
575
                return None
576
577
578
579
580
581
        finally:
            self.after_testfile()
            if dirname:
                os.chdir(here)


Adrien Di Mascio's avatar
Adrien Di Mascio committed
582
583
def make_parser():
    """creates the OptionParser instance
Adrien Di Mascio's avatar
Adrien Di Mascio committed
584
585
    """
    from optparse import OptionParser
Laurent Peuch's avatar
Laurent Peuch committed
586

Adrien Di Mascio's avatar
Adrien Di Mascio committed
587
    parser = OptionParser(usage=PYTEST_DOC)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
588

Adrien Di Mascio's avatar
Adrien Di Mascio committed
589
    parser.newargs = []
Laurent Peuch's avatar
Laurent Peuch committed
590

Adrien Di Mascio's avatar
Adrien Di Mascio committed
591
592
    def rebuild_cmdline(option, opt, value, parser):
        """carry the option to unittest_main"""
Adrien Di Mascio's avatar
Adrien Di Mascio committed
593
        parser.newargs.append(opt)
594

Adrien Di Mascio's avatar
Adrien Di Mascio committed
595
596
597
598
    def rebuild_and_store(option, opt, value, parser):
        """carry the option to unittest_main and store
        the value on current parser
        """
Adrien Di Mascio's avatar
Adrien Di Mascio committed
599
        parser.newargs.append(opt)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
600
601
        setattr(parser.values, option.dest, True)

602
    def capture_and_rebuild(option, opt, value, parser):
Laurent Peuch's avatar
Laurent Peuch committed
603
        warnings.simplefilter("ignore", DeprecationWarning)
604
605
        rebuild_cmdline(option, opt, value, parser)

606
    # logilab-pytest options
Laurent Peuch's avatar
Laurent Peuch committed
607
608
609
610
611
612
    parser.add_option(
        "-t", dest="testdir", default=None, help="directory where the tests will be found"
    )
    parser.add_option(
        "-d", dest="dbc", default=False, action="store_true", help="enable design-by-contract"
    )
613
    # unittest_main options provided and passed through logilab-pytest
Laurent Peuch's avatar
Laurent Peuch committed
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
    parser.add_option(
        "-v", "--verbose", callback=rebuild_cmdline, action="callback", help="Verbose output"
    )
    parser.add_option(
        "-i",
        "--pdb",
        callback=rebuild_and_store,
        dest="pdb",
        action="callback",
        help="Enable test failure inspection",
    )
    parser.add_option(
        "-x",
        "--exitfirst",
        callback=rebuild_and_store,
        dest="exitfirst",
        default=False,
        action="callback",
        help="Exit on first failure " "(only make sense when logilab-pytest run one test file)",
    )
    parser.add_option(
        "-R",
        "--restart",
        callback=rebuild_and_store,
        dest="restart",
        default=False,
        action="callback",
        help="Restart tests from where it failed (implies exitfirst) "
        "(only make sense if tests previously ran with exitfirst only)",
    )
    parser.add_option(
        "--color", callback=rebuild_cmdline, action="callback", help="colorize tracebacks"
    )
    parser.add_option(
        "-s",
        "--skip",
        # XXX: I wish I could use the callback action but it
        #      doesn't seem to be able to get the value
        #      associated to the option
        action="store",
        dest="skipped",
        default=None,
        help="test names matching this name will be skipped "
        "to skip several patterns, use commas",
    )
    parser.add_option(
        "-q", "--quiet", callback=rebuild_cmdline, action="callback", help="Minimal output"
    )
    parser.add_option(
        "-P",
        "--profile",
        default=None,
        dest="profile",
        help="Profile execution and store data in the given file",
    )
    parser.add_option(
        "-m",
        "--match",
        default=None,
        dest="tags_pattern",
        help="only execute test whose tag match the current pattern",
    )
Adrien Di Mascio's avatar
Adrien Di Mascio committed
676

677
    if DJANGO_FOUND:
Laurent Peuch's avatar
Laurent Peuch committed
678
679
680
681
682
683
684
685
        parser.add_option(
            "-J",
            "--django",
            dest="django",
            default=False,
            action="store_true",
            help="use logilab-pytest for django test cases",
        )
Adrien Di Mascio's avatar
Adrien Di Mascio committed
686
    return parser
687

Adrien Di Mascio's avatar
Adrien Di Mascio committed
688
689
690
691
692

def parseargs(parser):
    """Parse the command line and return (options processed), (options to pass to
    unittest_main()), (explicitfile or None).
    """
Adrien Di Mascio's avatar
Adrien Di Mascio committed
693
694
    # parse the command line
    options, args = parser.parse_args()
Laurent Peuch's avatar
Laurent Peuch committed
695
    filenames = [arg for arg in args if arg.endswith(".py")]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
696
697
    if filenames:
        if len(filenames) > 1:
Nicolas Chauvat's avatar
Nicolas Chauvat committed
698
            parser.error("only one filename is acceptable")
Adrien Di Mascio's avatar
Adrien Di Mascio committed
699
700
701
702
703
704
        explicitfile = filenames[0]
        args.remove(explicitfile)
    else:
        explicitfile = None
    # someone wants DBC
    testlib.ENABLE_DBC = options.dbc
Adrien Di Mascio's avatar
Adrien Di Mascio committed
705
    newargs = parser.newargs
Adrien Di Mascio's avatar
Adrien Di Mascio committed
706
    if options.skipped:
Laurent Peuch's avatar
Laurent Peuch committed
707
        newargs.extend(["--skip", options.skipped])
708
709
710
    # restart implies exitfirst
    if options.restart:
        options.exitfirst = True
Adrien Di Mascio's avatar
Adrien Di Mascio committed
711
712
713
    # append additional args to the new sys.argv and let unittest_main
    # do the rest
    newargs += args
Pierre-Yves David's avatar
Pierre-Yves David committed
714
    return options, explicitfile
Adrien Di Mascio's avatar
Adrien Di Mascio committed
715

716

717
@callable_deprecated("[logilab-common 1.3] logilab-pytest is deprecated, use another test runner")
Adrien Di Mascio's avatar
Adrien Di Mascio committed
718
def run():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
719
720
721
    parser = make_parser()
    rootdir, testercls = project_root(parser)
    options, explicitfile = parseargs(parser)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
722
    # mock a new command line
Adrien Di Mascio's avatar
Adrien Di Mascio committed
723
    sys.argv[1:] = parser.newargs
Adrien Di Mascio's avatar
Adrien Di Mascio committed
724
    cvg = None
Laurent Peuch's avatar
Laurent Peuch committed
725
    if "" not in sys.path:
Laurent Peuch's avatar
Laurent Peuch committed
726
        sys.path.insert(0, "")
727
    if DJANGO_FOUND and options.django:
Adrien Di Mascio's avatar
Adrien Di Mascio committed
728
        tester = DjangoTester(cvg, options)
729
    else:
Adrien Di Mascio's avatar
Adrien Di Mascio committed
730
        tester = testercls(cvg, options)
Sylvain's avatar
Sylvain committed
731
732
733
734
735
736
    if explicitfile:
        cmd, args = tester.testfile, (explicitfile,)
    elif options.testdir:
        cmd, args = tester.testonedir, (options.testdir, options.exitfirst)
    else:
        cmd, args = tester.testall, (options.exitfirst,)
737
    try:
738
        try:
Sylvain's avatar
Sylvain committed
739
740
            if options.profile:
                import hotshot
Laurent Peuch's avatar
Laurent Peuch committed
741

Sylvain's avatar
Sylvain committed
742
743
744
                prof = hotshot.Profile(options.profile)
                prof.runcall(cmd, *args)
                prof.close()
Laurent Peuch's avatar
Laurent Peuch committed
745
                print("profile data saved in", options.profile)
746
            else:
Stephanie Marcu's avatar
Stephanie Marcu committed
747
                cmd(*args)
748
749
        except SystemExit:
            raise
Laurent Peuch's avatar
Laurent Peuch committed
750
        except Exception:
751
            import traceback
Laurent Peuch's avatar
Laurent Peuch committed
752

753
            traceback.print_exc()
754
    finally:
755
        tester.show_report()
756
        sys.exit(tester.errcode)
757

Laurent Peuch's avatar
Laurent Peuch committed
758

759
760
761
762
763
764
765
766
767
768
769
770
class SkipAwareTestProgram(unittest.TestProgram):
    # XXX: don't try to stay close to unittest.py, use optparse
    USAGE = """\
Usage: %(progName)s [options] [test] [...]

Options:
  -h, --help       Show this message
  -v, --verbose    Verbose output
  -i, --pdb        Enable test failure inspection
  -x, --exitfirst  Exit on first failure
  -s, --skip       skip test matching this pattern (no regexp for now)
  -q, --quiet      Minimal output
771
  --color          colorize tracebacks
772
773
774
775
776
777
778
779
780
781
782
783
784

  -m, --match      Run only test whose tag match this pattern

  -P, --profile    FILE: Run the tests using cProfile and saving results
                   in FILE

Examples:
  %(progName)s                               - run default set of tests
  %(progName)s MyTestSuite                   - run suite 'MyTestSuite'
  %(progName)s MyTestCase.testSomething      - run MyTestCase.testSomething
  %(progName)s MyTestCase                    - run all 'test*' test methods
                                               in MyTestCase
"""
Laurent Peuch's avatar
Laurent Peuch committed
785
786
787
788
789
790
791
792
793
794

    def __init__(
        self,
        module="__main__",
        defaultTest=None,
        batchmode=False,
        cvg=None,
        options=None,
        outstream=sys.stderr,
    ):
795
796
797
798
799
        self.batchmode = batchmode
        self.cvg = cvg
        self.options = options
        self.outstream = outstream
        super(SkipAwareTestProgram, self).__init__(
Laurent Peuch's avatar
Laurent Peuch committed
800
801
            module=module, defaultTest=defaultTest, testLoader=NonStrictTestLoader()
        )
802
803
804
805
806
807
808

    def parseArgs(self, argv):
        self.pdbmode = False
        self.exitfirst = False
        self.skipped_patterns = []
        self.test_pattern = None
        self.tags_pattern = None
809
        self.colorize = False
810
811
        self.profile_name = None
        import getopt
Laurent Peuch's avatar
Laurent Peuch committed
812

813
        try:
Laurent Peuch's avatar
Laurent Peuch committed
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
            options, args = getopt.getopt(
                argv[1:],
                "hHvixrqcp:s:m:P:",
                [
                    "help",
                    "verbose",
                    "quiet",
                    "pdb",
                    "exitfirst",
                    "restart",
                    "skip=",
                    "color",
                    "match=",
                    "profile=",
                ],
            )
830
            for opt, value in options:
Laurent Peuch's avatar
Laurent Peuch committed
831
                if opt in ("-h", "-H", "--help"):
832
                    self.usageExit()
Laurent Peuch's avatar
Laurent Peuch committed
833
                if opt in ("-i", "--pdb"):
834
                    self.pdbmode = True
Laurent Peuch's avatar
Laurent Peuch committed
835
                if opt in ("-x", "--exitfirst"):
836
                    self.exitfirst = True
Laurent Peuch's avatar
Laurent Peuch committed
837
                if opt in ("-r", "--restart"):
838
839
                    self.restart = True
                    self.exitfirst = True
Laurent Peuch's avatar
Laurent Peuch committed
840
                if opt in ("-q", "--quiet"):
841
                    self.verbosity = 0
Laurent Peuch's avatar
Laurent Peuch committed
842
                if opt in ("-v", "--verbose"):
843
                    self.verbosity = 2
Laurent Peuch's avatar
Laurent Peuch committed
844
845
846
                if opt in ("-s", "--skip"):
                    self.skipped_patterns = [pat.strip() for pat in value.split(", ")]
                if opt == "--color":
847
                    self.colorize = True
Laurent Peuch's avatar
Laurent Peuch committed
848
849
                if opt in ("-m", "--match"):
                    # self.tags_pattern = value
850
                    self.options["tag_pattern"] = value
Laurent Peuch's avatar
Laurent Peuch committed
851
                if opt in ("-P", "--profile"):
852
853
854
                    self.profile_name = value
            self.testLoader.skipped_patterns = self.skipped_patterns
            if len(args) == 0 and self.defaultTest is None:
Laurent Peuch's avatar
Laurent Peuch committed
855
856
                suitefunc = getattr(self.module, "suite", None)
                if isinstance(suitefunc, (types.FunctionType, types.MethodType)):
857
858
859
860
861
862
863
864
                    self.test = self.module.suite()
                else:
                    self.test = self.testLoader.loadTestsFromModule(self.module)
                return
            if len(args) > 0:
                self.test_pattern = args[0]
                self.testNames = args
            else:
Laurent Peuch's avatar
Laurent Peuch committed
865
                self.testNames = (self.defaultTest,)
866
            self.createTests()
867
        except getopt.error as msg:
868
869
870
871
872
            self.usageExit(msg)

    def runTests(self):
        if self.profile_name:
            import cProfile
Laurent Peuch's avatar
Laurent Peuch committed
873
874

            cProfile.runctx("self._runTests()", globals(), locals(), self.profile_name)
875
876
877
878
        else:
            return self._runTests()

    def _runTests(self):
Laurent Peuch's avatar
Laurent Peuch committed
879
880
881
882
883
884
885
886
887
888
889
890
        self.testRunner = SkipAwareTextTestRunner(
            verbosity=self.verbosity,
            stream=self.outstream,
            exitfirst=self.exitfirst,
            pdbmode=self.pdbmode,
            cvg=self.cvg,
            test_pattern=self.test_pattern,
            skipped_patterns=self.skipped_patterns,
            colorize=self.colorize,
            batchmode=self.batchmode,
            options=self.options,
        )
891
892
893
894
895

        def removeSucceededTests(obj, succTests):
            """ Recursive function that removes succTests from
            a TestSuite or TestCase
            """
896
            if isinstance(obj, unittest.TestSuite):
897
898
899
                removeSucceededTests(obj._tests, succTests)
            if isinstance(obj, list):
                for el in obj[:]:
900
                    if isinstance(el, unittest.TestSuite):
901
                        removeSucceededTests(el, succTests)
902
                    elif isinstance(el, unittest.TestCase):
Laurent Peuch's avatar
Laurent Peuch committed
903
904
905
                        descr = ".".join(
                            (el.__class__.__module__, el.__class__.__name__, el._testMethodName)
                        )
906
907
                        if descr in succTests:
                            obj.remove(el)
Laurent Peuch's avatar
Laurent Peuch committed
908

909
        # take care, self.options may be None
Laurent Peuch's avatar
Laurent Peuch committed
910
        if getattr(self.options, "restart", False):
911
912
            # retrieve succeeded tests from FILE_RESTART
            try:
Laurent Peuch's avatar
Laurent Peuch committed
913
                restartfile = open(FILE_RESTART, "r")
914
                try:
Laurent Peuch's avatar
Laurent Peuch committed
915
                    succeededtests = list(elem.rstrip("\n\r") for elem in restartfile.readlines())
916
917
918
                    removeSucceededTests(self.test, succeededtests)
                finally:
                    restartfile.close()
919
            except Exception as ex:
Laurent Peuch's avatar
Laurent Peuch committed
920
921
922
923
                raise Exception(
                    "Error while reading succeeded tests into %s: %s"
                    % (osp.join(os.getcwd(), FILE_RESTART), ex)
                )
924
925
926
927
928

        result = self.testRunner.run(self.test)
        # help garbage collection: we want TestSuite, which hold refs to every
        # executed TestCase, to be gc'ed
        del self.test
Laurent Peuch's avatar
Laurent Peuch committed
929
        if getattr(result, "debuggers", None) and getattr(self, "pdbmode", None):
930
931
932
933
934
935
936
            start_interactive_mode(result)
        if not getattr(self, "batchmode", None):
            sys.exit(not result.wasSuccessful())
        self.result = result


class SkipAwareTextTestRunner(unittest.TextTestRunner):
Laurent Peuch's avatar
Laurent Peuch committed
937
938
939
940
941
942
943
944
945
946
947
948
949
950
    def __init__(
        self,
        stream=sys.stderr,
        verbosity=1,
        exitfirst=False,
        pdbmode=False,
        cvg=None,
        test_pattern=None,
        skipped_patterns=(),
        colorize=False,
        batchmode=False,
        options=None,
    ):
        super(SkipAwareTextTestRunner, self).__init__(stream=stream, verbosity=verbosity)
951
952
953
954
955
        self.exitfirst = exitfirst
        self.pdbmode = pdbmode
        self.cvg = cvg
        self.test_pattern = test_pattern
        self.skipped_patterns = skipped_patterns
956
        self.colorize = colorize
957
958
959
        self.batchmode = batchmode
        self.options = options

960
    def _this_is_skipped(self, testedname: str) -> bool:
961
962
        return any([(pat in testedname) for pat in self.skipped_patterns])

963
    def _runcondition(self, test: Callable, skipgenerator: bool = True) -> bool:
964
965
966
967
968
        if isinstance(test, testlib.InnerTest):
            testname = test.name
        else:
            if isinstance(test, testlib.TestCase):
                meth = test._get_test_method()
Laurent Peuch's avatar
Laurent Peuch committed
969
                testname = "%s.%s" % (test.__name__, meth.__name__)
970
971
972
973
            elif isinstance(test, types.FunctionType):
                func = test
                testname = func.__name__
            elif isinstance(test, types.MethodType):
974
                cls = test.__self__.__class__
Laurent Peuch's avatar
Laurent Peuch committed
975
                testname = "%s.%s" % (cls.__name__, test.__name__)
976
            else:
Laurent Peuch's avatar
Laurent Peuch committed
977
                return True  # Not sure when this happens
978
            if isgeneratorfunction(test) and skipgenerator:
Laurent Peuch's avatar
Laurent Peuch committed
979
                return self.does_match_tags(test)  # Let inner tests decide at run time
980
        if self._this_is_skipped(testname):
Laurent Peuch's avatar
Laurent Peuch committed
981
            return False  # this was explicitly skipped
982
983
        if self.test_pattern is not None:
            try:
Laurent Peuch's avatar
Laurent Peuch committed
984
985
                classpattern, testpattern = self.test_pattern.split(".")
                klass, name = testname.split(".")
986
987
988
989
990
991
992
993
                if classpattern not in klass or testpattern not in name:
                    return False
            except ValueError:
                if self.test_pattern not in testname:
                    return False

        return self.does_match_tags(test)

994
    def does_match_tags(self, test: Callable) -> bool:
995
        if self.options is not None:
Laurent Peuch's avatar
Laurent Peuch committed
996
            tags_pattern = getattr(self.options, "tags_pattern", None)
997
            if tags_pattern is not None:
Laurent Peuch's avatar
Laurent Peuch committed
998
                tags = getattr(test, "tags", testlib.Tags())
999
                if tags.inherit and isinstance(test, types.MethodType):
Laurent Peuch's avatar
Laurent Peuch committed
1000
                    tags = tags | getattr(test.__self__.__class__, "tags", testlib.Tags())