Commit 9f14b1da authored by Philippe Pepiot's avatar Philippe Pepiot
Browse files

Track worker exited prematurely issues

In case of exited worker (segfault), AsyncResult.traceback is None and
information is in AsyncResult.result, record it in the failed transition
comment.

Add a dedicated test class, this require to run the worker in prefork mode,
otherwise, in solo mode the information about worker lost isn't tracked.
Use "import ctypes; ctypes.string_at(0)" to simulate a segfault.
parent d748ec59b5d2
......@@ -331,7 +331,12 @@ class CeleryTaskAdapter(ICeleryTask):
self.info('<CeleryTask %s (task_id %s)> %s', self.entity.eid,
task_id, transition)
wf = self.entity.cw_adapt_to('IWorkflowable')
wf.fire_transition_if_possible(transition, result.traceback)
if (result.traceback is None and result.state == STATES.FAILURE
and result.result is not None):
comment = six.text_type(result.result)
else:
comment = result.traceback
wf.fire_transition_if_possible(transition, comment)
else:
self.info('<CeleryTask %s (task_id %s)> no transition found for '
'state %s', self.entity.eid, task_id, result.state)
......
......@@ -3,6 +3,7 @@ from __future__ import print_function
import sys
import logging
import os
import time
import six
......@@ -89,3 +90,8 @@ def spawn(self):
"celerytask_subtasks": chord((success.s(i) for i in range(10)),
tsum.s() | add.s(-45))().as_tuple(),
}
@app.task(name="segfault")
def segfault():
import ctypes
ctypes.string_at(0)
......@@ -296,6 +296,21 @@ class StartAsyncTaskTC(testlib.CubicWebTC):
[r.get() for r in results.values()], ['a', 'b', 'c'])
class SegfaultTC(BaseCeleryTaskTC):
worker_args = ['-P', 'prefork', '-c', '1', '-l', 'info']
def test_segfault(self):
with self.admin_access.cnx() as cnx:
task = start_async_task(cnx, 'segfault')
cnx.commit()
run_all_tasks(cnx)
self.wait_async_task(cnx, task.task_id, timeout=15)
wf = cnx.entity_from_eid(task.eid).cw_adapt_to('IWorkflowable')
self.assertEqual(wf.state, 'failed')
self.assertEqual(wf.latest_trinfo().comment, (
'Worker exited prematurely: signal 11 (SIGSEGV).'))
if __name__ == '__main__':
from unittest import main
main()
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment