Commit df974571 authored by Philippe Pepiot's avatar Philippe Pepiot
Browse files

[ccplugin] handle redis & database restart

We should not stop celery-monitor in case of temporary unavailability of redis
or cubicweb database.
Handle errors and automatically retry each 5 seconds.
parent ae5462a6f276
......@@ -16,6 +16,10 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import json
import logging
import time
import redis.exceptions
from cubicweb.toolsutils import Command
from cubicweb.server.serverconfig import ServerConfiguration
......@@ -24,6 +28,8 @@ from cubicweb.cwctl import CWCTL
from cw_celerytask_helpers.monitor import MONITOR_KEY
from cw_celerytask_helpers.utils import get_redis_client
logger = logging.getLogger(__name__)
class CeleryMonitorCommand(Command):
"""Synchronize celery task statuses"""
......@@ -47,12 +53,33 @@ class CeleryMonitorCommand(Command):
init_cmdline_log_threshold(config, self['loglevel'])
repo = repoapi.get_repository(config=config)
repo.hm.call_hooks('server_maintenance', repo=repo)
with repo.internal_cnx() as cnx:
self.loop(cnx)
source = repo.system_source
while True:
try:
with repo.internal_cnx() as cnx:
while True:
try:
self.loop(cnx)
except redis.exceptions.ConnectionError:
logger.error('redis ConnectionError, retry in 5s')
time.sleep(5)
except (source.OperationalError, source.InterfaceError):
while True:
source.exception('connection lost, retry in 5s')
time.sleep(5)
try:
for cnxset in repo.cnxsets:
cnxset.reconnect()
except (source.OperationalError, source.InterfaceError):
continue
else:
break
@staticmethod
def loop(cnx, timeout=None):
client = get_redis_client()
client.ping()
logger.info('Connected to redis')
test = (cnx.repo.config.mode == "test")
while True:
data = client.brpop(MONITOR_KEY, timeout=timeout)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment