Commit 844071b9 authored by Colomban Wendling's avatar Colomban Wendling

Mails: don't drop mails upon temporary relay error

We used to drop any mail we tried to send, regardless of whether
sending it actually succeeded or not.  This could result in lost mails
if the server failed to accept some mails for any temporary reason.

Fix this by re-queuing emails that failed to send and wait a few
moments before retrying.
parent 4487e522
...@@ -31,6 +31,8 @@ config.install_attr('emails.addr_from', ...@@ -31,6 +31,8 @@ config.install_attr('emails.addr_from',
config.install_attr('emails.smtp_host', 'localhost:25') config.install_attr('emails.smtp_host', 'localhost:25')
# The inactive timeout after which to close the SMTP connection # The inactive timeout after which to close the SMTP connection
config.install_attr('emails.smtp_keepalive_timeout', 60) config.install_attr('emails.smtp_keepalive_timeout', 60)
# Timeout after which to retry sending emails after a failure
config.install_attr('emails.smtp_retry_timeout', 60)
# Interval in seconds between global reports when some checks are in error # Interval in seconds between global reports when some checks are in error
# 0 disables reports # 0 disables reports
config.install_attr('emails.report.every', 0) config.install_attr('emails.report.every', 0)
......
...@@ -7,7 +7,7 @@ from sys import stderr ...@@ -7,7 +7,7 @@ from sys import stderr
from time import strftime from time import strftime
from datetime import datetime, timedelta from datetime import datetime, timedelta
import email.charset import email.charset
from threading import Thread from threading import Thread, Event
import queue import queue
import atexit import atexit
...@@ -21,7 +21,7 @@ class ThreadedSMTP(object): ...@@ -21,7 +21,7 @@ class ThreadedSMTP(object):
def __init__(self): def __init__(self):
self._queue = queue.Queue() self._queue = queue.Queue()
self._loop = True self._quit = Event()
self._thread = Thread(target=self.__loop) self._thread = Thread(target=self.__loop)
self._thread.daemon = True self._thread.daemon = True
self._thread.start() self._thread.start()
...@@ -29,8 +29,8 @@ class ThreadedSMTP(object): ...@@ -29,8 +29,8 @@ class ThreadedSMTP(object):
atexit.register(self.quit) atexit.register(self.quit)
def quit(self): def quit(self):
if self._loop: if not self._quit.is_set():
self._loop = False self._quit.set()
self._queue.put(((), {})) # put a dummy item to wake up the thread self._queue.put(((), {})) # put a dummy item to wake up the thread
self._queue.join() self._queue.join()
self._thread.join() self._thread.join()
...@@ -44,7 +44,7 @@ class ThreadedSMTP(object): ...@@ -44,7 +44,7 @@ class ThreadedSMTP(object):
from . import config from . import config
server = None server = None
while self._loop or not self._queue.empty(): while not self._quit.is_set() or not self._queue.empty():
try: try:
timeout = config.emails.smtp_keepalive_timeout timeout = config.emails.smtp_keepalive_timeout
args, kwargs = self._queue.get(timeout=timeout) args, kwargs = self._queue.get(timeout=timeout)
...@@ -57,7 +57,21 @@ class ThreadedSMTP(object): ...@@ -57,7 +57,21 @@ class ThreadedSMTP(object):
server = smtplib.SMTP(config.emails.smtp_host) server = smtplib.SMTP(config.emails.smtp_host)
server.sendmail(*args, **kwargs) server.sendmail(*args, **kwargs)
except Exception as e: except Exception as e:
logging.warning("Couldn't send email: %s" % str(e)) # message couldn't be sent, but assume it's a temporary
# server error and try again in a moment (unless when
# we're quitting, not to block). If it is a permanent
# error it most likely means the configuration is wrong
# anyway so the user has to fix it and restart.
if self._quit.is_set():
logging.warning("Couldn't send email: %s" % str(e))
else:
logging.warning(
"Couldn't send email (will retry in %ds): %s" %
(config.emails.smtp_retry_timeout, str(e)))
# put the item back and wait
self._queue.put_nowait((args, kwargs))
self._quit.wait(
timeout=config.emails.smtp_retry_timeout)
self._queue.task_done() self._queue.task_done()
self.__server_quit(server) self.__server_quit(server)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment