From 1c1f7cc619d5a0484da3f3962039e4ac7dd26d6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Jochum?= Date: Thu, 28 Oct 2021 09:47:15 +0200 Subject: [PATCH] 2.3.16-r13 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: René Jochum --- Dockerfile | 31 +++++++++- conf/conf.d/10-logging.conf | 6 +- conf/conf.d/10-ssl.conf | 2 +- conf/conf.d/20-imap.conf | 2 +- conf/conf.d/20-submission.conf.jinja | 88 ---------------------------- conf/conf.d/90-sieve.conf | 20 ++++++- conf/dovecot-sql.conf.ext.jinja | 30 ++++++---- docker-entrypoint.py | 34 +++++++++-- sieve/bin/rspamd-learn-fuzzy-ham.sh | 4 ++ sieve/bin/rspamd-learn-fuzzy-spam.sh | 4 ++ sieve/bin/rspamd-learn-ham.sh | 4 ++ sieve/bin/rspamd-learn-spam.sh | 4 ++ sieve/report-ham.sieve | 12 ++++ sieve/report-spam.sieve | 15 +++++ 14 files changed, 144 insertions(+), 112 deletions(-) delete mode 100644 conf/conf.d/20-submission.conf.jinja create mode 100755 sieve/bin/rspamd-learn-fuzzy-ham.sh create mode 100755 sieve/bin/rspamd-learn-fuzzy-spam.sh create mode 100755 sieve/bin/rspamd-learn-ham.sh create mode 100755 sieve/bin/rspamd-learn-spam.sh create mode 100644 sieve/report-ham.sieve create mode 100644 sieve/report-spam.sieve diff --git a/Dockerfile b/Dockerfile index a553119..8ac5d59 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,20 @@ -FROM registry.jochum.dev/jochum/debian-apt-cacher:buster-slim +FROM reg.jochum.dev/library/debian:buster-slim LABEL maintainer "René Jochum " +ENV DEBIAN_FRONTEND noninteractive + +# Configure Proxy +RUN set -ex; \ + sed -i 's/deb.debian.org/ftp.ch.debian.org/g' /etc/apt/sources.list && \ + apt-get update --allow-releaseinfo-change && \ + apt-get install -qy netcat-openbsd && \ + sed -i 's/ftp.ch.debian.org/deb.debian.org/g' /etc/apt/sources.list || exit 0 && \ + if [ -z "$APT_CACHER" ]; then APT_CACHER="apt-cacher.apt-cacher.svc.cluster.local"; fi && \ + if nc -w1 -z $APT_CACHER 3142 1>/dev/null 2>&1; then \ + echo 'Acquire::HTTP::Proxy "http://'$APT_CACHER':3142";' >> /etc/apt/apt.conf.d/01proxy && \ + echo 'Acquire::HTTPS::Proxy "false";' >> /etc/apt/apt.conf.d/01proxy; else exit 0; fi + RUN set -ex; \ apt-get update --allow-releaseinfo-change && \ apt-get dist-upgrade -y -o 'DPkg::Options::=--force-confold' -o 'DPkg::Options::=--force-confdef' && \ @@ -12,15 +25,27 @@ RUN set -ex; \ RUN set -ex; \ apt-get update --allow-releaseinfo-change && \ - apt-get install --no-install-recommends -y -o 'DPkg::Options::=--force-confold' -o 'DPkg::Options::=--force-confdef' dovecot-imapd dovecot-pop3d dovecot-lmtpd dovecot-submissiond dovecot-managesieved dovecot-sieve dovecot-pgsql dovecot-mysql python3-minimal python3-jinja2 && \ + apt-get install --no-install-recommends -y -o 'DPkg::Options::=--force-confold' -o 'DPkg::Options::=--force-confdef' dovecot-imapd dovecot-pop3d dovecot-lmtpd dovecot-managesieved dovecot-sieve dovecot-pgsql dovecot-mysql python3-minimal python3-jinja2 + +# Install rspamc to submit SPAM/HAM to rspamd +RUN set -ex; \ + apt-get update --allow-releaseinfo-change && \ + apt-get install --no-install-recommends -y -o 'DPkg::Options::=--force-confold' -o 'DPkg::Options::=--force-confdef' lsb-release wget ca-certificates gpg && \ + mkdir -p /etc/apt/keyrings && \ + wget -O- https://rspamd.com/apt-stable/gpg.key | gpg --dearmor > /etc/apt/keyrings/rspamd.gpg && \ + echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rspamd.gpg] http://rspamd.com/apt-stable/ `lsb_release -c -s` main" > /etc/apt/sources.list.d/rspamd.list && \ + echo "deb-src [arch=amd64 signed-by=/etc/apt/keyrings/rspamd.gpg] http://rspamd.com/apt-stable/ `lsb_release -c -s` main" >> /etc/apt/sources.list.d/rspamd.list && \ + apt-get update --allow-releaseinfo-change && \ + apt-get --no-install-recommends -y -o 'DPkg::Options::=--force-confold' -o 'DPkg::Options::=--force-confdef' install rspamd && \ rm -rf /var/lib/apt/lists/* RUN rm -f /etc/apt/apt.conf.d/01proxy COPY conf /conf +COPY sieve /sieve COPY docker-entrypoint.py / -EXPOSE 110/tcp 143/tcp 587/tcp 995/tcp 993/tcp 2525/tcp 4190/tcp 12345/tcp +EXPOSE 110/tcp 143/tcp 995/tcp 993/tcp 2525/tcp 4190/tcp 12345/tcp VOLUME ["/data", "/overrides", "/cert"] diff --git a/conf/conf.d/10-logging.conf b/conf/conf.d/10-logging.conf index d325255..c575f04 100644 --- a/conf/conf.d/10-logging.conf +++ b/conf/conf.d/10-logging.conf @@ -4,12 +4,12 @@ # Log file to use for error messages. "syslog" logs to syslog, # /dev/stderr logs to stderr. -log_path = /dev/stderr +log_path = /logs/dovecot.log # Log file to use for informational messages. Defaults to log_path. -#info_log_path = +# info_log_path = # Log file to use for debug messages. Defaults to info_log_path. -#debug_log_path = +# debug_log_path = # Syslog facility to use if you're logging to syslog. Usually if you don't # want to use "mail", you'll use local0..local7. Also other standard diff --git a/conf/conf.d/10-ssl.conf b/conf/conf.d/10-ssl.conf index 2add05e..4939b99 100644 --- a/conf/conf.d/10-ssl.conf +++ b/conf/conf.d/10-ssl.conf @@ -3,7 +3,7 @@ ## # SSL/TLS support: yes, no, required. -ssl = yes +ssl = required # PEM encoded X.509 SSL/TLS certificate and private key. They're opened before # dropping root privileges, so keep the key file unreadable by anyone but diff --git a/conf/conf.d/20-imap.conf b/conf/conf.d/20-imap.conf index 170ba25..f9170ee 100644 --- a/conf/conf.d/20-imap.conf +++ b/conf/conf.d/20-imap.conf @@ -90,7 +90,7 @@ protocol imap { # Space separated list of plugins to load (default is global mail_plugins). - mail_plugins = $mail_plugins imap_quota + mail_plugins = $mail_plugins imap_quota imap_sieve # Maximum number of IMAP connections allowed for a user from each IP address. # NOTE: The username is compared case-sensitively. diff --git a/conf/conf.d/20-submission.conf.jinja b/conf/conf.d/20-submission.conf.jinja deleted file mode 100644 index ba31c4d..0000000 --- a/conf/conf.d/20-submission.conf.jinja +++ /dev/null @@ -1,88 +0,0 @@ -## -## Settings specific to SMTP Submission -## - -# SMTP Submission logout format string: -# %i - total number of bytes read from client -# %o - total number of bytes sent to client -# %{command_count} - Number of commands received from client -# %{reply_count} - Number of replies sent to client -# %{session} - Session ID of the login session -# %{transaction_id} - ID of the current transaction, if any -#submission_logout_format = in=%i out=%o - -# Host name reported by the SMTP service, for example to the client in the -# initial greeting and to the relay server in the HELO/EHLO command. -# Default is the system's real hostname@domain. -hostname = {{ HOSTNAME }} - -# Maximum size of messages accepted for relay. This announced in the SIZE -# capability. If not configured, this is either determined from the relay -# server or left unlimited if no limit is known (relay will reply with error -# if some unknown limit exists there, which is duly passed to our client). -#submission_max_mail_size = - -# Maximum number of recipients accepted per connection (default: unlimited) -#submission_max_recipients = - -# Workarounds for various client bugs: -# whitespace-before-path: -# Allow one or more spaces or tabs between `MAIL FROM:' and path and between -# `RCPT TO:' and path. -# mailbox-for-path: -# Allow using bare Mailbox syntax (i.e., without <...>) instead of full path -# syntax. -# -# The list is space-separated. -#submission_client_workarounds = - -# Relay server configuration: -# -# The Dovecot SMTP submission service directly proxies the mail transaction -# to the SMTP relay configured here. - -# Host name for the relay server (required) -submission_relay_host = {{ RELAY_HOST }} - -# Port for the relay server -submission_relay_port = {{ RELAY_PORT }} - -# Is the relay server trusted? This determines whether we try to send -# (Postfix-specific) XCLIENT data to the relay server -submission_relay_trusted = {{ RELAY_TRUSTED }} - -# Authentication data for the relay server if authentication is required -#submission_relay_user = -#submission_relay_master_user = -#submission_relay_password = - -# SSL configuration for connection to relay server -# -# submission_relay_ssl: -# Indicates whether SSL is used for the connection to the relay server. The -# following values are defined for this setting: -# -# no - No SSL is used -# smtps - An SMTPS connection (immediate SSL) is used -# starttls - The STARTTLS command is used to establish SSL layer -#submission_relay_ssl = no - -# submission_relay_ssl_verify: -# Configures whether the SSL certificate of the relay server is to be -# verified. -#submission_relay_ssl_verify = yes - -# Write protocol logs for relay connection to this directory for debugging -#submission_relay_rawlog_dir = - -# BURL is configured implicitly by IMAP URLAUTH - -protocol submission { - # Space-separated list of plugins to load (default is global mail_plugins). - #mail_plugins = $mail_plugins - - # Maximum number of SMTP submission connections allowed for a user from - # each IP address. - # NOTE: The username is compared case-sensitively. - #mail_max_userip_connections = 10 -} diff --git a/conf/conf.d/90-sieve.conf b/conf/conf.d/90-sieve.conf index 0ccef47..b3beb34 100644 --- a/conf/conf.d/90-sieve.conf +++ b/conf/conf.d/90-sieve.conf @@ -1,6 +1,24 @@ +# https://www.kernel-error.de/2020/05/04/rspamd-automatisch-spam-ham-lernen-mit-dovecot-und-imapsieve/ + plugin { + sieve_plugins = sieve_imapsieve sieve_extprograms + sieve = ~/sieve/.dovecot.sieve - sieve_global_path = /data/sieve/default.sieve sieve_dir = ~/sieve + sieve_global_path = /data/sieve/default.sieve sieve_global_dir = /data/sieve/global/ + + # From elsewhere to Spam folder or flag changed in Spam folder + imapsieve_mailbox1_name = Junk + imapsieve_mailbox1_causes = COPY FLAG + imapsieve_mailbox1_before = file:/data/sieve/global/report-spam.sieve + + # From Spam folder to elsewhere + imapsieve_mailbox2_name = * + imapsieve_mailbox2_from = Junk + imapsieve_mailbox2_causes = COPY + imapsieve_mailbox2_before = file:/data/sieve/global/report-ham.sieve + + sieve_pipe_bin_dir = /usr/lib/dovecot/sieve + sieve_global_extensions = +vnd.dovecot.pipe +vnd.dovecot.environment } diff --git a/conf/dovecot-sql.conf.ext.jinja b/conf/dovecot-sql.conf.ext.jinja index b904ac7..acd551c 100644 --- a/conf/dovecot-sql.conf.ext.jinja +++ b/conf/dovecot-sql.conf.ext.jinja @@ -1,16 +1,26 @@ driver = {{ SQL_TYPE }} connect = host={{ SQL_HOST }} dbname={{ SQL_DATABASE }} user={{ SQL_USER }} password={{ SQL_PASSWORD }} default_pass_scheme = MD5 -user_query = \ - SELECT '/data/vmail/' || mb.maildir as home, '*:bytes=' || mb.quota AS quota_rule \ - FROM alias al \ - LEFT JOIN mailbox mb ON al.goto = mb.username \ - WHERE (al.address = '%u' OR al.address = SUBSTR('%u', POSITION('@' in '%u'))) and mb.active = TRUE \ +# user_query = \ +# SELECT '/data/vmail/' || mb.maildir as home, '*:bytes=' || mb.quota AS quota_rule \ +# FROM alias al \ +# LEFT JOIN mailbox mb ON al.goto = mb.username \ +# WHERE (al.address = '%u' OR al.address = SUBSTR('%u', POSITION('@' in '%u'))) and mb.active = TRUE \ +# LIMIT 1 + +# password_query = \ +# SELECT mb.username as user, mb.password as password, '/data/vmail/' || mb.maildir as userdb_home, '*:bytes=' || mb.quota AS userdb_quota_rule \ +# FROM alias al \ +# LEFT JOIN mailbox mb ON al.goto = mb.username \ +# WHERE (al.address = '%u' OR al.address = SUBSTR('%u', POSITION('@' in '%u'))) and mb.active = TRUE \ +# LIMIT 1 + +user_query = SELECT '/data/vmail/' || mb.maildir as home, '*:bytes=' || mb.quota AS quota_rule \ + FROM mailbox AS mb \ + WHERE mb.username = '%u' and mb.active = TRUE \ LIMIT 1 -password_query = \ - SELECT mb.username as user, mb.password as password, '/data/vmail/' || mb.maildir as userdb_home, '*:bytes=' || mb.quota AS userdb_quota_rule \ - FROM alias al \ - LEFT JOIN mailbox mb ON al.goto = mb.username \ - WHERE (al.address = '%u' OR al.address = SUBSTR('%u', POSITION('@' in '%u'))) and mb.active = TRUE \ +password_query = SELECT mb.username as user, mb.password as password, '/data/vmail/' || mb.maildir as userdb_home, '*:bytes=' || mb.quota AS userdb_quota_rule \ + FROM mailbox AS mb \ + WHERE mb.username = '%u' and mb.active = TRUE \ LIMIT 1 \ No newline at end of file diff --git a/docker-entrypoint.py b/docker-entrypoint.py index 6f59410..17a0faf 100755 --- a/docker-entrypoint.py +++ b/docker-entrypoint.py @@ -17,8 +17,8 @@ def jinja_render_file(in_path, data, out_path): rf.write(output) for dovecot_file in glob.glob("/conf/**/*.conf", recursive=True): - destination = os.path.join("/etc/dovecot", dovecot_file[6:]) - shutil.copyfile(dovecot_file, destination) + out_path = os.path.join("/etc/dovecot", dovecot_file[6:]) + shutil.copyfile(dovecot_file, out_path) for dovecot_file in glob.glob("/conf/**/*.jinja", recursive=True): out_path = os.path.join("/etc/dovecot", dovecot_file[6:-6]) @@ -26,17 +26,41 @@ for dovecot_file in glob.glob("/conf/**/*.jinja", recursive=True): os.chmod(out_path, 600) for dovecot_file in glob.glob("/overrides/**/*.conf", recursive=True): - destination = os.path.join("/etc/dovecot", dovecot_file[6:]) - shutil.copyfile(dovecot_file, destination) + out_path = os.path.join("/etc/dovecot", dovecot_file[11:]) + shutil.copyfile(dovecot_file, out_path) for dovecot_file in glob.glob("/overrides/**/*.jinja", recursive=True): - out_path = os.path.join("/etc/dovecot", dovecot_file[6:-6]) + out_path = os.path.join("/etc/dovecot", dovecot_file[11:-6]) jinja_render_file(dovecot_file, os.environ, out_path) os.chmod(out_path, 600) if not os.path.isfile("/data/dh.pem"): subprocess.call(["/usr/bin/openssl", "dhparam", "-dsaparam", "-out", "/data/dh.pem", "4096"]) +for sieve_file in glob.glob("/sieve/**/*.sieve", recursive=True): + out_path = os.path.join("/data/sieve/global/", sieve_file[7:]) + if os.path.exists(out_path): + continue + + out_dir = os.path.dirname(out_path) + if not os.path.exists(out_dir): + os.makedirs(out_dir) + + shutil.copyfile(sieve_file, out_path) + subprocess.call(["sievec", out_path]) + +for sieve_bin_file in glob.glob("/sieve/bin/**/*", recursive=True): + out_path = os.path.join("/usr/lib/dovecot/sieve/", sieve_bin_file[11:]) + if os.path.exists(out_path): + continue + + out_dir = os.path.dirname(out_path) + if not os.path.exists(out_dir): + os.makedirs(out_dir) + + shutil.copyfile(sieve_bin_file, out_path) + os.chmod(out_path, 0o755) + subprocess.call(["/bin/mkdir", "-p", "/data/vmail"]) subprocess.call(["/bin/chmod", "u=rwX,g=rX,o=rX", "/data"]) subprocess.call(["/bin/chown", "mail:", "/data/vmail"]) diff --git a/sieve/bin/rspamd-learn-fuzzy-ham.sh b/sieve/bin/rspamd-learn-fuzzy-ham.sh new file mode 100755 index 0000000..b059b38 --- /dev/null +++ b/sieve/bin/rspamd-learn-fuzzy-ham.sh @@ -0,0 +1,4 @@ +#!/bin/sh +RSPAMD_HOSTPORT=$(cat /run/secrets/rspamd/HOSTPORT) +RSPAMD_PASSWORD=$(cat /run/secrets/rspamd/PASSWORD) +exec /usr/bin/rspamc --password $RSPAMD_PASSWORD -h $RSPAMD_HOSTPORT -S LOCAL_FUZZY_WHITE -w -2 fuzzy_add \ No newline at end of file diff --git a/sieve/bin/rspamd-learn-fuzzy-spam.sh b/sieve/bin/rspamd-learn-fuzzy-spam.sh new file mode 100755 index 0000000..8d156b3 --- /dev/null +++ b/sieve/bin/rspamd-learn-fuzzy-spam.sh @@ -0,0 +1,4 @@ +#!/bin/sh +RSPAMD_HOSTPORT=$(cat /run/secrets/rspamd/HOSTPORT) +RSPAMD_PASSWORD=$(cat /run/secrets/rspamd/PASSWORD) +exec /usr/bin/rspamc --password $RSPAMD_PASSWORD -h $RSPAMD_HOSTPORT -S LOCAL_FUZZY_DENIED -w 15 fuzzy_add \ No newline at end of file diff --git a/sieve/bin/rspamd-learn-ham.sh b/sieve/bin/rspamd-learn-ham.sh new file mode 100755 index 0000000..66a2344 --- /dev/null +++ b/sieve/bin/rspamd-learn-ham.sh @@ -0,0 +1,4 @@ +#!/bin/sh +RSPAMD_HOSTPORT=$(cat /run/secrets/rspamd/HOSTPORT) +RSPAMD_PASSWORD=$(cat /run/secrets/rspamd/PASSWORD) +exec /usr/bin/rspamc --password $RSPAMD_PASSWORD -h $RSPAMD_HOSTPORT learn_ham \ No newline at end of file diff --git a/sieve/bin/rspamd-learn-spam.sh b/sieve/bin/rspamd-learn-spam.sh new file mode 100755 index 0000000..91538b5 --- /dev/null +++ b/sieve/bin/rspamd-learn-spam.sh @@ -0,0 +1,4 @@ +#!/bin/sh +RSPAMD_HOSTPORT=$(cat /run/secrets/rspamd/HOSTPORT) +RSPAMD_PASSWORD=$(cat /run/secrets/rspamd/PASSWORD) +exec /usr/bin/rspamc --password $RSPAMD_PASSWORD -h $RSPAMD_HOSTPORT learn_spam \ No newline at end of file diff --git a/sieve/report-ham.sieve b/sieve/report-ham.sieve new file mode 100644 index 0000000..00fab5a --- /dev/null +++ b/sieve/report-ham.sieve @@ -0,0 +1,12 @@ +require ["vnd.dovecot.pipe", "copy", "imapsieve", "environment", "variables"]; + +if environment :matches "imap.mailbox" "*" { + set "mailbox" "${1}"; +} + +if string "${mailbox}" [ "Trash", "train_ham", "train_prob", "train_spam" ] { + stop; +} + +pipe :copy "rspamd-learn-ham.sh"; +pipe :copy "rspamd-learn-fuzzy-ham.sh"; \ No newline at end of file diff --git a/sieve/report-spam.sieve b/sieve/report-spam.sieve new file mode 100644 index 0000000..2f9ada4 --- /dev/null +++ b/sieve/report-spam.sieve @@ -0,0 +1,15 @@ +require ["vnd.dovecot.pipe", "copy", "imapsieve", "environment", "imap4flags"]; + +if environment :is "imap.cause" "COPY" { + pipe :copy "rspamd-learn-spam.sh"; + pipe :copy "rspamd-learn-fuzzy-spam.sh"; +} + +# Catch replied or forwarded spam +elsif anyof (allof (hasflag "\\Answered", + environment :contains "imap.changedflags" "\\Answered"), + allof (hasflag "$Forwarded", + environment :contains "imap.changedflags" "$Forwarded")) { + pipe :copy "rspamd-learn-spam.sh"; + pipe :copy "rspamd-learn-fuzzy-spam.sh"; +} \ No newline at end of file