diff --git a/mail/k8s/dockerfile/exim4/Dockerfile b/mail/k8s/dockerfile/exim4/Dockerfile index 4ace045caa7d6224dad56d3734b4d998b650f3c8..2e2e2810ad8ff9906ab9fb45f373a34451d4a933 100644 --- a/mail/k8s/dockerfile/exim4/Dockerfile +++ b/mail/k8s/dockerfile/exim4/Dockerfile @@ -3,7 +3,9 @@ FROM ubuntu:14.04 LABEL version="1.0.0" LABEL maintainer="tommylikehu@gmail.com" -ENV TINI_VERSION v0.14.0 +COPY docker-entrypoint.sh /usr/local/bin/ + +ENV TINI_VERSION v0.18.0 RUN set -x \ && apt-get update \ && apt-get install -y ca-certificates curl --no-install-recommends \ @@ -24,4 +26,7 @@ ADD update-exim4.conf.conf /etc/exim4/ RUN sudo update-exim4.conf EXPOSE 25 + +ENTRYPOINT ["docker-entrypoint.sh"] + CMD ["tini", "--", "exim", "-bd", "-v"] \ No newline at end of file diff --git a/mail/k8s/dockerfile/exim4/docker-entrypoint.sh b/mail/k8s/dockerfile/exim4/docker-entrypoint.sh new file mode 100755 index 0000000000000000000000000000000000000000..62fb039c695c34abbf1e027d697d95e0c355563e --- /dev/null +++ b/mail/k8s/dockerfile/exim4/docker-entrypoint.sh @@ -0,0 +1,6 @@ +#! /bin/bash +set -e +#Update exim4 service first +sudo update-exim4.conf + +exec $@ diff --git a/mail/k8s/dockerfile/web/mailman.py b/mail/k8s/dockerfile/web/mailman.py new file mode 100644 index 0000000000000000000000000000000000000000..385ee9b32a53897f5593a507433d8d67a6510bb3 --- /dev/null +++ b/mail/k8s/dockerfile/web/mailman.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2014-2019 by the Free Software Foundation, Inc. +# +# This file is part of HyperKitty. +# +# HyperKitty is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# HyperKitty is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# HyperKitty. If not, see . +# +# Author: Aurelien Bompard +# + +import json +from email import message_from_binary_file +from email.message import EmailMessage +from email.policy import default +from functools import wraps + +from django.conf import settings +from django.core.exceptions import SuspiciousOperation, ImproperlyConfigured +from django.urls import reverse +from django.http import HttpResponse +from django.utils.http import urlunquote +from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_POST +from django_mailman3.models import MailDomain +from urllib.parse import urljoin + +from hyperkitty.lib.incoming import add_to_list, DuplicateMessage +from hyperkitty.lib.utils import get_message_id_hash + +import logging +logger = logging.getLogger(__name__) + + +def key_and_ip_auth(func): + @wraps(func) + def _decorator(request, *args, **kwargs): + for attr in ('MAILMAN_ARCHIVER_KEY', 'MAILMAN_ARCHIVER_FROM'): + if not hasattr(settings, attr): + msg = "Missing setting: %s" % attr + logger.error(msg) + raise ImproperlyConfigured(msg) + # if (request.META.get("REMOTE_ADDR") not in + # settings.MAILMAN_ARCHIVER_FROM): + # logger.error( + # "Access to the archiving API endpoint was forbidden from " + # "IP {}, your MAILMAN_ARCHIVER_FROM setting may be " + # "misconfigured".format(request.META["REMOTE_ADDR"])) + # return HttpResponse( + # """Forbidden + #

Access is forbidden

""", + # content_type="text/html", status=403) + if request.GET.get("key") != settings.MAILMAN_ARCHIVER_KEY: + return HttpResponse( + """Auth required +

Authorization Required

""", + content_type="text/html", status=401) + return func(request, *args, **kwargs) + return _decorator + + +def _get_url(mlist_fqdn, msg_id=None): + # We can't use HttpRequest.build_absolute_uri() because the mailman API may + # be accessed via localhost. + # https://docs.djangoproject.com/en/dev/ref/request-response/#django.http.HttpRequest.build_absolute_uri + # https://docs.djangoproject.com/en/dev/ref/contrib/sites/#getting-the-current-domain-for-full-urls + # result = urljoin(public_url, urlunquote( + # reverse('hk_list_overview', args=[mlist_fqdn]))) + # We use the MailDomain association from django_mailman3 to find out the + # proper domain. + if msg_id is None: + url = reverse('hk_list_overview', args=[mlist_fqdn]) + else: + msg_hash = get_message_id_hash(msg_id.strip().strip("<>")) + url = reverse('hk_message_index', kwargs={ + "mlist_fqdn": mlist_fqdn, "message_id_hash": msg_hash}) + relative_url = urlunquote(url) + mail_domain = mlist_fqdn.split("@")[1] + try: + domain = MailDomain.objects.get( + mail_domain=mail_domain).site.domain + except MailDomain.DoesNotExist: + domain = mail_domain + return urljoin("https://%s" % domain, relative_url) + + +@key_and_ip_auth +def urls(request): + result = _get_url(request.GET["mlist"], request.GET.get("msgid")) + return HttpResponse(json.dumps({"url": result}), + content_type='application/javascript') + + +@require_POST +@key_and_ip_auth +@csrf_exempt +def archive(request): + mlist_fqdn = request.POST["mlist"] + if "message" not in request.FILES: + raise SuspiciousOperation + msg = message_from_binary_file( + request.FILES['message'], _class=EmailMessage, policy=default) + try: + add_to_list(mlist_fqdn, msg) + except DuplicateMessage as e: + logger.info("Duplicate email with message-id '%s'", e.args[0]) + except ValueError as e: + logger.warning("Could not archive the email with message-id '%s': %s", + msg.get("Message-Id", None), e) + return HttpResponse(json.dumps({"error": str(e)}), + content_type='application/javascript') + url = _get_url(mlist_fqdn, msg['Message-Id']) + logger.info("Archived message %s to %s", msg['Message-Id'], url) + return HttpResponse(json.dumps({"url": url}), + content_type='application/javascript') \ No newline at end of file diff --git a/mail/k8s/local_development/kind-config.yaml b/mail/k8s/local_development/kind-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c6c76541e3635191dbca417555df1bdf7018dd8c --- /dev/null +++ b/mail/k8s/local_development/kind-config.yaml @@ -0,0 +1,18 @@ +kind: Cluster +apiVersion: kind.sigs.k8s.io/v1alpha3 +# 1 control plane node and 3 workers +nodes: + # the control plane node config + - role: control-plane + # the three workers + - role: worker + # The node mapping used to export mailman website and exim4 service + extraPortMappings: + - containerPort: 30000 + hostPort: 8000 + - containerPort: 30080 + hostPort: 8080 + - containerPort: 30025 + hostPort: 25 + - role: worker + - role: worker diff --git a/mail/k8s/local_development/local_down.sh b/mail/k8s/local_development/local_down.sh new file mode 100755 index 0000000000000000000000000000000000000000..6d60284d17dc6400d60dabf8a80e665b8d2b3444 --- /dev/null +++ b/mail/k8s/local_development/local_down.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +export CURRENT_ROOT=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +if [[ "${CLUSTER_NAME}xxx" == "xxx" ]];then + CLUSTER_NAME="integration" +fi +export CLUSTER_CONTEXT="--name ${CLUSTER_NAME}" + +# clean up +function cleanup { + echo "Uninstall mailman services" + kubectl delete -f ${CURRENT_ROOT}/mailman-with-postgres.yaml + echo "Deleting helm services" + helm delete mailman-nfs +# echo "Uninstall nfs common utils into kind nodes" +# NODES=$(kind get nodes --name ${CLUSTER_NAME}) +# NODES_ARY=($NODES) +# for key in "${!NODES_ARY[@]}" +# do +# echo "starting to patch node ${NODES_ARY[$key]} and uninstall nfs-common utils" +# docker exec -it ${NODES_ARY[$key]} bin/bash -c "apt remove nfs-common -y" +# done + echo "Running kind: [kind delete cluster ${CLUSTER_CONTEXT}]" + kind delete cluster ${CLUSTER_CONTEXT} + +} + +export KUBECONFIG="$(kind get kubeconfig-path ${CLUSTER_CONTEXT})" + +cleanup diff --git a/mail/k8s/local_development/local_up.sh b/mail/k8s/local_development/local_up.sh new file mode 100755 index 0000000000000000000000000000000000000000..f66f5ea590e31e874a29b10c07946012ca5f9ef4 --- /dev/null +++ b/mail/k8s/local_development/local_up.sh @@ -0,0 +1,120 @@ +#!/bin/bash + +export CURRENT_ROOT=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +export LOG_LEVEL=3 +export CLEANUP_CLUSTER=${CLEANUP_CLUSTER:-0} + +if [[ "${CLUSTER_NAME}xxx" == "xxx" ]];then + CLUSTER_NAME="integration" +fi + +export CLUSTER_CONTEXT="--name ${CLUSTER_NAME}" + +export KIND_OPT=${KIND_OPT:=" --config ${CURRENT_ROOT}/kind-config.yaml"} + +# spin up cluster with kind command +function kind-up-cluster { + check-prerequisites + check-kind + echo "Running kind: [kind create cluster ${CLUSTER_CONTEXT} ${KIND_OPT}]" + kind create cluster ${CLUSTER_CONTEXT} ${KIND_OPT} +} + +# install helm if not installed +function install-helm { + echo "checking helm" + which helm >/dev/null 2>&1 + if [[ $? -ne 0 ]]; then + echo "Install helm via script" + HELM_TEMP_DIR=`mktemp -d` + curl https://raw.githubusercontent.com/helm/helm/master/scripts/get > ${HELM_TEMP_DIR}/get_helm.sh + #TODO: There are some issue with helm's latest version, remove '--version' when it get fixed. + chmod 700 ${HELM_TEMP_DIR}/get_helm.sh && ${HELM_TEMP_DIR}/get_helm.sh --version v2.13.0 + else + echo -n "found helm, version: " && helm version + fi +} + +# check if kubectl installed +function check-prerequisites { + echo "checking prerequisites" + which kubectl >/dev/null 2>&1 + if [[ $? -ne 0 ]]; then + echo "kubectl not installed, exiting." + exit 1 + else + echo -n "found kubectl, " && kubectl version --short --client + fi +} + +# check if kind installed +function check-kind { + echo "checking kind" + which kind >/dev/null 2>&1 + if [[ $? -ne 0 ]]; then + echo "installing kind ." + GO111MODULE="on" go get sigs.k8s.io/kind@v0.4.0 + else + echo -n "found kind, version: " && kind version + fi +} + + +function install-mailman-service { + echo "installing helm service" + kubectl create serviceaccount --namespace kube-system tiller + kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller + + install-helm + helm init --service-account tiller --kubeconfig ${KUBECONFIG} --wait + + + echo "Install nfs common utils into kind nodes" + NODES=$(kind get nodes --name ${CLUSTER_NAME}) + NODES_ARY=($NODES) + for key in "${!NODES_ARY[@]}" + do + echo "starting to patch node ${NODES_ARY[$key]} and install nfs-common utils" + docker exec -it ${NODES_ARY[$key]} bin/bash -c "apt update && apt install nfs-common -y" + done + + echo "Install nfs provisioner" + kubectl create clusterrolebinding default-cluster-rule --clusterrole=cluster-admin --serviceaccount=default:default + helm repo add cloudposse-incubator https://charts.cloudposse.com/incubator + helm install --name mailman-nfs cloudposse-incubator/nfs-provisioner --set persistence.storageClass=standard --set persistence.size=1Gi --wait + + + echo "Install mailman services" + kubectl apply -f ${CURRENT_ROOT}/mailman-with-postgres.yaml + +} + +# clean up +function cleanup { + echo "Running kind: [kind delete cluster ${CLUSTER_CONTEXT}]" + kind delete cluster ${CLUSTER_CONTEXT} + +} + +echo $* | grep -E -q "\-\-help|\-h" +if [[ $? -eq 0 ]]; then + echo "Customize the kind-cluster name: + + export CLUSTER_NAME= # default: integration + +Customize kind options other than --name: + + export KIND_OPT= +" + exit 0 +fi + +if [[ $CLEANUP_CLUSTER -eq 1 ]]; then + trap cleanup EXIT +fi + +kind-up-cluster + +export KUBECONFIG="$(kind get kubeconfig-path ${CLUSTER_CONTEXT})" + +install-mailman-service diff --git a/mail/k8s/local_development/mailman-with-postgres.yaml b/mail/k8s/local_development/mailman-with-postgres.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72425ecdf63c51c616b1d5f1475b498fa7acfbcc --- /dev/null +++ b/mail/k8s/local_development/mailman-with-postgres.yaml @@ -0,0 +1,390 @@ +#Deployment for mailman suit services + +# PVC (nfs share) used for mailman-core and mailman MTA service +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: core-and-mta-volume + annotations: + volume.beta.kubernetes.io/storage-class: "local-nfs" +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 200Mi + storageClassName: local-nfs + + +# Headless Service for mailman suit service +--- +apiVersion: v1 +kind: Service +metadata: + name: mail-suit-service + labels: + app: mail-suit-service +spec: + selector: + app: mail-suit-service + clusterIP: None + +# StatefulSet for mail core service +--- +kind: StatefulSet +apiVersion: apps/v1beta1 +metadata: + name: mailman-core + namespace: default + labels: + app: mail-suit-service +spec: + serviceName: mail-suit-service + replicas: 1 + selector: + matchLabels: + app: mail-suit-service + template: + metadata: + labels: + app: mail-suit-service + spec: + containers: + - name: mailman-core + image: maxking/mailman-core:0.2.3 + imagePullPolicy: "IfNotPresent" + volumeMounts: + - mountPath: /opt/mailman/ + name: mailman-core-volume + env: + - name: DATABASE_URL + value: postgres://mailman:mailmanpass@mailman-database-0.mail-suit-service.default.svc.cluster.local/mailmandb + - name: DATABASE_TYPE + value: postgres + - name: DATABASE_CLASS + value: mailman.database.postgresql.PostgreSQLDatabase + # NOTE: Please update the HYPERKITTY_API_KEY + - name: HYPERKITTY_API_KEY + value: someapikey + # NOTE: Please update the HYPERKITTY_URL + - name: HYPERKITTY_URL + value: http://mail-web-service.default.svc.cluster.local:8000/hyperkitty + # NOTE: We must use the cluster service here, since NodePort or NodeBalance will SNAT client address + - name: SMTP_HOST + value: mailman-exim4-0.mail-suit-service.default.svc.cluster.local + volumes: + - name: mailman-core-volume + persistentVolumeClaim: + claimName: core-and-mta-volume + +# StatefulSet for postgres database service +--- +kind: StatefulSet +apiVersion: apps/v1beta1 +metadata: + name: mailman-database + namespace: default + labels: + app: mail-suit-service +spec: + serviceName: mail-suit-service + replicas: 1 + selector: + matchLabels: + app: mail-suit-service + template: + metadata: + labels: + app: mail-suit-service + spec: + containers: + - name: mailman-database + image: postgres:9.6-alpine + imagePullPolicy: "IfNotPresent" + volumeMounts: + - mountPath: /var/lib/postgresql/data + name: mailman-database-volume + env: + - name: POSTGRES_DB + value: mailmandb + - name: POSTGRES_USER + value: mailman + - name: POSTGRES_PASSWORD + value: mailmanpass + #NOTE: Empty dir can't be used in a production dir. Please upgrade it before using. + volumes: + - name: mailman-database-volume + emptyDir: {} + +# configmap for mail exim4 service, these three files are directly read from exim config folder +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mailman-exim4-configmap + namespace: default +data: + 25_mm3_macros: | + # Place this file at + # /etc/exim4/conf.d/main/25_mm3_macros + + domainlist mm3_domains=tommylike.me + MM3_LMTP_HOST=mailman-core-0.mail-suit-service.default.svc.cluster.local + MM3_LMTP_PORT=8024 + # According to the configuration of: https://mailman.readthedocs.io/en/release-3.0/src/mailman/docs/MTA.html + # We need updating this, for the purpose of delivering emails to the mailman + MM3_HOME=/opt/mailman/var + + ################################################################ + # The configuration below is boilerplate: + # you should not need to change it. + + # The path to the list receipt (used as the required file when + # matching list addresses) + MM3_LISTCHK=MM3_HOME/lists/${local_part}.${domain} + + 55_mm3_transport: | + # Place this file at + # /etc/exim4/conf.d/transport/55_mm3_transport + + mailman3_transport: + debug_print = "Email for mailman" + driver = smtp + protocol = lmtp + allow_localhost + hosts = MM3_LMTP_HOST + port = MM3_LMTP_PORT + rcpt_include_affixes = true + + 455_mm3_router: | + # Place this file at + # /etc/exim4/conf.d/router/455_mm3_router + + mailman3_router: + driver = accept + domains = +mm3_domains + require_files = MM3_LISTCHK + local_part_suffix_optional + local_part_suffix = -admin : \ + -bounces : -bounces+* : \ + -confirm : -confirm+* : \ + -join : -leave : \ + -owner : -request : \ + -subscribe : -unsubscribe + transport = mailman3_transport + + update-exim4-conf.conf: | + dc_eximconfig_configtype='internet' + dc_other_hostnames='tommylike.me;' + dc_local_interfaces='' + dc_readhost='' + # NOTE: wildchart is used here, but it's not safe at all. + dc_relay_domains='*' + dc_minimaldns='false' + # NOTE: wildchart is used here, but it's not safe at all. + dc_relay_nets='*' + dc_smarthost='' + CFILEMODE='644' + dc_use_split_config='true' + dc_hide_mailname='' + dc_mailname_in_oh='true' + dc_localdelivery='mail_spool' + +#Service for exim4 pods to export Port 25 via NodePort +--- +apiVersion: v1 +kind: Service +metadata: + name: mail-exim4-service + labels: + app: mail-suit-service +spec: + type: NodePort + ports: + - port: 25 + name: exim4-port + nodePort: 30025 + selector: + component: mail-exim4-service + + +# StatefulSet for exim4 services +--- +kind: StatefulSet +apiVersion: apps/v1beta1 +metadata: + name: mailman-exim4 + namespace: default + labels: + app: mail-suit-service + component: mail-exim4-service +spec: + serviceName: mail-suit-service + replicas: 1 + selector: + matchLabels: + app: mail-suit-service + component: mail-exim4-service + template: + metadata: + labels: + app: mail-suit-service + component: mail-exim4-service + spec: + containers: + - name: mailman-exim4 + #NOTE: This image is directly built from our dockerfile located in exim4 folder + image: tommylike/mailman-exim4:0.0.1 + imagePullPolicy: "IfNotPresent" + volumeMounts: + - mountPath: /etc/exim4/conf.d/main/25_mm3_macros + name: mailman-exim4-configmap-volume + subPath: 25_mm3_macros + - mountPath: /etc/exim4/conf.d/transport/55_mm3_transport + name: mailman-exim4-configmap-volume + subPath: 55_mm3_transport + - mountPath: /etc/exim4/conf.d/router/455_mm3_router + name: mailman-exim4-configmap-volume + subPath: 455_mm3_router + - mountPath: /etc/exim4/update-exim4.conf.conf + name: mailman-exim4-configmap-volume + subPath: update-exim4-conf.conf + - mountPath: /opt/mailman/ + name: mailman-exim4-volume + #NOTE: Empty dir can't be used in a production dir. Please upgrade it before using. + volumes: + - name: mailman-exim4-configmap-volume + configMap: + name: mailman-exim4-configmap + - name: mailman-exim4-volume + persistentVolumeClaim: + claimName: core-and-mta-volume + + +# Service for mail web service to export NodePort +--- +apiVersion: v1 +kind: Service +metadata: + name: mail-web-service + labels: + app: mail-suit-service +spec: + type: NodePort + ports: + - port: 8080 + name: website-port-uwsgi + nodePort: 30080 + - port: 8000 + name: website-port-http + nodePort: 30000 + selector: + component: mail-web-service + +# configmap for mail web service +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mailman-web-configmap + namespace: default +data: + settings_local.py: | + import os + import socket + + DEBUG = True + + EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' + #NOTE: this is the MTA host, we need to update it. + EMAIL_HOST = 'mailman-exim4-0.mail-suit-service.default.svc.cluster.local' + EMAIL_PORT = 25 + + MAILMAN_ARCHIVER_FROM = socket.gethostbyname(os.environ.get('MAILMAN_HOST_IP')) + + ALLOWED_HOSTS = [ + "localhost", # Archiving API from Mailman, keep it. + # Add here all production URLs you may have. + "mailman-database-0.mail-suit-service.default.svc.cluster.local", + "mailman-core-0.mail-suit-service.default.svc.cluster.local", + "mailman-web-0.mail-suit-service.default.svc.cluster.local", + "mail-web-service.default.svc.cluster.local", + #NOTE: This is the public ip address of the served host + "159.138.26.163", + "tommylike.me", + os.environ.get('SERVE_FROM_DOMAIN'), + os.environ.get('DJANGO_ALLOWED_HOSTS'), + ] + +# Deployment for mail web service +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: mailman-web + namespace: default + labels: + component: mail-web-service + app: mail-suit-service +spec: + replicas: 2 + selector: + matchLabels: + component: mail-web-service + app: mail-suit-service + template: + metadata: + labels: + component: mail-web-service + app: mail-suit-service + spec: + hostname: mailman-web + containers: + - name: mailman-web + # We modified the mail-web image to add static folder. + image: tommylike/mailman-web:0.2.3 + imagePullPolicy: "IfNotPresent" + volumeMounts: + - mountPath: /opt/mailman-web-config + name: mailman-web-configmap-volume + - mountPath: /opt/mailman-web-data + name: mailman-web-volume + env: + - name: DATABASE_TYPE + value: postgres + - name: DATABASE_URL + value: postgres://mailman:mailmanpass@mailman-database-0.mail-suit-service.default.svc.cluster.local/mailmandb + - name: HYPERKITTY_API_KEY + # NOTE: Please update the HYPERKITTY_API_KEY + value: someapikey + - name: SECRET_KEY + # NOTE: Please update the SECRET_KEY + value: community_key + - name: UWSGI_STATIC_MAP + # NOTE: This static folder has been added into docker image located at /opt/mailman-web/static + value: /static=/opt/mailman-web-data/static + - name: MAILMAN_REST_URL + value: http://mailman-core-0.mail-suit-service.default.svc.cluster.local:8001 + - name: MAILMAN_HOST_IP + value: mailman-core-0.mail-suit-service.default.svc.cluster.local + - name: MAILMAN_ADMIN_USER + value: tommylike + - name: MAILMAN_ADMIN_EMAIL + value: tommylikehu@gmail.com + #NOTE: this is the domain name that mailman web will serve + - name: SERVE_FROM_DOMAIN + value: tommylike.me + #NOTE: Command is overwritten for the purpose of copy config file into dest folder + command: + - /bin/sh + - -c + - | + cp /opt/mailman-web-config/settings_local.py /opt/mailman-web-data; + docker-entrypoint.sh uwsgi --ini /opt/mailman-web/uwsgi.ini; + #NOTE: Empty dir can't be used in a production dir. Please upgrade it before using. + volumes: + - name: mailman-web-volume + emptyDir: {} + - name: mailman-web-configmap-volume + configMap: + name: mailman-web-configmap \ No newline at end of file