Beberapa hari yang lalu mail server kantor ter blacklist gmail, sehingga mail dari domain kami tidak bisa mengirim ke gmail karena di anggap mengirim spam , setelah saya lakukan cek , beberapa langkah untuk menangani nya adalah menghentikan spam ,membersihkan ip mail dari blacklist domain  , pemesangan SPF, DKIM dan PTR record.
pada tahap kali ini akan coba saya sedikit jelaskan , cara untuk penangkal spam dan brute force pada mail server dalam kasus saya di ubuntu server , beikut contoh log bruteforce
.  sebelum nya kita harus Mengecek dan Mengatasi Account yang Terindikasi Spam pada Zimbra Mail Server , saya mengikuti tutorial di http://vavai.com/2015/04/11/tips-mengecek-dan-mengatasi-account-yang-terindikasi-spam-pada-zimbra-mail-server/   ,

fail2ban sendiri adaalh aplikasi yang menggunakan IPTables untuk melakukan proses banned terhadap kegagalan akses. Fungsinya bukan hanya sebatas aplikasi SSH seperti yang dilakukan oleh aplikasi Denyhosts, melainkan juga terhadap semua aplikasi lain yang menggunakan port TCP, seperti SMTP, POP3, IMAP, FTP, Apache dan lain-lain.
berikut langkah – langkah install dan integrasi dengan zimbra mail server,

Install Fail2ban

root@mail:~# apt-get install fail2ban

Konfigurasi

buka file jail.conf

root@mail:~# cd /etc/fail2ban/jail.conf

Silahkan Copas dan sesuaikan

# Fail2Ban configuration file
#
# Author: Cyril Jaquier
#
# $Revision: 747 $
## The DEFAULT allows a global definition of the options. They can be overridden
# in each jail afterwards.
[DEFAULT]
# "ignoreip" can be an IP address, a CIDR mask or a DNS host. Fail2ban will not
# ban a host which matches an address in this list. Several addresses can be
# defined using space separator.
ignoreip = 127.0.0.1/8 ip public server/28
# "bantime" is the number of seconds that a host is banned.
bantime = 1800
# A host is banned if it has generated "maxretry" during the last "findtime"
# seconds.
findtime = 600

# "maxretry" is the number of failures before a host get banned.
maxretry = 3

# "backend" specifies the backend used to get files modification. Available
# options are "gamin", "polling" and "auto". This option can be overridden in
# each jail too (use "gamin" for a jail and "polling" for another).
#
# gamin: requires Gamin (a file alteration monitor) to be installed. If Gamin
# is not installed, Fail2ban will use polling.
# polling: uses a polling algorithm which does not require external libraries.
# auto: will choose Gamin if available and polling otherwise.
backend = auto

# This jail corresponds to the standard configuration in Fail2ban 0.6.
# The mail-whois action send a notification e-mail with a whois request
# in the body.

[ssh-iptables]

enabled = true
filter = sshd
action = iptables[name=SSH, port=ssh, protocol=tcp]
sendmail-whois[name=SSH, dest=aldykustyandi@alshifacharity.com, sender=fail2ban@alshifacharity.com]
logpath = /var/log/auth.log
maxretry = 4

# This jail forces the backend to "polling".

[sasl-iptables]

enabled = false
filter = sasl
backend = polling
action = iptables[name=sasl, port=smtp, protocol=tcp]
sendmail-whois[name=sasl, dest=aldykustyandi@alshifacharity.com]
logpath = /var/log/zimbra.log

# Here we use TCP-Wrappers instead of Netfilter/Iptables. "ignoreregex" is
# used to avoid banning the user "myuser".

[ssh-tcpwrapper]

enabled = false
filter = sshd
action = hostsdeny
sendmail-whois[name=SSH, dest=aldykustyandi@alshifacharity.com]
ignoreregex = for myuser from
logpath = /var/log/messages

# This jail uses ipfw, the standard firewall on FreeBSD. The "ignoreip"
# option is overridden in this jail. Moreover, the action "mail-whois" defines
# the variable "name" which contains a comma using "". The characters '' are
# valid too.

[zimbra-account]
enabled = true
filter = zimbra
action = iptables-allports[name=zimbra-account]
sendmail[name=zimbra-account, dest=aldykustyandi@alshifacharity.com]
logpath = /opt/zimbra/log/mailbox.log
bantime = 1800
maxretry = 5

[zimbra-audit]
enabled = true
filter = zimbra
action = iptables-allports[name=zimbra-audit]
sendmail[name=Zimbra-audit, dest=aldykustyandi@alshifacharity.com]
logpath = /opt/zimbra/log/audit.log
bantime = 1800
maxretry = 5

[zimbra-recipient]
enabled = true
filter = zimbra
action = iptables-allports[name=zimbra-recipient]
sendmail[name=Zimbra-recipient, dest=aldykustyandi@alshifacharity.com]
logpath = /var/log/zimbra.log
#findtime = 604800
bantime = 172800
maxretry = 5

[postfix]
enabled = true
filter = postfix
action = iptables-multiport[name=postfix, port=smtp, protocol=tcp]
sendmail-buffered[name=Postfix, dest=aldykustyandi@alshifacharity.com]
logpath = /var/log/zimbra.log
bantime = -1
maxretry = 5

#[sasl]
#enabled = true
#port = smtp,ssmtp,imap2,imap3,imaps,pop3,pop3s
#filter = sasl
# You might consider monitoring /var/log/warn.log instead
# if you are running postfix. See http://bugs.debian.org/507990
#logpath = /var/log/zimbra.log

buat file zimbra.conf di /etc/fail2ban/filter.d/

root@mail:~# pico /etc/fail2ban/filter.d/zimbra.conf

isi file seperti berikut

# Fail2Ban configuration file
#
# Author:
#
# $Revision: 1 $
#

[Definition]

# Option:  failregex
# Notes.:  regex to match the password failures messages in the logfile. The
#          host must be matched by a group named "host". The tag "" can
#          be used for standard IP/hostname matching and is only an alias for
#          (?:::f{4,6}:)?(?P[\w\-.^_]+)
# Values:  TEXT
#
failregex = \[ip=;\] account - authentication failed for .* \(no such account\)$
                        \[ip=;\] security - cmd=Auth; .* error=authentication failed for .*, invalid password;$
                        ;oip=;.* security - cmd=Auth; .* protocol=soap; error=authentication failed for .* invalid password;$
                        \[oip=;.* SoapEngine - handler exception: authentication failed for .*, account not found$
                        WARN .*;ip=;ua=ZimbraWebClient .* security - cmd=AdminAuth; .* error=authentication failed for .*;$
                        NOQUEUE: reject: RCPT from .*\[\]: 550 5.1.1 .*: Recipient address rejected:

# .*\[ip=;\] .* - authentication failed for .* \(invalid password\)
#
# Option:  ignoreregex
# Notes.:  regex to ignore. If this regex matches, the line is ignored.
# Values:  TEXT
#
ignoreregex =

Reload Fail2ban

root@mail:~# fail2ban-client reload

cek status

root@mail:~# fail2ban-client status
Status
|- Number of jail:	5
`- Jail list:		postfix, zimbra-audit, zimbra-recipient, zimbra-account, ssh-iptables

contoh ip yang ter blok

root@mail:~# iptables -L
Chain INPUT (policy ACCEPT)
target     prot opt source               destination         
fail2ban-zimbra-audit  tcp  --  anywhere             anywhere            
fail2ban-zimbra-account  tcp  --  anywhere             anywhere            
fail2ban-SSH  tcp  --  anywhere             anywhere            tcp dpt:ssh 
fail2ban-postfix  tcp  --  anywhere             anywhere            multiport dports smtp 
fail2ban-zimbra-recipient  tcp  --  anywhere             anywhere            

Chain FORWARD (policy ACCEPT)
target     prot opt source               destination         

Chain OUTPUT (policy ACCEPT)
target     prot opt source               destination         

Chain fail2ban-SSH (1 references)
target     prot opt source               destination         
DROP       all  --  221.194.44.227       anywhere            
DROP       all  --  221.229.172.35       anywhere            
DROP       all  --  91.201.236.50        anywhere            
DROP       all  --  221.194.44.218       anywhere            
DROP       all  --  121.18.238.19        anywhere            
RETURN     all  --  anywhere             anywhere            

Chain fail2ban-postfix (1 references)
target     prot opt source               destination         
RETURN     all  --  anywhere             anywhere            

Chain fail2ban-zimbra-account (1 references)
target     prot opt source               destination         
RETURN     all  --  anywhere             anywhere            

Chain fail2ban-zimbra-audit (1 references)
target     prot opt source               destination         
RETURN     all  --  anywhere             anywhere            

Chain fail2ban-zimbra-recipient (1 references)
target     prot opt source               destination         
DROP       all  --  securemail-pl-mx18.synaq.com  anywhere            
RETURN     all  --  anywhere             anywhere            
root@mail:~#

contoh log pada saat banned dan unbanned ip

root@mail:~# tail -f /var/log/fail2ban.log
2016-08-18 11:20:58,249 fail2ban.actions: WARNING [ssh-iptables] Unban 103.207.37.31
2016-08-18 11:24:02,481 fail2ban.actions: WARNING [ssh-iptables] Ban 91.224.160.106
2016-08-18 11:38:24,506 fail2ban.actions: WARNING [ssh-iptables] Unban 114.80.116.204
2016-08-18 11:41:29,738 fail2ban.actions: WARNING [ssh-iptables] Ban 121.18.238.19
2016-08-18 11:42:05,785 fail2ban.actions: WARNING [ssh-iptables] Ban 221.194.44.218
2016-08-18 11:42:29,845 fail2ban.actions: WARNING [ssh-iptables] Ban 91.201.236.50
2016-08-18 11:47:44,240 fail2ban.actions: WARNING [ssh-iptables] Unban 221.229.172.35
2016-08-18 11:49:13,355 fail2ban.actions: WARNING [ssh-iptables] Ban 221.229.172.35
2016-08-18 11:54:02,753 fail2ban.actions: WARNING [ssh-iptables] Unban 91.224.160.106
2016-08-18 11:58:02,188 fail2ban.actions: WARNING [ssh-iptables] Ban 221.194.44.227

referensi : http://www.excellent.co.id/product-services/zimbra/implementasi-fail2ban-untuk-meningkatkan-performa-keamanan-zimbra-mail-server-2/