Tuning rabbitmq for hiload systems

Tune limits

/etc/security/limits.conf
* - nofile 999999
* soft nofile 1000000
* hard nofile 1000000
root soft nofile 1000000
root hard nofile 1000000
root soft nproc unlimited
root hard nproc unlimited

ulimit -a

Tune sysctl

vm.swappiness=1
vm.max_map_count=262144
net.ipv4.tcp_wmem = 4096 65536 16777216
net.ipv4.tcp_wmem = 4096 262144 4194304
net.ipv4.tcp_window_scaling = 1
net.ipv4.tcp_tw_reuse  =  0
#net.ipv4.tcp_tw_recycle  =  0
net.ipv4.tcp_timestamps = 1
net.ipv4.tcp_syncookies = 0
net.ipv4.tcp_synack_retries = 1
net.ipv4.tcp_sack = 1
net.ipv4.tcp_rmem = 4096 262144 4194304
net.ipv4.tcp_rfc1337 = 1
net.ipv4.tcp_orphan_retries = 0
net.ipv4.tcp_no_metrics_save = 1
net.ipv4.tcp_moderate_rcvbuf = 1
net.ipv4.tcp_mem = 50576   64768   98152
net.ipv4.tcp_max_syn_backlog = 65536
net.ipv4.tcp_max_orphans = 65536
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 20
net.ipv4.tcp_keepalive_intvl = 60
net.ipv4.tcp_fin_timeout = 10
net.ipv4.tcp_congestion_control = htcp
net.ipv4.tcp_adv_win_scale = 2
net.ipv4.route.flush = 1
net.ipv4.ip_local_port_range = 10240 65535
net.ipv4.ip_forward = 1
net.ipv4.icmp_ignore_bogus_error_responses = 1
net.ipv4.icmp_echo_ignore_broadcasts = 1
net.ipv4.icmp_echo_ignore_all = 1
net.ipv4.conf.lo.rp_filter = 1
net.ipv4.conf.lo.accept_source_route = 0
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.default.accept_source_route = 0
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.all.secure_redirects = 0
net.ipv4.conf.all.rp_filter = 1
net.ipv4.conf.all.accept_source_route = 0
net.ipv4.conf.all.accept_redirects = 0
kernel.sem = 350 358400 64 1024
fs.inotify.max_user_watches = 67108864
fs.file-max = 518144
net.core.somaxconn = 15000
#net.core.netdev_max_backlog = 1000
net.core.rmem_default = 65536
net.core.wmem_default = 65536
net.core.rmem_max = 67108864
net.core.wmem_max = 67108864
kernel.pid_max = 4194303
net/bridge/bridge-nf-call-ip6tables = 1
net/bridge/bridge-nf-call-iptables = 1
net/bridge/bridge-nf-call-arptables = 1

sysctl -p

Create dir

mkdir -p /etc/systemd/system/rabbitmq-server.service.d

add file

nano /etc/systemd/system/rabbitmq-server.service.d/override.conf

add line

[Service]
LimitNOFILE=65536

systemctl daemon-reload

Edit rabbitmq config

nano /etc/rabbitmq/rabbitmq.config

[
  {kernel, [
    {inet_default_connect_options, [{nodelay, true}]},
    {inet_default_listen_options,  [{nodelay, true}]}
  ]},
    {rabbit, [
     {loopback_users, []},
       {tcp_listen_options, [
                          {backlog,       4096},
                          {nodelay,       true},
                          {linger,        {true,0}},
                          {exit_on_close, false},
                          {sndbuf,        196608},
                          {recbuf,        196608}
                         ]}
]}
].

Restart rabbitmq

/etc/init.d/rabbitmq-server restart

Check

rabbitmqctl status | grep -A 4 file_descriptors

Output

{file_descriptors,
     [{total_limit,99900},
      {total_used,3},
      {sockets_limit,89908},
      {sockets_used,1}]},