I’ve been evaluating routing-test for the purpose of implementing OSPF and BGP on our core networks. I was curious how many prefixes an RB112 could carry with BGP before exhausting the free memory. Here’s what I found.
The network layout follows…
-----[RB112, 1B]------[RB112, 2B]-----
[3com 5232]----| |------[x86 MikroTik, 1C]
-----[RB112, 1A]------[RB112, 2A]-----
1A and 2B are RR clients of 3com, with inter-client relection disabled on 3com.
2A and 2B are RR clients of 1A and 1B, respectively.
1C is a route reflector client of 2A and 2B.
The links between 1A->2A, and 1B->2B, are running 5.8GHz nstreme on NMP-8602s.
All nodes are running OSPF for the IGP.
Each of the RB112s had approximately 4MB of free memory after startup. Loading 4000 prefixes on 1C using “/routing bgp network add” caused the free memory on each RB112 to drop by about 512KB-1MB, or, a total of ~3MB free. In an attempt to make this a somewhat more “real life” test, I wrote a perl script (linked below) to generate random route flaps on 1C, that would propogate through the rest of the network.
At a rate of about 1 flap per second with 4000+ prefixes, each of the RB112s loses about 20-30KB of free memory per second. When they reach about 1-2MB of free memory, the become unresponsive and have to be reset.
The overall conclusion is that running BGP on RB112s is a bad idea. Particularly because of the limited amount of RAM. Even running ~100 prefixes over BGP, the free memory can become exhausted quickly forcing the router to be reset.
As soon as someone has some RB532A boards in stock, I’ll be testing the same configuration on that hardware as well. 20-30KB memory loss per route flap seems like a lot, but there’s no way to tell if this will eventually even out given the small amout of memory that was available during testing.
MT, can you check for the possibility of a memory leak from route flaps in 2.9.33 routing-test? Script and configuration attached below. Comments/criticisms welcome and appreciated.
-Kristian
BGP route flap generator:
http://users.fire2wire.com/~khoff/mikrotik/bgp_burn.txt
1A Cofnig:
/ ip address
add address=10.0.1.101/24 network=10.0.1.0 broadcast=10.0.1.255 \
interface=ether1 comment="" disabled=no
add address=10.0.2.1/30 network=10.0.2.0 broadcast=10.0.2.3 interface=wlan1 \
comment="" disabled=no
add address=10.16.0.2/32 network=10.16.0.2 broadcast=10.16.0.2 \
interface=bridge1 comment="" disabled=no
/ routing ospf
set router-id=10.16.0.2 distribute-default=never redistribute-connected=no \
redistribute-static=no redistribute-rip=no redistribute-bgp=no \
metric-default=1 metric-connected=20 metric-static=20 metric-rip=20 \
metric-bgp=20
/ routing ospf area
set backbone area-id=0.0.0.0 type=default translator-role=translate-candidate \
authentication=none disabled=no
add name="area10" area-id=0.0.0.10 type=default \
translator-role=translate-never authentication=none default-cost=1 \
disabled=no
/ routing ospf interface
set FIXME interface=ether1 cost=10 priority=1 authentication-key="" \
network-type=broadcast retransmit-interval=5s transmit-delay=1s \
hello-interval=10s dead-interval=40s
set FIXME interface=wlan1 cost=51 priority=1 authentication-key="" \
network-type=broadcast retransmit-interval=5s transmit-delay=1s \
hello-interval=10s dead-interval=40s
/ routing ospf network
add network=10.0.1.0/24 area=backbone disabled=no
add network=10.0.2.0/24 area=backbone disabled=no
add network=10.16.0.2/32 area=backbone disabled=no
add network=10.0.0.0/24 area=backbone disabled=yes
/ routing bgp instance
set default name="default" as=65500 router-id=10.16.0.2 \
redistribute-connected=no redistribute-static=no redistribute-rip=no \
redistribute-ospf=no redistribute-other-bgp=no out-filter="" \
client-to-client-reflection=yes ignore-as-path-len=no comment="" \
disabled=no
/ routing bgp peer
add name="3com" instance=default remote-address=10.0.1.1 remote-as=65500 \
tcp-md5-key="" nexthop-choice=default multihop=no route-reflect=no \
hold-time=3m ttl=1 in-filter="" out-filter="" comment="" disabled=no
add name="1B" instance=default remote-address=10.0.1.102 remote-as=65500 \
tcp-md5-key="" nexthop-choice=default multihop=no route-reflect=no \
hold-time=3m ttl=1 in-filter="" out-filter="" comment="" disabled=yes
add name="2A" instance=default remote-address=10.0.2.2 remote-as=65500 \
tcp-md5-key="" nexthop-choice=default multihop=no route-reflect=yes \
hold-time=3m ttl=1 in-filter="" out-filter="" comment="" disabled=no
/ routing rip
set distribute-default=never redistribute-static=no redistribute-connected=no \
redistribute-ospf=no redistribute-bgp=no metric-default=1 metric-static=1 \
metric-connected=1 metric-ospf=1 metric-bgp=1 update-timer=30s \
timeout-timer=3m garbage-timer=2m
/ routing rip interface
add interface=all receive=v2 send=v2 authentication=none authentication-key="" \
key-chain="" in-filter="" out-filter="" disabled=no
2A Config:
/ ip address
add address=10.0.2.2/30 network=10.0.2.0 broadcast=10.0.2.3 interface=wlan1 \
comment="" disabled=no
add address=10.16.0.4/32 network=10.16.0.4 broadcast=10.16.0.4 \
interface=bridge1 comment="" disabled=no
add address=10.0.2.130/29 network=10.0.2.128 broadcast=10.0.2.135 \
interface=ether1 comment="" disabled=no
/ routing ospf
set router-id=10.16.0.4 distribute-default=never redistribute-connected=no \
redistribute-static=no redistribute-rip=no redistribute-bgp=no \
metric-default=1 metric-connected=20 metric-static=20 metric-rip=20 \
metric-bgp=20
/ routing ospf area
set backbone area-id=0.0.0.0 type=default translator-role=translate-candidate \
authentication=none disabled=no
add name="area10" area-id=0.0.0.10 type=default \
translator-role=translate-never authentication=none default-cost=1 \
disabled=no
/ routing ospf interface
set FIXME interface=wlan1 cost=50 priority=1 authentication-key="" \
network-type=broadcast retransmit-interval=5s transmit-delay=1s \
hello-interval=10s dead-interval=40s
set FIXME interface=ether1 cost=10 priority=1 authentication-key="" \
network-type=broadcast retransmit-interval=5s transmit-delay=1s \
hello-interval=10s dead-interval=40s
/ routing ospf network
add network=10.0.1.0/24 area=backbone disabled=no
add network=10.0.2.0/24 area=backbone disabled=no
add network=10.16.0.4/32 area=backbone disabled=no
add network=10.0.0.0/24 area=backbone disabled=yes
/ routing bgp instance
set default name="default" as=65500 router-id=10.16.0.4 \
redistribute-connected=no redistribute-static=no redistribute-rip=no \
redistribute-ospf=no redistribute-other-bgp=no out-filter="" \
client-to-client-reflection=yes ignore-as-path-len=no comment="" \
disabled=no
/ routing bgp peer
add name="2B" instance=default remote-address=10.0.2.131 remote-as=65500 \
tcp-md5-key="" nexthop-choice=default multihop=no route-reflect=no \
hold-time=3m ttl=1 in-filter="" out-filter="" comment="" disabled=yes
add name="1C" instance=default remote-address=10.0.2.129 remote-as=65500 \
tcp-md5-key="" nexthop-choice=default multihop=no route-reflect=yes \
hold-time=3m ttl=1 in-filter="" out-filter="" comment="" disabled=no
add name="1A" instance=default remote-address=10.0.2.1 remote-as=65500 \
tcp-md5-key="" nexthop-choice=default multihop=no route-reflect=no \
hold-time=3m ttl=1 in-filter="" out-filter="" comment="" disabled=no
/ routing rip
set distribute-default=never redistribute-static=no redistribute-connected=no \
redistribute-ospf=no redistribute-bgp=no metric-default=1 metric-static=1 \
metric-connected=1 metric-ospf=1 metric-bgp=1 update-timer=30s \
timeout-timer=3m garbage-timer=2m
/ routing rip interface
add interface=all receive=v2 send=v2 authentication=none authentication-key="" \
key-chain="" in-filter="" out-filter="" disabled=no
1B Config:
/ ip address
add address=10.0.1.102/24 network=10.0.1.0 broadcast=10.0.1.255 \
interface=ether1 comment="" disabled=no
add address=10.0.2.5/30 network=10.0.2.4 broadcast=10.0.2.7 interface=wlan1 \
comment="" disabled=no
add address=10.16.0.3/32 network=10.16.0.3 broadcast=10.16.0.3 \
interface=bridge1 comment="" disabled=no
/ routing ospf
set router-id=10.16.0.3 distribute-default=never redistribute-connected=no \
redistribute-static=no redistribute-rip=no redistribute-bgp=no \
metric-default=1 metric-connected=20 metric-static=20 metric-rip=20 \
metric-bgp=20
/ routing ospf area
set backbone area-id=0.0.0.0 type=default translator-role=translate-candidate \
authentication=none disabled=no
add name="area10" area-id=0.0.0.10 type=default \
translator-role=translate-never authentication=none default-cost=1 \
disabled=no
/ routing ospf interface
set FIXME interface=ether1 cost=10 priority=1 authentication-key="" \
network-type=broadcast retransmit-interval=5s transmit-delay=1s \
hello-interval=10s dead-interval=40s
set FIXME interface=wlan1 cost=50 priority=1 authentication-key="" \
network-type=broadcast retransmit-interval=5s transmit-delay=1s \
hello-interval=10s dead-interval=40s
/ routing ospf network
add network=10.0.1.0/24 area=backbone disabled=no
add network=10.0.2.0/24 area=backbone disabled=no
add network=10.16.0.3/32 area=backbone disabled=no
add network=10.0.0.0/24 area=backbone disabled=yes
/ routing bgp instance
set default name="default" as=65500 router-id=10.16.0.3 \
redistribute-connected=no redistribute-static=no redistribute-rip=no \
redistribute-ospf=no redistribute-other-bgp=no out-filter="" \
client-to-client-reflection=yes ignore-as-path-len=no comment="" \
disabled=no
/ routing bgp peer
add name="2B" instance=default remote-address=10.0.2.6 remote-as=65500 \
tcp-md5-key="" nexthop-choice=default multihop=no route-reflect=yes \
hold-time=3m ttl=1 in-filter="" out-filter="" comment="" disabled=no
add name="1A" instance=default remote-address=10.0.1.101 remote-as=65500 \
tcp-md5-key="" nexthop-choice=default multihop=no route-reflect=no \
hold-time=3m ttl=1 in-filter="" out-filter="" comment="" disabled=yes
add name="3com" instance=default remote-address=10.0.1.1 remote-as=65500 \
tcp-md5-key="" nexthop-choice=default multihop=no route-reflect=no \
hold-time=3m ttl=1 in-filter="" out-filter="" comment="" disabled=no
/ routing rip
set distribute-default=never redistribute-static=no redistribute-connected=no \
redistribute-ospf=no redistribute-bgp=no metric-default=1 metric-static=1 \
metric-connected=1 metric-ospf=1 metric-bgp=1 update-timer=30s \
timeout-timer=3m garbage-timer=2m
/ routing rip interface
add interface=all receive=v2 send=v2 authentication=none authentication-key="" \
key-chain="" in-filter="" out-filter="" disabled=no
2B Config:
/ ip address
add address=10.0.2.6/30 network=10.0.2.4 broadcast=10.0.2.7 interface=wlan1 \
comment="" disabled=no
add address=10.16.0.5/32 network=10.16.0.5 broadcast=10.16.0.5 \
interface=bridge1 comment="" disabled=no
add address=10.0.2.131/29 network=10.0.2.128 broadcast=10.0.2.135 \
interface=ether1 comment="" disabled=no
/ routing ospf
set router-id=10.16.0.5 distribute-default=never redistribute-connected=no \
redistribute-static=no redistribute-rip=no redistribute-bgp=no \
metric-default=1 metric-connected=20 metric-static=20 metric-rip=20 \
metric-bgp=20
/ routing ospf area
set backbone area-id=0.0.0.0 type=default translator-role=translate-candidate \
authentication=none disabled=no
add name="area10" area-id=0.0.0.10 type=default \
translator-role=translate-never authentication=none default-cost=1 \
disabled=no
/ routing ospf interface
set FIXME interface=wlan1 cost=51 priority=1 authentication-key="" \
network-type=broadcast retransmit-interval=5s transmit-delay=1s \
hello-interval=10s dead-interval=40s
set FIXME interface=ether1 cost=10 priority=1 authentication-key="" \
network-type=broadcast retransmit-interval=5s transmit-delay=1s \
hello-interval=10s dead-interval=40s
/ routing ospf network
add network=10.0.1.0/24 area=backbone disabled=no
add network=10.16.0.5/32 area=backbone disabled=no
add network=10.0.2.0/24 area=backbone disabled=no
add network=10.0.0.0/24 area=backbone disabled=yes
/ routing bgp instance
set default name="default" as=65500 router-id=10.16.0.5 \
redistribute-connected=no redistribute-static=no redistribute-rip=no \
redistribute-ospf=no redistribute-other-bgp=no out-filter="" \
client-to-client-reflection=yes ignore-as-path-len=no comment="" \
disabled=no
/ routing bgp peer
add name="2A" instance=default remote-address=10.0.2.130 remote-as=65500 \
tcp-md5-key="" nexthop-choice=default multihop=no route-reflect=no \
hold-time=3m ttl=1 in-filter="" out-filter="" comment="" disabled=yes
add name="1C" instance=default remote-address=10.0.2.129 remote-as=65500 \
tcp-md5-key="" nexthop-choice=default multihop=no route-reflect=yes \
hold-time=3m ttl=1 in-filter="" out-filter="" comment="" disabled=no
add name="1B" instance=default remote-address=10.0.2.5 remote-as=65500 \
tcp-md5-key="" nexthop-choice=default multihop=no route-reflect=no \
hold-time=3m ttl=1 in-filter="" out-filter="" comment="" disabled=no
/ routing rip
set distribute-default=never redistribute-static=no redistribute-connected=no \
redistribute-ospf=no redistribute-bgp=no metric-default=1 metric-static=1 \
metric-connected=1 metric-ospf=1 metric-bgp=1 update-timer=30s \
timeout-timer=3m garbage-timer=2m
/ routing rip interface
add interface=all receive=v2 send=v2 authentication=none authentication-key="" \
key-chain="" in-filter="" out-filter="" disabled=no
1C Config:
/ ip address
add address=10.0.2.129/29 network=10.0.2.128 broadcast=10.0.2.135 \
interface=ether1 comment="" disabled=no
add address=10.16.0.6/32 network=10.16.0.6 broadcast=10.16.0.6 \
interface=bridge1 comment="" disabled=no
add address=10.0.2.193/28 network=10.0.2.192 broadcast=10.0.2.207 \
interface=ether2 comment="" disabled=no
/ routing ospf
set router-id=10.16.0.6 distribute-default=never redistribute-connected=no \
redistribute-static=no redistribute-rip=no redistribute-bgp=no \
metric-default=1 metric-connected=20 metric-static=20 metric-rip=20 \
metric-bgp=20
/ routing ospf area
set backbone area-id=0.0.0.0 type=default translator-role=translate-candidate \
authentication=none disabled=no
/ routing ospf interface
set FIXME interface=ether1 cost=10 priority=1 authentication-key="" \
network-type=broadcast retransmit-interval=5s transmit-delay=1s \
hello-interval=10s dead-interval=40s
/ routing ospf network
add network=10.0.1.0/24 area=backbone disabled=no
add network=10.0.2.0/24 area=backbone disabled=no
add network=10.16.0.6/32 area=backbone disabled=no
/ routing bgp instance
set default name="default" as=65500 router-id=10.16.0.6 \
redistribute-connected=no redistribute-static=no redistribute-rip=no \
redistribute-ospf=no redistribute-other-bgp=no out-filter="" \
client-to-client-reflection=yes ignore-as-path-len=no comment="" \
disabled=no
/ routing bgp peer
add name="2A" instance=default remote-address=10.0.2.130 remote-as=65500 \
tcp-md5-key="" nexthop-choice=default multihop=no route-reflect=no \
hold-time=3m ttl=1 in-filter="" out-filter="" comment="" disabled=no
add name="2B" instance=default remote-address=10.0.2.131 remote-as=65500 \
tcp-md5-key="" nexthop-choice=default multihop=no route-reflect=no \
hold-time=3m ttl=1 in-filter="" out-filter="" comment="" disabled=no
/ routing rip
set distribute-default=never redistribute-static=no redistribute-connected=no \
redistribute-ospf=no redistribute-bgp=no metric-default=1 metric-static=1 \
metric-connected=1 metric-ospf=1 metric-bgp=1 update-timer=30s \
timeout-timer=3m garbage-timer=2m
/ routing rip interface
add interface=all receive=v2 send=v2 authentication=none authentication-key="" \
key-chain="" in-filter="" out-filter="" disabled=no