Confusing routing behavior CCR1009-7G-1C-1S+

deadpete · Wed May 15, 2024 7:24 pm

Hi folks,

I have setup a router according to my best knowledge. There are a number of separate internal LANs connected to the router. There are 3 services reachable from the internet, SMTP (25) on one public IP address, and HTTP (80) / HTTPS (443) on another public IP. The web server is also accessible internally using it's internet name. This works.

To reach certain services like SMTP (587), printers, RDP across different LANs, I have configured simple forwarding rules to allow traffic from one LAN to another. Those rules work, except for one specific instance.

There is for example the following forwarding rule that works without problem.

/ip firewall filter
add action=accept chain=forward dst-address=192.168.0.0/23 in-interface-list=LAN src-address=172.16.10.0/24

while this forwarding rule does not work without srcnat masquerade

/ip firewall filter
add action=accept chain=forward dst-address=172.16.10.93 in-interface=mainnet out-interface=konnet src-address=192.168.0.0/23

To make it work I had to add this

/ip firewall nat
add action=masquerade chain=srcnat dst-address=172.16.10.93 src-address=192.168.0.0/23

I would be grateful if somebody could explain this odd behavior. It really beats me. The whole configuration is included below.

Best regards,

Peter

/interface bridge
add frame-types=admit-only-vlan-tagged name=bridge pvid=40 vlan-filtering=yes
/interface ethernet
set [ find default-name=combo1 ] combo-mode=sfp
/interface vlan
add interface=bridge name=admnet vlan-id=190
add interface=bridge name=guestnet vlan-id=60
add interface=bridge name=konnet vlan-id=10
add interface=bridge name=mainnet vlan-id=1
add interface=bridge name=mgmtnet vlan-id=40
add interface=bridge name=persnet vlan-id=110
add interface=bridge name=proxynet vlan-id=150
add interface=bridge name=testnet vlan-id=220
/interface list
add name=WAN
add name=LAN
add name=MGMT
add name=DHCP-LAN
add name=USERLAN
/ip pool
add name=konpool ranges=172.16.10.128-172.16.10.254
add name=guestpool ranges=192.168.6.128-192.168.6.254
add name=testpool ranges=192.168.22.201-192.168.22.254
/ip dhcp-server
add address-pool=konpool interface=konnet lease-time=12h name=kondhcp
add address-pool=guestpool interface=guestnet name=guestdhcp
add address-pool=testpool interface=testnet lease-time=1d name=testdhcp
/port
set 0 name=serial0
set 1 name=serial1
/interface bridge port
add bridge=bridge frame-types=admit-only-vlan-tagged interface=ether1 pvid=\
    974
add bridge=bridge interface=ether2
add bridge=bridge interface=ether3
add bridge=bridge interface=ether4
add bridge=bridge interface=ether5
add bridge=bridge interface=ether6
add bridge=bridge frame-types=admit-only-untagged-and-priority-tagged \
    interface=ether7 pvid=40
add bridge=bridge frame-types=admit-only-vlan-tagged interface=sfp-sfpplus1 \
    pvid=975
/ip firewall connection tracking
set loose-tcp-tracking=no
/ip neighbor discovery-settings
set discover-interface-list=MGMT
/ip settings
set tcp-syncookies=yes
/ipv6 settings
set disable-ipv6=yes
/interface bridge vlan
add bridge=bridge tagged=bridge,sfp-sfpplus1,ether1 vlan-ids=1
add bridge=bridge tagged=bridge,sfp-sfpplus1,ether1 vlan-ids=10
add bridge=bridge tagged=bridge,sfp-sfpplus1,ether1 untagged=ether7 vlan-ids=\
    40
add bridge=bridge tagged=bridge,sfp-sfpplus1,ether1 vlan-ids=60
add bridge=bridge tagged=bridge,sfp-sfpplus1,ether1 vlan-ids=110
add bridge=bridge tagged=bridge,sfp-sfpplus1,ether1 vlan-ids=150
add bridge=bridge tagged=bridge,sfp-sfpplus1,ether1 vlan-ids=190
add bridge=bridge tagged=bridge,sfp-sfpplus1,ether1 vlan-ids=220
/interface list member
add interface=combo1 list=WAN
add interface=mainnet list=LAN
add interface=konnet list=LAN
add interface=mgmtnet list=LAN
add interface=guestnet list=LAN
add interface=persnet list=LAN
add interface=proxynet list=LAN
add interface=admnet list=LAN
add interface=testnet list=LAN
add interface=mgmtnet list=MGMT
add interface=konnet list=DHCP-LAN
add interface=guestnet list=DHCP-LAN
add interface=testnet list=DHCP-LAN
add interface=mainnet list=USERLAN
add interface=konnet list=USERLAN
add interface=guestnet list=USERLAN
add interface=testnet list=USERLAN
/ip address
add address=192.168.4.100/24 interface=mgmtnet network=192.168.4.0
add address=aaa.aaa.aaa.aaa/29 interface=combo1 network=aaa.aaa.aaa.xxx
add address=192.168.6.100/24 interface=guestnet network=192.168.6.0
add address=172.16.10.100/24 interface=konnet network=172.16.10.0
add address=192.168.11.100/24 interface=persnet network=192.168.11.0
add address=192.168.15.100/24 interface=proxynet network=192.168.15.0
add address=192.168.19.100/24 interface=admnet network=192.168.19.0
add address=192.168.22.100/24 interface=testnet network=192.168.22.0
add address=192.168.0.101/23 interface=mainnet network=192.168.0.0
add address=bbb.bbb.bbb.1/29 interface=combo1 network=bbb.bbb.bbb.yyy
add address=bbb.bbb.bbb.2/29 interface=combo1 network=bbb.bbb.bbb.yyy
add address=bbb.bbb.bbb.3/29 interface=combo1 network=bbb.bbb.bbb.yyy
/ip dhcp-server network
add address=172.16.10.0/24 dns-server=172.16.10.11,172.16.10.10 domain=\
    kon.talps gateway=172.16.10.100 ntp-server=172.16.10.98
add address=192.168.6.0/24 dns-server=192.168.6.100 gateway=192.168.6.100
add address=192.168.22.0/24 dns-server=192.168.22.10 domain=testdom.talps \
    gateway=192.168.22.100 ntp-server=192.168.22.98
/ip dns
set allow-remote-requests=yes servers=123.45.67.8,123.45.67.9



/ip firewall address-list
add address=192.168.4.0/24 list=isolatedLAN
add address=192.168.6.0/24 list=isolatedLAN
add address=192.168.11.0/24 list=isolatedLAN
add address=192.168.15.0/24 list=isolatedLAN
add address=192.168.19.0/24 list=isolatedLAN
add address=192.168.22.0/24 list=isolatedLAN
add address=192.168.0.101 list=routerAddresses
add address=192.168.4.100 list=routerAddresses
add address=192.168.6.100 list=routerAddresses
add address=192.168.11.100 list=routerAddresses
add address=192.168.15.100 list=routerAddresses
add address=192.168.19.100 list=routerAddresses
add address=192.168.22.100 list=routerAddresses
add address=172.16.10.100 list=routerAddresses


# Some incoming packets are first filtered by the RAW table
/ip firewall filter
# Classic start of the rule set from Mikrotik wiki
add action=accept chain=input protocol=icmp
add action=accept chain=input connection-state=established,related,untracked
add action=drop chain=input connection-state=invalid

# Allow input from local LAN interfaces that were not dropped by the RAW filter
add action=accept chain=input in-interface-list=LAN
add action=accept chain=input in-interface=mgmtnet
# Drop the rest
add action=drop chain=input

# Forward packets not dropped by the RAW table
add action=fasttrack-connection chain=forward connection-state=\
    established,related hw-offload=yes
add action=accept chain=forward connection-state=\
    established,related,untracked

# Forward packets that are routed elswhere
add action=accept chain=forward dst-address=10.0.10.0/23 in-interface-list=LAN
add action=accept chain=forward dst-address=10.0.14.0/24 in-interface-list=LAN

# Forward packets from/to limited hosts (SMTP server, RDP)
add action=accept chain=forward dst-address=192.168.0.111 dst-port=587 \
    in-interface=guestnet out-interface=mainnet protocol=tcp src-address=\
    192.168.6.13
add action=accept chain=forward dst-address=192.168.0.51 dst-port=3389 \
    in-interface=guestnet out-interface=mainnet protocol=udp src-address=\
    192.168.6.13
add action=accept chain=forward dst-address=192.168.0.51 dst-port=3389 \
    in-interface=guestnet out-interface=mainnet protocol=tcp src-address=\
    192.168.6.13
add action=accept chain=forward dst-address=192.168.6.13 dst-port=23389 \
    in-interface=mainnet out-interface=guestnet protocol=udp src-address=\
    192.168.0.51
add action=accept chain=forward dst-address=192.168.6.13 dst-port=23389 \
    in-interface=mainnet out-interface=guestnet protocol=tcp src-address=\
    192.168.0.51

# !!! This is problematic, does not work without masquerade!
add action=accept chain=forward dst-address=172.16.10.93 in-interface=mainnet \
    out-interface=konnet src-address=192.168.0.0/23

# Forward packets to webserver
add action=accept chain=forward dst-address=192.168.15.0/24 \
    in-interface-list=LAN src-address=192.168.0.0/23
add action=accept chain=forward dst-address=192.168.15.0/24 \
    in-interface-list=LAN src-address=172.16.10.0/24

# Forward packets from network A to network B, traffic from B to A is prohibited
add action=accept chain=forward dst-address=192.168.0.0/23 in-interface-list=\
    LAN src-address=172.16.10.0/24

# Allow all hosts on 2 different networks to use SMTP server
add action=accept chain=forward dst-address=192.168.0.111 dst-port=587 \
    in-interface=admnet out-interface=mainnet protocol=tcp src-address=\
    192.168.19.0/24
add action=accept chain=forward dst-address=192.168.0.111 dst-port=587 \
    in-interface=persnet out-interface=mainnet protocol=tcp src-address=\
    192.168.11.0/24

# Allow RDP to one specific host from a single PC
add action=accept chain=forward dst-address=192.168.11.231 dst-port=3389 \
    in-interface=mainnet out-interface=persnet protocol=tcp src-address=\
    192.168.0.58

# Drop invalid packets
add action=drop chain=forward connection-state=invalid

# Allow from LAN to WAN
add action=accept chain=forward in-interface-list=LAN out-interface-list=WAN

# Allow dst-nated packages to servers behind the router (SMTP, http, etc.)
add action=accept chain=forward connection-nat-state=dstnat \
    in-interface-list=WAN

# Drop everything else
add action=drop chain=forward


# NAT rules, specific rules are on top
/ip firewall nat

# !!! I had to resort to this "trick" to be able to reach the device 172.16.10.9
# from the network 192.168.0.0/23, see forward rule above
# This is strange, as the forward rule from the network 172.16.10.0/24 to 192.168.0.0/23
# does not need masquerading. There are other equivalent rules in the forward chain also
# that do not need masquerading.
add action=masquerade chain=srcnat dst-address=172.16.10.93 \
    src-address=192.168.0.0/23

# All outgoing traffic to the internet from the internal networks through aaa.aaa.aaa.aaa
add action=src-nat chain=srcnat out-interface-list=WAN to-addresses=\
    aaa.aaa.aaa.aaa

# dstnat rules to reach internal servers
add action=dst-nat chain=dstnat dst-address=aaa.aaa.aaa.aaa dst-port=25 \
    in-interface-list=WAN protocol=tcp to-addresses=192.168.0.112 to-ports=25
add action=dst-nat chain=dstnat dst-address=aaa.aaa.aaa.aaa dst-port=80 \
    in-interface-list=WAN protocol=tcp to-addresses=192.168.0.112 to-ports=80
add action=dst-nat chain=dstnat dst-address=aaa.aaa.aaa.aaa dst-port=443 \
    in-interface-list=WAN protocol=tcp to-addresses=192.168.0.112 to-ports=\
    443
add action=dst-nat chain=dstnat dst-address=bbb.bbb.bbb.3 dst-port=80 \
    protocol=tcp to-addresses=192.168.15.99 to-ports=80
add action=dst-nat chain=dstnat dst-address=bbb.bbb.bbb.3 dst-port=443 \
    protocol=tcp to-addresses=192.168.15.99 to-ports=443

/ip firewall raw
# Classic RAW icmp filter from Mikrotik wiki
add action=jump chain=prerouting jump-target=icmp4 protocol=icmp
add action=accept chain=icmp4 icmp-options=0:0 limit=5,10:packet protocol=icmp
add action=accept chain=icmp4 icmp-options=3:0 protocol=icmp
add action=accept chain=icmp4 icmp-options=3:1 protocol=icmp
add action=accept chain=icmp4 icmp-options=3:2 protocol=icmp
add action=accept chain=icmp4 icmp-options=3:3 protocol=icmp
add action=accept chain=icmp4 icmp-options=3:4 protocol=icmp
add action=accept chain=icmp4 icmp-options=8:0 limit=5,10:packet protocol=icmp
add action=accept chain=icmp4 icmp-options=11:0-255 protocol=icmp
add action=drop chain=icmp4 protocol=icmp

# Allow public web server also from 2 internal networks
add action=accept chain=prerouting dst-address=192.168.15.99 src-address=\
    192.168.0.0/23
add action=accept chain=prerouting dst-address=192.168.15.99 src-address=\
    172.16.10.0/24

# Drop packets from some internal networks destined to router's WebFig
add action=drop chain=prerouting dst-address-list=routerAddresses dst-port=80 \
    log=yes log-prefix=RACC protocol=tcp

# Allow DNS requests from addresses isolatedLAN list, but not from 192.168.4.0/24
add action=accept chain=prerouting dst-port=53 protocol=udp src-address=\
    !192.168.4.0/24 src-address-list=isolatedLAN
add action=accept chain=prerouting dst-port=53 protocol=tcp src-address=\
    !192.168.4.0/24 src-address-list=isolatedLAN

# Allow DHCP requests from hosts in the interface list DHCP-LAN
add action=accept chain=prerouting dst-address=255.255.255.255 dst-port=67 \
    in-interface-list=DHCP-LAN protocol=udp src-address=0.0.0.0 src-port=68

# Drop traffic between addresses in isolatedLAN list
add action=drop chain=prerouting dst-address-list=isolatedLAN src-address=\
    !192.168.4.0/24 src-address-list=isolatedLAN

# Route away packets that should be handled elsewhere
/ip route
add disabled=no dst-address=10.0.10.0/24 gateway=192.168.0.216 routing-table=\
    main suppress-hw-offload=no
add disabled=no dst-address=10.0.11.0/24 gateway=192.168.0.216 routing-table=\
    main suppress-hw-offload=no
add disabled=no dst-address=10.0.14.0/24 gateway=172.16.10.97 routing-table=\
    main suppress-hw-offload=no

anav · Wed May 15, 2024 7:43 pm

I wouldnt even begin to assess the config without a much clearer set of requirements and a detailed network diagram

a. identify all the user(s)/device(s), groups of users/devices ( including admin )
b. identify all the traffic they need to execute.

deadpete · Wed May 15, 2024 7:50 pm

Hi anav,

It really boils down to those 2 forwarding clauses I described above. The rest works without any quirks.

All users in network 172.16.10.0/24 can reach everything in network 192.168.0.0/23.

Nobody from network 192.168.0.0/23 can reach the printer 172.16.10.93, unless I add the srcnat rule.

There are several working forward rules in the configuration that are analogous to the non working rule.

That puzzles me a lot.

Best regards,

Peter

deadpete · Wed May 15, 2024 9:15 pm

Hi anav,

Well, fulfilling your request would take at least the better part of a week, and the volume of text would be a couple of hundred A4 pages

Doing something like that is of course complete madness, just to satisfy ones curiosity. I had the choice of letting it be, everything is working, so just leave it, or start to think. I just didn't want to give up, it's acknowledging defeat. So, a good thinking session always starts with a long walk.

The probable cause came to my mind quickly, after about 1 km. The printer in question had the wrong gateway set, which was very quickly confirmed.

The lesson learned here: If similar things work in the configuration, but there is one particular item sticking out that *should* work, then the problem is most probably elsewhere.

Sorry for the noise folks.

Peter

anav · Wed May 15, 2024 9:29 pm

No actually your configuration is hosed......

So for some reason the subnet 192.168.0.0/23 is blocked from reaching the printer at 172.16.10.93.

The first thing I would do is get rid of vlan1 for any data traffic. VLAN1 is used in the background by the router already and should NOT be used to somehow cover off BRIDGE or normal LAN traffic,
Such traffic should be defined as lets say VLAN11 and given its own subnet etc or use an existing subnet, but hey you dont even have that.
In other words as I suspected without the requested information your configuration makes no sense, especially without context.

For example I note ports 2-6 have no definition, if using vlans they are either access, hybrid or trunk ports !!

It is now clearer to me that there are issues.........
add bridge=bridge frame-types=admit-only-vlan-tagged interface=sfp-sfpplus1 \
pvid=975

So, a perfect case of conflict, only vlan tags are allowed ( aka a trunk port ) but then you have a PVID, which is used for an untagged access port.
Cant have both.
The bridge vlans also compound the error.

I would get rid of all the jump ICMP rules, not worth the hassle,

Possible reasons.

a. device has internal blocking ( printer not likely )
b. raw rules are blocking this traffic by accident
c. something else???

What I find strange is that the subnet 192.168.0.0/23 is NOT an existing subnet on the router???
Although the printer is outside the DHCP pool this should not cause any issues.

/ip pool
add name=konpool ranges=172.16.10.128-172.16.10.254
add name=guestpool ranges=192.168.6.128-192.168.6.254
add name=testpool ranges=192.168.22.201-192.168.22.254

Where are the rest of the vlans??

Applicable rules I see.
Forward chain:

Why are you giving the printer subnet [color=#008000])(konnet[/color] , access to the main subnet ( mainnet )
# Forward packets from network A to network B, traffic from B to A is prohibited
add action=accept chain=forward dst-address=192.168.0.0/23 in-interface-list=\
LAN src-address=172.16.10.0/24

THIS IS REVERSED FROM WHAT I THINK YOU WANT?

Should be:
add action=accept chain=forward in-interface-list=LAN src-address=192.168.0.0/23 dst-address=172.16.10.93

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

Now lets look at RAW RULES........

Nothing I see here............

deadpete · Wed May 15, 2024 10:16 pm

Hi anav,

I set the correct gateway on the printer 172.16.10.93 to 172.16.10.100 and removed the srcnat rule. It works now, I can print from the 192.168.0.0/23 network.

There are only 2 ports used on the router (3 if you count ether7 as an emergency port for configuration). I could probably skip it and use the console port.

Port sfp-sfpplus1 is the trunk port connecting the router to the internal networks through a switch, the PVID can be whatever, as only tagged traffic is allowed anyway.

The combo1 port is where internet comes in, several routed public IPs. So ports ether1-ether6 are not used. I should probably set ether1 to PVID=1 and remove it completely from further configuration, it was used for testing. Maybe it's even better to remove all the unused ports from the bridge.

Yes, the subnet 192.168.0.0/23 exists, maybe it got accidentally deleted when I sanitized the configuration. Otherwise I would have been noticed about loss of connectivity within 10 seconds.

Vlans without DHCP services consist just of a bunch of fixed IP devices. They have been split up in different vlans to make the chunks manageable. Using DHCP there would create a plain mess. But those devices require internet connection anyway.

Access from the 172.16.10.0/24 network to the 192.168.0.0/23 should be allowed, but not the reverse, except for the printer 172.16.10.93.

Except for explicitly permitted connections, all connections between different lans must not be permitted.

After correcting, everything seems to work as it should.

Thanks for taking your time anav, very appreciated.

Best regards,

Peter

Confusing routing behavior CCR1009-7G-1C-1S+ [SOLVED]

Confusing routing behavior CCR1009-7G-1C-1S+

Re: Confusing routing behavior CCR1009-7G-1C-1S+

Re: Confusing routing behavior CCR1009-7G-1C-1S+

Re: Confusing routing behavior CCR1009-7G-1C-1S+ [SOLVED]

Re: Confusing routing behavior CCR1009-7G-1C-1S+

Re: Confusing routing behavior CCR1009-7G-1C-1S+

Who is online