MultiWAN + LAN through vlans

coreshock · March 18, 2024, 9:05pm

Hello!
at first - my English is not enough for long explanations, but I hope You understand me )
second - special thanks to @pcunite for these posts Using RouterOS to VLAN your network and MultiWAN with RouterOS. Also thanks to @anav for many helpful posts and answers on this forum.

I have the next network topology

All ISP are connected as ethernet and obtain IP from DHCP. (one real public IP)
Currently, I have started to configure this for a test solution.
For tests, I use hAP ac^3 instead rb4011, and my other network simulates one ISP.
And I have some questions
for pk-r00

all ports placed in one bridge (wan, lan, trunk). Is it correct?
without firewall. (as this device does not terminate any public traffic). correct?
All ISP live in one bridge, and potentially dhcp from one is visible to the other. Do I need to correct this? and how?

for pk-wt01 (on diagram pk-r01) (for test I use hAP ac^3 instead rb4011 on real)

ether1 in bridge. Correct? through this port passes public(WAN) and private(LAN) traffic. Is it secure?
WAN vlans configured on ether1, LAN vlans on bridge. Correct?

pk-r00

# mar/18/2024 21:48:07 by RouterOS 7.8
#
# model = RB760iGS

/interface bridge
add ingress-filtering=no name=bridge1 vlan-filtering=yes

/interface ethernet
set [ find default-name=ether5 ] poe-out=off
set [ find default-name=sfp1 ] disabled=yes

/interface vlan
# ISP VLANs
add interface=bridge1 name=vlan_isp_fn_12 vlan-id=12
add interface=bridge1 name=vlan_isp_ks_13 vlan-id=13
add interface=bridge1 name=vlan_isp_vg_11 vlan-id=11
# local VLANs
add interface=bridge1 name=vlan_mngt_100 vlan-id=100
add interface=bridge1 name=vlan_pako_101 vlan-id=101

/interface list
add name=LAN
add name=ISP

/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip hotspot profile
set [ find default=yes ] html-directory=hotspot
/port
set 0 name=serial0

/interface bridge port
# WANs
add bridge=bridge1 interface=ether1 pvid=11
add bridge=bridge1 interface=ether2 pvid=12
add bridge=bridge1 interface=ether3 pvid=13
# trunk for WAN and LAN
add bridge=bridge1 frame-types=admit-only-vlan-tagged interface=ether4

/ip neighbor discovery-settings
set discover-interface-list=LAN

/interface bridge vlan
# WAN (for test env use only 2)
add bridge=bridge1 tagged=ether4 untagged=ether1 vlan-ids=11
add bridge=bridge1 tagged=ether4 untagged=ether2 vlan-ids=12
add bridge=bridge1 tagged=bridge1,ether4 vlan-ids=100
add bridge=bridge1 tagged=bridge1,ether4 vlan-ids=101

/interface list member
add interface=ether4 list=LAN
add interface=ether5 list=LAN
add interface=vlan_mngt_100 list=LAN
add interface=bridge1 list=LAN
add interface=vlan_pako_101 list=LAN

/ip dhcp-client
add interface=vlan_mngt_100

/ip service
set telnet disabled=yes
set ftp disabled=yes
set www disabled=yes
set ssh disabled=no
set api disabled=yes
set api-ssl disabled=yes

/system clock
set time-zone-name=Europe/Kiev
/system identity
set name=pk-r00

pk-wt01

# 2024-03-18 21:49:32 by RouterOS 7.12.1
#
# model = RBD53iG-5HacD2HnD

/interface bridge
add admin-mac=xx:xx:xx:xx:xx:xx auto-mac=no comment=defconf \
    ingress-filtering=no name=bridge vlan-filtering=yes

/interface ethernet
set [ find default-name=ether1 ] mac-address=yy:yy:yy:yy:yy:yy
set [ find default-name=ether5 ] poe-out=off

/interface wireless
set [ find default-name=wlan1 ] band=2ghz-b/g/n channel-width=20/40mhz-XX \
    distance=indoors frequency=auto installation=indoor mode=ap-bridge ssid=\
    MikroTik-EF0AC4 wireless-protocol=802.11
set [ find default-name=wlan2 ] band=5ghz-a/n/ac channel-width=\
    20/40/80mhz-XXXX distance=indoors frequency=auto installation=indoor \
    mode=ap-bridge ssid=pk-wt_0x24v wireless-protocol=802.11

/interface vlan
# WAN
add interface=ether1 name=vlan_isp_fn_12 vlan-id=12
add interface=ether1 name=vlan_isp_ks_13 vlan-id=13
add interface=ether1 name=vlan_isp_vg_11 vlan-id=11
# LAN
add interface=bridge name=vlan_mngt_100 vlan-id=100
add interface=bridge name=vlan_pako_101 vlan-id=101

/interface list
add comment=defconf name=WAN
add comment=defconf name=LAN

/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik

/ip pool
add name=pool_mngt ranges=192.168.100.2-192.168.100.10
add name=pool_pako ranges=192.168.101.100-192.168.101.150

/ip dhcp-server
add address-pool=pool_pako interface=vlan_pako_101 lease-time=521w3d23h59m59s name=dhcp_pako
add address-pool=pool_mngt interface=vlan_mngt_100 lease-time=521w3d10m name=dhcp_mngt

/routing table
add fib name=isp_vg
add fib name=isp_fn
add fib name=isp_ks

/interface bridge port
add bridge=bridge interface=ether2 pvid=101
add bridge=bridge interface=ether5
add bridge=bridge frame-types=admit-only-vlan-tagged interface=ether1

/ip firewall connection tracking
set loose-tcp-tracking=no

/ip neighbor discovery-settings
set discover-interface-list=LAN

/ip settings
set rp-filter=loose

/interface bridge vlan
# only local VLANs, no WAN VALNs
add bridge=bridge tagged=bridge,ether1 untagged=ether2 vlan-ids=101
add bridge=bridge tagged=bridge,ether1 untagged=ether2 vlan-ids=100

/interface list member
add comment=defconf interface=bridge list=LAN
add interface=vlan_isp_vg_11 list=WAN
add interface=vlan_isp_fn_12 list=WAN
add interface=vlan_isp_ks_13 list=WAN
add interface=ether2 list=LAN
add interface=vlan_pako_101 list=LAN
add interface=vlan_mngt_100 list=LAN

/ip address
add address=192.168.100.1/24 interface=vlan_mngt_100 network=192.168.100.0
add address=192.168.101.1/24 interface=vlan_pako_101 network=192.168.101.0

/ip dhcp-client
# ISP with static adress, but obtain from DHCP (by ISP rules)
add add-default-route=no interface=vlan_isp_fn_12 use-peer-dns=no use-peer-ntp=no
# ISP DHCP. script for change routing
add add-default-route=no interface=vlan_isp_vg_11 script=":if (\$bound=1) do={\
    \r\
    \n    /ip/route/set [find gateway!=\$\"gateway-address\" and comment=\"isp\
    _vg_monitor\"] gateway=\$\"gateway-address\"\r\
    \n    :local msg (\"isp_vg_monitor:: ip has been changed. ip: \" . \$\"lea\
    se-address\" . \"; gw:\" . \$\"gateway-address\");\r\
    \n    :log info \$msg;\r\
    \n}\r\
    \n" use-peer-dns=no use-peer-ntp=
    no
/ip dhcp-server network
add address=192.168.100.0/24 gateway=192.168.100.1 netmask=24
add address=192.168.101.0/24 dns-server=192.168.101.1 gateway=192.168.101.1 netmask=24

/ip dns
set allow-remote-requests=yes servers=8.8.8.8,9.9.9.9

/ip dns static
add address=192.168.100.1 comment=defconf name=r01.pako.lan

# default config
/ip firewall filter
add action=accept chain=input comment=\
    "defconf: accept established,related,untracked" connection-state=\
    established,related,untracked
add action=drop chain=input comment="defconf: drop invalid" connection-state=\
    invalid
add action=accept chain=input comment="defconf: accept ICMP" protocol=icmp
add action=accept chain=input comment=\
    "defconf: accept to local loopback (for CAPsMAN)" dst-address=127.0.0.1
add action=drop chain=input comment="defconf: drop all not coming from LAN" \
    in-interface-list=!LAN
add action=accept chain=forward comment="defconf: accept in ipsec policy" \
    ipsec-policy=in,ipsec
add action=accept chain=forward comment="defconf: accept out ipsec policy" \
    ipsec-policy=out,ipsec
add action=fasttrack-connection chain=forward comment="defconf: fasttrack" \
    connection-state=established,related hw-offload=yes
add action=accept chain=forward comment=\
    "defconf: accept established,related, untracked" connection-state=\
    established,related,untracked
add action=drop chain=forward comment="defconf: drop invalid" \
    connection-state=invalid
add action=drop chain=forward comment=\
    "defconf: drop all from WAN not DSTNATed" connection-nat-state=!dstnat \
    connection-state=new in-interface-list=WAN

# "copypaste" from @pcunit forum topic http://forum.mikrotik.com/t/multiwan-with-routeros/163698/1
/ip firewall mangle
add action=mark-connection chain=prerouting connection-state=new \
    in-interface=vlan_isp_vg_11 new-connection-mark=isp_vg_wan passthrough=\
    yes
add action=mark-routing chain=prerouting connection-mark=isp_vg_wan \
    in-interface-list=LAN new-routing-mark=isp_vg passthrough=yes
add action=mark-connection chain=prerouting connection-state=new \
    in-interface=vlan_isp_fn_12 new-connection-mark=isp_fn_wan passthrough=\
    yes
add action=mark-routing chain=prerouting connection-mark=isp_fn_wan \
    in-interface-list=LAN new-routing-mark=isp_fn passthrough=yes
add action=mark-connection chain=input connection-state=new in-interface=\
    vlan_isp_vg_11 new-connection-mark=isp_vg_wan passthrough=yes
add action=mark-routing chain=output connection-mark=isp_vg_wan \
    new-routing-mark=isp_vg passthrough=yes
add action=mark-connection chain=input connection-state=new in-interface=\
    vlan_isp_fn_12 new-connection-mark=isp_fn_wan passthrough=yes
add action=mark-routing chain=output connection-mark=isp_fn_wan \
    new-routing-mark=isp_fn passthrough=yes
/ip firewall nat
add action=masquerade chain=srcnat comment="defconf: masquerade" \
    ipsec-policy=out,none out-interface-list=WAN

# "copypaste" from @pcunit forum topic http://forum.mikrotik.com/t/multiwan-with-routeros/163698/1
/ip route
# real ISP with static IP
add comment=isp_fn_monitor disabled=no distance=2 dst-address=1.1.1.1/32 \
    gateway=xxx.xxx.xxx.xxx pref-src="" routing-table=main scope=10 \
    suppress-hw-offload=no target-scope=11
add check-gateway=ping comment=isp_fn_gw distance=2 dst-address=0.0.0.0/0 \
    gateway=1.1.1.1 scope=10 target-scope=12
add comment=isp_fn_wan distance=2 dst-address=0.0.0.0/0 gateway=1.1.1.1 \
    routing-table=isp_fn scope=10 target-scope=12
# for a test for second ISP i use my other network, and OpenDNS IP for check internet
add comment=isp_vg_monitor disabled=no distance=1 dst-address=\
    208.67.222.222/32 gateway=192.168.76.1 pref-src="" routing-table=main \
    scope=10 suppress-hw-offload=no target-scope=11
add check-gateway=ping comment=isp_vg_gw distance=1 dst-address=0.0.0.0/0 \
    gateway=208.67.222.222 scope=10 target-scope=12
add comment=isp_vg_wan disabled=no distance=1 dst-address=0.0.0.0/0 gateway=\
    208.67.222.222 pref-src="" routing-table=isp_vg scope=10 \
    suppress-hw-offload=no target-scope=12

# default config
/ipv6 firewall address-list
add address=::/128 comment="defconf: unspecified address" list=bad_ipv6
add address=::1/128 comment="defconf: lo" list=bad_ipv6
add address=fec0::/10 comment="defconf: site-local" list=bad_ipv6
add address=::ffff:0.0.0.0/96 comment="defconf: ipv4-mapped" list=bad_ipv6
add address=::/96 comment="defconf: ipv4 compat" list=bad_ipv6
add address=100::/64 comment="defconf: discard only " list=bad_ipv6
add address=2001:db8::/32 comment="defconf: documentation" list=bad_ipv6
add address=2001:10::/28 comment="defconf: ORCHID" list=bad_ipv6
add address=3ffe::/16 comment="defconf: 6bone" list=bad_ipv6

# default config
/ipv6 firewall filter
add action=accept chain=input comment=\
    "defconf: accept established,related,untracked" connection-state=\
    established,related,untracked
add action=drop chain=input comment="defconf: drop invalid" connection-state=\
    invalid
add action=accept chain=input comment="defconf: accept ICMPv6" protocol=\
    icmpv6
add action=accept chain=input comment="defconf: accept UDP traceroute" port=\
    33434-33534 protocol=udp
add action=accept chain=input comment=\
    "defconf: accept DHCPv6-Client prefix delegation." dst-port=546 protocol=\
    udp src-address=fe80::/10
add action=accept chain=input comment="defconf: accept IKE" dst-port=500,4500 \
    protocol=udp
add action=accept chain=input comment="defconf: accept ipsec AH" protocol=\
    ipsec-ah
add action=accept chain=input comment="defconf: accept ipsec ESP" protocol=\
    ipsec-esp
add action=accept chain=input comment=\
    "defconf: accept all that matches ipsec policy" ipsec-policy=in,ipsec
add action=drop chain=input comment=\
    "defconf: drop everything else not coming from LAN" in-interface-list=\
    !LAN
add action=accept chain=forward comment=\
    "defconf: accept established,related,untracked" connection-state=\
    established,related,untracked
add action=drop chain=forward comment="defconf: drop invalid" \
    connection-state=invalid
add action=drop chain=forward comment=\
    "defconf: drop packets with bad src ipv6" src-address-list=bad_ipv6
add action=drop chain=forward comment=\
    "defconf: drop packets with bad dst ipv6" dst-address-list=bad_ipv6
add action=drop chain=forward comment="defconf: rfc4890 drop hop-limit=1" \
    hop-limit=equal:1 protocol=icmpv6
add action=accept chain=forward comment="defconf: accept ICMPv6" protocol=\
    icmpv6
add action=accept chain=forward comment="defconf: accept HIP" protocol=139
add action=accept chain=forward comment="defconf: accept IKE" dst-port=\
    500,4500 protocol=udp
add action=accept chain=forward comment="defconf: accept ipsec AH" protocol=\
    ipsec-ah
add action=accept chain=forward comment="defconf: accept ipsec ESP" protocol=\
    ipsec-esp
add action=accept chain=forward comment=\
    "defconf: accept all that matches ipsec policy" ipsec-policy=in,ipsec
add action=drop chain=forward comment=\
    "defconf: drop everything else not coming from LAN" in-interface-list=\
    !LAN


# "copypaste" from @pcunit forum topic http://forum.mikrotik.com/t/multiwan-with-routeros/163698/1
/routing rule
add action=lookup-only-in-table disabled=no dst-address=192.168.101.0/24 table=main
add action=lookup-only-in-table disabled=no dst-address=192.168.100.0/24 table=main
# I should disable this rule by netwatch, because if ISP2 is down 192.168.101.0 not switch to other ISPs
add action=lookup comment=pako_route_rule_fn disabled=no dst-address="" src-address=192.168.101.0/24 table=isp_fn

/system clock
set time-zone-name=Europe/Kiev
/system identity
set name=pk-wt01
/system note
set show-at-login=no
/tool mac-server
set allowed-interface-list=LAN
/tool mac-server mac-winbox
set allowed-interface-list=LAN

/tool netwatch
add disabled=no down-script="/routing/rule/set [find comment=\"pako_route_rule\
    _fn\"] disabled=yes\r\
    \n:log info \"fn_down\"" host=1.1.1.1 http-codes="" interval=10s \
    test-script="" type=simple up-script="/routing/rule/set [find comment=\"pa\
    ko_route_rule_fn\"] disabled=no\r\
    \n:log info \"fn_up\"\r\
    \n"

anav · March 19, 2024, 11:38am

Just to be clear the RB4001 r2 is acting as a router and has its own SUbnets with DHCP.

Your router settings for the ISPs make no sense to me. Why are you using vlans?
Why is there only one IP DHCP client when you said all three get assigned that way.
why is your single IP DHCP client have v100 mgmt…

Nothing makes sense at the moment, are you sure you get dynamic IPs from all three ISPs for example
What type of connection are they ( LTE, Starlink etc…)

Looking bridge ports, why is ether1,ether2 there but not ether3.

As I said, there is no logic so I cannot progress.

Why is vlan 11,12,13 going to the RB4011… ??

How does 140 popup when not created at either router ???
Which port does v101 go out on the first router…another missing diagram item
I am assuming vlan150 is coming from the RB4011 and the associated device is connected to the R1 port ether5

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

My conclusion thus far is you have two routers when you should only be using one router.

coreshock · March 19, 2024, 6:13pm

I’m trying to build a solution as on diagram step by step. In the first phase, I want to build all vlans for ISP.

How does 140 popup when not created at either router ???

A private network will be built in the next phases.

Which port does v101 go out on the first router…another missing diagram item

sorry, 101 is missing on the diagram. it’s one of the private vlans.

So, private vlans are absent on the attached configs.

The main goal of this post evaluate the config for ISP vlans.

Why is vlan 11,12,13 going to the RB4011… ??

I use two routers because physical ethernet from ISPs placed in one room, and all local ethernet is placed in other. And I have only one uplink between these places. And I want to use rb4011 as the main router.
All three ISP connect as ethernet (currently, for a test i have shown in config only two)

So, in my idea, I consolidate all ISP in the first router in one uplink and translate to the second (rb4011) router through vlans.

I am assuming vlan150 is coming from the RB4011 and the associated device is connected to the R1 port ether5

Also, ether5 on the first router will be used as an access port for the local network.

in a real environment, vlan_mngt_100 will be used for two AP.
My main question - it is possible to place traffic from ISP and my local traffic in one eher port and one bridge?

My conclusion thus far is you have two routers when you should only be using one router.

Yes (and no .
Honestly, the first router is not a “router”, in this schema, it works as a switch. But I don’t have a clear switch, I use what I have.

due to physical cabling
I have 10+ devices has connect via UTP
rb4011 more powerful instead rb760. In the feature on the router will be configured VPN server and a few VPN tunnels to other nets

anav · March 19, 2024, 6:18pm

Okay thanks for the clarification, by the way if you dont want to mention private subnets, then dont put them in the config or diagram, otherwise terribly confusing.
I prefer seeing it all and understanding what you have in mind ref;
a. identify all users/devices
b. identifying all traffic flows they should have.

Need more info on ISPs.

Are you saying you have three ISP modems in one room ( where 750 is also located) with one wired connection to another room where RB4100 and network are located??

Can you confirm three separate ISP modems, or one ISP modem and three IP addresses ( aka a block of IPs)
ISP1 static/dynamic, public non-public
ISP2 same??
ISP3 same??

ISP1 throughput??
ISP2 throughput??
ISP3 throughput??

++++++++++++++++++++++++ based on a bunch of factors I am thinking its best you put the RB4100 first and then hexS as a switch.
Another possibility if you can replace the existing cable ( what is it, cat5, cat5e, cat6, cat6a ???) and use the spf+ port on the RB4011 and get another small switch… with one SFP+ port and some 1gig ports.

coreshock · March 19, 2024, 6:57pm

Yes.
ISP1 - ethernet (dynamic/public/dhcp)
ISP2 - ethernet (static/ but from DHCP)
ISP3 - ethernet (dynamic/private/dhcp)
Three physical cables.
All these in one place (where 750)

All local cables in another place (where rb4011)

between place one and place two I have only one UTP cable

anav · March 19, 2024, 7:16pm

UTP cable of what standard ?
What are the throughputs of each ISP??
What is your plan
Use 1, the other 2 are backup. OR . PCC all three used all the time. ??

coreshock · March 19, 2024, 7:27pm

cat 5e (1gb possible)
1gb(but it as document, reality about 700 at max ) + 100 + 100

use all ISP at the same time.
each segment local network has our ISP.
but if ISP is down then segment switches to ISP that is UP
It’s my plan

Sorry
I do not want to waste your time on the details.
Maybe you look at the vlans, bridge, ports, that are used between two devices?
I need to get three interfaces with ISP on the rb4011, and make that secures.
All other - parts of another season this serial I mean local networks, balancing etc.

And sorry again - my English is not clear, and some phases may not be polite. But it is only for a reason I do have not enough English

anav · March 19, 2024, 8:54pm

You will not be able to maximize your throughputs as the hex will be one bottleneck and the cable between the two devices the other bottleneck.

Suggest you configure ether5 off the bridge first to setup the hex as per the below.
Then connect to the hex when behind the RB4011 via ether4, and then modify ether5 for the camera VLAN.

…

/interface bridge
add ingress-filtering=no name=bridge vlan-filtering=yes
/interface ethernet
set [ find default-name=ether5 ] name=emergaccess
/interface vlan
add interface=bridge name=WAN1-vlan11  vlan-ids=11
add interface=bridge name=WAN2-vlan22  vlan-ids=22
add interface=bridge name=WAN3-vlan33  vlan-ids=33
add interface=bridge name=MGMT-vlan100  vlan-ids=100
/interface list
add name=MGMT
/interface bridge port
add bridge=bridge edge=yes ingress-filtering=yes frame-types=admit-only-untagged-and-priority-tagged interface=ether1  pvid=11
add bridge=bridge edge=yes ingress-filtering=yes frame-types=admit-only-untagged-and-priority-tagged interface=ether2  pvid=22
add bridge=bridge edge=yes ingress-filtering=yes frame-types=admit-only-untagged-and-priority-tagged interface=ether3  pvid=33
add bridge=bridge ingress-filtering=yes frame-types=admit-only-vlan-tagged  interface=ether4
/ip neighbor discovery-settings
set discover-interface-list=MGMT
/interface bridge vlan
add bridge=bridge tagged=ether4,  untagged=ether1 vlan-ids=11
add bridge=bridge tagged=ether4,  untagged=ether2 vlan-ids=22
add bridge=bridge tagged=ether4,  untagged=ether3 vlan-ids=33
add bridge=bridge tagged=bridge,ether4 vlan-ids=100
/interface list member
add interface=MGMT-vlan100 list=MGMT
add interface=emergaccess list=MGMT
/ip dns
set allow-remote-requests=yes servers=192.168.100.
/ip address
add address=192.168.100.2/24 interface=MGMT-vlan100 network=192.168.100.0
add address=192.168.55.1/24 interface=emergaccess network=192.168.55.0
/ip dns
set allow-remote-requests=yes servers=192.168.100.1
/ip route
add dst-address=0.0.0.0/0 gateway=192.168.100.1 routing-table=main
/ip service
/system ntp client
set enabled=yes
/system ntp client servers
add address=192.168.100.1
/tool mac-server mac-winbox
set allowed-interface-list=MGMT


Then to change ether5  Modify /Bridge ports and /Bridge Vlans after the RB4011 is setup and you can confirm winbox access via the trunk connection on hex ether4 to RB4011 ether1.


/interface bridge port
add bridge=bridge edge=yes ingress-filtering=yes frame-types=admit-only-untagged-and-priority-tagged interface=ether1  pvid=11
add bridge=bridge edge=yes ingress-filtering=yes frame-types=admit-only-untagged-and-priority-tagged interface=ether2  pvid=22
add bridge=bridge edge=yes ingress-filtering=yes frame-types=admit-only-untagged-and-priority-tagged interface=ether3  pvid=33
add bridge=bridge ingress-filtering=yes frame-types=admit-only-vlan-tagged  interface=ether4
add bridge=bridge ingress-filtering=yes frame-types=admit-only-untagged-and-priority-tagged interface=ether5  pvid=150

/interface bridge vlan
add bridge=bridge tagged=ether4  untagged=ether1 vlan-ids=11
add bridge=bridge tagged=ether4  untagged=ether2 vlan-ids=22
add bridge=bridge tagged=ether4  untagged=ether3 vlan-ids=33
add bridge=bridge tagged=ether4  untagged=ether5 vlan-ids=150
add bridge=bridge tagged=bridge,ether4 vlan-ids=100

coreshock · March 19, 2024, 9:39pm

Thank you!
I understand it. and it works for me.

But my favorite question about this section

/interface bridge port
add bridge=bridge edge=yes ingress-filtering=yes frame-types=admit-only-untagged-and-priority-tagged interface=ether1  pvid=11
add bridge=bridge edge=yes ingress-filtering=yes frame-types=admit-only-untagged-and-priority-tagged interface=ether2  pvid=22
add bridge=bridge edge=yes ingress-filtering=yes frame-types=admit-only-untagged-and-priority-tagged interface=ether3  pvid=33
add bridge=bridge ingress-filtering=yes frame-types=admit-only-vlan-tagged  interface=ether4
add bridge=bridge ingress-filtering=yes frame-types=admit-only-untagged-and-priority-tagged interface=ether5  pvid=150

in other words - we have two (public and private) traffic at the same bridge

similar situation on the second router (rb4011) - ether1 (directly connected to ether4 hex) will be in the same bridge with locals

is it normal for security reasons?
Or do I need to configure firewall on hex? if yes - how rules do I need?

anav · March 19, 2024, 9:43pm

Nope, no firewall rules are required. The idea of accepting the bridge default protocol mode of RTSP and setting ports to EDGE, should ensure no possible cross talk between WAN ports ( and of course the vlan isolation as well ).

anav · March 19, 2024, 9:44pm

The tougher setup is on the RB4011, and how to setup up two switch chips efficiently in a bridge that contains both wan and lan elements.

coreshock · March 19, 2024, 10:02pm

Thank you so much!!

Nope, no firewall rules are required. The idea of accepting the bridge default protocol mode of RTSP and setting ports to EDGE, should ensure no possible cross talk between WAN ports ( and of course the vlan isolation as well ).

My first and main question is resolved!

about two switch chip on 4011 - currently its very difficult question for me. And I will return to it (or not) after full setup and look at CPU utilization.

so - this topic may be marked as resolved!
after full setup, i post these full configs and a new diagram. maybe someone will be interested

anav · March 19, 2024, 10:40pm

You didnt provide your IP DHCP setup for WAN3 ???
Is isp2 public IP???

Also its not clear why you are mangling.

Reasons to mangle.
Ensure VPN going to a specific WAN leaves the same WAN ( aka proper router services handling )
Ensure external users going to a server on a specific WAN have their traffic leave on the SAME wan, aka port forwarding
Ensure PCC load balancing works.

Thus far I have seen neither on your config.
Because WAN2 and WAN3 are so small, it would make sense to use them as backups only
If you use pCC load balancing the mangles you have are wrong and you would need probably 10/0 10/9 LB to approximate roughly an 8 to 1 ratio

There is also load balancing by bandwidth so when iSP1 gets overloaded the spillover goes to WAN2, but in your case this is not helpful because if you are maxing out WAN1, then there is no more pipe room anyway for anymore traffic either on the hex or on the ethernet cable.

The best bet is to have Ether1 as primary and if it fails then use wan2, and wan3 for traffic in a LB setup… this should be interesting…

would also setup a remote VPN access via ISP1… and if ISP2 is also public a back-up wireguard vpn via that connection.

coreshock · March 20, 2024, 2:18pm

You didnt provide your IP DHCP setup for WAN3 ???

I’m not configured this currently. In real ISP3 obtains IP from DCHP. It’s no public IP

Is isp2 public IP???

ISP2 is a public IP. and only ISP2 has a public IP. ISP1 and ISP3 - non public.

Also its not clear why you are mangling.

You are right.
Mangling in my configs is a “monkey” copypaste from @pcunit post (sorry)
I plan to study it in detail later.

I haven’t goal to classic load balancing. I planned the next scenario:

All users used the internet via ISP1 (has the highest speed)
Few pc/laptop used ISP2. Because ISP have a public IP, and this IP needs to be used to access specific resources.
TVs used ISP3, because ISP3 gives some contents
Inbound VPN will be configured on ISP2 (because it has public IP)

In this scenario, do I need to mangle only ISP2 traffic? In other words - mangle only inbound traffic?

Load balancing is not necessary for me. Failover - yes.
Also, throughput is not critical. For all my local networks 100mb is enough but we always want to more and more

anav · March 20, 2024, 6:43pm

Yes, some requirements I can chew on.
All good info, and I would add that my approach would be an interesting deviation.

PRIMARY IS WAN1
if WAN1 fails, BACKUP goes to BOTH WAN2 and WAN3 doing PCC load balancing for standard traffic that was using WAN1.

However you have provided good info in that
WAN2 needs to be used even when WAN1 is up
a. for VPN
b. for something else please elaborate ( do you have servers on the LAN that you need external users to access via WAN2, access or do you just mean outbound traffic , do you have one or two users or devices on the LAN that require to only go outbound on WAN2 ???

Final question, which ISP gateway needs a script to be kept updated when it changes.
For example my ethernet cable dynamic public IP, when it changes WANIP or gateway, both automatically change in my manual routes
My fiber connection, gets the new IP and new gateway in IP DHCP client but it doesnt auto update my manual routes so I script that based on the usual = detecting comment text.

coreshock · March 21, 2024, 7:44pm

b. for something else please elaborate

I’m working from home.
For access to some resources I have two ways

Up on my PC OpenVPN/CiscoVPN
The owner of these resources can give me access by IP.
I’m oversimplifying, but that’s pretty much is
second way more convenient for me.

if WAN1 fails, BACKUP goes to BOTH WAN2 and WAN3 doing PCC load balancing for standard traffic that was using WAN1.

Thanks, It’s a good and interesting idea. I will think about this, but I lack knowledge about pcc currently (need to rtfm )

Final question, which ISP gateway needs a script to be kept updated when it changes.

ISP1 & ISP3
config from the first post contains the script to update the gateway for ISP1.

# ISP DHCP. script for change routing
add add-default-route=no interface=vlan_isp_vg_11 script=":if (\$bound=1) do={\
    \r\
    \n    /ip/route/set [find gateway!=\$\"gateway-address\" and comment=\"isp\
    _vg_monitor\"] gateway=\$\"gateway-address\"\r\
    \n    :local msg (\"isp_vg_monitor:: ip has been changed. ip: \" . \$\"lea\
    se-address\" . \"; gw:\" . \$\"gateway-address\");\r\
    \n    :log info \$msg;\r\
    \n}\r\
    \n" use-peer-dns=no use-peer-ntp=
    no

if You about this

Amm0 · March 21, 2024, 8:08pm

Depending on how you’re using the ports on the RB4011… But if you can use two bridges, one with the ether1-5, other with ether6-10, and you’d for sure keep HW offloading this way.

Use one bridge for WANs and another bridge for LANs is pretty easy/clean on RB4011 (and matches the physical port grouping on device). Routed traffic is always going to use CPU anyway, so the two bridges does make a clean separation. Only down side is does mean VLAN trunks cannot [easily] include both WAN and LAN – since you’d have two different bridges.