Post bellow is addressing the question when AIO RDO Liberty Node has to have external networks of VLAN type with predefined vlan tags. Straight forward packstack –allinone install doesn’t allow to achieve desired network configuration. External network provider of vlan type appears to be required. In particular case, office networks 10.10.10.0/24 vlan tagged (157) ,10.10.57.0/24 vlan tagged (172), 10.10.32.0/24 vlan tagged (200) already exists when RDO install is running. If demo_provision was “y” , then delete router1 and created external network of VXLAN type.
I got back to this writing due to recent post
https://ask.openstack.org/en/question/91611/how-to-configure-multiple-external-networks-in-rdo-libertymitaka/
answer provided contains several misleading steps in configuration vlan enabled bridges.
First
***********************************************************
Update /etc/neutron/plugins/ml2/ml2_conf.ini
***********************************************************
[root@ip-192-169-142-52 ml2(keystone_demo)]# cat ml2_conf.ini
[ml2]
type_drivers = flat,vlan,vxlan
tenant_network_types = vlan,vxlan
mechanism_drivers =openvswitch
path_mtu = 0
[ml2_type_flat]
[ml2_type_vlan]
network_vlan_ranges = vlan157:157:157,vlan172:172:172,vlan200:200:200
[ml2_type_gre]
[ml2_type_vxlan]
vni_ranges =10:100
vxlan_group =224.0.0.1
[ml2_type_geneve]
[securitygroup]
enable_security_group = True
**************
Then
**************
# openstack-service restart neutron
***************************************************
Invoke external network provider
***************************************************
[root@ip-192-169-142-52 ~(keystone_admin]#neutron net-create vlan157 –shared –provider:network_type vlan –provider:segmentation_id 157 –provider:physical_network vlan157 –router:external
[root@ip-192-169-142-52 ~(keystone_admin]# neutron subnet-create –name sub-vlan157 –gateway 10.10.10.1 –allocation-pool start=10.10.10.100,end=10.10.10.200 vlan157 10.10.10.0/24
***********************************************
Create second external network
***********************************************
[root@ip-192-169-142-52 ~(keystone_admin]# neutron net-create vlan172 --shared --provider:network_type vlan --provider:segmentation_id 172 --provider:physical_network vlan172 --router:external
[root@ip-192-169-142-52 ~(keystone_admin]# neutron subnet-create --name sub-vlan172 --gateway 10.10.57.1 --allocation-pool start=10.10.57.100,end=10.10.57.200 vlan172 10.10.57.0/24
***********************************************
Create third external network
***********************************************
[root@ip-192-169-142-52 ~(keystone_admin]# neutron net-create vlan200 --shared --provider:network_type vlan --provider:segmentation_id 200 --provider:physical_network vlan200 --router:external
[root@ip-192-169-142-52 ~(keystone_admin]# neutron subnet-create --name sub-vlan200 --gateway 10.10.32.1 --allocation-pool start=10.10.32.100,end=10.10.57.200 vlan172 10.10.32.0/24
***********************************************************************
No need to update sub-net (vs [ 1 ]). No switch to "enable_isolataed_metadata=True"
Neutron L3 agent configuration results attaching qg-<port-id> interfaces to br-int
***********************************************************************
[root@ip-192-169-142-52 ~(keystone_admin)]# neutron net-show vlan157
+—————————+————————————–+
| Field | Value |
+—————————+————————————–+
| admin_state_up | True |
| id | b41e4d36-9a63-4631-abb0-6436f2f50e2e |
| mtu | 0 |
| name | vlan157 |
| provider:network_type | vlan |
| provider:physical_network | vlan157 |
| provider:segmentation_id | 157 |
| router:external | True |
| shared | True |
| status | ACTIVE |
| subnets | bb753fc3-f257-4ce5-aa7c-56648648056b |
| tenant_id | b18d25d66bbc48b1ad4b855a9c14da70 |
+—————————+————————————–+
[root@ip-192-169-142-52 ~(keystone_admin)]# neutron subnet-show sub-vlan157
+——————-+——————————————————————+
| Field | Value |
+——————-+——————————————————————+
| allocation_pools | {“start”: “10.10.10.100”, “end”: “10.10.10.200”} |
| cidr | 10.10.10.0/24 |
| dns_nameservers | |
| enable_dhcp | True |
| gateway_ip | 10.10.10.1 |
| host_routes | {“destination”: “169.254.169.254/32”, “nexthop”: “10.10.10.151”} |
| id | bb753fc3-f257-4ce5-aa7c-56648648056b |
| ip_version | 4 |
| ipv6_address_mode | |
| ipv6_ra_mode | |
| name | sub-vlan157 |
| network_id | b41e4d36-9a63-4631-abb0-6436f2f50e2e |
| subnetpool_id | |
| tenant_id | b18d25d66bbc48b1ad4b855a9c14da70 |
+——————-+——————————————————————+
[root@ip-192-169-142-52 ~(keystone_admin)]# neutron net-show vlan172
+—————————+————————————–+
| Field | Value |
+—————————+————————————–+
| admin_state_up | True |
| id | 3714adc9-ab17-4f96-9df2-48a6c0b64513 |
| mtu | 0 |
| name | vlan172 |
| provider:network_type | vlan |
| provider:physical_network | vlan172 |
| provider:segmentation_id | 172 |
| router:external | True |
| shared | True |
| status | ACTIVE |
| subnets | 21419f2f-212b-409a-8021-2b4a2ba6532f |
| tenant_id | b18d25d66bbc48b1ad4b855a9c14da70 |
+—————————+————————————–+
[root@ip-192-169-142-52 ~(keystone_admin)]# neutron subnet-show sub-vlan172
+——————-+——————————————————————+
| Field | Value |
+——————-+——————————————————————+
| allocation_pools | {“start”: “10.10.57.100”, “end”: “10.10.57.200”} |
| cidr | 10.10.57.0/24 |
| dns_nameservers | |
| enable_dhcp | True |
| gateway_ip | 10.10.57.1 |
| host_routes | {“destination”: “169.254.169.254/32”, “nexthop”: “10.10.57.151”} |
| id | 21419f2f-212b-409a-8021-2b4a2ba6532f |
| ip_version | 4 |
| ipv6_address_mode | |
| ipv6_ra_mode | |
| name | sub-vlan172 |
| network_id | 3714adc9-ab17-4f96-9df2-48a6c0b64513 |
| subnetpool_id | |
| tenant_id | b18d25d66bbc48b1ad4b855a9c14da70 |
+——————-+——————————————————————+
[root@ip-192-169-142-52 ~(keystone_admin)]# neutron net-show vlan200
+—————————+————————————–+
| Field | Value |
+—————————+————————————–+
| admin_state_up | True |
| id | 3dc90ff7-b1df-4079-aca1-cceedb23f440 |
| mtu | 0 |
| name | vlan200 |
| provider:network_type | vlan |
| provider:physical_network | vlan200 |
| provider:segmentation_id | 200 |
| router:external | True |
| shared | True |
| status | ACTIVE |
| subnets | 60181211-ea36-4e4e-8781-f13f743baa19 |
| tenant_id | b18d25d66bbc48b1ad4b855a9c14da70 |
+—————————+————————————–+
[root@ip-192-169-142-52 ~(keystone_admin)]# neutron subnet-show sub-vlan200
+——————-+————————————————–+
| Field | Value |
+——————-+————————————————–+
| allocation_pools | {“start”: “10.10.32.100”, “end”: “10.10.32.200”} |
| cidr | 10.10.32.0/24 |
| dns_nameservers | |
| enable_dhcp | True |
| gateway_ip | 10.10.32.1 |
| host_routes | |
| id | 60181211-ea36-4e4e-8781-f13f743baa19 |
| ip_version | 4 |
| ipv6_address_mode | |
| ipv6_ra_mode | |
| name | sub-vlan200 |
| network_id | 3dc90ff7-b1df-4079-aca1-cceedb23f440 |
| subnetpool_id | |
| tenant_id | b18d25d66bbc48b1ad4b855a9c14da70 |
+——————-+————————————————–+
**************
Next Step
**************
# modprobe 8021q
# ovs-vsctl add-br br-vlan
# ovs-vsctl add-port br-vlan eth1
# vconfig add br-vlan 157
# ovs-vsctl add-br br-vlan2
# ovs-vsctl add-port br-vlan2 eth2
# vconfig add br-vlan2 172
# ovs-vsctl add-br br-vlan3
# ovs-vsctl add-port br-vlan3 eth3
# vconfig add br-vlan3 200
******************************
Update l3_agent.ini file
******************************
external_network_bridge =
gateway_external_network_id =
**********************************************
/etc/neutron/plugins/ml2/openvswitch_agent.ini
**********************************************
bridge_mappings = vlan157:br-vlan,vlan172:br-vlan2,vlan200:br-vlan3
*************************************
Update Neutron Configuration
*************************************
# openstack-service restart neutron
*******************************************
Set up config persistent between reboots
*******************************************
/etc/sysconfig/network-scripts/ifcfg-eth1
DEVICE=”eth1″
ONBOOT=yes
OVS_BRIDGE=br-vlan
TYPE=OVSPort
DEVICETYPE=”ovs”
/etc/sysconfig/network-scripts/ifcfg-br-vlan
DEVICE=br-vlan
BOOTPROTO=none
ONBOOT=yes
TYPE=OVSBridge
DEVICETYPE=”ovs”
/etc/sysconfig/network-scripts/ifcfg-br-vlan.157
BOOTPROTO=”none”
DEVICE=”br-vlan.157″
ONBOOT=”yes”
IPADDR=”10.10.10.150″
PREFIX=”24″
GATEWAY=”10.10.10.1″
DNS1=”83.221.202.254″
VLAN=yes
NOZEROCONF=yes
USERCTL=no
/etc/sysconfig/network-scripts/ifcfg-eth2
DEVICE=”eth2″
ONBOOT=yes
OVS_BRIDGE=br-vlan2
TYPE=OVSPort
DEVICETYPE=”ovs”
/etc/sysconfig/network-scripts/ifcfg-br-vlan2
DEVICE=br-vlan2
BOOTPROTO=none
ONBOOT=yes
TYPE=OVSBridge
DEVICETYPE=”ovs”
/etc/sysconfig/network-scripts/ifcfg-br-vlan2.172
BOOTPROTO=”none”
DEVICE=”br-vlan2.172″
ONBOOT=”yes”
IPADDR=”10.10.57.150″
PREFIX=”24″
GATEWAY=”10.10.57.1″
DNS1=”83.221.202.254″
VLAN=yes
NOZEROCONF=yes
/etc/sysconfig/network-scripts/ifcfg-br-vlan3
DEVICE=br-vlan3
BOOTPROTO=none
ONBOOT=yes
TYPE=OVSBridge
DEVICETYPE=”ovs”
/etc/sysconfig/network-scripts/ifcfg-br-vlan3.200
BOOTPROTO=”none”
DEVICE=”br-vlan3.200″
ONBOOT=”yes”
IPADDR=”10.10.32.150″
PREFIX=”24″
GATEWAY=”10.10.32.1″
DNS1=”83.221.202.254″
VLAN=yes
NOZEROCONF=yes
USERCTL=no
/etc/sysconfig/network-scripts/ifcfg-eth3
DEVICE=”eth3″
ONBOOT=yes
OVS_BRIDGE=br-vlan3
TYPE=OVSPort
DEVICETYPE=”ovs”
********************************************
Routing table on AIO RDO Liberty Node
********************************************
[root@ip-192-169-142-52 ~(keystone_admin)]# ip route
default via 10.10.10.1 dev br-vlan.157
10.10.10.0/24 dev br-vlan.157 proto kernel scope link src 10.10.10.150
10.10.32.0/24 dev br-vlan3.200 proto kernel scope link src 10.10.32.150
10.10.57.0/24 dev br-vlan2.172 proto kernel scope link src 10.10.57.150
169.254.0.0/16 dev eth0 scope link metric 1002
169.254.0.0/16 dev eth1 scope link metric 1003
169.254.0.0/16 dev eth2 scope link metric 1004
169.254.0.0/16 dev eth3 scope link metric 1005
169.254.0.0/16 dev br-vlan3 scope link metric 1008
169.254.0.0/16 dev br-vlan2 scope link metric 1009
169.254.0.0/16 dev br-vlan scope link metric 1011
192.169.142.0/24 dev eth0 proto kernel scope link src 192.169.142.52
****************************************************************************
Notice that both qrouter-namespaces are attached to br-int.
No switch to “enable_isolated_metadata=True” vs [ 1 ]
*****************************************************************************
[root@ip-192-169-142-52 ~(keystone_admin)]# neutron net-list | grep vlan
| 3dc90ff7-b1df-4079-aca1-cceedb23f440 | vlan200 | 60181211-ea36-4e4e-8781-f13f743baa19 10.10.32.0/24 |
| 235c8173-d3f8-407e-ad6a-c1d3d423c763 | vlan172 | c7588239-4941-419b-8d27-ccd970acc4ce 10.10.57.0/24 |
| b41e4d36-9a63-4631-abb0-6436f2f50e2e | vlan157 | bb753fc3-f257-4ce5-aa7c-56648648056b 10.10.10.0/24 |
[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-vsctl show
40286423-e174-4714-9c82-32d026ef47ca
Bridge br-vlan
Port “eth1”
Interface “eth1”
Port br-vlan
Interface br-vlan
type: internal
Port phy-br-vlan
Interface phy-br-vlan
type: patch
options: {peer=int-br-vlan}
Bridge br-ex
Port br-ex
Interface br-ex
type: internal
Bridge br-tun
fail_mode: secure
Port br-tun
Interface br-tun
type: internal
Port patch-int
Interface patch-int
type: patch
options: {peer=patch-tun}
Bridge “br-vlan2”
Port “phy-br-vlan2”
Interface “phy-br-vlan2”
type: patch
options: {peer=”int-br-vlan2″}
Port “eth2”
Interface “eth2”
Port “br-vlan2”
Interface “br-vlan2”
type: internal
Bridge “br-vlan3”
Port “br-vlan3”
Interface “br-vlan3”
type: internal
Port “phy-br-vlan3”
Interface “phy-br-vlan3”
type: patch
options: {peer=”int-br-vlan3″}
Port “eth3”
Interface “eth3”
Bridge br-int
fail_mode: secure
Port “qr-4e77c7a3-b5”
tag: 3
Interface “qr-4e77c7a3-b5”
type: internal
Port “int-br-vlan3”
Interface “int-br-vlan3″
type: patch
options: {peer=”phy-br-vlan3”}
Port “tap8e684c78-a3”
tag: 2
Interface “tap8e684c78-a3”
type: internal
Port patch-tun
Interface patch-tun
type: patch
options: {peer=patch-int}
Port “qvoe2761636-b5”
tag: 4
Interface “qvoe2761636-b5”
Port “tap6cd6fadf-31”
tag: 1
Interface “tap6cd6fadf-31”
type: internal
Port “qg-02f7ff0d-6d”
tag: 2
Interface “qg-02f7ff0d-6d”
type: internal
Port “qg-943f7831-46”
tag: 1
Interface “qg-943f7831-46”
type: internal
Port “tap4ef27b41-be”
tag: 5
Interface “tap4ef27b41-be”
type: internal
Port “qr-f0fd3793-4e”
tag: 8
Interface “qr-f0fd3793-4e”
type: internal
Port “tapb1435e62-8b”
tag: 7
Interface “tapb1435e62-8b”
type: internal
Port “qvo1bb76476-05”
tag: 3
Interface “qvo1bb76476-05”
Port “qvocf68fcd8-68”
tag: 8
Interface “qvocf68fcd8-68”
Port “qvo8605f075-25”
tag: 4
Interface “qvo8605f075-25”
Port “qg-08ccc224-1e”
tag: 7
Interface “qg-08ccc224-1e”
type: internal
Port “tapbb485628-0b”
tag: 4
Interface “tapbb485628-0b”
type: internal
Port “int-br-vlan2”
Interface “int-br-vlan2″
type: patch
options: {peer=”phy-br-vlan2”}
Port “tapee030534-da”
tag: 8
Interface “tapee030534-da”
type: internal
Port “qr-4d679697-39”
tag: 4
Interface “qr-4d679697-39”
type: internal
Port br-int
Interface br-int
type: internal
Port “tap9b38c69e-46”
tag: 6
Interface “tap9b38c69e-46”
type: internal
Port “tapc166022a-54”
tag: 3
Interface “tapc166022a-54”
type: internal
Port “qvo66d8f235-d4”
tag: 8
Interface “qvo66d8f235-d4”
Port int-br-vlan
Interface int-br-vlan
type: patch
options: {peer=phy-br-vlan}
ovs_version: “2.4.0”
[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns
qdhcp-e826aa22-dee0-478d-8bd7-721336e3824a
qrouter-a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b
qdhcp-3dc90ff7-b1df-4079-aca1-cceedb23f440
qdhcp-4481aee1-ef86-4997-bf52-e435aafb9c20
qdhcp-eda69965-c6ee-42be-944f-2d61498e4bea
qdhcp-6768214b-b71c-4178-a0fc-774b2a5d59ef
qdhcp-b41e4d36-9a63-4631-abb0-6436f2f50e2e
qdhcp-03812cc9-69c5-492a-9995-985bf6e1ff13
qdhcp-235c8173-d3f8-407e-ad6a-c1d3d423c763
qdhcp-d958a059-f7bd-4f9f-93a3-3499d20a1fe2
qrouter-c1900dab-447f-4f87-80e7-b4c8ca087d28
qrouter-71237c84-59ca-45dc-a6ec-23eb94c4249d
********************************************************************************
Access to Nova Metadata Server provided via neutron-ns-metadata-proxy
running in corresponding qrouter namespaces (Neutron L3 Configuration)
********************************************************************************
[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns exec qrouter-a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b netstat -antp
Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp 0 0 0.0.0.0:9697 0.0.0.0:* LISTEN 12548/python2
[root@ip-192-169-142-52 ~(keystone_admin)]# ps aux | grep 12548
neutron 12548 0.0 0.4 281028 35992 ? S 18:34 0:00 /usr/bin/python2 /bin/neutron-ns-metadata-proxy –pid_file=/var/lib/neutron/external/pids/a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b.pid –metadata_proxy_socket=/var/lib/neutron/metadata_proxy –router_id=a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b –state_path=/var/lib/neutron –metadata_port=9697 –metadata_proxy_user=990 –metadata_proxy_group=988 –verbose –log-file=neutron-ns-metadata-proxy-a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b.log –log-dir=/var/log/neutron
root 32665 0.0 0.0 112644 960 pts/8 S+ 19:29 0:00 grep –color=auto 12548
******************************************************************************
OVS flow verification on br-vlan3,br-vlan2. On each external network vlan172,
vlan200 two VMs (on each one of vlan networks) are pinging each other
******************************************************************************
[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-flows br-vlan3 | grep NORMAL
cookie=0x0, duration=3554.739s, table=0, n_packets=33, n_bytes=2074, idle_age=2137, priority=4,in_port=2,dl_vlan=7 actions=mod_vlan_vid:200,NORMAL
cookie=0x0, duration=4204.459s, table=0, n_packets=2102, n_bytes=109304, idle_age=1, priority=0 actions=NORMAL
[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-flows br-vlan3 | grep NORMAL
cookie=0x0, duration=3557.643s, table=0, n_packets=33, n_bytes=2074, idle_age=2140, priority=4,in_port=2,dl_vlan=7 actions=mod_vlan_vid:200,NORMAL
cookie=0x0, duration=4207.363s, table=0, n_packets=2103, n_bytes=109356, idle_age=2, priority=0 actions=NORMAL
[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-flows br-vlan3 | grep NORMAL
cookie=0x0, duration=3568.225s, table=0, n_packets=33, n_bytes=2074, idle_age=2151, priority=4,in_port=2,dl_vlan=7 actions=mod_vlan_vid:200,NORMAL
cookie=0x0, duration=4217.945s, table=0, n_packets=2109, n_bytes=109668, idle_age=0, priority=0 actions=NORMAL
[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-flows br-vlan2 | grep NORMAL
cookie=0x0, duration=4140.528s, table=0, n_packets=11, n_bytes=642, idle_age=695, priority=4,in_port=2,dl_vlan=1 actions=mod_vlan_vid:172,NORMAL
cookie=0x0, duration=4225.918s, table=0, n_packets=2113, n_bytes=109876, idle_age=1, priority=0 actions=NORMAL
[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-flows br-vlan2 | grep NORMAL
cookie=0x0, duration=4143.600s, table=0, n_packets=11, n_bytes=642, idle_age=698, priority=4,in_port=2,dl_vlan=1 actions=mod_vlan_vid:172,NORMAL
cookie=0x0, duration=4228.990s, table=0, n_packets=2115, n_bytes=109980, idle_age=0, priority=0 actions=NORMAL
[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-flows br-vlan2 | grep NORMAL
cookie=0x0, duration=4145.912s, table=0, n_packets=11, n_bytes=642, idle_age=700, priority=4,in_port=2,dl_vlan=1 actions=mod_vlan_vid:172,NORMAL
cookie=0x0, duration=4231.302s, table=0, n_packets=2116, n_bytes=110032, idle_age=0, priority=0 actions=NORMAL
********************************************************************************
Next question how local vlan tag 7 gets generated
Run following commands :-
********************************************************************************
[root@ip-192-169-142-52 ~(keystone_admin)]# neutron net-show vlan200
+—————————+————————————–+
| Field | Value |
+—————————+————————————–+
| admin_state_up | True |
| id | 3dc90ff7-b1df-4079-aca1-cceedb23f440 |
| mtu | 0 |
| name | vlan200 |
| provider:network_type | vlan |
| provider:physical_network | vlan200 |
| provider:segmentation_id | 200 |
| router:external | True |
| shared | True |
| status | ACTIVE |
| subnets | 60181211-ea36-4e4e-8781-f13f743baa19 |
| tenant_id | b18d25d66bbc48b1ad4b855a9c14da70 |
+—————————+————————————–+
[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns | grep 3dc90ff7-b1df-4079-aca1-cceedb23f440
qdhcp-3dc90ff7-b1df-4079-aca1-cceedb23f440
[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns exec qdhcp-3dc90ff7-b1df-4079-aca1-cceedb23f440 ifconfig
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 0 (Local Loopback)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
tapb1435e62-8b: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 10.10.32.100 netmask 255.255.255.0 broadcast 10.10.32.255
inet6 fe80::f816:3eff:fee3:19f2 prefixlen 64 scopeid 0x20<link>
ether fa:16:3e:e3:19:f2 txqueuelen 0 (Ethernet)
RX packets 27 bytes 1526 (1.4 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 8 bytes 648 (648.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns exec qdhcp-3dc90ff7-b1df-4079-aca1-cceedb23f440 route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 10.10.32.1 0.0.0.0 UG 0 0 0 tapb1435e62-8b
10.10.32.0 0.0.0.0 255.255.255.0 U 0 0 0 tapb1435e62-8b
[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-vsctl show | grep b1435e62-8b
Port “tapb1435e62-8b”
Interface “tapb1435e62-8b”
**************************************************************************
Actually, directives mentioned in [ 1 ]
**************************************************************************
# neutron subnet-create –name vlan100 –gateway 192.168.0.1 –allocation-pool \
start=192.168.0.150,end=192.168.0.200 –enable-dhcp \
–dns-nameserver 192.168.0.1 vlan100 192.168.0.0/24
# neutron subnet-update –host-route destination=169.254.169.254/32,nexthop=192.168.0.151 vlan100
along with switch to “enable_isolated_metadata=True” are targeting launching VMs to external_fixed_ip pool in qdhcp-3dc90ff7-b1df-4079-aca1-cceedb23f440 without creating Neutron router, spiting tenants with vlan tag IDs. I might be missing somesing , but [ 1 ] configures system where each vlan(XXX) external network would belong the only one tenant supposed identified by tag (XXX).
Unless RBAC policies will be created to control who has access to the provider network.
That is not what I intend to do. Neutron work flow on br-int won’t touch mentioned qdhcp-namespace at all. Any external vlan(XXX) network might be used by several tenants each one having it ownVXLAN subnet (identified in system by VXLAN ID) and it’s own neutron router(XXX) to external network vlan(XXX). AIO RDO set up is just a sample, I am talking about Network Node in multi node RDO Liberty depoyment.
*********************************************
Fragment from `ovs-vsct show `
*********************************************
Port “tapb1435e62-8b”
tag: 7
Interface “tapb1435e62-8b”
*************************************************************************
Next appearance of vlan tag 7, as expected is qg-08ccc224-1e.
Outgoing interface of qrouter-a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b
namespace.
*************************************************************************
[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns exec qrouter-a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b ifconfig
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 0 (Local Loopback)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
qg-08ccc224-1e: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 10.10.32.101 netmask 255.255.255.0 broadcast 10.10.32.255
inet6 fe80::f816:3eff:fed4:e7d prefixlen 64 scopeid 0x20<link>
ether fa:16:3e:d4:0e:7d txqueuelen 0 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 28 bytes 1704 (1.6 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
qr-f0fd3793-4e: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 30.0.0.1 netmask 255.255.255.0 broadcast 30.0.0.255
inet6 fe80::f816:3eff:fea9:5422 prefixlen 64 scopeid 0x20<link>
ether fa:16:3e:a9:54:22 txqueuelen 0 (Ethernet)
RX packets 68948 bytes 7192868 (6.8 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 68859 bytes 7185051 (6.8 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns exec qrouter-a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 10.10.32.1 0.0.0.0 UG 0 0 0 qg-08ccc224-1e
10.10.32.0 0.0.0.0 255.255.255.0 U 0 0 0 qg-08ccc224-1e
30.0.0.0 0.0.0.0 255.255.255.0 U 0 0 0 qr-f0fd3793-4e
*******************************************************************************************************
Now verify Neutron router connecting qrouter-namespace, having interface with tag 7 and qdhcp namespace, been create to launch the instances.
RoutesDSA has been created with external gateway to vlan200 and internal interface to subnet private07 (30.0.0.0/24) having dhcp enabled and DNS server defined.
vlan157,vlan172 are configured as external networks for theirs coresponding routers as well.
*******************************************************************************************************
[root@ip-192-169-142-52 ~(keystone_admin)]# neutron router-list | grep RoutesDSA
| a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b | RoutesDSA | {“network_id”: “3dc90ff7-b1df-4079-aca1-cceedb23f440“, “enable_snat”: true, “external_fixed_ips”: [{“subnet_id”: “60181211-ea36-4e4e-8781-f13f743baa19”, “ip_address”: “10.10.32.101”}]} | False | False |
[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns | grep a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b
qrouter-a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b
[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns | grep 3dc90ff7-b1df-4079-aca1-cceedb23f440
qdhcp-3dc90ff7-b1df-4079-aca1-cceedb23f440
[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns exec qdhcp-3dc90ff7-b1df-4079-aca1-cceedb23f440 ifconfig
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 0 (Local Loopback)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
tapb1435e62-8b: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 10.10.32.100 netmask 255.255.255.0 broadcast 10.10.32.255
inet6 fe80::f816:3eff:fee3:19f2 prefixlen 64 scopeid 0x20<link>
ether fa:16:3e:e3:19:f2 txqueuelen 0 (Ethernet)
RX packets 27 bytes 1526 (1.4 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 8 bytes 648 (648.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
**************************
Finally run:-
**************************
[root@ip-192-169-142-52 ~(keystone_admin)]# neutron router-port-list RoutesDSA
+————————————–+——+——————-+————————————————————————————-+
| id | name | mac_address | fixed_ips |
+————————————–+——+——————-+————————————————————————————-+
| 08ccc224-1e23-491a-8eec-c4db0ec00f02 | | fa:16:3e:d4:0e:7d | {“subnet_id”: “60181211-ea36-4e4e-8781-f13f743baa19“, “ip_address”: “10.10.32.101”} |
| f0fd3793-4e5a-467a-bd3c-e87bc9063d26 | | fa:16:3e:a9:54:22 | {“subnet_id”: “0c962484-3e48-4d86-a17f-16b0b1e5fc4d“, “ip_address”: “30.0.0.1”} |
+————————————–+——+——————-+————————————————————————————-+
[root@ip-192-169-142-52 ~(keystone_admin)]# neutron subnet-list | grep 0c962484-3e48-4d86-a17f-16b0b1e5fc4d
| 0c962484-3e48-4d86-a17f-16b0b1e5fc4d | | 30.0.0.0/24 | {“start”: “30.0.0.2”, “end”: “30.0.0.254”} |
[root@ip-192-169-142-52 ~(keystone_admin)]# neutron subnet-list | grep 60181211-ea36-4e4e-8781-f13f743baa19
| 60181211-ea36-4e4e-8781-f13f743baa19 | sub-vlan200 | 10.10.32.0/24 | {“start”: “10.10.32.100”, “end”: “10.10.32.200”} |
************************************
OVS Flows at br-vlan3
************************************
[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-flows br-vlan3 | grep NORMAL
cookie=0x0, duration=15793.182s, table=0, n_packets=33, n_bytes=2074, idle_age=14376, priority=4,in_port=2,dl_vlan=7 actions=mod_vlan_vid:200,NORMAL
cookie=0x0, duration=16442.902s, table=0, n_packets=8221, n_bytes=427492, idle_age=1, priority=0 actions=NORMAL
[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-flows br-vlan3 | grep NORMAL
сookie=0x0, duration=15796.300s, table=0, n_packets=33, n_bytes=2074, idle_age=14379, priority=4,in_port=2,dl_vlan=7 actions=mod_vlan_vid:200,NORMAL
cookie=0x0, duration=16446.020s, table=0, n_packets=8223, n_bytes=427596, idle_age=0, priority=0 actions=NORMAL
************************************************************
OVS Flow for {phy-br-vlan3,in-br-vlan3} veth pair
************************************************************
[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl show br-vlan3 | grep phy-br-vlan3
2(phy-br-vlan3): addr:da:e4:fb:ba:8b:1a
[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl show br-int | grep int-br-vlan3
19(int-br-vlan3): addr:b2:a9:9e:89:07:1b
[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-ports br-vlan3 2
OFPST_PORT reply (xid=0x2): 1 ports
port 2: rx pkts=6977, bytes=304270, drop=0, errs=0, frame=0, over=0, crc=0
tx pkts=55, bytes=7037, drop=0, errs=0, coll=0
[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-ports br-vlan3 2
OFPST_PORT reply (xid=0x2): 1 ports
port 2: rx pkts=6979, bytes=304354, drop=0, errs=0, frame=0, over=0, crc=0
tx pkts=55, bytes=7037, drop=0, errs=0, coll=0
[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-ports br-vlan3 2
OFPST_PORT reply (xid=0x2): 1 ports
port 2: rx pkts=6981, bytes=304438, drop=0, errs=0, frame=0, over=0, crc=0
tx pkts=55, bytes=7037, drop=0, errs=0, coll=0
[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-ports br-int 19
OFPST_PORT reply (xid=0x2): 1 ports
port 19: rx pkts=55, bytes=7037, drop=0, errs=0, frame=0, over=0, crc=0
tx pkts=6991, bytes=304858, drop=0, errs=0, coll=0
[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-ports br-int 19
OFPST_PORT reply (xid=0x2): 1 ports
port 19: rx pkts=55, bytes=7037, drop=0, errs=0, frame=0, over=0, crc=0
tx pkts=6994, bytes=304984, drop=0, errs=0, coll=0
[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-ports br-int 19
OFPST_PORT reply (xid=0x2): 1 ports
port 19: rx pkts=55, bytes=7037, drop=0, errs=0, frame=0, over=0, crc=0
tx pkts=7450, bytes=324136, drop=0, errs=0, coll=0
****************************************************************
Another OVS flow on test br-int for vlan157
****************************************************************
[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns exec qdhcp-4481aee1-ef86-4997-bf52-e435aafb9c20 ssh -i oskeyvls.pem cirros@10.10.10.101
$ ping -c 5 10.10.10.108
PING 10.10.10.108 (10.10.10.108): 56 data bytes
64 bytes from 10.10.10.108: seq=0 ttl=63 time=0.706 ms
64 bytes from 10.10.10.108: seq=1 ttl=63 time=0.772 ms
64 bytes from 10.10.10.108: seq=2 ttl=63 time=0.734 ms
64 bytes from 10.10.10.108: seq=3 ttl=63 time=0.740 ms
64 bytes from 10.10.10.108: seq=4 ttl=63 time=0.785 ms
— 10.10.10.108 ping statistics —
5 packets transmitted, 5 packets received, 0% packet loss
round-trip min/avg/max = 0.706/0.747/0.785 ms
******************************************************************************
Testing VM1<=>VM2 via floating IPs on external vlan net 10.10.10.0/24
*******************************************************************************
[root@ip-192-169-142-52 ~(keystone_admin)]# nova list –all
+————————————–+————–+———————————-+——–+————+————-+———————————+
| ID | Name | Tenant ID | Status | Task State | Power State | Networks |
+————————————–+————–+———————————-+——–+————+————-+———————————+
| a3d5ecf6-0fdb-4aa3-815f-171871eccb77 | CirrOSDevs01 | f16de8f8497d4f92961018ed836dee19 | ACTIVE | – | Running | private=40.0.0.17, 10.10.10.101 |
| 1b65f5db-d7d5-4e92-9a7c-60e7866ff8e5 | CirrOSDevs02 | f16de8f8497d4f92961018ed836dee19 | ACTIVE | – | Running | private=40.0.0.18, 10.10.10.110 |
| 46b7dad1-3a7d-4d94-8407-a654cca42750 | VF23Devs01 | f16de8f8497d4f92961018ed836dee19 | ACTIVE | – | Running | private=40.0.0.19, 10.10.10.111 |
+————————————–+————–+———————————-+——–+————+————-+———————————+
[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns
qdhcp-4481aee1-ef86-4997-bf52-e435aafb9c20
qdhcp-b41e4d36-9a63-4631-abb0-6436f2f50e2e
qrouter-c1900dab-447f-4f87-80e7-b4c8ca087d28
[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns exec qdhcp-4481aee1-ef86-4997-bf52-e435aafb9c20 ssh cirros@10.10.10.110
The authenticity of host ‘10.10.10.110 (10.10.10.110)’ can’t be established.
RSA key fingerprint is b8:d3:ec:10:70:a7:da:d4:50:13:a8:2d:01:ba:e4:83.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added ‘10.10.10.110’ (RSA) to the list of known hosts.
cirros@10.10.10.110’s password:
$ ifconfig
eth0 Link encap:Ethernet HWaddr FA:16:3E:F1:6E:E5
inet addr:40.0.0.18 Bcast:40.0.0.255 Mask:255.255.255.0
inet6 addr: fe80::f816:3eff:fef1:6ee5/64 Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1400 Metric:1
RX packets:367 errors:0 dropped:0 overruns:0 frame:0
TX packets:291 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:36442 (35.5 KiB) TX bytes:32019 (31.2 KiB)
lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
inet6 addr: ::1/128 Scope:Host
UP LOOPBACK RUNNING MTU:16436 Metric:1
RX packets:0 errors:0 dropped:0 overruns:0 frame:0
TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:0 (0.0 B) TX bytes:0 (0.0 B)
$ curl http://169.254.169.254/latest/meta-data/public-ipv4
10.10.10.110$
$ ssh fedora@10.10.10.111
Host ‘10.10.10.111’ is not in the trusted hosts file.
(fingerprint md5 23:c0:fb:fd:74:80:2f:12:d3:09:2f:9e:dd:19:f1:74)
Do you want to continue connecting? (y/n) y
fedora@10.10.10.111’s password:
Last login: Sun Dec 13 15:52:43 2015 from 10.10.10.101
[fedora@vf23devs01 ~]$ ifconfig
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1400
inet 40.0.0.19 netmask 255.255.255.0 broadcast 40.0.0.255
inet6 fe80::f816:3eff:fea4:1a52 prefixlen 64 scopeid 0x20<link>
ether fa:16:3e:a4:1a:52 txqueuelen 1000 (Ethernet)
RX packets 283 bytes 30213 (29.5 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 303 bytes 35022 (34.2 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 0 (Local Loopback)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
[fedora@vf23devs01 ~]$ curl http://169.254.169.254/latest/meta-data/public-ipv4
10.10.10.111[fedora@vf23devs01 ~]$
[fedora@vf23devs01 ~]$ curl http://169.254.169.254/latest/meta-data/instance-id
i-00000009[fedora@vf23devs01 ~]$
[fedora@vf23devs01 ~]$
Posted by dbaxps 
Xen Virtualization on Linux and Solaris 