@sebastian-roth Thanks for clearing up the ‘default interface’ question. I was maybe rabbit holing that a bit.
For certain ip_forward is set on all fog hosts that I have spun up in the past week:
cat /proc/sys/net/ipv4/ip_forward
1
As a sanity check, I installed fog on a fedora host as well. And low and behold the fedora host worked the very first time! That solves my needs and I can move forward. However, I do still have the ubuntu VMs dormant that I can spin up and do testing on, and I would like to contribute any knowledge gained here to the wiki.
Here is the script I use to setup masquerading on iptables:
#!/bin/bash
WAN_INTERFACE=eth2
check_ip_forwarding () {
if [[ $(cat /proc/sys/net/ipv4/ip_forward) ]]; then
echo 'ip forwarding already enabled'
else
#echo 1 > /proc/sys/net/ipv4/ip_forward
sysctl -w net.ipv4.ip_forward=1
echo 'net.ipv4.ip_forward = 1' >> /etc/sysctl.d/net.ipv4.ip_forward.conf
echo 'ip forwarding enabled'
fi
}
add_network_to_forwards () {
LOCAL_NETWORK_CIDR=$1
echo "Adding $LOCAL_NETWORK_CIDR to network forwards."
sudo iptables -I FORWARD -o $WAN_INTERFACE -s $LOCAL_NETWORK_CIDR -j ACCEPT
sudo iptables -I INPUT -s $LOCAL_NETWORK_CIDR -j ACCEPT
}
check_ip_forwarding
sudo iptables -t nat -A POSTROUTING -o $WAN_INTERFACE -j MASQUERADE
add_network_to_forwards 10.88.88.0/24
add_network_to_forwards 10.88.89.0/24
add_network_to_forwards 192.168.89.0/24
and here is my current netplan from the ubuntu host:
network:
version: 2
ethernets:
eth0:
dhcp4: true
eth1:
addresses:
- 192.168.89.2/24
eth2:
dhcp4: true
eth3:
addresses:
- 10.88.88.1/24
eth4:
addresses:
- 10.88.89.1/24
where:
eth0 = default virtual network on KVM
eth1 = fog virtual network on KVM
eth2 = ISP/router LAN
eth3 = br2 on KVM host hardware interface
eth4 = br3 on KVM host hardware interface
/etc/dhcp/dhcpd.conf:
option space PXE;
option PXE.mtftp-ip code 1 = ip-address;
option PXE.mtftp-cport code 2 = unsigned integer 16;
option PXE.mtftp-sport code 3 = unsigned integer 16;
option PXE.mtftp-tmout code 4 = unsigned integer 8;
option PXE.mtftp-delay code 5 = unsigned integer 8;
option arch code 93 = unsigned integer 16;
use-host-decl-names on;
ddns-update-style interim;
ignore client-updates;
subnet 10.1.2.0 netmask 255.255.255.0 {}
subnet 192.168.89.0 netmask 255.255.255.0 {
option subnet-mask 255.255.255.0;
range dynamic-bootp 192.168.89.10 192.168.89.254;
default-lease-time 21600;
max-lease-time 43200;
option routers 192.168.122.1;
option domain-name-servers 127.0.0.53;
next-server 192.168.89.2;
class "Legacy" {
match if substring(option vendor-class-identifier, 0, 20) = "PXEClient:Arch:00000";
filename "undionly.kkpxe";
}
class "UEFI-32-2" {
match if substring(option vendor-class-identifier, 0, 20) = "PXEClient:Arch:00002";
filename "i386-efi/ipxe.efi";
}
class "UEFI-32-1" {
match if substring(option vendor-class-identifier, 0, 20) = "PXEClient:Arch:00006";
filename "i386-efi/ipxe.efi";
}
class "UEFI-64-1" {
match if substring(option vendor-class-identifier, 0, 20) = "PXEClient:Arch:00007";
filename "ipxe.efi";
}
class "UEFI-64-2" {
match if substring(option vendor-class-identifier, 0, 20) = "PXEClient:Arch:00008";
filename "ipxe.efi";
}
class "UEFI-64-3" {
match if substring(option vendor-class-identifier, 0, 20) = "PXEClient:Arch:00009";
filename "ipxe.efi";
}
class "SURFACE-PRO-4" {
match if substring(option vendor-class-identifier, 0, 32) = "PXEClient:Arch:00007:UNDI:003016";
filename "ipxe.efi";
}
class "Apple-Intel-Netboot" {
match if substring(option vendor-class-identifier, 0, 14) = "AAPLBSDPC/i386";
option dhcp-parameter-request-list 1,3,17,43,60;
if (option dhcp-message-type = 8) {
option vendor-class-identifier "AAPLBSDPC";
if (substring(option vendor-encapsulated-options, 0, 3) = 01:01:01) {
# BSDP List
option vendor-encapsulated-options 01:01:01:04:02:80:00:07:04:81:00:05:2a:09:0D:81:00:05:2a:08:69:50:58:45:2d:46:4f:47;
filename "ipxe.efi";
}
}
}
}
subnet 10.88.88.0 netmask 255.255.255.0 {
option subnet-mask 255.255.255.0;
range dynamic-bootp 10.88.88.3 10.88.88.254;
default-lease-time 21600;
max-lease-time 43200;
option routers 10.88.88.1;
option domain-name-servers 8.8.8.8;
next-server 10.88.88.1;
class "Legacy" {
match if substring(option vendor-class-identifier, 0, 20) = "PXEClient:Arch:00000";
filename "undionly.kkpxe";
}
class "UEFI-32-2" {
match if substring(option vendor-class-identifier, 0, 20) = "PXEClient:Arch:00002";
filename "i386-efi/ipxe.efi";
}
class "UEFI-32-1" {
match if substring(option vendor-class-identifier, 0, 20) = "PXEClient:Arch:00006";
filename "i386-efi/ipxe.efi";
}
class "UEFI-64-1" {
match if substring(option vendor-class-identifier, 0, 20) = "PXEClient:Arch:00007";
filename "ipxe.efi";
}
class "UEFI-64-2" {
match if substring(option vendor-class-identifier, 0, 20) = "PXEClient:Arch:00008";
filename "ipxe.efi";
}
class "UEFI-64-3" {
match if substring(option vendor-class-identifier, 0, 20) = "PXEClient:Arch:00009";
filename "ipxe.efi";
}
class "SURFACE-PRO-4" {
match if substring(option vendor-class-identifier, 0, 32) = "PXEClient:Arch:00007:UNDI:003016";
filename "ipxe.efi";
}
class "Apple-Intel-Netboot" {
match if substring(option vendor-class-identifier, 0, 14) = "AAPLBSDPC/i386";
option dhcp-parameter-request-list 1,3,17,43,60;
if (option dhcp-message-type = 8) {
option vendor-class-identifier "AAPLBSDPC";
if (substring(option vendor-encapsulated-options, 0, 3) = 01:01:01) {
# BSDP List
option vendor-encapsulated-options 01:01:01:04:02:80:00:07:04:81:00:05:2a:09:0D:81:00:05:2a:08:69:50:58:45:2d:46:4f:47;
filename "ipxe.efi";
}
}
}
}
subnet 10.88.89.0 netmask 255.255.255.0 {
option subnet-mask 255.255.255.0;
range dynamic-bootp 10.88.89.3 10.88.89.254;
default-lease-time 21600;
max-lease-time 43200;
option routers 10.88.89.1;
option domain-name-servers 8.8.8.8;
next-server 10.88.89.1;
class "Legacy" {
match if substring(option vendor-class-identifier, 0, 20) = "PXEClient:Arch:00000";
filename "undionly.kkpxe";
}
class "UEFI-32-2" {
match if substring(option vendor-class-identifier, 0, 20) = "PXEClient:Arch:00002";
filename "i386-efi/ipxe.efi";
}
class "UEFI-32-1" {
match if substring(option vendor-class-identifier, 0, 20) = "PXEClient:Arch:00006";
filename "i386-efi/ipxe.efi";
}
class "UEFI-64-1" {
match if substring(option vendor-class-identifier, 0, 20) = "PXEClient:Arch:00007";
filename "ipxe.efi";
}
class "UEFI-64-2" {
match if substring(option vendor-class-identifier, 0, 20) = "PXEClient:Arch:00008";
filename "ipxe.efi";
}
class "UEFI-64-3" {
match if substring(option vendor-class-identifier, 0, 20) = "PXEClient:Arch:00009";
filename "ipxe.efi";
}
class "SURFACE-PRO-4" {
match if substring(option vendor-class-identifier, 0, 32) = "PXEClient:Arch:00007:UNDI:003016";
filename "ipxe.efi";
}
class "Apple-Intel-Netboot" {
match if substring(option vendor-class-identifier, 0, 14) = "AAPLBSDPC/i386";
option dhcp-parameter-request-list 1,3,17,43,60;
if (option dhcp-message-type = 8) {
option vendor-class-identifier "AAPLBSDPC";
if (substring(option vendor-encapsulated-options, 0, 3) = 01:01:01) {
# BSDP List
option vendor-encapsulated-options 01:01:01:04:02:80:00:07:04:81:00:05:2a:09:0D:81:00:05:2a:08:69:50:58:45:2d:46:4f:47;
filename "ipxe.efi";
}
}
}
}
VMs joining the network do get DHCP leases, and they can communicate with other machines on the network they join, but the ubuntu VM does not seem to be routing upstream despite me setting the above rules in iptables and ensuring ip_forward is on, and I’ve even chucked in a iptables -F
before everything to bang on things with a bigger hammer.
EDIT: the original ubuntu VM was perhaps laden with too much experimentation, after creating a new VM and applying the above finalized configs everything is working now.
Cheers, I have a fully functioning fog to assimilate all the things and I will contribute to the wiki once I repeat this a few times to ensure I’ve got everything working properly.