Examples
slow-link
Two sites joined by a slow link: the user at the branch reaches the web server at the far site through Isoloom's router, whose interface into far adds 80 ms of delay, jitter, 1% loss and a 10 Mbit cap (networks.far.tc). The check measures the delay; isoloom tc changes it live.
Tested: Runs in CI on Docker with isoloom test: the check measures the delay the router adds.
version: 1
name: slow-link
networks:
branch: { cidr: 10.75.1.0/24 }
far:
cidr: 10.75.2.0/24
tc: { delay: 80ms, jitter: 10ms, loss: 1, rate: 10mbit }
reach:
- { from: branch, to: far, ports: [80] }
machines:
web:
networks: { far: 10 }
services: [{ port: 80, http: true }]
docker:
build: build/web
vm:
os: debian-12
provision: [provision/web.sh]
user:
access: true
networks: { branch: 10 }
vm:
os: debian-12
tools:
shell: {} # a toolbox on both networks to watch the link from (isoloom connect shell)
checks:
- name: the far site answers, slowly
from: user
script: checks/slow-but-there.sh
Targets
docker, hosted, docker-vm, cloud-docker, vagrant, proxmox. See Targets for what each means.
Checks
The spec's own:
- the far site answers (
script)
Plus the checks Isoloom derives from its services and reach rules, run from every machine. See Checks.
Generated files
What isoloom generate writes for this spec, as committed next to it. Don't edit them: change
the spec and generate again (isoloom check fails in CI when they're out of date).
.isoloom/docker/compose.yml
# Generated by isoloom from isoloom.yml. Don't edit: change isoloom.yml and run
# `isoloom generate`. `isoloom check` fails when this file is out of date.
# Start: docker compose -f .isoloom/docker/compose.yml up -d --wait
# Checks: isoloom test docker (or: docker compose -f .isoloom/docker/compose.yml --profile check run --rm isoloom-check, and isoloom-check-<machine>)
# Stop: docker compose -f .isoloom/docker/compose.yml down -v
name: slow-link
services:
isoloom-router:
image: alpine:3.20
hostname: isoloom-router
cap_add:
- NET_ADMIN
sysctls:
net.ipv4.ip_forward: '1'
networks:
branch:
ipv4_address: 10.75.1.254
far:
ipv4_address: 10.75.2.254
environment:
RULES: |
table inet isoloom {
chain forward {
type filter hook forward priority 0; policy drop;
ct state established,related accept
ip saddr 10.75.1.0/24 ip daddr 10.75.2.0/24 meta l4proto { tcp, udp } th dport { 80 } accept
ip saddr 10.75.1.0/24 ip daddr 10.75.2.0/24 icmp type echo-request accept
}
}
TC: |
IF=$$(ip -o -4 addr show | awk '$$4 ~ /^10\.75\.2\.254\//{print $$2}' | head -n 1); [ -n "$$IF" ] && tc qdisc replace dev "$$IF" root netem delay 80ms 10ms loss 1% rate 10mbit
entrypoint:
- /bin/sh
- -c
- apk add --no-cache nftables iproute2 >/dev/null && printf '%s' "$$RULES" | nft -f - && printf '%s' "$$TC" | sh && exec sleep infinity
healthcheck:
test:
- CMD-SHELL
- nft list table inet isoloom >/dev/null 2>&1
interval: 3s
timeout: 3s
retries: 60
restart: unless-stopped
web:
build:
context: ../../build/web
image: isoloom/slow-link-web
platform: linux/amd64
hostname: web
networks:
far:
ipv4_address: 10.75.2.10
extra_hosts:
- user:10.75.1.10
healthcheck:
test:
- CMD-SHELL
- (nc -z 127.0.0.1 80 2>/dev/null || bash -c '</dev/tcp/127.0.0.1/80' 2>/dev/null)
interval: 5s
timeout: 3s
retries: 60
start_period: 10s
labels:
isoloom.service.80: http:80
restart: unless-stopped
web-routes:
image: alpine:3.20
network_mode: service:web
cap_add:
- NET_ADMIN
entrypoint:
- /bin/sh
- -c
- ip route replace 10.75.1.0/24 via 10.75.2.254 && exec sleep infinity
healthcheck:
test:
- CMD-SHELL
- ip route show 10.75.1.0/24 | grep -q via
interval: 2s
timeout: 2s
retries: 30
depends_on:
web:
condition: service_started
isoloom-router:
condition: service_healthy
restart: unless-stopped
isoloom-access:
image: alpine:3.20
hostname: user
networks:
branch:
ipv4_address: 10.75.1.10
extra_hosts:
- web:10.75.2.10
entrypoint:
- sleep
- infinity
profiles:
- check
isoloom-access-routes:
image: alpine:3.20
network_mode: service:isoloom-access
cap_add:
- NET_ADMIN
entrypoint:
- /bin/sh
- -c
- ip route replace 10.75.2.0/24 via 10.75.1.254 && exec sleep infinity
healthcheck:
test:
- CMD-SHELL
- ip route show 10.75.2.0/24 | grep -q via
interval: 2s
timeout: 2s
retries: 30
depends_on:
isoloom-access:
condition: service_started
isoloom-router:
condition: service_healthy
restart: unless-stopped
profiles:
- check
isoloom-check:
image: curlimages/curl:8.11.1
profiles:
- check
entrypoint:
- /bin/sh
- /isoloom/run.sh
volumes:
- ./checks/user.sh:/isoloom/run.sh:ro
- ../..:/isoloom/project:ro
network_mode: service:isoloom-access
depends_on:
web:
condition: service_healthy
web-routes:
condition: service_healthy
isoloom-access:
condition: service_started
isoloom-access-routes:
condition: service_healthy
isoloom-tool-shell:
image: nicolaka/netshoot
hostname: shell
entrypoint:
- sleep
- infinity
cap_add:
- NET_ADMIN
- NET_RAW
networks:
branch:
ipv4_address: 10.75.1.252
far:
ipv4_address: 10.75.2.252
restart: unless-stopped
networks:
branch:
ipam:
config:
- subnet: 10.75.1.0/24
gateway: 10.75.1.1
far:
ipam:
config:
- subnet: 10.75.2.0/24
gateway: 10.75.2.1
.isoloom/vagrant/Vagrantfile
# Generated by isoloom from isoloom.yml. Don't edit: change isoloom.yml and run
# `isoloom generate`. `isoloom check` fails when this file is out of date.
# Start: cd .isoloom/vagrant && vagrant up
# Stop: cd .isoloom/vagrant && vagrant destroy -f
ROOT = File.expand_path("../..", __dir__)
# Copied into each VM: the project, without version control or generated files.
PROJECT = Dir.children(ROOT).reject { |e| [".git", ".vagrant"].include?(e) || e.start_with?(".isoloom") }.sort
# ESXi (vagrant-vmware-esxi): ESXI_VIRTUAL_NETWORK lists port groups, comma-separated: the
# management network first, then one per network, in order (branch, far); a missing one reuses the last.
ESXI_NETWORKS = ENV.fetch("ESXI_VIRTUAL_NETWORK", "VM Network").split(",").map(&:strip)
LAB_NETWORKS = ["branch", "far"]
def esxi_networks(nets)
[ESXI_NETWORKS[0]] + nets.map { |n| ESXI_NETWORKS[1 + LAB_NETWORKS.index(n)] || ESXI_NETWORKS[-1] }
end
Vagrant.configure("2") do |config|
config.vm.synced_folder ".", "/vagrant", disabled: true
config.vm.boot_timeout = 900
config.vm.define "isoloom-router" do |m|
m.vm.box = "bento/debian-12"
m.vm.hostname = "isoloom-router"
m.vm.network "private_network", ip: "10.75.1.254", netmask: "255.255.255.0", virtualbox__intnet: "isoloom-slow-link-branch", libvirt__network_name: "isoloom-slow-link-branch", libvirt__dhcp_enabled: false
m.vm.network "private_network", ip: "10.75.2.254", netmask: "255.255.255.0", virtualbox__intnet: "isoloom-slow-link-far", libvirt__network_name: "isoloom-slow-link-far", libvirt__dhcp_enabled: false
m.vm.provider "virtualbox" do |v|
v.name = "slow-link · router"
v.linked_clone = true
v.cpus = 1
v.memory = 512
end
m.vm.provider "vmware_desktop" do |v|
v.vmx["displayName"] = "slow-link · router"
v.vmx["numvcpus"] = "1"
v.vmx["memsize"] = "512"
end
m.vm.provider "parallels" do |v|
v.name = "slow-link · router"
v.linked_clone = true
v.cpus = 1
v.memory = 512
end
m.vm.provider "libvirt" do |v, o|
o.vm.box = "generic/debian12"
v.cpus = 1
v.memory = 512
end
m.vm.provider "vmware_esxi" do |v|
v.esxi_hostname = ENV.fetch("ESXI_HOSTNAME", "")
v.esxi_hostport = ENV.fetch("ESXI_HOSTPORT", "22").to_i
v.esxi_username = ENV.fetch("ESXI_USERNAME", "root")
v.esxi_password = "env:ESXI_PASSWORD"
v.esxi_disk_store = ENV["ESXI_DATASTORE"] if ENV["ESXI_DATASTORE"]
v.esxi_virtual_network = esxi_networks(["branch", "far"])
v.guest_name = "slow-link-router"
v.guest_numvcpus = 1
v.guest_memsize = 512
end
m.vm.provision "shell", name: "router", inline: <<~'SH'
set -e
export DEBIAN_FRONTEND=noninteractive
apt-get update -qq
apt-get install -y -qq nftables >/dev/null
echo 'net.ipv4.ip_forward=1' > /etc/sysctl.d/90-isoloom.conf
sysctl -q -p /etc/sysctl.d/90-isoloom.conf
cat > /etc/nftables.conf <<'NFT'
flush ruleset
table inet isoloom {
chain forward {
type filter hook forward priority 0; policy drop;
ct state established,related accept
ip saddr 10.75.1.0/24 ip daddr 10.75.2.0/24 meta l4proto { tcp, udp } th dport { 80 } accept
ip saddr 10.75.1.0/24 ip daddr 10.75.2.0/24 icmp type echo-request accept
}
}
NFT
systemctl enable nftables
nft -f /etc/nftables.conf
mkdir -p /etc/isoloom
cat > /etc/isoloom/tc.sh <<'TC'
#!/bin/sh
# Link impairment (networks.*.tc) on this router's interfaces.
IF=$(ip -o -4 addr show | awk '$4 ~ /^10\.75\.2\.254\//{print $2}' | head -n 1); [ -n "$IF" ] && tc qdisc replace dev "$IF" root netem delay 80ms 10ms loss 1% rate 10mbit
TC
chmod +x /etc/isoloom/tc.sh
cat > /etc/systemd/system/isoloom-tc.service <<'UNIT'
[Unit]
Description=Link impairment on the isoloom router
After=network-online.target
Wants=network-online.target
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/etc/isoloom/tc.sh
[Install]
WantedBy=multi-user.target
UNIT
systemctl daemon-reload
systemctl enable isoloom-tc.service
systemctl restart isoloom-tc.service
SH
end
config.vm.define "web" do |m|
m.vm.box = "bento/debian-12"
m.vm.hostname = "web"
m.vm.network "private_network", ip: "10.75.2.10", netmask: "255.255.255.0", virtualbox__intnet: "isoloom-slow-link-far", libvirt__network_name: "isoloom-slow-link-far", libvirt__dhcp_enabled: false
m.vm.provider "virtualbox" do |v|
v.name = "slow-link · web"
v.linked_clone = true
v.cpus = 1
v.memory = 1024
end
m.vm.provider "vmware_desktop" do |v|
v.vmx["displayName"] = "slow-link · web"
v.vmx["numvcpus"] = "1"
v.vmx["memsize"] = "1024"
end
m.vm.provider "parallels" do |v|
v.name = "slow-link · web"
v.linked_clone = true
v.cpus = 1
v.memory = 1024
end
m.vm.provider "utm" do |v|
v.name = "slow-link · web"
v.cpus = 1
v.memory = 1024
end
m.vm.provider "qemu" do |v|
v.smp = "cpus=1"
v.memory = "1024M"
end
m.vm.provider "vmware_esxi" do |v|
v.esxi_hostname = ENV.fetch("ESXI_HOSTNAME", "")
v.esxi_hostport = ENV.fetch("ESXI_HOSTPORT", "22").to_i
v.esxi_username = ENV.fetch("ESXI_USERNAME", "root")
v.esxi_password = "env:ESXI_PASSWORD"
v.esxi_disk_store = ENV["ESXI_DATASTORE"] if ENV["ESXI_DATASTORE"]
v.esxi_virtual_network = esxi_networks(["far"])
v.guest_name = "slow-link-web"
v.guest_numvcpus = 1
v.guest_memsize = 1024
end
m.vm.provider "libvirt" do |v, o|
o.vm.box = "generic/debian12"
v.cpus = 1
v.memory = 1024
end
m.vm.provision "shell", name: "hosts", inline: "for l in '10.75.1.10 user'; do grep -qxF \"$l\" /etc/hosts || echo \"$l\" >> /etc/hosts; done"
m.vm.provision "shell", name: "routes", inline: <<~'SH'
mkdir -p /etc/isoloom
cat > /etc/isoloom/routes.sh <<'ROUTES'
#!/bin/sh
# Routes to the other isoloom networks.
ip route replace 10.75.1.0/24 via 10.75.2.254 2>/dev/null || true
ROUTES
chmod +x /etc/isoloom/routes.sh
for d in /etc/network/if-up.d /etc/networkd-dispatcher/routable.d; do
if [ -d "$d" ]; then ln -sf /etc/isoloom/routes.sh "$d/zz-isoloom-routes"; fi
done
cat > /etc/systemd/system/isoloom-routes.service <<'UNIT'
[Unit]
Description=Routes to the other isoloom networks
After=network-online.target
Wants=network-online.target
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/etc/isoloom/routes.sh
[Install]
WantedBy=multi-user.target
UNIT
systemctl daemon-reload
systemctl enable isoloom-routes.service
systemctl restart isoloom-routes.service
SH
PROJECT.each do |entry|
m.vm.provision "file", source: File.join(ROOT, entry), destination: "/tmp/isoloom-project/#{entry}"
end
m.vm.provision "shell", name: "project", inline: "rm -rf /opt/isoloom && mv /tmp/isoloom-project /opt/isoloom"
m.vm.provision "shell", name: "provision/web.sh", inline: "cd /opt/isoloom && sh provision/web.sh"
end
config.vm.define "user" do |m|
m.vm.box = "bento/debian-12"
m.vm.hostname = "user"
m.vm.network "private_network", ip: "10.75.1.10", netmask: "255.255.255.0", virtualbox__intnet: "isoloom-slow-link-branch", libvirt__network_name: "isoloom-slow-link-branch", libvirt__dhcp_enabled: false
m.vm.provider "virtualbox" do |v|
v.name = "slow-link · user"
v.linked_clone = true
v.cpus = 1
v.memory = 1024
end
m.vm.provider "vmware_desktop" do |v|
v.vmx["displayName"] = "slow-link · user"
v.vmx["numvcpus"] = "1"
v.vmx["memsize"] = "1024"
end
m.vm.provider "parallels" do |v|
v.name = "slow-link · user"
v.linked_clone = true
v.cpus = 1
v.memory = 1024
end
m.vm.provider "utm" do |v|
v.name = "slow-link · user"
v.cpus = 1
v.memory = 1024
end
m.vm.provider "qemu" do |v|
v.smp = "cpus=1"
v.memory = "1024M"
end
m.vm.provider "vmware_esxi" do |v|
v.esxi_hostname = ENV.fetch("ESXI_HOSTNAME", "")
v.esxi_hostport = ENV.fetch("ESXI_HOSTPORT", "22").to_i
v.esxi_username = ENV.fetch("ESXI_USERNAME", "root")
v.esxi_password = "env:ESXI_PASSWORD"
v.esxi_disk_store = ENV["ESXI_DATASTORE"] if ENV["ESXI_DATASTORE"]
v.esxi_virtual_network = esxi_networks(["branch"])
v.guest_name = "slow-link-user"
v.guest_numvcpus = 1
v.guest_memsize = 1024
end
m.vm.provider "libvirt" do |v, o|
o.vm.box = "generic/debian12"
v.cpus = 1
v.memory = 1024
end
m.vm.provision "shell", name: "hosts", inline: "for l in '10.75.2.10 web'; do grep -qxF \"$l\" /etc/hosts || echo \"$l\" >> /etc/hosts; done"
m.vm.provision "shell", name: "routes", inline: <<~'SH'
mkdir -p /etc/isoloom
cat > /etc/isoloom/routes.sh <<'ROUTES'
#!/bin/sh
# Routes to the other isoloom networks.
ip route replace 10.75.2.0/24 via 10.75.1.254 2>/dev/null || true
ROUTES
chmod +x /etc/isoloom/routes.sh
for d in /etc/network/if-up.d /etc/networkd-dispatcher/routable.d; do
if [ -d "$d" ]; then ln -sf /etc/isoloom/routes.sh "$d/zz-isoloom-routes"; fi
done
cat > /etc/systemd/system/isoloom-routes.service <<'UNIT'
[Unit]
Description=Routes to the other isoloom networks
After=network-online.target
Wants=network-online.target
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/etc/isoloom/routes.sh
[Install]
WantedBy=multi-user.target
UNIT
systemctl daemon-reload
systemctl enable isoloom-routes.service
systemctl restart isoloom-routes.service
SH
m.vm.provision "shell", name: "checks", run: "never", path: "checks/user.sh", env: { "ISOLOOM_DERIVED" => ENV.fetch("ISOLOOM_DERIVED", "1") }
end
# Tool `shell`: a toolbox on every network (tcpdump, nmap, curl, dig, netcat).
config.vm.define "isoloom-tool-shell" do |m|
m.vm.box = "bento/debian-12"
m.vm.hostname = "shell"
m.vm.network "private_network", ip: "10.75.1.252", netmask: "255.255.255.0", virtualbox__intnet: "isoloom-slow-link-branch", libvirt__network_name: "isoloom-slow-link-branch", libvirt__dhcp_enabled: false
m.vm.network "private_network", ip: "10.75.2.252", netmask: "255.255.255.0", virtualbox__intnet: "isoloom-slow-link-far", libvirt__network_name: "isoloom-slow-link-far", libvirt__dhcp_enabled: false
m.vm.provider "virtualbox" do |v|
v.name = "slow-link · tool shell"
v.linked_clone = true
v.cpus = 1
v.memory = 512
end
m.vm.provider "libvirt" do |v, o|
o.vm.box = "generic/debian12"
v.cpus = 1
v.memory = 512
end
m.vm.provider "vmware_esxi" do |v|
v.esxi_hostname = ENV.fetch("ESXI_HOSTNAME", "")
v.esxi_hostport = ENV.fetch("ESXI_HOSTPORT", "22").to_i
v.esxi_username = ENV.fetch("ESXI_USERNAME", "root")
v.esxi_password = "env:ESXI_PASSWORD"
v.esxi_disk_store = ENV["ESXI_DATASTORE"] if ENV["ESXI_DATASTORE"]
v.esxi_virtual_network = esxi_networks(["branch", "far"])
v.guest_name = "slow-link-tool-shell"
v.guest_numvcpus = 1
v.guest_memsize = 512
end
m.vm.provision "shell", name: "hosts", inline: "for l in '10.75.2.10 web' '10.75.1.10 user'; do grep -qxF \"$l\" /etc/hosts || echo \"$l\" >> /etc/hosts; done"
m.vm.provision "shell", name: "toolbox", inline: "export DEBIAN_FRONTEND=noninteractive; apt-get update -qq && apt-get install -y -qq tcpdump nmap curl dnsutils netcat-openbsd iproute2 >/dev/null"
end
end
.isoloom/docker-vm/Vagrantfile
# Generated by isoloom from isoloom.yml. Don't edit: change isoloom.yml and run
# `isoloom generate`. `isoloom check` fails when this file is out of date.
# Start: cd .isoloom/docker-vm && vagrant up
# Checks: cd .isoloom/docker-vm && vagrant provision --provision-with checks
# Stop: cd .isoloom/docker-vm && vagrant destroy -f
ROOT = File.expand_path("../..", __dir__)
# Copied into the VM: the project, its generated Compose file included.
PROJECT = Dir.children(ROOT).reject { |e| [".git", ".vagrant"].include?(e) }.sort
Vagrant.configure("2") do |config|
config.vm.box = "bento/debian-12"
config.vm.hostname = "slow-link"
config.vm.synced_folder ".", "/vagrant", disabled: true
config.vm.boot_timeout = 600
config.vm.provider "virtualbox" do |v|
v.name = "slow-link · docker"
v.cpus = 2
v.memory = 2048
end
config.vm.provider "vmware_desktop" do |v|
v.vmx["displayName"] = "slow-link · docker"
v.vmx["numvcpus"] = "2"
v.vmx["memsize"] = "2048"
end
config.vm.provider "parallels" do |v|
v.name = "slow-link · docker"
v.cpus = 2
v.memory = 2048
end
config.vm.provider "libvirt" do |v, o|
o.vm.box = "generic/debian12"
v.cpus = 2
v.memory = 2048
end
config.vm.provider "vmware_esxi" do |v|
v.esxi_hostname = ENV.fetch("ESXI_HOSTNAME", "")
v.esxi_hostport = ENV.fetch("ESXI_HOSTPORT", "22").to_i
v.esxi_username = ENV.fetch("ESXI_USERNAME", "root")
v.esxi_password = "env:ESXI_PASSWORD"
v.esxi_disk_store = ENV["ESXI_DATASTORE"] if ENV["ESXI_DATASTORE"]
v.esxi_virtual_network = [ENV.fetch("ESXI_VIRTUAL_NETWORK", "VM Network").split(",").first.strip]
v.guest_name = "slow-link-docker"
v.guest_numvcpus = 2
v.guest_memsize = 2048
end
config.vm.provision "shell", name: "docker", inline: "command -v docker >/dev/null || curl -fsSL https://get.docker.com | sh"
PROJECT.each do |entry|
config.vm.provision "file", source: File.join(ROOT, entry), destination: "/tmp/isoloom-project/#{entry}"
end
config.vm.provision "shell", name: "project", inline: "rm -rf /opt/isoloom && mv /tmp/isoloom-project /opt/isoloom"
config.vm.provision "shell", name: "environment", inline: "cd /opt/isoloom && ISOLOOM_PUBLISH_ADDRESS=0.0.0.0 ISOLOOM_PUBLISH_FIXED=1 docker compose -f .isoloom/docker/compose.yml up -d --build --wait --wait-timeout 900 && mkdir -p /var/lib/isoloom && echo ready > /var/lib/isoloom/ready"
config.vm.provision "shell", name: "checks", run: "never", inline: "cd /opt/isoloom && failed=0; for s in $(docker compose -f .isoloom/docker/compose.yml --profile check config --services | grep '^isoloom-check'); do docker compose -f .isoloom/docker/compose.yml --profile check run --rm -e ISOLOOM_DERIVED \"$s\" || failed=1; done; exit $failed", env: { "ISOLOOM_DERIVED" => ENV.fetch("ISOLOOM_DERIVED", "1") }
end
.isoloom/proxmox/main.tf
# Generated by isoloom from isoloom.yml. Don't edit: change isoloom.yml and run
# `isoloom generate`. `isoloom check` fails when this file is out of date.
# Start: terraform -chdir=.isoloom/proxmox init && terraform -chdir=.isoloom/proxmox apply -var proxmox_endpoint=https://<server>:8006/ -var proxmox_api_token=<user@realm!name=secret>
# Stop: terraform -chdir=.isoloom/proxmox destroy (same variables)
terraform {
required_version = ">= 1.6"
backend "local" {}
required_providers {
proxmox = {
source = "bpg/proxmox"
version = "~> 0.115"
}
# The controller's SSH key (environment-level provisioning).
tls = {
source = "hashicorp/tls"
version = "~> 4.0"
}
}
}
variable "proxmox_endpoint" {
type = string
description = "https://<server>:8006/"
}
variable "proxmox_api_token" {
type = string
default = ""
sensitive = true
description = "user@realm!name=secret; or use proxmox_username and proxmox_password"
}
variable "proxmox_username" {
type = string
default = "root@pam"
}
variable "proxmox_password" {
type = string
default = ""
sensitive = true
}
variable "proxmox_insecure" {
type = bool
default = true
description = "Accept the server's self-signed certificate"
}
variable "proxmox_ssh_username" {
type = string
default = "root"
description = "Uploading cloud-init snippets goes over SSH to the node"
}
variable "proxmox_ssh_private_key_file" {
type = string
default = ""
}
variable "proxmox_ssh_address" {
type = string
default = ""
description = "The node's SSH address, when the API reports one this machine can't reach"
}
variable "node" {
type = string
default = "pve"
}
variable "datastore" {
type = string
default = "local-lvm"
description = "Where VM disks go"
}
variable "image_datastore" {
type = string
default = "local"
description = "A datastore with 'iso' content, for cloud images"
}
variable "snippets_datastore" {
type = string
default = "local"
description = "A datastore with 'snippets' content, for cloud-init"
}
variable "uplink_bridge" {
type = string
default = "vmbr0"
description = "The bridge the router reaches the internet through"
}
variable "slot" {
type = number
default = 1
description = "1 to 99, unique per environment on this server (SDN ids are short)"
}
variable "ssh_public_key" {
type = string
default = ""
description = "Installed for the user `isoloom` on every VM"
}
provider "proxmox" {
endpoint = var.proxmox_endpoint
api_token = var.proxmox_api_token != "" ? var.proxmox_api_token : null
username = var.proxmox_api_token != "" ? null : var.proxmox_username
password = var.proxmox_api_token != "" ? null : var.proxmox_password
insecure = var.proxmox_insecure
ssh {
agent = false
username = var.proxmox_ssh_username
password = var.proxmox_ssh_private_key_file != "" ? null : var.proxmox_password
private_key = var.proxmox_ssh_private_key_file != "" ? file(var.proxmox_ssh_private_key_file) : null
dynamic "node" {
for_each = var.proxmox_ssh_address == "" ? [] : [1]
content {
name = var.node
address = var.proxmox_ssh_address
}
}
}
}
locals {
zone = "iso${var.slot}"
# The project, written to /opt/isoloom in each machine that has steps.
root = abspath("${path.module}/../..")
project = [for f in fileset(local.root, "**") : f if !startswith(f, ".git/") && !startswith(f, ".isoloom/") && !startswith(f, ".vagrant/")]
project_files = [for f in local.project : {
path = "/opt/isoloom/${f}"
encoding = "b64"
content = filebase64("${local.root}/${f}")
}]
users = var.ssh_public_key == "" ? [] : [{
name = "isoloom"
sudo = "ALL=(ALL) NOPASSWD:ALL"
shell = "/bin/bash"
ssh_authorized_keys = [var.ssh_public_key]
}]
}
# The environment's networks: an SDN simple zone, a VNet per network.
resource "proxmox_sdn_zone_simple" "env" {
id = local.zone
nodes = [var.node]
}
# Network `branch`: 10.75.1.0/24
resource "proxmox_sdn_vnet" "branch" {
id = "i${var.slot}n0"
zone = proxmox_sdn_zone_simple.env.id
}
# Network `far`: 10.75.2.0/24
resource "proxmox_sdn_vnet" "far" {
id = "i${var.slot}n1"
zone = proxmox_sdn_zone_simple.env.id
}
resource "proxmox_sdn_applier" "env" {
depends_on = [proxmox_sdn_vnet.branch, proxmox_sdn_vnet.far]
}
resource "proxmox_download_file" "debian_12" {
node_name = var.node
datastore_id = var.image_datastore
content_type = "iso"
url = "https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-amd64.qcow2"
# Per environment: a shared file would go with whichever environment is destroyed first.
file_name = "iso${var.slot}-debian-12.img"
overwrite_unmanaged = true
}
# The router: forwards between networks with the reach rules, and to the internet.
resource "proxmox_virtual_environment_file" "router" {
node_name = var.node
datastore_id = var.snippets_datastore
content_type = "snippets"
source_raw {
file_name = "iso${var.slot}-router.yaml"
data = "#cloud-config\n${yamlencode({
hostname = "isoloom-router"
users = local.users
packages = ["nftables", "qemu-guest-agent"]
write_files = [
{ path = "/etc/systemd/network/10-wan.network", content = join("\n", ["[Match]", "MACAddress=${format("02:15:%02x:00:00:00", var.slot)}", "", "[Network]", "DHCP=yes"]) },
{ path = "/etc/nftables.conf", content = "flush ruleset\ntable inet isoloom {\n chain forward {\n type filter hook forward priority 0; policy drop;\n ct state established,related accept\n ip saddr 10.75.1.0/24 ip daddr 10.75.2.0/24 meta l4proto { tcp, udp } th dport { 80 } accept\n ip saddr 10.75.1.0/24 ip daddr 10.75.2.0/24 icmp type echo-request accept\n ip saddr { 10.75.1.0/24, 10.75.2.0/24 } ip daddr != { 10.75.1.0/24, 10.75.2.0/24 } accept\n }\n chain prerouting {\n type nat hook prerouting priority -100;\n }\n chain postrouting {\n type nat hook postrouting priority 100;\n ip saddr { 10.75.1.0/24, 10.75.2.0/24 } ip daddr != { 10.75.1.0/24, 10.75.2.0/24 } masquerade\n }\n}\n" },
{ path = "/etc/sysctl.d/90-isoloom.conf", content = "net.ipv4.ip_forward=1\n" },
{ path = "/etc/systemd/network/20-branch.network", content = join("\n", ["[Match]", "MACAddress=${format("02:15:%02x:01:%02x:00", var.slot, 0)}", "", "[Network]", "Address=10.75.1.254/24", "ConfigureWithoutCarrier=yes"]) },
{ path = "/etc/systemd/network/20-far.network", content = join("\n", ["[Match]", "MACAddress=${format("02:15:%02x:01:%02x:00", var.slot, 1)}", "", "[Network]", "Address=10.75.2.254/24", "ConfigureWithoutCarrier=yes"]) }
]
runcmd = [
["sysctl", "-p", "/etc/sysctl.d/90-isoloom.conf"],
["systemctl", "enable", "systemd-networkd"],
["systemctl", "restart", "systemd-networkd"],
["systemctl", "enable", "--now", "nftables"],
["nft", "-f", "/etc/nftables.conf"],
["systemctl", "enable", "--now", "qemu-guest-agent"],
]
})}"
}
}
resource "proxmox_virtual_environment_vm" "isoloom_router" {
name = "iso${var.slot}-router"
node_name = var.node
tags = ["isoloom", "slow-link"]
on_boot = false
# Its uplink address (DHCP), for the published ports.
agent {
enabled = true
}
cpu {
cores = 1
type = "host"
}
memory {
dedicated = 512
}
disk {
datastore_id = var.datastore
file_id = proxmox_download_file.debian_12.id
interface = "virtio0"
size = 8
}
network_device {
bridge = var.uplink_bridge
mac_address = upper(format("02:15:%02x:00:00:00", var.slot))
}
network_device {
bridge = proxmox_sdn_vnet.branch.id
mac_address = upper(format("02:15:%02x:01:%02x:00", var.slot, 0))
}
network_device {
bridge = proxmox_sdn_vnet.far.id
mac_address = upper(format("02:15:%02x:01:%02x:00", var.slot, 1))
}
initialization {
datastore_id = var.datastore
user_data_file_id = proxmox_virtual_environment_file.router.id
ip_config {
ipv4 {
address = "dhcp"
}
}
}
operating_system {
type = "l26"
}
serial_device {}
depends_on = [proxmox_sdn_applier.env]
}
# Machine `web`.
resource "proxmox_virtual_environment_file" "web" {
node_name = var.node
datastore_id = var.snippets_datastore
content_type = "snippets"
source_raw {
file_name = "iso${var.slot}-web.yaml"
data = "#cloud-config\n${yamlencode({
hostname = "web"
users = local.users
packages = ["nftables", "curl", "netcat-openbsd"]
write_files = local.project_files
runcmd = [
["sh", "-c", "printf '%s\\n' '10.75.1.10 user' >> /etc/hosts"],
["sh", "-c", "cd /opt/isoloom && sh provision/web.sh"],
["sh", "-c", "mkdir -p /var/lib/isoloom && echo ready > /var/lib/isoloom/ready"]
]
})}"
}
}
resource "proxmox_virtual_environment_vm" "web" {
name = "iso${var.slot}-web"
node_name = var.node
tags = ["isoloom", "slow-link"]
on_boot = false
cpu {
cores = 1
type = "host"
}
memory {
dedicated = 1024
}
disk {
datastore_id = var.datastore
file_id = proxmox_download_file.debian_12.id
interface = "virtio0"
size = 20
}
network_device {
bridge = proxmox_sdn_vnet.far.id
}
initialization {
datastore_id = var.datastore
user_data_file_id = proxmox_virtual_environment_file.web.id
dns {
servers = ["1.1.1.1"]
}
ip_config {
ipv4 {
address = "10.75.2.10/24"
gateway = "10.75.2.254"
}
}
}
operating_system {
type = "l26"
}
serial_device {}
depends_on = [proxmox_virtual_environment_vm.isoloom_router]
}
# Machine `user`.
resource "proxmox_virtual_environment_file" "user" {
node_name = var.node
datastore_id = var.snippets_datastore
content_type = "snippets"
source_raw {
file_name = "iso${var.slot}-user.yaml"
data = "#cloud-config\n${yamlencode({
hostname = "user"
users = local.users
packages = ["nftables", "curl", "netcat-openbsd"]
write_files = local.project_files
runcmd = [
["sh", "-c", "printf '%s\\n' '10.75.2.10 web' >> /etc/hosts"],
["sh", "-c", "mkdir -p /var/lib/isoloom && echo ready > /var/lib/isoloom/ready"]
]
})}"
}
}
resource "proxmox_virtual_environment_vm" "user" {
name = "iso${var.slot}-user"
node_name = var.node
tags = ["isoloom", "slow-link"]
on_boot = false
cpu {
cores = 1
type = "host"
}
memory {
dedicated = 1024
}
disk {
datastore_id = var.datastore
file_id = proxmox_download_file.debian_12.id
interface = "virtio0"
size = 20
}
network_device {
bridge = proxmox_sdn_vnet.branch.id
}
initialization {
datastore_id = var.datastore
user_data_file_id = proxmox_virtual_environment_file.user.id
dns {
servers = ["1.1.1.1"]
}
ip_config {
ipv4 {
address = "10.75.1.10/24"
gateway = "10.75.1.254"
}
}
}
operating_system {
type = "l26"
}
serial_device {}
depends_on = [proxmox_virtual_environment_vm.isoloom_router]
}
locals {
router_address = [for a in flatten(proxmox_virtual_environment_vm.isoloom_router.ipv4_addresses) : a if a != "127.0.0.1" && !contains(["10.75.1.254", "10.75.2.254"], a)][0]
}
output "address" {
value = local.router_address
}
# The machines, through the router: ssh -J isoloom@<address> isoloom@<machine>.
output "machines" {
value = {
"web" = "10.75.2.10"
"user" = "10.75.1.10"
}
}
output "ssh_user" {
value = "isoloom"
}
# The checks: each runner piped to its machine (ssh -J isoloom@<address> isoloom@<host> sh -s < <script>), or `isoloom test proxmox`.
output "checks" {
value = [
{ position = "user", machine = "user", host = "10.75.1.10", user = "isoloom", script = ".isoloom/proxmox/checks/user.sh" }
]
}
.isoloom/docker-vm/proxmox/main.tf
# Generated by isoloom from isoloom.yml. Don't edit: change isoloom.yml and run
# `isoloom generate`. `isoloom check` fails when this file is out of date.
# Start: terraform -chdir=.isoloom/docker-vm/proxmox init && terraform -chdir=.isoloom/docker-vm/proxmox apply \
# -var proxmox_endpoint=https://<server>:8006/ -var proxmox_api_token=… -var ssh_public_key="$(cat ~/.ssh/id_ed25519.pub)" -var ssh_private_key_file=~/.ssh/id_ed25519
# Stop: terraform -chdir=.isoloom/docker-vm/proxmox destroy (same variables)
terraform {
required_version = ">= 1.6"
backend "local" {}
required_providers {
proxmox = {
source = "bpg/proxmox"
version = "~> 0.84"
}
}
}
variable "proxmox_endpoint" {
type = string
description = "https://<server>:8006/"
}
variable "proxmox_api_token" {
type = string
default = ""
sensitive = true
description = "user@realm!name=secret; or use proxmox_username and proxmox_password"
}
variable "proxmox_username" {
type = string
default = "root@pam"
}
variable "proxmox_password" {
type = string
default = ""
sensitive = true
}
variable "proxmox_insecure" {
type = bool
default = true
description = "Accept the server's self-signed certificate"
}
variable "proxmox_ssh_username" {
type = string
default = "root"
description = "Uploading the cloud-init snippet goes over SSH to the node"
}
variable "proxmox_ssh_private_key_file" {
type = string
default = ""
}
variable "proxmox_ssh_address" {
type = string
default = ""
description = "The node's SSH address, when the API reports one this machine can't reach"
}
variable "node" {
type = string
default = "pve"
}
variable "datastore" {
type = string
default = "local-lvm"
description = "Where the VM's disk goes"
}
variable "image_datastore" {
type = string
default = "local"
description = "A datastore with 'iso' content, for the cloud image"
}
variable "snippets_datastore" {
type = string
default = "local"
description = "A datastore with 'snippets' content, for cloud-init"
}
variable "uplink_bridge" {
type = string
default = "vmbr0"
description = "The bridge the VM gets its address (DHCP) and the internet from"
}
variable "slot" {
type = number
default = 1
description = "1 to 99, unique per environment on this server"
}
provider "proxmox" {
endpoint = var.proxmox_endpoint
api_token = var.proxmox_api_token != "" ? var.proxmox_api_token : null
username = var.proxmox_api_token != "" ? null : var.proxmox_username
password = var.proxmox_api_token != "" ? null : var.proxmox_password
insecure = var.proxmox_insecure
ssh {
agent = false
username = var.proxmox_ssh_username
password = var.proxmox_ssh_private_key_file != "" ? null : var.proxmox_password
private_key = var.proxmox_ssh_private_key_file != "" ? file(var.proxmox_ssh_private_key_file) : null
dynamic "node" {
for_each = var.proxmox_ssh_address == "" ? [] : [1]
content {
name = var.node
address = var.proxmox_ssh_address
}
}
}
}
locals {
root = abspath("${path.module}/../../..")
# The VM's address on the uplink, from the guest agent (not the loopback).
ip = [for a in flatten(proxmox_virtual_environment_vm.env.ipv4_addresses) : a if a != "127.0.0.1"][0]
}
resource "proxmox_download_file" "debian" {
node_name = var.node
datastore_id = var.image_datastore
content_type = "iso"
url = "https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-amd64.qcow2"
file_name = "iso${var.slot}-docker-debian-12.img"
overwrite_unmanaged = true
}
resource "proxmox_virtual_environment_file" "env" {
node_name = var.node
datastore_id = var.snippets_datastore
content_type = "snippets"
source_raw {
file_name = "iso${var.slot}-docker.yaml"
data = "#cloud-config\n${yamlencode({
hostname = "slow-link"
users = [{
name = "isoloom"
sudo = "ALL=(ALL) NOPASSWD:ALL"
shell = "/bin/bash"
ssh_authorized_keys = [var.ssh_public_key]
}]
packages = ["qemu-guest-agent", "curl"]
runcmd = [["systemctl", "enable", "--now", "qemu-guest-agent"]]
})}"
}
}
resource "proxmox_virtual_environment_vm" "env" {
name = "iso${var.slot}-slow-link"
node_name = var.node
tags = ["isoloom", "slow-link"]
on_boot = false
agent {
enabled = true
}
cpu {
cores = 2
type = "host"
}
memory {
dedicated = 1536
}
disk {
datastore_id = var.datastore
file_id = proxmox_download_file.debian.id
interface = "virtio0"
size = 30
}
network_device {
bridge = var.uplink_bridge
}
initialization {
datastore_id = var.datastore
user_data_file_id = proxmox_virtual_environment_file.env.id
ip_config {
ipv4 {
address = "dhcp"
}
}
}
operating_system {
type = "l26"
}
serial_device {}
}
variable "ssh_public_key" {
type = string
}
variable "ssh_private_key_file" {
type = string
description = "The private key of ssh_public_key: Terraform copies the project over SSH"
}
variable "auto_stop_minutes" {
type = number
default = 0
description = "Shut the VM down after this many minutes (0: never). Destroy still ends the billing of disks and addresses"
}
# The environment, over SSH: the project, Docker, then the Compose file.
resource "terraform_data" "environment" {
triggers_replace = [proxmox_virtual_environment_vm.env.id]
connection {
type = "ssh"
host = local.ip
user = "isoloom"
private_key = file(pathexpand(var.ssh_private_key_file))
timeout = "10m"
}
provisioner "remote-exec" {
inline = [
"cloud-init status --wait >/dev/null 2>&1 || true",
var.auto_stop_minutes > 0 ? "sudo shutdown -h +${var.auto_stop_minutes} >/dev/null 2>&1" : "true",
"sudo mkdir -p /opt/isoloom && sudo chown isoloom /opt/isoloom",
]
}
# The project as an archive: a plain copy drops the executable bits (entrypoint scripts).
provisioner "local-exec" {
command = "tar -czf \"${path.module}/.isoloom-project.tgz\" --exclude=.git --exclude=.vagrant --exclude=.terraform --exclude=.isoloom-project.tgz -C \"${local.root}\" ."
}
provisioner "file" {
source = "${path.module}/.isoloom-project.tgz"
destination = "/tmp/isoloom-project.tgz"
}
provisioner "remote-exec" {
inline = [
"set -e",
"tar -xzf /tmp/isoloom-project.tgz -C /opt/isoloom && rm -f /tmp/isoloom-project.tgz",
"command -v docker >/dev/null || curl -fsSL https://get.docker.com | sudo sh",
"cd /opt/isoloom && sudo -E env ISOLOOM_PUBLISH_ADDRESS=0.0.0.0 ISOLOOM_PUBLISH_FIXED=1 docker compose -f .isoloom/docker/compose.yml up -d --build --wait --wait-timeout 900",
"sudo mkdir -p /var/lib/isoloom && echo ready | sudo tee /var/lib/isoloom/ready >/dev/null",
]
}
}
output "ip" {
value = local.ip
}
output "ssh_user" {
value = "isoloom"
}
output "ready_file" {
value = "/var/lib/isoloom/ready"
}
.isoloom/cloud-docker/aws/main.tf
# Generated by isoloom from isoloom.yml. Don't edit: change isoloom.yml and run
# `isoloom generate`. `isoloom check` fails when this file is out of date.
# Start: terraform -chdir=.isoloom/cloud-docker/aws init && terraform -chdir=.isoloom/cloud-docker/aws apply \
# -var allowed_cidr=<your IP>/32 -var ssh_public_key="$(cat ~/.ssh/id_ed25519.pub)" -var ssh_private_key_file=~/.ssh/id_ed25519
# Stop: terraform -chdir=.isoloom/cloud-docker/aws destroy (same variables)
terraform {
required_version = ">= 1.6"
backend "local" {}
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 6.0"
}
}
}
variable "region" {
type = string
default = "eu-west-3"
}
variable "allowed_cidr" {
type = string
description = "Who may reach the VM (SSH and the published ports), e.g. your IP/32"
}
variable "ssh_public_key" {
type = string
}
variable "ssh_private_key_file" {
type = string
description = "The private key of ssh_public_key: Terraform copies the project over SSH"
}
variable "instance_type" {
type = string
default = "t3.small"
}
variable "auto_stop_minutes" {
type = number
default = 0
description = "Shut the VM down (and terminate it) after this long; 0 = never"
}
provider "aws" {
region = var.region
default_tags {
tags = {
"isoloom:environment" = "slow-link"
"managed-by" = "isoloom"
}
}
}
# A suffix, so two copies of the environment in one account don't collide.
resource "terraform_data" "id" {
input = substr(replace(uuid(), "-", ""), 0, 8)
lifecycle {
ignore_changes = [input]
}
}
locals {
name = "isoloom-slow-link-${terraform_data.id.output}"
root = abspath("${path.module}/../../..")
}
data "aws_availability_zones" "available" {
state = "available"
}
data "aws_ami" "debian" {
most_recent = true
owners = ["136693071363"]
filter {
name = "name"
values = ["debian-12-amd64-*"]
}
}
resource "aws_vpc" "env" {
cidr_block = "10.42.0.0/16"
tags = { Name = local.name }
}
resource "aws_internet_gateway" "env" {
vpc_id = aws_vpc.env.id
tags = { Name = local.name }
}
resource "aws_subnet" "env" {
vpc_id = aws_vpc.env.id
cidr_block = "10.42.1.0/24"
availability_zone = data.aws_availability_zones.available.names[0]
map_public_ip_on_launch = true
tags = { Name = local.name }
}
resource "aws_route_table" "env" {
vpc_id = aws_vpc.env.id
route {
cidr_block = "0.0.0.0/0"
gateway_id = aws_internet_gateway.env.id
}
tags = { Name = local.name }
}
resource "aws_route_table_association" "env" {
subnet_id = aws_subnet.env.id
route_table_id = aws_route_table.env.id
}
resource "aws_key_pair" "env" {
key_name = local.name
public_key = var.ssh_public_key
}
resource "aws_security_group" "env" {
name = local.name
vpc_id = aws_vpc.env.id
ingress {
description = "SSH"
from_port = 22
to_port = 22
protocol = "tcp"
cidr_blocks = [var.allowed_cidr]
}
egress {
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}
}
resource "aws_instance" "env" {
ami = data.aws_ami.debian.id
instance_type = var.instance_type
subnet_id = aws_subnet.env.id
key_name = aws_key_pair.env.key_name
vpc_security_group_ids = [aws_security_group.env.id]
instance_initiated_shutdown_behavior = "terminate"
user_data = var.auto_stop_minutes > 0 ? "#!/bin/sh\nshutdown -h +${var.auto_stop_minutes}\n" : null
root_block_device {
volume_size = 20
volume_type = "gp3"
encrypted = true
}
metadata_options {
http_tokens = "required"
}
depends_on = [aws_route_table_association.env]
tags = { Name = local.name }
}
# The environment, over SSH: the project, Docker, then the Compose file.
resource "terraform_data" "environment" {
triggers_replace = [aws_instance.env.id]
connection {
type = "ssh"
host = aws_instance.env.public_ip
user = "admin"
private_key = file(pathexpand(var.ssh_private_key_file))
timeout = "10m"
}
provisioner "remote-exec" {
inline = ["cloud-init status --wait >/dev/null 2>&1 || true", "sudo mkdir -p /opt/isoloom && sudo chown admin /opt/isoloom"]
}
# The project as an archive: a plain copy drops the executable bits (entrypoint scripts).
provisioner "local-exec" {
command = "tar -czf \"${path.module}/.isoloom-project.tgz\" --exclude=.git --exclude=.vagrant --exclude=.terraform --exclude=.isoloom-project.tgz -C \"${local.root}\" ."
}
provisioner "file" {
source = "${path.module}/.isoloom-project.tgz"
destination = "/tmp/isoloom-project.tgz"
}
provisioner "remote-exec" {
inline = [
"set -e",
"tar -xzf /tmp/isoloom-project.tgz -C /opt/isoloom && rm -f /tmp/isoloom-project.tgz",
"command -v docker >/dev/null || curl -fsSL https://get.docker.com | sudo sh",
"cd /opt/isoloom && sudo -E env ISOLOOM_PUBLISH_ADDRESS=0.0.0.0 ISOLOOM_PUBLISH_FIXED=1 docker compose -f .isoloom/docker/compose.yml up -d --build --wait --wait-timeout 900",
"sudo mkdir -p /var/lib/isoloom && echo ready | sudo tee /var/lib/isoloom/ready >/dev/null",
]
}
}
output "ip" {
value = aws_instance.env.public_ip
}
output "ssh_user" {
value = "admin"
}
output "ready_file" {
value = "/var/lib/isoloom/ready"
}
The other clouds, the same shape as AWS: azure, gcp, digitalocean, linode, oci.